diff --git a/.golangci.yaml b/.golangci.yaml index 9cd608150..8d45a4cb4 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -227,6 +227,12 @@ linters-settings: require-explanation: true # Enable to require nolint directives to mention the specific linter being suppressed. Default is false. require-specific: true + perfsprint: + strconcat: false + sprintf1: false + errorf: false + int-conversion: true + fiximports: true revive: rules: - name: unexported-return @@ -310,6 +316,7 @@ linters: - nilerr - noctx - nolintlint + - perfsprint - revive - staticcheck - stylecheck @@ -331,7 +338,6 @@ issues: text: "DefaultVolumesToRestic" # No need to report deprecate for DefaultVolumesToRestic. - path: ".*_test.go$" linters: - - dupword - errcheck - goconst - gosec @@ -342,7 +348,6 @@ issues: - unused - path: test/ linters: - - dupword - errcheck - goconst - gosec @@ -351,6 +356,14 @@ issues: - stylecheck - unparam - unused + - path: ".*data_upload_controller_test.go$" + linters: + - dupword + text: "type" + - path: ".*config_test.go$" + linters: + - dupword + text: "bucket" # The list of ids of default excludes to include or disable. By default it's empty. include: diff --git a/changelogs/unreleased/8550-Lyndon-Li b/changelogs/unreleased/8550-Lyndon-Li new file mode 100644 index 000000000..9bca175a2 --- /dev/null +++ b/changelogs/unreleased/8550-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #8044, allow users to ignore delay binding the restorePVC of data mover when it is in WaitForFirstConsumer mode \ No newline at end of file diff --git a/changelogs/unreleased/8590-Lyndon-Li b/changelogs/unreleased/8590-Lyndon-Li new file mode 100644 index 000000000..ee46f3c5c --- /dev/null +++ b/changelogs/unreleased/8590-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #8579, set event burst to block event broadcaster from filtering events \ No newline at end of file diff --git a/changelogs/unreleased/8591-reasonerjt b/changelogs/unreleased/8591-reasonerjt new file mode 100644 index 000000000..05c0e20dd --- /dev/null +++ b/changelogs/unreleased/8591-reasonerjt @@ -0,0 +1 @@ +Skip patching the PV in finalization for failed operation \ No newline at end of file diff --git a/changelogs/unreleased/8594-Lyndon-Li b/changelogs/unreleased/8594-Lyndon-Li new file mode 100644 index 000000000..0241e3482 --- /dev/null +++ b/changelogs/unreleased/8594-Lyndon-Li @@ -0,0 +1 @@ +Data mover restore for Windows \ No newline at end of file diff --git a/changelogs/unreleased/8602-Lyndon-Li b/changelogs/unreleased/8602-Lyndon-Li new file mode 100644 index 000000000..ed32e0901 --- /dev/null +++ b/changelogs/unreleased/8602-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #8067, add tmp folder (/tmp for linux, C:\Windows\Temp for Windows) as an alternative of udmrepo's config file location \ No newline at end of file diff --git a/changelogs/unreleased/8603-ywk253100 b/changelogs/unreleased/8603-ywk253100 new file mode 100644 index 000000000..7c7510e82 --- /dev/null +++ b/changelogs/unreleased/8603-ywk253100 @@ -0,0 +1 @@ +Check the PVB status via podvolume Backupper rather than calling API server to avoid API server issue \ No newline at end of file diff --git a/changelogs/unreleased/8606-Lyndon-Li b/changelogs/unreleased/8606-Lyndon-Li new file mode 100644 index 000000000..ca3c7f873 --- /dev/null +++ b/changelogs/unreleased/8606-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #8418, add Windows toleration to data mover pods \ No newline at end of file diff --git a/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml index c322a2385..2ccfb3055 100644 --- a/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml +++ b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml @@ -92,6 +92,13 @@ spec: DataMover specifies the data mover to be used by the backup. If DataMover is "" or "velero", the built-in data mover will be used. type: string + nodeOS: + description: NodeOS is OS of the node where the DataDownload is processed. + enum: + - auto + - linux + - windows + type: string operationTimeout: description: |- OperationTimeout specifies the time used to wait internal operations, diff --git a/config/crd/v2alpha1/bases/velero.io_datauploads.yaml b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml index 005b11e5f..02f6cdcbb 100644 --- a/config/crd/v2alpha1/bases/velero.io_datauploads.yaml +++ b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml @@ -144,7 +144,8 @@ spec: description: DataUploadStatus is the current status of a DataUpload. properties: acceptedByNode: - description: Node is name of the node where the DataUpload is prepared. + description: AcceptedByNode is name of the node where the DataUpload + is prepared. type: string acceptedTimestamp: description: |- @@ -175,6 +176,13 @@ spec: node: description: Node is name of the node where the DataUpload is processed. type: string + nodeOS: + description: NodeOS is OS of the node where the DataUpload is processed. + enum: + - auto + - linux + - windows + type: string path: description: Path is the full path of the snapshot volume being backed up. diff --git a/config/crd/v2alpha1/crds/crds.go b/config/crd/v2alpha1/crds/crds.go index d67770b45..2c292fc42 100644 --- a/config/crd/v2alpha1/crds/crds.go +++ b/config/crd/v2alpha1/crds/crds.go @@ -29,8 +29,8 @@ import ( ) var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYKs\xe3\xb8\x11\xbe\xfbWtM\x0es\x19\xc9;\xc9V*\xa5\xdbXN\xaa\\ٙ\xa8֎\xef Ѣ\xb0\x06\x01\x04\x0f9\xca㿧\x1a )\x90\x84,K\xbb\x19\xdd\x044>|\xe8n\xf4\x03\\,\x167̈g\xb4Nh\xb5\x02f\x04\xfeӣ\xa2\x7fn\xf9\xf2'\xb7\x14\xfav\xff\xf9\xe6E(\xbe\x82up^\xb7?\xa3\xd3\xc1\xd6x\x8f[\xa1\x84\x17Zݴ\xe8\x19g\x9e\xadn\x00\x98R\xda3\x1av\xf4\x17\xa0\xd6\xca[-%\xdaE\x83j\xf9\x12*\xac\x82\x90\x1cm\x04\xef\xb7\xde\xff\xb0\xfc\xfc\xe3\xf2\x87\x1b\x00\xc5Z\\\x01\xe1q\xfd\xaa\xa4f\xdc-\xf7(\xd1\xea\xa5\xd07\xce`M\xc0\x8d\xd5\xc1\xac\xe08\x91\x16v\x9b&\xc2\xf7̳\xfb\x0e#\x0eK\xe1\xfc_gS?\t\xe7㴑\xc129\xd9;\xce8\xa1\x9a \x99\x1d\xcf\xdd\x00\xb8Z\x1b\\\xc17\xdaڰ\x1ai\xac;S\xa4\xb2\x00\xc6y\xd4\x12\x93\x1b+\x94G\xbb\xd62\xb4\xbdv\x16\xc0\xd1\xd5V\x18\x1f\xb5\x90\xd3\x02\xe7\x99\x0f\x0e\\\xa8w\xc0\x1c|\xc3\xd7\xdb\a\xb5\xb1\xba\xb1\xe8\x12-\x80_\x9cV\x1b\xe6w+X&\xf1\xa5\xd91\x87\xddlR\xe5c\x9c\xe8\x86\xfc\x81\xf8:o\x85jJ\f\x9eD\x8b\xc0\x83\x8d&\xa4s\xd7\b~'ܘ\xda+sD\xcfz\xe4'\x89\xc4y\x82s\x9e\xb5f\xca([\x9a(q\xe6\xb1Dh\xad[#\xd1#\x87\xea\xe0\xb1?\xc6Vۖ\xf9\x15\b\xe5\xff\xf8\xe3i]t\xcaZƥ\xf7Z\x8d\x15sG\xa3\x90\r'&d\xa5\x06mQ;\xda3\xf9k\x88x\x02\xb8\xcb\xd6'&\t7\x1f?K\x85\\\x0e\xf4\x16\xfc\x0e\xe1\x8e\xd5/\xc1\xc0\xa3ז5\b?\xe9:\x99\xefu\x87\x16\xa3D\x95$\xc8{A\x90\xed\xb4-\x9a\xce`\xbdL\xb2\x1dX\x8f5\xb1\xdfx\xa3\xdfܷj\x8b\xac\xe8[}\xa8YF\t\xa1U\xd9\xc1\xbe4\xf8.\xe7ʕ\xa84\xc7Lc#N\u0081\xb1\xbaF\xe7\xdepx\x02\x18\xb1\xf8v\x1c\x98\xa9&I\xec\x7fϤٱ\xcf)\xc8\xd4;l٪[\xa1\r\xaa/\x9b\x87\xe7?<\x8e\x86ፀ\xc1j\xef(R\x10}c\xb5\u05f5\x96P\xa1\x7fET\xc9\xf4\xadޣ\xa58\xd7\b\xe5\x06D\x8a\xda<\x178\xc6l\xf2\xef\x88G\xb3i\xd2b\xf4\x1e\"hs\xeb\x03\xedi\xd0z\xd1G\xe1\x0e\xfb\x98`\xb2\xd1\xc99\xfe\xb3\x18\xcd\x01\xd0\xd1\xd3*\xe0\x94i0\x1d\xab\x8b\xad\xc8;m%\xe3\t\a\x16\x8dE\x87*\xe5\x1e\x1af\nt\xf5\v\xd6~9\x81~DK0\xe0v:HN\x87ݣ\xf5`\xb1֍\x12\xff\x1a\xb0\x1dx\x1d7\x95̣\xf3\xf12Z\xc5$\xec\x99\f\xf8\x89\x946An\xd9\x01,Ҟ\x10T\x86\x17\x17\xb8)\x8f\xaf\xa4E\xa1\xb6z\x05;\xef\x8d[\xdd\xde6\xc2\xf7i\xb7\xd6m\x1b\x94\xf0\x87\xdbh\rQ\x05\xaf\xad\xbb\xe5\xb8Gy\xebD\xb3`\xb6\xde\t\x8f\xb5\x0f\x16o\x99\x11\x8bx\x10\x15S\xef\xb2忳]\xa2v\xa3mg\x8e\x98~1a^`\x1eʢt+X\a\x95\x8ex\xb4\x02\r\x91\xea~\xfe\xf3\xe3\x13\xf4L\x92\xa5\x92Q\x8e\xa23\xbd\xf4\xf6!m\n\xb5E\x9b\xd6m\xadn#&*n\xb4P>\xfe\xa9\xa5@\xe5\xc1\x85\xaa\x15\x9e\xdc\xe0\x1f\x01\x9d'\xd3Maױ4\x81\n!\x18\x8a\a|*\xf0\xa0`\xcdZ\x94k\xe6\xf0;ۊ\xac\xe2\x16d\x84wY+/\xb8\xa6\xc2I\xbd\xd9D_1\x9d0m\x1eA\x1e\r\xd6dUR,-\x13[\xd1e\x12\n\x03l$;\xd6P\xf9\xeaӯ\x98M\xa6B\xe7܍~w%\xa0\x9e\xad\xca\x02y\x97\xeb\\\x97\xa4\xe48I\xe5\xbfY~\xb4h\xb4\x13^\xdb\xc31KN]\xe1\xa4U\xe8W3U\xa3\xbc\xe6x\xeb\xb8\x12\x84\xe2\xa4s\x1c\\\x99\x82PB\x8dD\xb5j4]\xae\x91)\xe0\xc1\x93\f\xf9\xb6C_>\xa8*f5\xa1\xe0XSB^;N\x8f[i-\x91M\xb5H^\xf8\x95\xd2\xc2Z\xab\xadh\xe6\a\xcf\xcb\xdfS.rF\xa7\x05\x87Ͷ\xa4S\x90w\x12\x93E\xccP\x8b\xdeu)\xb4oE\x13\xec)\xfbo\x05J>\x8b?'oR\x7f\xe0\xb8\xcb56\x1e\xa8\xf7\xb7\xab\xcbjY\xea\xf5:F(\x17\xeb\xdd\xcc5\xe7$\x01\x1e\xb6\x19\xa2p\xf0\xe1\x03h\v\x1fRO\xf4\xe1SZ\x1d\x84\xf4\v1\xca\xff\xafB\xca~\x97\x8b\xbc{H\xf9Tu\xe9\xe0\xaf\xd1\xc1\xdf&\x18\x13Ux\xaa\x11\xe3\xf1\xbd\x86W&\xb2\xb4;\xec\xee>\x15p+\xdcR\x8c\xb6\xe8\x83UtC\xd0Z\nZ.B\xea0+\x03\xde<\xa9S̸\x9d\xf6\x0f\xf7g\xce\xf88\b\xf6\xa1\xe8\xe1\xbe\x0fD\xcf\xd1\x10C<\xea$\xc1\xeb\"\xfd\xbe\xb0\xe21\xd3]\xc66\xa6ס\t\xbd\xc6,\x8fc\x88\xfe0ڊF\x90\xf2\xd50s\f\x9a{\xead\xa3(\x1d\x119\x04s\x82;P\x84\xa2|^!p\xb1ݢ\xa5\xa4\x1d3z\xdax\xf3\xbc\xfe\xe8\xb2M\xc46\xffC\xc1\xb0e\xc6 \xa7\xf6\x81\x8c\xdb\xe9\xea\"-yf\x1b\xf4ϑ\xf4\x19\x15=e\xa2\xbd*(\xfbS\xafו\x97\xd1Y\xa3\x18l\x9eׅb\x90~\x9b\xe79\xc3ө\x12\xba\xbe\xe0\x84\x11g,g\xd6\xea\xf8\f\x18E\x887#-\x80ٿc\xe7\xcds)\xf1\x0e\xea\x00\xbfc\x9e$\xba>\x0e\xaaC\x11\x13\xfa+ҙ\xf3:\xbe\xf5\xbb\b\xaf\xdfd\xbc\x9eR>\xc1\xb7:\xfcjʔׅE>g\xbdx\xc3r\v0\xfb\xe2`\xfd\xfe\xecU\xdeyQ.\xd1&2\xd3\xd0?\x99>\xc6\xcb\xe9\xc48\xaeLf\xf3+\xf9\xaeZ6v\xda\xef\xadf\xd3\xfbYg\xf6:\xd8\x18t\xbaW5j\x10\xaf\xaagY]\xa3\xf1\xc8\xef\x0e\xd4ޟ\x89#$B\x04\xd4\xdb\xef\f\x7f7\xc7W\x064\xecҢ\xb3\xa74\xbc\x85\\\x93\x00\xbeLAbCly\x96\x96\xe7tS\xb5r\x9a4\xc0\x13\xb5\x12\xb1\xa1\xfb\x9821-\x8b\xf9\x9d\x8a\xb6٦3\x84\xfe}\x8d:\xb6\x05\xad\x9fI\xa8 %\xab$\xae\xc0\xdbp\xaaz-\x17\xeb\xe9i1\x7fE\xba\xaar\x9f\xc3\xccudžw\x93\xf8\xbe\xd5?j\x96Tv\xc4\x1b\x14\x96\xe0\x90\x03\xeeQ\x01\xf5cLH\xe4=f\xa1\x84=\xa7\xf9\x02i\xf7]\x95ߢs\xac9w\x81\xbe&\xa9\xf4\xd4\xd0-\x01VQ\xdd8mg>\xba\xeen_tw\xd4ov\x89\x8b\x8f\x85\x17q\x89\xed\xd7\x192\x1b\x92)Ŵ\x81\xda\xe9\xa0F?T\xa1-e\x9eo\xf8Z\x18\xed\xefgaj\xd3]\xfa\xc2\xd4\xec+E>\x99\xfa\xdcRb\xec犘\xc3g\x80\xc2\xdc_\xe2e\xb8H\xd3\x1d\xbfk\xae\xfb\xd0-\xef\xb4\xecox|\xbeW\xa1\xadВ\x19\xe2\a\x82\xde\x1eC\xdd\xcf\x14ϭV*\xfe\x06\x84\xa1\x17\x88PKx\xdaQi\x92Z\xfc\xbe;\xe2\xc2\x19\xc9\x0e\xc3a\xf2\n\xb5\x00~\xbc5\xb3\x17\xdcK\x8b\xd4\xe1sJ\xb9\xf2*}\x13\x19\xff\xe6_7&\xf3\xc3g\x92\xff\xcf\x0eo4\xf8\xe3\xcfVW\xb5R#\x84s\xa9\xa0\xfb\x8cvy\x04\x1fo\xf3=\x83wQ{\xb3\xc1Ȝg\xd8݃\\>\x12\xaa\xe1\x95z\x05\xff\xfe\xef\xcd\xff\x02\x00\x00\xff\xff\x84s\xba\x82\x91\x1e\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYIs\xe3\xb8\x15\xbe\xfbW\xbc\xea\x1c\xe6b\xc9\xd3\xc9T*\xa5[[N\xaa\\\x99vT-\xc7w\x88|\"1\x06\x01\x06\x8b\x14g\xf9\xef\xa9\a\x80\x14HBk\xa6\x87\a\x97\x85\xe5\xe1m\xf8ނ\xd9lv\xc7Z\xfe\x86\xdap%\x17\xc0Z\x8e\xff\xb4(闙\xbf\xff\xc9̹z\xd8}\xbe{\xe7\xb2\\\xc0\xd2\x19\xab\x9aoh\x94\xd3\x05>\xe1\x96Kn\xb9\x92w\rZV2\xcb\x16w\x00LJe\x19\r\x1b\xfa\tP(i\xb5\x12\x02\xf5\xacB9\x7fw\x1b\xdc8.JԞxw\xf4\xee\xc7\xf9\xe7\x9f\xe6?\xde\x01H\xd6\xe0\x02\x88\x9ek\x85b\xa5\x99\xefP\xa0Vs\xae\xeeL\x8b\x05\x91\xad\xb4r\xed\x02\x0e\x13a[<2\xb0\xfb\xc4,\xfb\xbb\xa7\xe0\a\x057\xf6\xaf\xa3\x89\x9f\xb9\xb1~\xb2\x15N318Տ\x1b.+'\x98Ng\xee\x00L\xa1Z\\\xc0\v\x1dٲ\x02i,J\xe2Y\x98\x01+K\xaf\x1b&V\x9aK\x8bz\xa9\x84k:\x9d̠DSh\xdeZ/\xfb\x81!0\x96Yg\xc0\xb8\xa2\x06f\xe0\x05\xf7\x0f\xcfr\xa5U\xa5\xd1\x04\x96\x00~1J\xae\x98\xad\x170\x0f\xcb\xe7m\xcd\f\xc6٠\xbe\xb5\x9f\x88C\xf6\x83\xb85VsY\xe5\xce\x7f\xe5\rB\xe9\xb47\x1b\xc9\\ ؚ\x9b\x94\xb1=3Ĝ\xb6X\x1ee\xc3\xcf\x131cYӎ\xf9I\xb6\x06\x86Jf1\xc7\xceR5\xad@\x8b%l>,vBl\x95n\x98]\x00\x97\xf6\x8f?\x1d\xd7DT\xd5\xdco}Rr\xa8\x96G\x1a\x85d8pB\x16\xaaPgu\xa3,\x13\xff\x0f#\x96\b<&\xfb\x03'\x81n:~\x96\x15r7P[\xb05\xc2#+\xde]\vk\xab4\xab\x10~VE0\u07beF\x1d\x8d\xb7\tKL\xad\x9c(a\xd3I\f`\xac\xd2Y+\xb6X\xccîH\xb7#;2\xe5\xf0\xcc_\xd9\xc9\n\x8d,\xebd\x1d\xca\xcc\xfd\n\xaed\xdeӾTx\x91\x97\xa5ڔ\xaa\xc4^u\x98r\xc4\r\xb4Z\x15h\xcc\t\xbf\xa7\xed\x03\x1e^\x0e\x03\x13\xb5\x84\x15\xbb\xdf3\xd1\xd6\xecs@\x99\xa2Ɔ-\xe2\x0eբ\xfc\xb2z~\xfb\xc3z0\fG1\x83\x15\xd6\x10X\x10\xeb\xadVV\x15J\xc0\x06\xed\x1eQz܂F\xedP\x13\xc8U\\\x1a`\xb2\xeciB\xba\xe0\x00\xd5\xe4\xe4\x9e\x1e͆\xc9\xe8N\xaaE\x9d\x9a\x1d\xe8\xc8\x16\xb5\xe5\x1d\xfa\x86/\t+\xc9\xe8H\x88\xff\xcc\x06s\x00$w\xd8\x05%\xc5\x17\fREl\xc52\xaa*؍\x1b\xd0\xd8j4(Cġa&Am~\xc1\xc2\xceG\xa4ר\x89Lw\x1f\n%w\xa8-h,T%\xf9\xbfz\xda\x06\xac\xf2\x87\nf\xd1X\x7f!\xb5d\x02vL8\xbc\x1fi\x8f\xbe\x86}\x80F:\x13\x9cL\xe8\xf9\rf\xcc\xc7W\xa5\x11\xb8ܪ\x05\xd4ֶf\xf1\xf0Pq\xdb\x05\xdbB5\x8d\x93\xdc~\xe0Λ\xf2w:\x86g38v\xe2\x85\xe1\xf3\x81\xf2\n\xf3P\xfc\xa4+\xc1\"\xa9 \xe2\xc1\n4D\xaa\xfb\xf6\xe7\xf5+t\x9c\x04K\x05\xa3\x1c\x96N\xf4\xd2ه\xb4\xc9\xe5\x16uطժ\xf14Q\x96\xad\xe2\xd2\xfa\x1f\x85\xe0(-\x18\xb7i\xb8%7\xf8\x87Cc\xc9tc\xb2K\x9f\x90\xc0\x06\xc1\xb5\x04\x05\xe5x\xc1\xb3\x84%kP,\x99\xc1\xdf\xd8Vd\x153##\\d\xad4\xcd\x1a/\x0e\xeaM&\xbaL\xe9\x88i\x0f\xf0\xb1n\xb1 \x9b\x92Zi\x13\xdf\xf2\x18K\b\x03X\xb2r\xa8\x9d\xfc\xb5\xa7/\x1bBƋι\x1a}\x8f9B\x1d\xaf2\xc1\xef.\xd4\xc5\xc8$\x86\x91)\xfd\x0e \x1f\xf7hl\x95\xe1V\xe9\x0f\"\x1cB\xe3\xd8\r\x8eZ\x84\xbe\x82\xc9\x02\xc5-\xe2-\xfdN\xe0\xb2$\x8dc\xef\xc6\x04@\x81\xaagT\xc9J\xd1\xc5J\f\x01ϖV\x90W\x1b\xb4y1e&\x94q\t\x87l\x12Ҭq,\xeaF)\x81l\xac\xc1\xc2\xf0\xb5d\xad\xa9\x95=#\xf0\xf3\x16\xba\x95\xaf\x1f-\xd2\xe1\xcb\xf5\xf3=\xfd\xe9\xc6Ƀv\xbc\x8c\x10O\xb7\x8c\xf2\xaa\xbc٢\x9d\x97\xebg0q\xfb\xd4H\xd2\t\xc16\x02\x17`\xb5\x9b\nv\xdca\xe9\xeb\xc8.\x053\xd9\x05#\x01\xd7\xe9\xfa\x9cOv\x04\xa1\xf0+l\xcdr\x86\xf2\x1a\xa7\bG\xe5A\xb2\x89\xf7\x89\x10칭\xb3;O8%\xc44\x8fUx\xb1@\xc9\xf2\xac<\xf1r\x05q\xd4\xf6\x840\xab\xb7\xa5\x97\xf7\x9cd\x84\xed\xb7H\x16H\x1e\xf7ĉlo\x83\r9\xe9F\\\x1e\x13Nѕ#\xe4\xc0\x12\\{=\xeftù\xc6r\xca\xf3l`\xaf\xcc\xf4P\xe8#\xd7v\x12\x06 fx_)\x87[*\xb9\xe5\xd5\xf4\xec\xb4X=uGN\x8a6\t/ɑ\xa4q\x8a&\xc4\xc9̧\x93\xb3.\xd4P\"\xb6\xe5\x95\xd3Ǯ\xfe\x96\xa3('\xd9\xc2\xd9\xdb~F\x1f\x9e\x89[@\xbb\x97\xac\v\x96\x11\xbf\x924:x\x893\xbe\x80Mb\xcdT\x06 \x9c}\x02\xa5\xe1Shl|\xba\x0f\xbb\x1d\x17v\xc6\a\xb9\xfc\x9e\vѝrU\xb8\xea\xf3w\xaa\x9e\x94;\x87\xe3Y\x1d\xfcmDc\xa4\nK\x95\x9e\x17\xdf*\xd83\x9e\xe4\xd0\xfd\xe9\xe6>Cw\x83[J\xb84Z\xa7%\x85<Ԛr\x10\xe3I*\x97\xc1\xfc\x13\x92\x9a$\xfe\x9c\x91r\x1c\xaa\xbc\x14\xf4\xff\x18\xcbS\x00\xc8\b\x90\xb3\xf1)\x0e}~\xdcw\x91n1\xc5zH\xa2c^i^qR\xb8\xecg\x0e\x99Oĺ\xd8\"\xf0H\xe6\xa18\xeb\x9f=Z\x1aB\xcb\x039\xba\xce\xe1pB{&K\x1f\x9c\xfb\xf92^\xbd\xcc\xc5=\xab\x90\xd5\xdb\xf2\x9c\xbd\xfa\x833PN\xc3\xfb\x9a\x17\xf5\xd0t|\n\xaa\x00\x96\xbd\xa3Ot\xaf`3\x8f\xe1\xb3|\xda;Z3\xbe}\xa3\xe9\xd4e\xc7SCCggWoˋJ\x03ߵ\xb8\xac8\b\xedȨ\xe5\xc2i\xedˮ0J\xd5\xf6\r\xe5\x01+\nl-\x96\x8f\x1f/\xaa<\xe7\xf4\xb4\x84\x8e\x97\x97wk\xb0e\xd7\xe6\xf0\x1dK}G閫\xf8eL\xc4\xf7\x16t\x99\x80\xe2\x94\xdd\x00(Ǚ\x06x%'\xf6\xb5\xf1\x0f\x01\ai\x9bGW\xba\x82\x93C'\x14\xbav%\x15\xbf3\xda\x7f[$\xcd\xd7>\xa1S\x9b\xf6\xe2n*\x84\xa6d\xa6\xbac]\xc5曄]\x8b8\xa7\xb1\x03\xb9^_\x81\x1a\x96\x80;\x94@\xb5-\xe3\x82\xe2\xb3'\x99\x01\xa9\xd3Tb\xa0\n\xef\x01]ӣk\x90e\xbbO\xe7-\x99Q\xc2\x14\xb1\xbe\xa71\xfb4\xf1\x1b\x1a'2\x89\xc1wL\x13Ñ\xa1\xfc6\xd94\xf1t}\xc8\f0ЁHD\x89c\xc0t\xb1\x92\xb2\xb9c\x83ư\xea\x1cj}\r\xabB\xab,n\x01\xb6\xa1Ti\xc8\xda\x0f&\x82\xe9Up%\xbf\vn\xc6.\xf7U\x9c\xb4\xcc\xd6g8Y1[w\x01d\xeb\x84\xf0{&\x89U\xccI6H\xb7\xe9\xd7ʯ|\x83\xe3\x1c{\xb4&\x17\xe0\xf0\x12GB\xe9\x9a\\-\xf7\x82\xfb\xcch\a֙\xa9U\x8c\x00\x99\xa9\xc9\xfb_:\x19zH\xb9\x8b\xd6\xcdei\xf6Ol\x99\xb9\xbfxh\xbcJϑ\xbf[\xb0\xbf\xefF\xd5Jtp\xef\x9fƤk6\xa8\xc9\b\xfe\xf1mT\xa8S\u0099X,C8\xd9\xdfg\xb9\x9e\xd2\x1c^kn\xba\xfeYW\xa7\x94ܴ\x82}\xf4\xb2\x9c\x03\x9c\xfe2\x8f\xdfE\xa6Nr\xba\xf1\xd4?T\xe6\xfb\x18\xb9\xd7\xc6\xe17}7\x1c\xcd\xf7\x0f\x90\xdf\xe7\x84\x13h\xd9]\xef\xe7\xa7\v\v\xb0\xe7\xa7\xee*\xf2\x12\xa5\xa5\x9a\xf2\xf0\x16uH\xe5}o3\xa7\xcbqO\xf7\xba\xeac\xf0|}S56\xa0p&\x87\x89\xaf\xe9\xb9LaM`@\x10\xe4_?\x96\xe3\xf7\xce\xfb\xfe\xf9\x94\xd9\xf8\x04S\xd4LV\x98+q\x94\xa4\xc0\xe8\x03\xeb\xf5I\xc9P\xa0\xdf2\x1f\xc9z\xd5d\xd0s^&\xb4c\x13-\x1dq\x9b\xfeMl\x01\xff\xfe\xef\xdd\xff\x02\x00\x00\xff\xff*b\xfd\xb1\xf5\"\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcYɒ\x1b\xb9\x11\xbd\xf7Wd\xc8\a]D\xf6Ȟp8x\x93\xd8vD\x87G-ư\xddwT!Y\xc4\b\x05\xc0Xȡ\x97\x7f\x9fH\xa0\xaaX\v\xb8\xce\xc2\x1b\x81D\xe2\xe5\x82\xcc\a\xd4l6{`F\xbc\xa1uB\xab\x050#\xf0g\x8f\x8a\xfe\xb9\xf9\xb7\xbf\xb9\xb9Џ\xbb\x8f\x0f߄\xe2\vX\x06\xe7u\xfd#:\x1dl\x89O\xb8\x11Jx\xa1\xd5C\x8d\x9eq\xe6\xd9\xe2\x01\x80)\xa5=\xa3aG\x7f\x01J\xad\xbc\xd5R\xa2\x9dU\xa8\xe6\xdfB\x81E\x10\x92\xa3\x8d\xcaۭw\xdf\xcd?~?\xff\xee\x01@\xb1\x1a\x17@\xfa\xb8\xde+\xa9\x19w\xf3\x1dJ\xb4z.\xf4\x833X\x92\xe2\xca\xea`\x16p\x9cH\v\x9bM\x13\xe0'\xe6\xd9S\xa3#\x0eK\xe1\xfc?'S?\b\xe7㴑\xc129\xda;\xce8\xa1\xaa \x99\x1d\xce=\x00\xb8R\x1b\\\xc0\vmmX\x894\xd6\xd8\x14\xa1̀q\x1e\xbd\xc4\xe4\xca\n\xe5\xd1.\xb5\fu\xeb\x9d\x19pt\xa5\x15\xc6G/\xf4a\x81\xf3\xcc\a\a.\x94[`\x0e^p\xff\xf8\xacVVW\x16]\x82\x05\xf0\x93\xd3j\xc5\xfcv\x01\xf3$>7[氙M\xae\\ljf\xc8\x1f\b\xaf\xf3V\xa8*\x87\xe0U\xd4\b<\xd8\x18B\xb2\xbbD\xf0[\xe1\x86\xd0\xf6\xcc\x11<둟\x04\x12\xe7I\x9d\xf3\xac6cD\xbd\xa5\t\x12g\x1es\x80\x96\xba6\x12=r(\x0e\x1e[36\xda\xd6\xcc/@(\xff\xd7\xefO\xfb\xa2q\xd6<.}\xd2j\xe8\x98\xcf4\n\xbdᄄ\xa2T\xa1\xcdzG{&\x7f\r\x10O\n>\xf7\xd6'$Io\x7f\xfc\"\x14J9\xd0\x1b\xf0[\x84Ϭ\xfc\x16\f\xac\xbd\xb6\xacB\xf8A\x97)|\xfb-Z\x8c\x12E\x92\xa0\xec\x05A\xb1\xd36\x1b:\x83\xe5<\xc96\xcaZ]\xa3\xf8\r7\xfa\xcds\xab\xb4Ȳ\xb9Ֆ\x9ay\x94\x10Z\xe5\x13\xecS\x85W%W߉Js\xecyl\x80I80V\x97\xe8ܙ\x84'\x05\x03\x14/ǁ\x89k\x92\xc4\xee\xcfL\x9a-\xfb\x98\x8aL\xb9Ś-\x9a\x15ڠ\xfa\xb4z~\xfb\xcbz0\fg\n\x06+\xbd\xa3JA\xf0\x8d\xd5^\x97ZB\x81~\x8f\xa8R\xe8k\xbdCKu\xae\x12\xcau\x1a\xa9j\xf3\xbe\xc0\xb1fS~G}4\x9b&-\xc6\xec!\x80\xb6\x1f}\xa0=\rZ/\xda*\xdc\xe8>6\x98\xde\xe8Ȏ\xff\xcd\x06s\x00dzZ\x05\x9c:\r&\xb3\x9aڊ\xbc\xf1V\n\x9ep`\xd1Xt\xa8R\xef\xa1a\xa6@\x17?a\xe9\xe7#\xd5k\xb4\xa4\x06\xdcV\a\xc9\xc9\xd8\x1dZ\x0f\x16K])\xf1\x9fN\xb7\x03\xaf㦒yt>\x1eF\xab\x98\x84\x1d\x93\x01?\x90\xd3F\x9akv\x00\x8b\xb4'\x04\xd5\xd3\x17\x17\xb81\x8e/\xe4E\xa16z\x01[\xef\x8d[<>V·m\xb7\xd4u\x1d\x94\xf0\x87\xc7\x18\rQ\x04\xaf\xad{\xe4\xb8C\xf9\xe8D5c\xb6\xdc\n\x8f\xa5\x0f\x16\x1f\x99\x11\xb3h\x88\x8a\xadw^\xf3?٦Q\xbb\xc1\xb6\x93DL\xbf\xd80o\b\x0fuQ:\x15\xacQ\x95L\x16xV\xb0d5\xca%s\xf8\aNJ\xa2\xe2f\x14\x84\xab\xa2\xd5'\\c\xe1\xe4\xde\xdeD˘N\x84\xb6_A\xd6\x06K\x8a*9\x96\x96\x89\x8dh:\t\x95\x016\x90\x1dz(\x7f\xf4\xe9\x97\xed&c\xa1K\xe9F\xbf\xcf9E-Z\xd5+\xe4M\xafsM\x93\x92\xc3&\xd5\xffM\xfa\xa3E\xa3\x9d\xf0\xda\x1e\x8e]r\x9c\n'\xa3B\xbf\x92\xa9\x12\xe5=\xe6-\xe3J\x10\x8a\x93ϱKe*BIk\x04\xaaU\xa5\xe9p\rB\x01Ϟd(\xb7\x1d\xfa\xbc\xa1*\xdbՄ\x82#\xa7\x84>w\x1c\x9b[h-\x91\x8d\xbdHY\xf8\x85\xda\xc2R\xab\x8d\xa8\xa6\x86\xf7\xe9\xef\xa9\x14\xb9\xe0\xd3L\xc2\xf6\xb6$+(;\t\xc9,v\xa8Y\x9b\xbaT\xda7\xa2\n\xf6T\xfc7\x02%\x9fԟ\x93'\xa958\xeerO\x8c;\xe8\xed\xe9j\xbaZ\xaf\xf5z\x1d+\x94\x8b|\xb7\x97\x9aS\x90\x00ϛ\x9eF\xe1\xe0\xdd;\xd0\x16ޥ;ѻ\x0fiu\x10\xd2\xcfĠ\xff\uf154\xed.7e71\x9c\xaf\xeb\v\x96\xbfD!\xc2\xf3u}+\xb7\x9a\xa2A\x15\xea\xe9\x863`\xc1\xeb̰\x14*\xfc\x9c\x19\xdf\v\xc5\xf5\xde\xddbl\xc7o\x88b\xea\xe0\xef\t\xf8ב\x8eQ\xdc=\x11\xe2\x18k\xafa\xcfD\x8fct\xbb\xbb\x0f\x19\xbd\x05n\xa8!Y\xf4\xc1**\ah-Uh\x17U\xea0\xe1:\x92\x19\x97\xfe\xd1\xf4\xb1^\x8e'\x86ue4\xdb?\x92W\x11\xf7\xf8\xacp-uO\x8f\x85M\xd8\xcb`c\xd1i\x9e\x10\xe96|\x17yge\x89\xc6#\xff|\xa0\xae~E\xe3'\x00\xea\xfc\xa3ʿ̱\xed\xa3a\xb72\xec\x16R\xf7\xf0sO\x03\xf84V\x12o\xff\x96\xf7\xda\xf2\x14n\xa2f\xa7A\x03\xbcҽ)\xde^ߧNL\xcbb\x7f'\x86:\xd9t\xa2\xa1}L\xa4\xeb\xe9\x8c\xd6O$T\x90\x92\x15\x12\x17\xe0m8E\xd5\xf37\x93\xf4\x8e\xda\x7f2\xbb\xeb\x9a2U3\xf5\x1d\xeb\x1e\x89\xe2c^\xfb\x82\x9bs\xd9Q_簤\x0e9\xe0\x0e\x15\xd0\xe5\x93\t\x89\xbcՙ\xe1\xeb\x97<\x9f\x01=\xa5\x82\xbf\xa7\xf3kt\x8eU\x97\x0eЗ$\x95\xdeU\x9a%\xc0\n\xe2\x8dc\xd6\xfc\xde5g\xfbf\xfe\xfe\xdb\x1c\xe2+\xd9\xfb\x19,\xf1\xaey\x01̊dr5\xad\x83v\xba\xa8\xc1\x99\xcb\xc3\v\xee3\xa3\xed\xf9\xccL\xad\x9aC\x9f\x99\x9a|\x92\xe9O\xa6K}\xae1\xb6sY\x9d\xdd7\x8f\xcc\xdc?\xe2a\xb8\xc9\xd3\r\xbe{\x8e{\xf74\xb0ղ=\xe1\xf1[\x85\nu\x81\x96\xc2\x10\xbf\x86\xb4\xf1\xe8x?S\xbc\x1f\xb5\x1c\xf9\xeb4tw\x81\xa8j\x0e\xaf[\xa2&\xe9=\xa3\xbd\x1dq\xe1\x8cd\x87Θ>C\xcd(?\x9e\x9a\xc9s\xf5\xad$\xb5\xfbv\x94g^\xb9\x0f@\xc3\xdf\xf4S\xceh\xbe\xfb&\xf4\xfb\xecp\xe65c\xf8\x8d\uebab\xd4@åV\xd0|3\xbc\xbd\x82\x0f\xb7\xf9#\x8bw\xd6{\x93\xc1\x88\x9c\xf7t7\xaf\x8f\xfd\x91PtO\xf2\v\xf8\xef\xff\x1f~\t\x00\x00\xff\xff\xd9ߐm~\x1f\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcZIs\xe3\xb8\x15\xbe\xfbW\xbc\xea\x1c\xe6b\xc9\xd3\xc9T*\xa5[[N\xaa\\\x99v\xbbZ\x8e\xef\x10\xf9Db\f\x02\f\x16i\x9c\xe5\xbfO=\x00\xa4@\x12Zg\xbay\xe8jayx\x1b\xbe\xb7\xc0\xb3\xd9솵\xfc\x15\xb5\xe1J.\x80\xb5\x1c\x7f\xb5(闙\xbf\xfd\xcd̹\xba\xdb~\xbcy\xe3\xb2\\\xc0\xd2\x19\xab\x9a\xafh\x94\xd3\x05>\xe0\x86Kn\xb9\x927\rZV2\xcb\x167\x00LJe\x19\r\x1b\xfa\tP(i\xb5\x12\x02\xf5\xacB9\x7fsk\\;.JԞxw\xf4\xf6\xc7\xf9ǟ\xe6?\xde\x00H\xd6\xe0\x02\x88\x9ek\x85b\xa5\x99oQ\xa0Vs\xaenL\x8b\x05\x91\xad\xb4r\xed\x02\xf6\x13a[<2\xb0\xfb\xc0,\xfb\x97\xa7\xe0\a\x057\xf6\x9f\xa3\x89\x9f\xb9\xb1~\xb2\x15N318Տ\x1b.+'\x98Ngn\x00L\xa1Z\\\xc0\x13\x1dٲ\x02i,J\xe2Y\x98\x01+K\xaf\x1b&\x9e5\x97\x16\xf5R\t\xd7t:\x99A\x89\xa6м\xb5^\xf6=C`,\xb3\u0380qE\r\xcc\xc0\x13\xee\xee\x1e\xe5\xb3V\x95F\x13X\x02\xf8\xc5(\xf9\xccl\xbd\x80yX>okf0\xce\x06\xf5\xad\xfcD\x1c\xb2\xefĭ\xb1\x9a\xcb*w\xfe\vo\x10J\xa7\xbd\xd9H\xe6\x02\xc1\xd6ܤ\x8c\xed\x98!\xe6\xb4\xc5\xf2 \x1b~\x9e\x88\x19˚v\xccO\xb250T2\x8b9v\x96\xaai\x05Z,a\xfdn\xb1\x13b\xa3t\xc3\xec\x02\xb8\xb4\x7f\xfd\xe9\xb0&\xa2\xaa\xe6~냒C\xb5\xdc\xd3($Á\x13\xb2P\x85:\xab\x1be\x99\xf8=\x8cX\"p\x9f\xec\x0f\x9c\x04\xba\xe9\xf8IV\xc8\xdd@m\xc0\xd6\b\xf7\xacxs-\xac\xacҬB\xf8Y\x15\xc1x\xbb\x1au4\xde:,1\xb5r\xa2\x84u'1\x80\xb1Jg\xad\xd8b1\x0f\xbb\"ݎ\xecȔ\xc33\xff`'+4\xb2\xac\x93u(3\xf7+\xb8\x92yO\xfbT\xe1Y^\x96jS\xaa\x12{\xd5a\xca\x117\xd0jU\xa01G\xfc\x9e\xb6\x0fxx\xda\x0fL\xd4\x12Vl\xff\xccD[\xb3\x8f\x01e\x8a\x1a\x1b\xb6\x88;T\x8b\xf2\xd3\xf3\xe3\xeb_V\x83a8\x88\x19\xac\xb0\x86\xc0\x82Xo\xb5\xb2\xaaP\x02\xd6hw\x88\xd2\xe3\x164j\x8b\x9a@\xae\xe2\xd2\x00\x93eO\x13\xd2\x05{\xa8&'\xf7\xf4h6LFwR-\xea\xd4\xec@G\xb6\xa8-\xef\xd07|IXIFGB\xfco6\x98\x03 \xb9\xc3.()\xbe`\x90*b+\x96QU\xc1n܀\xc6V\xa3A\x19\"\x0e\r3\tj\xfd\v\x16v>\"\xbdBMd\xba\xfbP(\xb9EmAc\xa1*\xc9\xff\xd3\xd36`\x95?T0\x8b\xc6\xfa\v\xa9%\x13\xb0e\xc2\xe1\xedH{\xf45\xec\x1d4ҙ\xe0dB\xcfo0c>>+\x8d\xc0\xe5F-\xa0\xb6\xb65\x8b\xbb\xbb\x8a\xdb.\xd8\x16\xaai\x9c\xe4\xf6\xfd\xce\x1b\x83\xaf\x9dU\xdaܕ\xb8Eqgx5c\xba\xa8\xb9\xc5\xc2:\x8dw\xac\xe53/\x88\xf4\x01wޔ\x7f\xd21<\x9b\xc1\xb1\x13/\f\x9f\x0f\x94\x17\x98\x87\xe2']\t\x16I\x05\x11\xf7V\xa0!R\xdd\u05ff\xaf^\xa0\xe3$X*\x18e\xbft\xa2\x97\xce>\xa4M.7\xa8þ\x8dV\x8d\xa7\x89\xb2l\x15\x97\xd6\xff(\x04Gi\xc1\xb8u\xc3-\xb9\xc1\xbf\x1d\x1aK\xa6\x1b\x93]\xfa\x84\x04\xd6\b\xae%((\xc7\v\x1e%,Y\x83b\xc9\f~g[\x91Ǔ\x8cp\x96\xb5\xd24k\xbc8\xa87\x99\xe82\xa5\x03\xa6\xdd\xc3ǪłlJj\xa5M|\xc3c,!\f`\xc9ʡv\xf2מ\xbel\b\x19/:\xe5j\xf4\xdd\xe7\bu\xbc\xca\x04\xbf\xbbP\x17#\x93\x18F\xa6\xf4ۃ|ܣ\xb1U\x86[\xa5߉p\b\x8dc78h\x11\xfa\n&\v\x14\u05c8\xb7\xf4;\x81˒4\x8e\xbd\x1b\x13\x00\x05\xaa\x9eQ%+E\x17+1\x04pm'a\x00b\x86\xf7\x99r\xb8\xa5\x92\x1b^M\xcfN\x8b\xd5cw\xe4\xa8h\x93\xf0\x92\x1cI\x1a\xa7hB\x9c\xcc|:9\xebB\r%b\x1b^9}\xe8\xeao8\x8ar\x92-\x9c\xbc\xed'\xf4ᙸ\x06\xb4{ɺ`\x19\xf1+I\xa3\x83\x978\xe3\v\xd8$\xd6Le\x00\xc2\xc9=En\xe0\xc3\aP\x1a>\x84\xc6Ƈ۰\xdbqag|\x90\xcb\xef\xb8\x10\xdd)\x17\x85\xab>\x7f\xa7\xeaI\xb9S8\x9e\xd5\xc1\x97\x11\x8d\x91*,Uz^|\xab`\xc7x\x92C\xf7\xa7\x9b\xdb\f\xdd5n(\xe1\xd2h\x9d\x96\x14\xf2Pk\xcaA\x8c'\xa9\\\x06\xf3\x8fHj\x92\xf8sB\xcaq\xa8\xf2R\xd0\xff\xc7X\x9e\x02@F\x80\x9c\x8d\x8fq\xe8\xf3㾋t\x8d)VC\x12\x1d\xf3J\xf3\x8a\x93\xc2e?\xb3\xcf|\"\xd6\xc5\x16\x81G2\x0f\xc5Y\xff\xec\xd1\xd2\x10Z\xee\xc9\xd1u\x0e\x87\x13\xda3Y\xfa\xe0\xdcϗ\xf1\xeae.\xeeI\x85<\xbf.O٫?8\x03\xe54\xbc\xabyQ\x0fMǧ\xa0\n`\xd9\x1b\xfaD\xf7\x026\xf3\x18>˧\xbd\xa35\xe3\xdb7\x9aN]v<54tv\xf6\xf9uyVi\xe0\xbb\x16\xe7\x15\a\xa1\x1d\x19\xb5\\8\xad}\xd9\x15F\xa9ھ\xa2<`E\x81\xad\xc5\xf2\xfe\xfdI\x95\xa7\x9c\xfe\xd3`11\"\xcf\xe9\xdbdL\xed;9زK\xf3\xfb\x8eݾ\xdbt\xcd5\xfd4&\xe2\xfb\x0e\xbaL\x00s\x9a\xad\a\xb09\xcc4\xc0\v9\xb8\xaf\x9b\x7f\b\x18I\xdb<\xf2\xd2\xf5\x9c\x1c:\xa1е2\xa90\x9e\xd1\xfe\xeb\xa2l\xbe.\n]ܴOwU\x914%3\xd5\x1d\xeb\xaa9\xdf@\xec\xda\xc79\x8d\xed\xc9\xf5\xfa\n\u0530\x04ܢ\x04\xaa{\x19\x17\x14\xbb=\xc9\f\x80\x1d\xa7\x12\x83Xx+\xe8\x1a\"]\xf3,ۙ:mɌ\x12\xa6h\xf6-\x8d٧\x90_\xd18\x91I\x1a\xbea\n\x19\x8e\f\xa5\xb9ɦ\x90\xc7kGf\x80\x81\x0eD\"n\x1c\x02\xad\xb3\x95\x94\xcd+\x1b4\x86U\xa7\x10\xedsX\x15\xdahq\v\xb05\xa5QC\xd6~0\x11h/\x82+y\x1aS/B\xd2A\a\xfcbN\xbe\xac\xce\xe0\xe5ˊ\x0e\xf9\xb2\xfa\xbd\xbc\xa0tM\xae\xb0bΪ̰\xe0\xd2\xfd\x9a\x19\xdfqY\xaa\xdd\xf4~\x1d\x11\xb5e\xb6>!\xe83\xb3u\x17G7N\b\xbfg\x92_\xc6\xd4l\x8d\x04\x1c\x7fT\x9a\xe9\xfb<\xa7أ5\xb98\x8f\xe7ܙC\x9a\x7f\xc2]f\xb4\x8bK\x99\xa9\xe7\x18\xec2S\x93g\xd0t2\xb4\xd2r\x98\xd2\xcdei\xf6/\x8d\x99\xb9\x7f\xf8(p\x91\x9e#\x7fׄ\xb9\xbe)W+\xd1E6\xffB(]\xb3FMF\xf0o\x90\xa3~\x05\xe5݉\xc52\x84\x93\xfd}\xb2\xef)\xcd\xe1\xa5\xe6\xa6k#v\xe5Z\xc9M+\xd8{/\xcb)l\xedqk\xfc<4u\x92\xe3\xfd\xb7\xfe\xbd6\xdf\xce\xc9=\xba\x0e\xbf\xe9\xf3\xe9h\xbe\x7f\x87\xfd6'\x1c\t\f\xdd\xf5~|8\xb3\x0e}|\xe8\xae\"/QZ*\xad\xf7Or\xfb\x8aƷxs\xba\x1c\xb7\xb6/+\xc2\x06\xaf\xf8W\x15\xa5\x03\n'ҵ\xf8G\x05\xb9\xa4hE`@\x10\xe4\x1f\x81\x96\xe3g\xdf\xdb\xfe\x15\x99\xd9\xf8\x12U\xd4LV\x98\xab\xf4\x94\xa4\x1c\xc0\xe7\x10\x97\xe7_C\x81\xbeg\xea\x95\xf5\xaaɠ\xe7\xbcLh\xc7^b:\xe2\xd6\xfd\xd3\xe0\x02\xfe\xfb\xff\x9b\xdf\x02\x00\x00\xff\xff\xcc\b\u008b\xfc#\x00\x00"), } var CRDs = crds() diff --git a/internal/delete/delete_item_action_handler_test.go b/internal/delete/delete_item_action_handler_test.go index 8bd120602..a71de864d 100644 --- a/internal/delete/delete_item_action_handler_test.go +++ b/internal/delete/delete_item_action_handler_test.go @@ -230,7 +230,7 @@ func (h *harness) addResource(t *testing.T, resource *test.APIResource) { } // recordResourcesAction is a delete item action that can be configured to run -// for specific resources/namespaces and simply record the items that is is +// for specific resources/namespaces and simply record the items that is // executed for. type recordResourcesAction struct { selector velero.ResourceSelector diff --git a/pkg/apis/velero/v2alpha1/data_download_types.go b/pkg/apis/velero/v2alpha1/data_download_types.go index 3a700661a..f79af10a8 100644 --- a/pkg/apis/velero/v2alpha1/data_download_types.go +++ b/pkg/apis/velero/v2alpha1/data_download_types.go @@ -54,6 +54,10 @@ type DataDownloadSpec struct { // OperationTimeout specifies the time used to wait internal operations, // before returning error as timeout. OperationTimeout metav1.Duration `json:"operationTimeout"` + + // NodeOS is OS of the node where the DataDownload is processed. + // +optional + NodeOS NodeOS `json:"nodeOS,omitempty"` } // TargetVolumeSpec is the specification for a target PVC. diff --git a/pkg/apis/velero/v2alpha1/data_upload_types.go b/pkg/apis/velero/v2alpha1/data_upload_types.go index 546caa05e..140a94ec3 100644 --- a/pkg/apis/velero/v2alpha1/data_upload_types.go +++ b/pkg/apis/velero/v2alpha1/data_upload_types.go @@ -96,6 +96,16 @@ const ( DataUploadPhaseFailed DataUploadPhase = "Failed" ) +// NodeOS represents OS of a node. +// +kubebuilder:validation:Enum=auto;linux;windows +type NodeOS string + +const ( + NodeOSLinux NodeOS = "linux" + NodeOSWindows NodeOS = "windows" + NodeOSAuto NodeOS = "auto" +) + // DataUploadStatus is the current status of a DataUpload. type DataUploadStatus struct { // Phase is the current state of the DataUpload. @@ -144,7 +154,12 @@ type DataUploadStatus struct { // Node is name of the node where the DataUpload is processed. // +optional Node string `json:"node,omitempty"` - // Node is name of the node where the DataUpload is prepared. + + // NodeOS is OS of the node where the DataUpload is processed. + // +optional + NodeOS NodeOS `json:"nodeOS,omitempty"` + + // AcceptedByNode is name of the node where the DataUpload is prepared. // +optional AcceptedByNode string `json:"acceptedByNode,omitempty"` @@ -221,4 +236,8 @@ type DataUploadResult struct { // +optional // +nullable DataMoverResult *map[string]string `json:"dataMoverResult,omitempty"` + + // NodeOS is OS of the node where the DataUpload is processed. + // +optional + NodeOS NodeOS `json:"nodeOS,omitempty"` } diff --git a/pkg/backup/actions/csi/pvc_action.go b/pkg/backup/actions/csi/pvc_action.go index a078a2308..589cd61d7 100644 --- a/pkg/backup/actions/csi/pvc_action.go +++ b/pkg/backup/actions/csi/pvc_action.go @@ -19,6 +19,7 @@ package csi import ( "context" "fmt" + "strconv" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" "github.com/pkg/errors" @@ -484,9 +485,7 @@ func newDataUpload( if backup.Spec.UploaderConfig != nil && backup.Spec.UploaderConfig.ParallelFilesUpload > 0 { dataUpload.Spec.DataMoverConfig = make(map[string]string) - dataUpload.Spec.DataMoverConfig[uploaderUtil.ParallelFilesUpload] = fmt.Sprintf( - "%d", backup.Spec.UploaderConfig.ParallelFilesUpload, - ) + dataUpload.Spec.DataMoverConfig[uploaderUtil.ParallelFilesUpload] = strconv.Itoa(backup.Spec.UploaderConfig.ParallelFilesUpload) } return dataUpload diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index d30542887..280164cc4 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -35,10 +35,8 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/selection" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -315,7 +313,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers( var podVolumeBackupper podvolume.Backupper if kb.podVolumeBackupperFactory != nil { - podVolumeBackupper, err = kb.podVolumeBackupperFactory.NewBackupper(ctx, backupRequest.Backup, kb.uploaderType) + podVolumeBackupper, err = kb.podVolumeBackupperFactory.NewBackupper(ctx, log, backupRequest.Backup, kb.uploaderType) if err != nil { log.WithError(errors.WithStack(err)).Debugf("Error from NewBackupper") return errors.WithStack(err) @@ -745,6 +743,7 @@ func (kb *kubernetesBackupper) handleItemBlockPostHooks(ctx context.Context, ite log := itemBlock.Log defer itemBlock.itemBackupper.hookTracker.AsyncItemBlocks.Done() + // the post hooks will not execute until all PVBs of the item block pods are processed if err := kb.waitUntilPVBsProcessed(ctx, log, itemBlock, hookPods); err != nil { log.WithError(err).Error("failed to wait PVBs processed for the ItemBlock") return @@ -758,36 +757,19 @@ func (kb *kubernetesBackupper) handleItemBlockPostHooks(ctx context.Context, ite } } +// wait all PVBs of the item block pods to be processed func (kb *kubernetesBackupper) waitUntilPVBsProcessed(ctx context.Context, log logrus.FieldLogger, itemBlock BackupItemBlock, pods []itemblock.ItemBlockItem) error { - requirement, err := labels.NewRequirement(velerov1api.BackupUIDLabel, selection.Equals, []string{string(itemBlock.itemBackupper.backupRequest.UID)}) - if err != nil { - return errors.Wrapf(err, "failed to create label requirement") - } - options := &kbclient.ListOptions{ - LabelSelector: labels.NewSelector().Add(*requirement), - } - pvbList := &velerov1api.PodVolumeBackupList{} - if err := kb.kbClient.List(context.Background(), pvbList, options); err != nil { - return errors.Wrap(err, "failed to list PVBs") - } - - podMap := map[string]struct{}{} - for _, pod := range pods { - podMap[string(pod.Item.GetUID())] = struct{}{} - } - pvbMap := map[*velerov1api.PodVolumeBackup]bool{} - for i, pvb := range pvbList.Items { - if _, exist := podMap[string(pvb.Spec.Pod.UID)]; !exist { - continue + for _, pod := range pods { + namespace, name := pod.Item.GetNamespace(), pod.Item.GetName() + pvbs, err := itemBlock.itemBackupper.podVolumeBackupper.ListPodVolumeBackupsByPod(namespace, name) + if err != nil { + return errors.Wrapf(err, "failed to list PodVolumeBackups for pod %s/%s", namespace, name) } - - processed := false - if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted || - pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed { - processed = true + for _, pvb := range pvbs { + pvbMap[pvb] = pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted || + pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed } - pvbMap[&pvbList.Items[i]] = processed } checkFunc := func(context.Context) (done bool, err error) { @@ -796,8 +778,8 @@ func (kb *kubernetesBackupper) waitUntilPVBsProcessed(ctx context.Context, log l if processed { continue } - updatedPVB := &velerov1api.PodVolumeBackup{} - if err := kb.kbClient.Get(ctx, kbclient.ObjectKeyFromObject(pvb), updatedPVB); err != nil { + updatedPVB, err := itemBlock.itemBackupper.podVolumeBackupper.GetPodVolumeBackup(pvb.Namespace, pvb.Name) + if err != nil { allProcessed = false log.Infof("failed to get PVB: %v", err) continue diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index eb25d65f0..425fa827c 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -3945,7 +3945,7 @@ func TestBackupWithHooks(t *testing.T) { type fakePodVolumeBackupperFactory struct{} -func (f *fakePodVolumeBackupperFactory) NewBackupper(context.Context, *velerov1.Backup, string) (podvolume.Backupper, error) { +func (f *fakePodVolumeBackupperFactory) NewBackupper(context.Context, logrus.FieldLogger, *velerov1.Backup, string) (podvolume.Backupper, error) { return &fakePodVolumeBackupper{}, nil } @@ -3978,6 +3978,24 @@ func (b *fakePodVolumeBackupper) WaitAllPodVolumesProcessed(log logrus.FieldLogg return b.pvbs } +func (b *fakePodVolumeBackupper) GetPodVolumeBackup(namespace, name string) (*velerov1.PodVolumeBackup, error) { + for _, pvb := range b.pvbs { + if pvb.Namespace == namespace && pvb.Name == name { + return pvb, nil + } + } + return nil, nil +} +func (b *fakePodVolumeBackupper) ListPodVolumeBackupsByPod(podNamespace, podName string) ([]*velerov1.PodVolumeBackup, error) { + var pvbs []*velerov1.PodVolumeBackup + for _, pvb := range b.pvbs { + if pvb.Spec.Pod.Namespace == podNamespace && pvb.Spec.Pod.Name == podName { + pvbs = append(pvbs, pvb) + } + } + return pvbs, nil +} + // TestBackupWithPodVolume runs backups of pods that are annotated for PodVolume backup, // and ensures that the pod volume backupper is called, that the returned PodVolumeBackups // are added to the Request object, and that when PVCs are backed up with PodVolume, the diff --git a/pkg/builder/data_download_builder.go b/pkg/builder/data_download_builder.go index 9364022bd..5b23a9dcc 100644 --- a/pkg/builder/data_download_builder.go +++ b/pkg/builder/data_download_builder.go @@ -148,6 +148,12 @@ func (d *DataDownloadBuilder) Node(node string) *DataDownloadBuilder { return d } +// NodeOS sets the DataDownload's Node OS. +func (d *DataDownloadBuilder) NodeOS(nodeOS velerov2alpha1api.NodeOS) *DataDownloadBuilder { + d.object.Spec.NodeOS = nodeOS + return d +} + // AcceptedByNode sets the DataDownload's AcceptedByNode. func (d *DataDownloadBuilder) AcceptedByNode(node string) *DataDownloadBuilder { d.object.Status.AcceptedByNode = node diff --git a/pkg/builder/data_upload_builder.go b/pkg/builder/data_upload_builder.go index b4fa72e43..b77566bf6 100644 --- a/pkg/builder/data_upload_builder.go +++ b/pkg/builder/data_upload_builder.go @@ -151,6 +151,12 @@ func (d *DataUploadBuilder) Node(node string) *DataUploadBuilder { return d } +// NodeOS sets the DataUpload's Node OS. +func (d *DataUploadBuilder) NodeOS(nodeOS velerov2alpha1api.NodeOS) *DataUploadBuilder { + d.object.Status.NodeOS = nodeOS + return d +} + // AcceptedByNode sets the DataUpload's AcceptedByNode. func (d *DataUploadBuilder) AcceptedByNode(node string) *DataUploadBuilder { d.object.Status.AcceptedByNode = node diff --git a/pkg/cmd/cli/backup/create_test.go b/pkg/cmd/cli/backup/create_test.go index 60ce69ea4..b9be38e1e 100644 --- a/pkg/cmd/cli/backup/create_test.go +++ b/pkg/cmd/cli/backup/create_test.go @@ -19,6 +19,7 @@ package backup import ( "context" "fmt" + "strconv" "strings" "testing" "time" @@ -222,7 +223,7 @@ func TestCreateCommand(t *testing.T) { flags.Parse([]string{"--default-volumes-to-fs-backup", defaultVolumesToFsBackup}) flags.Parse([]string{"--resource-policies-configmap", resPoliciesConfigmap}) flags.Parse([]string{"--data-mover", dataMover}) - flags.Parse([]string{"--parallel-files-upload", fmt.Sprintf("%d", parallelFilesUpload)}) + flags.Parse([]string{"--parallel-files-upload", strconv.Itoa(parallelFilesUpload)}) //flags.Parse([]string{"--wait"}) client := velerotest.NewFakeControllerRuntimeClient(t).(kbclient.WithWatch) diff --git a/pkg/cmd/cli/datamover/restore.go b/pkg/cmd/cli/datamover/restore.go index 244060cc9..4730cf903 100644 --- a/pkg/cmd/cli/datamover/restore.go +++ b/pkg/cmd/cli/datamover/restore.go @@ -160,7 +160,24 @@ func newdataMoverRestore(logger logrus.FieldLogger, factory client.Factory, conf return nil, errors.Wrap(err, "error to create client") } - cache, err := ctlcache.New(clientConfig, cacheOption) + var cache ctlcache.Cache + retry := 10 + for { + cache, err = ctlcache.New(clientConfig, cacheOption) + if err == nil { + break + } + + retry-- + if retry == 0 { + break + } + + logger.WithError(err).Warn("Failed to create client cache, need retry") + + time.Sleep(time.Second) + } + if err != nil { cancelFunc() return nil, errors.Wrap(err, "error to create client cache") diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index 713d86359..d5e7193cc 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -353,7 +353,13 @@ func (s *nodeAgentServer) run() { s.logger.WithError(err).Fatal("Unable to create the data upload controller") } - dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, podResources, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics) + var restorePVCConfig nodeagent.RestorePVC + if s.dataPathConfigs != nil && s.dataPathConfigs.RestorePVCConfig != nil { + restorePVCConfig = *s.dataPathConfigs.RestorePVCConfig + s.logger.Infof("Using customized restorePVC config %v", restorePVCConfig) + } + + dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, restorePVCConfig, podResources, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics) if err = dataDownloadReconciler.SetupWithManager(s.mgr); err != nil { s.logger.WithError(err).Fatal("Unable to create the data download controller") } diff --git a/pkg/cmd/server/plugin/plugin.go b/pkg/cmd/server/plugin/plugin.go index b0f8aae3f..3fa174d81 100644 --- a/pkg/cmd/server/plugin/plugin.go +++ b/pkg/cmd/server/plugin/plugin.go @@ -21,6 +21,7 @@ import ( "github.com/spf13/cobra" apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/datamover" dia "github.com/vmware-tanzu/velero/internal/delete/actions/csi" @@ -162,7 +163,7 @@ func NewCommand(f client.Factory) *cobra.Command { newVolumeSnapshotClassBackupItemAction, ). RegisterRestoreItemActionV2( - "velero.io/csi-pvc-restorer", + constant.PluginCSIPVCRestoreRIA, newPvcRestoreItemAction(f), ). RegisterRestoreItemActionV2( diff --git a/pkg/constant/constant.go b/pkg/constant/constant.go index f8d5b191f..dc2c0e558 100644 --- a/pkg/constant/constant.go +++ b/pkg/constant/constant.go @@ -19,4 +19,6 @@ const ( ControllerSchedule = "schedule" ControllerServerStatusRequest = "server-status-request" ControllerRestoreFinalizer = "restore-finalizer" + + PluginCSIPVCRestoreRIA = "velero.io/csi-pvc-restorer" ) diff --git a/pkg/controller/data_download_controller.go b/pkg/controller/data_download_controller.go index 45b367cd8..3d5b2965a 100644 --- a/pkg/controller/data_download_controller.go +++ b/pkg/controller/data_download_controller.go @@ -64,13 +64,15 @@ type DataDownloadReconciler struct { restoreExposer exposer.GenericRestoreExposer nodeName string dataPathMgr *datapath.Manager + restorePVCConfig nodeagent.RestorePVC podResources v1.ResourceRequirements preparingTimeout time.Duration metrics *metrics.ServerMetrics } func NewDataDownloadReconciler(client client.Client, mgr manager.Manager, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager, - podResources v1.ResourceRequirements, nodeName string, preparingTimeout time.Duration, logger logrus.FieldLogger, metrics *metrics.ServerMetrics) *DataDownloadReconciler { + restorePVCConfig nodeagent.RestorePVC, podResources v1.ResourceRequirements, nodeName string, preparingTimeout time.Duration, + logger logrus.FieldLogger, metrics *metrics.ServerMetrics) *DataDownloadReconciler { return &DataDownloadReconciler{ client: client, kubeClient: kubeClient, @@ -79,6 +81,7 @@ func NewDataDownloadReconciler(client client.Client, mgr manager.Manager, kubeCl Clock: &clock.RealClock{}, nodeName: nodeName, restoreExposer: exposer.NewGenericRestoreExposer(kubeClient, logger), + restorePVCConfig: restorePVCConfig, dataPathMgr: dataPathMgr, podResources: podResources, preparingTimeout: preparingTimeout, @@ -180,21 +183,15 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{}, nil } - hostingPodLabels := map[string]string{velerov1api.DataDownloadLabel: dd.Name} - for _, k := range util.ThirdPartyLabels { - if v, err := nodeagent.GetLabelValue(ctx, r.kubeClient, dd.Namespace, k, kube.NodeOSLinux); err != nil { - if err != nodeagent.ErrNodeAgentLabelNotFound { - log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) - } - } else { - hostingPodLabels[k] = v - } + exposeParam, err := r.setupExposeParam(dd) + if err != nil { + return r.errorOut(ctx, dd, err, "failed to set exposer parameters", log) } // Expose() will trigger to create one pod whose volume is restored by a given volume snapshot, // but the pod maybe is not in the same node of the current controller, so we need to return it here. // And then only the controller who is in the same node could do the rest work. - err = r.restoreExposer.Expose(ctx, getDataDownloadOwnerObject(dd), dd.Spec.TargetVolume.PVC, dd.Spec.TargetVolume.Namespace, hostingPodLabels, r.podResources, dd.Spec.OperationTimeout.Duration) + err = r.restoreExposer.Expose(ctx, getDataDownloadOwnerObject(dd), exposeParam) if err != nil { if err := r.client.Get(ctx, req.NamespacedName, dd); err != nil { if !apierrors.IsNotFound(err) { @@ -233,7 +230,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request log.Debugf("Data download is been canceled %s in Phase %s", dd.GetName(), dd.Status.Phase) r.tryCancelAcceptedDataDownload(ctx, dd, "") } else if peekErr := r.restoreExposer.PeekExposed(ctx, getDataDownloadOwnerObject(dd)); peekErr != nil { - r.tryCancelAcceptedDataDownload(ctx, dd, fmt.Sprintf("found a dataupload %s/%s with expose error: %s. mark it as cancel", dd.Namespace, dd.Name, peekErr)) + r.tryCancelAcceptedDataDownload(ctx, dd, fmt.Sprintf("found a datadownload %s/%s with expose error: %s. mark it as cancel", dd.Namespace, dd.Name, peekErr)) log.Errorf("Cancel dd %s/%s because of expose error %s", dd.Namespace, dd.Name, peekErr) } else if dd.Status.AcceptedTimestamp != nil { if time.Since(dd.Status.AcceptedTimestamp.Time) >= r.preparingTimeout { @@ -727,6 +724,42 @@ func (r *DataDownloadReconciler) closeDataPath(ctx context.Context, ddName strin r.dataPathMgr.RemoveAsyncBR(ddName) } +func (r *DataDownloadReconciler) setupExposeParam(dd *velerov2alpha1api.DataDownload) (exposer.GenericRestoreExposeParam, error) { + log := r.logger.WithField("datadownload", dd.Name) + + nodeOS := string(dd.Spec.NodeOS) + if nodeOS == "" { + log.Info("nodeOS is empty in DD, fallback to linux") + nodeOS = kube.NodeOSLinux + } + + if err := kube.HasNodeWithOS(context.Background(), nodeOS, r.kubeClient.CoreV1()); err != nil { + return exposer.GenericRestoreExposeParam{}, errors.Wrapf(err, "no appropriate node to run datadownload %s/%s", dd.Namespace, dd.Name) + } + + hostingPodLabels := map[string]string{velerov1api.DataDownloadLabel: dd.Name} + for _, k := range util.ThirdPartyLabels { + if v, err := nodeagent.GetLabelValue(context.Background(), r.kubeClient, dd.Namespace, k, nodeOS); err != nil { + if err != nodeagent.ErrNodeAgentLabelNotFound { + log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) + } + } else { + hostingPodLabels[k] = v + } + } + + return exposer.GenericRestoreExposeParam{ + TargetPVCName: dd.Spec.TargetVolume.PVC, + TargetNamespace: dd.Spec.TargetVolume.Namespace, + HostingPodLabels: hostingPodLabels, + Resources: r.podResources, + OperationTimeout: dd.Spec.OperationTimeout.Duration, + ExposeTimeout: r.preparingTimeout, + NodeOS: nodeOS, + RestorePVCConfig: r.restorePVCConfig, + }, nil +} + func getDataDownloadOwnerObject(dd *velerov2alpha1api.DataDownload) v1.ObjectReference { return v1.ObjectReference{ Kind: dd.Kind, diff --git a/pkg/controller/data_download_controller_test.go b/pkg/controller/data_download_controller_test.go index 7bb224fa1..046103771 100644 --- a/pkg/controller/data_download_controller_test.go +++ b/pkg/controller/data_download_controller_test.go @@ -50,8 +50,10 @@ import ( datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks" "github.com/vmware-tanzu/velero/pkg/exposer" "github.com/vmware-tanzu/velero/pkg/metrics" + "github.com/vmware-tanzu/velero/pkg/nodeagent" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/uploader" + "github.com/vmware-tanzu/velero/pkg/util/kube" exposermockes "github.com/vmware-tanzu/velero/pkg/exposer/mocks" ) @@ -66,7 +68,7 @@ func dataDownloadBuilder() *builder.DataDownloadBuilder { PV: "test-pv", PVC: "test-pvc", Namespace: "test-ns", - }) + }).NodeOS(velerov2alpha1api.NodeOS("linux")) } func initDataDownloadReconciler(objects []runtime.Object, needError ...bool) (*DataDownloadReconciler, error) { @@ -140,7 +142,7 @@ func initDataDownloadReconcilerWithError(objects []runtime.Object, needError ... dataPathMgr := datapath.NewManager(1) - return NewDataDownloadReconciler(fakeClient, nil, fakeKubeClient, dataPathMgr, corev1.ResourceRequirements{}, "test-node", time.Minute*5, velerotest.NewLogger(), metrics.NewServerMetrics()), nil + return NewDataDownloadReconciler(fakeClient, nil, fakeKubeClient, dataPathMgr, nodeagent.RestorePVC{}, corev1.ResourceRequirements{}, "test-node", time.Minute*5, velerotest.NewLogger(), metrics.NewServerMetrics()), nil } func TestDataDownloadReconcile(t *testing.T) { @@ -166,6 +168,8 @@ func TestDataDownloadReconcile(t *testing.T) { }, } + node := builder.ForNode("fake-node").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() + tests := []struct { name string dd *velerov2alpha1api.DataDownload @@ -325,9 +329,15 @@ func TestDataDownloadReconcile(t *testing.T) { }, { name: "Restore is exposed", - dd: dataDownloadBuilder().Result(), + dd: dataDownloadBuilder().NodeOS(velerov2alpha1api.NodeOSLinux).Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), }, + { + name: "Expected node doesn't exist", + dd: dataDownloadBuilder().NodeOS(velerov2alpha1api.NodeOSWindows).Result(), + targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), + expectedStatusMsg: "no appropriate node to run datadownload", + }, { name: "Get empty restore exposer", dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhasePrepared).Result(), @@ -387,9 +397,9 @@ func TestDataDownloadReconcile(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - var objs []runtime.Object + objs := []runtime.Object{daemonSet, node} if test.targetPVC != nil { - objs = []runtime.Object{test.targetPVC, daemonSet} + objs = append(objs, test.targetPVC) } r, err := initDataDownloadReconciler(objs, test.needErrs...) require.NoError(t, err) @@ -959,7 +969,7 @@ func (dt *ddResumeTestHelper) resumeCancellableDataPath(_ *DataUploadReconciler, return dt.resumeErr } -func (dt *ddResumeTestHelper) Expose(context.Context, corev1.ObjectReference, string, string, map[string]string, corev1.ResourceRequirements, time.Duration) error { +func (dt *ddResumeTestHelper) Expose(context.Context, corev1.ObjectReference, exposer.GenericRestoreExposeParam) error { return nil } diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 66f5b67f7..3185373c1 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -285,6 +285,10 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + if res.ByPod.NodeOS == nil { + return r.errorOut(ctx, du, errors.New("unsupported ambiguous node OS"), "invalid expose result", log) + } + log.Info("Exposed snapshot is ready and creating data path routine") // Need to first create file system BR and get data path instance then update data upload status @@ -317,6 +321,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) original := du.DeepCopy() du.Status.Phase = velerov2alpha1api.DataUploadPhaseInProgress du.Status.StartTimestamp = &metav1.Time{Time: r.Clock.Now()} + du.Status.NodeOS = velerov2alpha1api.NodeOS(*res.ByPod.NodeOS) if err := r.client.Patch(ctx, du, client.MergeFrom(original)); err != nil { log.WithError(err).Warnf("Failed to update dataupload %s to InProgress, will data path close and retry", du.Name) @@ -792,6 +797,8 @@ func (r *DataUploadReconciler) closeDataPath(ctx context.Context, duName string) } func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload) (interface{}, error) { + log := r.logger.WithField("dataupload", du.Name) + if du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { pvc := &corev1.PersistentVolumeClaim{} err := r.client.Get(context.Background(), types.NamespacedName{ @@ -803,7 +810,7 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload return nil, errors.Wrapf(err, "failed to get PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } - nodeOS, err := kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), r.logger) + nodeOS, err := kube.GetPVCAttachingNodeOS(pvc, r.kubeClient.CoreV1(), r.kubeClient.StorageV1(), log) if err != nil { return nil, errors.Wrapf(err, "failed to get attaching node OS for PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) } @@ -821,7 +828,7 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload for _, k := range util.ThirdPartyLabels { if v, err := nodeagent.GetLabelValue(context.Background(), r.kubeClient, du.Namespace, k, nodeOS); err != nil { if err != nodeagent.ErrNodeAgentLabelNotFound { - r.logger.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) + log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) } } else { hostingPodLabels[k] = v @@ -843,6 +850,7 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload NodeOS: nodeOS, }, nil } + return nil, nil } diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index f480a692c..0a0210869 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -166,6 +166,7 @@ func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconci RestoreSize: &restoreSize, }, } + daemonSet := &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", @@ -265,9 +266,10 @@ func dataUploadBuilder() *builder.DataUploadBuilder { } type fakeSnapshotExposer struct { - kubeClient kbclient.Client - clock clock.WithTickerAndDelayedExecution - peekErr error + kubeClient kbclient.Client + clock clock.WithTickerAndDelayedExecution + ambiguousNodeOS bool + peekErr error } func (f *fakeSnapshotExposer) Expose(ctx context.Context, ownerObject corev1.ObjectReference, param interface{}) error { @@ -296,7 +298,13 @@ func (f *fakeSnapshotExposer) GetExposed(ctx context.Context, du corev1.ObjectRe if err != nil { return nil, err } - return &exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: pod, VolumeName: dataUploadName}}, nil + + nodeOS := "linux" + pNodeOS := &nodeOS + if f.ambiguousNodeOS { + pNodeOS = nil + } + return &exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: pod, VolumeName: dataUploadName, NodeOS: pNodeOS}}, nil } func (f *fakeSnapshotExposer) PeekExposed(ctx context.Context, ownerObject corev1.ObjectReference) error { @@ -350,6 +358,8 @@ func TestReconcile(t *testing.T) { expectedRequeue ctrl.Result expectedErrMsg string needErrs []bool + removeNode bool + ambiguousNodeOS bool peekErr error notCreateFSBR bool fsBRInitErr error @@ -359,25 +369,29 @@ func TestReconcile(t *testing.T) { name: "Dataupload is not initialized", du: builder.ForDataUpload("unknown-ns", "unknown-name").Result(), expectedRequeue: ctrl.Result{}, - }, { + }, + { name: "Error get Dataupload", du: builder.ForDataUpload(velerov1api.DefaultNamespace, "unknown-name").Result(), expectedRequeue: ctrl.Result{}, expectedErrMsg: "getting DataUpload: Get error", needErrs: []bool{true, false, false, false}, - }, { + }, + { name: "Unsupported data mover type", du: dataUploadBuilder().DataMover("unknown type").Result(), expected: dataUploadBuilder().Phase("").Result(), expectedRequeue: ctrl.Result{}, - }, { + }, + { name: "Unknown type of snapshot exposer is not initialized", du: dataUploadBuilder().SnapshotType("unknown type").Result(), expectedProcessed: true, expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), expectedRequeue: ctrl.Result{}, expectedErrMsg: "unknown type type of snapshot exposer is not exist", - }, { + }, + { name: "Dataupload should be accepted", du: dataUploadBuilder().Result(), pod: builder.ForPod("fake-ns", dataUploadName).Volumes(&corev1.Volume{Name: "test-pvc"}).Result(), @@ -394,6 +408,27 @@ func TestReconcile(t *testing.T) { expectedRequeue: ctrl.Result{}, expectedErrMsg: "failed to get PVC", }, + { + name: "Dataupload should fail to get PVC attaching node", + du: dataUploadBuilder().Result(), + pod: builder.ForPod("fake-ns", dataUploadName).Volumes(&corev1.Volume{Name: "test-pvc"}).Result(), + pvc: builder.ForPersistentVolumeClaim("fake-ns", "test-pvc").StorageClass("fake-sc").Result(), + expectedProcessed: true, + expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), + expectedRequeue: ctrl.Result{}, + expectedErrMsg: "error to get storage class", + }, + { + name: "Dataupload should fail because expected node doesn't exist", + du: dataUploadBuilder().Result(), + pod: builder.ForPod("fake-ns", dataUploadName).Volumes(&corev1.Volume{Name: "test-pvc"}).Result(), + pvc: builder.ForPersistentVolumeClaim("fake-ns", "test-pvc").Result(), + removeNode: true, + expectedProcessed: true, + expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), + expectedRequeue: ctrl.Result{}, + expectedErrMsg: "no appropriate node to run data upload", + }, { name: "Dataupload should be prepared", du: dataUploadBuilder().SnapshotType(fakeSnapshotType).Result(), @@ -407,6 +442,15 @@ func TestReconcile(t *testing.T) { expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseInProgress).Result(), expectedRequeue: ctrl.Result{}, }, + { + name: "Dataupload should fail if expose returns ambiguous nodeOS", + pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Volumes(&corev1.Volume{Name: "dataupload-1"}).Result(), + du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).SnapshotType(fakeSnapshotType).Result(), + ambiguousNodeOS: true, + expectedProcessed: true, + expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), + expectedErrMsg: "unsupported ambiguous node OS", + }, { name: "Dataupload with not enabled cancel", pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Volumes(&corev1.Volume{Name: "dataupload-1"}).Result(), @@ -557,6 +601,11 @@ func TestReconcile(t *testing.T) { require.NoError(t, err) } + if test.removeNode { + err = r.kubeClient.CoreV1().Nodes().Delete(ctx, "fake-node", metav1.DeleteOptions{}) + require.NoError(t, err) + } + if test.dataMgr != nil { r.dataPathMgr = test.dataMgr } else { @@ -564,7 +613,7 @@ func TestReconcile(t *testing.T) { } if test.du.Spec.SnapshotType == fakeSnapshotType { - r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{fakeSnapshotType: &fakeSnapshotExposer{r.client, r.Clock, test.peekErr}} + r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{fakeSnapshotType: &fakeSnapshotExposer{r.client, r.Clock, test.ambiguousNodeOS, test.peekErr}} } else if test.du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{velerov2alpha1api.SnapshotTypeCSI: exposer.NewCSISnapshotExposer(r.kubeClient, r.csiSnapshotClient, velerotest.NewLogger())} } diff --git a/pkg/controller/restore_finalizer_controller.go b/pkg/controller/restore_finalizer_controller.go index d9fa5797a..2caec1788 100644 --- a/pkg/controller/restore_finalizer_controller.go +++ b/pkg/controller/restore_finalizer_controller.go @@ -22,6 +22,12 @@ import ( "sync" "time" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/vmware-tanzu/velero/pkg/constant" + "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + storagev1api "k8s.io/api/storage/v1" "github.com/pkg/errors" @@ -155,6 +161,12 @@ func (r *restoreFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Req restoredPVCList := volume.RestoredPVCFromRestoredResourceList(restoredResourceList) + restoreItemOperations, err := backupStore.GetRestoreItemOperations(restore.Name) + if err != nil { + log.WithError(err).Error("error getting itemOperationList") + return ctrl.Result{}, errors.Wrap(err, "error getting itemOperationList") + } + finalizerCtx := &finalizerContext{ logger: log, restore: restore, @@ -163,6 +175,9 @@ func (r *restoreFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Req restoredPVCList: restoredPVCList, multiHookTracker: r.multiHookTracker, resourceTimeout: r.resourceTimeout, + restoreItemOperationList: restoreItemOperationList{ + items: restoreItemOperations, + }, } warnings, errs := finalizerCtx.execute() @@ -239,16 +254,44 @@ func (r *restoreFinalizerReconciler) finishProcessing(restorePhase velerov1api.R return kubeutil.PatchResourceWithRetriesOnErrors(r.resourceTimeout, original, restore, r.Client) } +type restoreItemOperationList struct { + items []*itemoperation.RestoreOperation +} + +func (r *restoreItemOperationList) selectByResource(group, resource, ns, name string) []*itemoperation.RestoreOperation { + var res []*itemoperation.RestoreOperation + rid := velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: group, + Resource: resource, + }, + Namespace: ns, + Name: name, + } + for _, item := range r.items { + if item != nil && item.Spec.ResourceIdentifier == rid { + res = append(res, item) + } + } + return res +} + +// SelectByPVC filters the restore item operation list by PVC namespace and name. +func (r *restoreItemOperationList) SelectByPVC(ns, name string) []*itemoperation.RestoreOperation { + return r.selectByResource("", "persistentvolumeclaims", ns, name) +} + // finalizerContext includes all the dependencies required by finalization tasks and // a function execute() to orderly implement task logic. type finalizerContext struct { - logger logrus.FieldLogger - restore *velerov1api.Restore - crClient client.Client - volumeInfo []*volume.BackupVolumeInfo - restoredPVCList map[string]struct{} - multiHookTracker *hook.MultiHookTracker - resourceTimeout time.Duration + logger logrus.FieldLogger + restore *velerov1api.Restore + crClient client.Client + volumeInfo []*volume.BackupVolumeInfo + restoredPVCList map[string]struct{} + restoreItemOperationList restoreItemOperationList + multiHookTracker *hook.MultiHookTracker + resourceTimeout time.Duration } func (ctx *finalizerContext) execute() (results.Result, results.Result) { //nolint:unparam //temporarily ignore the lint report: result 0 is always nil (unparam) @@ -310,6 +353,17 @@ func (ctx *finalizerContext) patchDynamicPVWithVolumeInfo() (errs results.Result return false, err } + // Check whether the async operation to populate the PVC is successful. If it's not, will skip patching the PV, instead of waiting. + operations := ctx.restoreItemOperationList.SelectByPVC(pvc.Namespace, pvc.Name) + for _, op := range operations { + if op.Spec.RestoreItemAction == constant.PluginCSIPVCRestoreRIA && + op.Status.Phase != itemoperation.OperationPhaseCompleted { + log.Warnf("skipping PV patch, because the operation to restore the PVC is not completed, "+ + "operation: %s, phase: %s", op.Spec.OperationID, op.Status.Phase) + return true, nil + } + } + // We are handling a common but specific scenario where a PVC is in a pending state and uses a storage class with // VolumeBindingMode set to WaitForFirstConsumer. In this case, the PV patch step is skipped to avoid // failures due to the PVC not being bound, which could cause a timeout and result in a failed restore. diff --git a/pkg/controller/restore_finalizer_controller_test.go b/pkg/controller/restore_finalizer_controller_test.go index 83738feb1..518b82424 100644 --- a/pkg/controller/restore_finalizer_controller_test.go +++ b/pkg/controller/restore_finalizer_controller_test.go @@ -23,6 +23,11 @@ import ( "testing" "time" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -145,6 +150,7 @@ func TestRestoreFinalizerReconcile(t *testing.T) { if test.restore != nil && test.restore.Namespace == velerov1api.DefaultNamespace { require.NoError(t, r.Client.Create(context.Background(), test.restore)) backupStore.On("GetRestoredResourceList", test.restore.Name).Return(map[string][]string{}, nil) + backupStore.On("GetRestoreItemOperations", test.restore.Name).Return([]*itemoperation.RestoreOperation{}, nil) } if test.backup != nil { assert.NoError(t, r.Client.Create(context.Background(), test.backup)) @@ -627,3 +633,112 @@ func Test_restoreFinalizerReconciler_finishProcessing(t *testing.T) { }) } } + +func TestRestoreOperationList(t *testing.T) { + var empty []*itemoperation.RestoreOperation + tests := []struct { + name string + items []*itemoperation.RestoreOperation + inputPVCNS string + inputPVCName string + expected []*itemoperation.RestoreOperation + }{ + { + name: "no restore operations", + items: []*itemoperation.RestoreOperation{}, + inputPVCNS: "ns-1", + inputPVCName: "pvc-1", + expected: empty, + }, + { + name: "one operation with matched info and a nil element", + items: []*itemoperation.RestoreOperation{ + nil, + { + Spec: itemoperation.RestoreOperationSpec{ + RestoreName: "restore-1", + RestoreUID: "uid-1", + RestoreItemAction: "velero.io/csi-pvc-restorer", + OperationID: "dd-abbb048d-7036-4855-bf50-ebba978b59a6.2426dd0e-b863-4222b5b2b", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "persistentvolumeclaims", + }, + Namespace: "ns-1", + Name: "pvc-1", + }, + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseCompleted, + OperationUnits: "Byte", + Description: "Completed", + }, + }, + }, + inputPVCNS: "ns-1", + inputPVCName: "pvc-1", + expected: []*itemoperation.RestoreOperation{ + { + Spec: itemoperation.RestoreOperationSpec{ + RestoreName: "restore-1", + RestoreUID: "uid-1", + RestoreItemAction: "velero.io/csi-pvc-restorer", + OperationID: "dd-abbb048d-7036-4855-bf50-ebba978b59a6.2426dd0e-b863-4222b5b2b", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "persistentvolumeclaims", + }, + Namespace: "ns-1", + Name: "pvc-1", + }, + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseCompleted, + OperationUnits: "Byte", + Description: "Completed", + }, + }, + }, + }, + { + name: "one operation with incorrect resource type", + items: []*itemoperation.RestoreOperation{ + { + Spec: itemoperation.RestoreOperationSpec{ + RestoreName: "restore-1", + RestoreUID: "uid-1", + RestoreItemAction: "velero.io/csi-pvc-restorer", + OperationID: "dd-abbb048d-7036-4855-bf50-ebba978b59a6.2426dd0e-b863-4222b5b2b", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "configmaps", + }, + Namespace: "ns-1", + Name: "pvc-1", + }, + }, + Status: itemoperation.OperationStatus{ + Phase: itemoperation.OperationPhaseCompleted, + OperationUnits: "Byte", + Description: "Completed", + }, + }, + }, + inputPVCNS: "ns-1", + inputPVCName: "pvc-1", + expected: empty, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := restoreItemOperationList{ + items: tt.items, + } + assert.Equal(t, tt.expected, l.SelectByPVC(tt.inputPVCNS, tt.inputPVCName)) + }) + } +} diff --git a/pkg/datamover/backup_micro_service_test.go b/pkg/datamover/backup_micro_service_test.go index 90aff37b3..417cf81cb 100644 --- a/pkg/datamover/backup_micro_service_test.go +++ b/pkg/datamover/backup_micro_service_test.go @@ -155,7 +155,7 @@ func TestOnDataUploadCompleted(t *testing.T) { { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), - expectedErr: "Failed to marshal backup result { false { }}: fake-marshal-error", + expectedErr: "Failed to marshal backup result { false { } 0}: fake-marshal-error", }, { name: "succeed", diff --git a/pkg/datamover/restore_micro_service_test.go b/pkg/datamover/restore_micro_service_test.go index c2e9ca4c2..f2328746c 100644 --- a/pkg/datamover/restore_micro_service_test.go +++ b/pkg/datamover/restore_micro_service_test.go @@ -101,7 +101,7 @@ func TestOnDataDownloadCompleted(t *testing.T) { { name: "marshal fail", marshalErr: errors.New("fake-marshal-error"), - expectedErr: "Failed to marshal restore result {{ }}: fake-marshal-error", + expectedErr: "Failed to marshal restore result {{ } 0}: fake-marshal-error", }, { name: "succeed", diff --git a/pkg/datapath/file_system_test.go b/pkg/datapath/file_system_test.go index fab33df1c..4e196595e 100644 --- a/pkg/datapath/file_system_test.go +++ b/pkg/datapath/file_system_test.go @@ -85,6 +85,7 @@ func TestAsyncBackup(t *testing.T) { SnapshotID: "fake-snapshot", EmptySnapshot: false, Source: AccessPoint{ByPath: "fake-path"}, + TotalBytes: 1000, }, }, path: "fake-path", @@ -95,7 +96,7 @@ func TestAsyncBackup(t *testing.T) { t.Run(test.name, func(t *testing.T) { fs := newFileSystemBR("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*fileSystemBR) mockProvider := providerMock.NewProvider(t) - mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.err) + mockProvider.On("RunBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Backup.SnapshotID, test.result.Backup.EmptySnapshot, test.result.Backup.TotalBytes, test.err) mockProvider.On("Close", mock.Anything).Return(nil) fs.uploaderProv = mockProvider fs.initialized = true @@ -167,7 +168,8 @@ func TestAsyncRestore(t *testing.T) { }, result: Result{ Restore: RestoreResult{ - Target: AccessPoint{ByPath: "fake-path"}, + Target: AccessPoint{ByPath: "fake-path"}, + TotalBytes: 1000, }, }, path: "fake-path", @@ -179,7 +181,7 @@ func TestAsyncRestore(t *testing.T) { t.Run(test.name, func(t *testing.T) { fs := newFileSystemBR("job-1", "test", nil, "velero", Callbacks{}, velerotest.NewLogger()).(*fileSystemBR) mockProvider := providerMock.NewProvider(t) - mockProvider.On("RunRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.err) + mockProvider.On("RunRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.result.Restore.TotalBytes, test.err) mockProvider.On("Close", mock.Anything).Return(nil) fs.uploaderProv = mockProvider fs.initialized = true diff --git a/pkg/datapath/micro_service_watcher_test.go b/pkg/datapath/micro_service_watcher_test.go index de71a0e9e..6d4d355a7 100644 --- a/pkg/datapath/micro_service_watcher_test.go +++ b/pkg/datapath/micro_service_watcher_test.go @@ -203,6 +203,7 @@ func TestStartWatch(t *testing.T) { expectStartEvent: true, expectTerminateEvent: true, expectComplete: true, + expectProgress: 1, }, { name: "completed", @@ -223,6 +224,7 @@ func TestStartWatch(t *testing.T) { expectStartEvent: true, expectTerminateEvent: true, expectComplete: true, + expectProgress: 1, }, { name: "completed with redirect error", @@ -244,6 +246,7 @@ func TestStartWatch(t *testing.T) { expectStartEvent: true, expectTerminateEvent: true, expectComplete: true, + expectProgress: 1, }, { name: "complete but terminated event not received in time", @@ -263,6 +266,7 @@ func TestStartWatch(t *testing.T) { }, expectStartEvent: true, expectComplete: true, + expectProgress: 1, }, { name: "complete but terminated event not received immediately", @@ -286,6 +290,7 @@ func TestStartWatch(t *testing.T) { expectStartEvent: true, expectTerminateEvent: true, expectComplete: true, + expectProgress: 1, }, { name: "completed with progress", @@ -313,7 +318,7 @@ func TestStartWatch(t *testing.T) { expectStartEvent: true, expectTerminateEvent: true, expectComplete: true, - expectProgress: 2, + expectProgress: 3, }, { name: "failed", diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 043462792..cfc34cc92 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -281,10 +281,16 @@ func (e *csiSnapshotExposer) GetExposed(ctx context.Context, ownerObject corev1. curLog.WithField("pod", pod.Name).Infof("Backup volume is found in pod at index %v", i) + var nodeOS *string + if os, found := pod.Spec.NodeSelector[kube.NodeOSLabel]; found { + nodeOS = &os + } + return &ExposeResult{ByPod: ExposeByPod{ HostingPod: pod, HostingContainer: containerName, VolumeName: volumeName, + NodeOS: nodeOS, }}, nil } @@ -580,6 +586,7 @@ func (e *csiSnapshotExposer) createBackupPod( var securityCtx *corev1.PodSecurityContext nodeSelector := map[string]string{} podOS := corev1.PodOS{} + toleration := []corev1.Toleration{} if nodeOS == kube.NodeOSWindows { userID := "ContainerAdministrator" securityCtx = &corev1.PodSecurityContext{ @@ -590,6 +597,13 @@ func (e *csiSnapshotExposer) createBackupPod( nodeSelector[kube.NodeOSLabel] = kube.NodeOSWindows podOS.Name = kube.NodeOSWindows + + toleration = append(toleration, corev1.Toleration{ + Key: "os", + Operator: "Equal", + Effect: "NoSchedule", + Value: "windows", + }) } else { userID := int64(0) securityCtx = &corev1.PodSecurityContext{ @@ -660,6 +674,7 @@ func (e *csiSnapshotExposer) createBackupPod( Volumes: volumes, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: securityCtx, + Tolerations: toleration, }, } diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index b85775389..5dc39e5a9 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -35,10 +35,37 @@ import ( "github.com/vmware-tanzu/velero/pkg/util/kube" ) +// GenericRestoreExposeParam define the input param for Generic Restore Expose +type GenericRestoreExposeParam struct { + // TargetPVCName is the target volume name to be restored + TargetPVCName string + + // TargetNamespace is the namespace of the volume to be restored + TargetNamespace string + + // HostingPodLabels is the labels that are going to apply to the hosting pod + HostingPodLabels map[string]string + + // Resources defines the resource requirements of the hosting pod + Resources corev1.ResourceRequirements + + // ExposeTimeout specifies the timeout for the entire expose process + ExposeTimeout time.Duration + + // OperationTimeout specifies the time wait for resources operations in Expose + OperationTimeout time.Duration + + // NodeOS specifies the OS of node that the volume should be attached + NodeOS string + + // RestorePVCConfig is the config for restorePVC (intermediate PVC) of generic restore + RestorePVCConfig nodeagent.RestorePVC +} + // GenericRestoreExposer is the interfaces for a generic restore exposer type GenericRestoreExposer interface { // Expose starts the process to a restore expose, the expose process may take long time - Expose(context.Context, corev1.ObjectReference, string, string, map[string]string, corev1.ResourceRequirements, time.Duration) error + Expose(context.Context, corev1.ObjectReference, GenericRestoreExposeParam) error // GetExposed polls the status of the expose. // If the expose is accessible by the current caller, it waits the expose ready and returns the expose result. @@ -74,25 +101,25 @@ type genericRestoreExposer struct { log logrus.FieldLogger } -func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1.ObjectReference, targetPVCName string, sourceNamespace string, hostingPodLabels map[string]string, resources corev1.ResourceRequirements, timeout time.Duration) error { +func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1.ObjectReference, param GenericRestoreExposeParam) error { curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, - "target PVC": targetPVCName, - "source namespace": sourceNamespace, + "target PVC": param.TargetPVCName, + "target namespace": param.TargetNamespace, }) - selectedNode, targetPVC, err := kube.WaitPVCConsumed(ctx, e.kubeClient.CoreV1(), targetPVCName, sourceNamespace, e.kubeClient.StorageV1(), timeout) + selectedNode, targetPVC, err := kube.WaitPVCConsumed(ctx, e.kubeClient.CoreV1(), param.TargetPVCName, param.TargetNamespace, e.kubeClient.StorageV1(), param.ExposeTimeout, param.RestorePVCConfig.IgnoreDelayBinding) if err != nil { - return errors.Wrapf(err, "error to wait target PVC consumed, %s/%s", sourceNamespace, targetPVCName) + return errors.Wrapf(err, "error to wait target PVC consumed, %s/%s", param.TargetNamespace, param.TargetPVCName) } - curLog.WithField("target PVC", targetPVCName).WithField("selected node", selectedNode).Info("Target PVC is consumed") + curLog.WithField("target PVC", param.TargetPVCName).WithField("selected node", selectedNode).Info("Target PVC is consumed") if kube.IsPVCBound(targetPVC) { - return errors.Errorf("Target PVC %s/%s has already been bound, abort", sourceNamespace, targetPVCName) + return errors.Errorf("Target PVC %s/%s has already been bound, abort", param.TargetNamespace, param.TargetPVCName) } - restorePod, err := e.createRestorePod(ctx, ownerObject, targetPVC, timeout, hostingPodLabels, selectedNode, resources) + restorePod, err := e.createRestorePod(ctx, ownerObject, targetPVC, param.OperationTimeout, param.HostingPodLabels, selectedNode, param.Resources, param.NodeOS) if err != nil { return errors.Wrapf(err, "error to create restore pod") } @@ -253,19 +280,19 @@ func (e *genericRestoreExposer) CleanUp(ctx context.Context, ownerObject corev1. kube.DeletePVAndPVCIfAny(ctx, e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, 0, e.log) } -func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject corev1.ObjectReference, targetPVCName string, sourceNamespace string, timeout time.Duration) error { +func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject corev1.ObjectReference, targetPVCName string, targetNamespace string, timeout time.Duration) error { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name curLog := e.log.WithFields(logrus.Fields{ "owner": ownerObject.Name, "target PVC": targetPVCName, - "source namespace": sourceNamespace, + "target namespace": targetNamespace, }) - targetPVC, err := e.kubeClient.CoreV1().PersistentVolumeClaims(sourceNamespace).Get(ctx, targetPVCName, metav1.GetOptions{}) + targetPVC, err := e.kubeClient.CoreV1().PersistentVolumeClaims(targetNamespace).Get(ctx, targetPVCName, metav1.GetOptions{}) if err != nil { - return errors.Wrapf(err, "error to get target PVC %s/%s", sourceNamespace, targetPVCName) + return errors.Wrapf(err, "error to get target PVC %s/%s", targetNamespace, targetPVCName) } restorePV, err := kube.WaitPVCBound(ctx, e.kubeClient.CoreV1(), e.kubeClient.CoreV1(), restorePVCName, ownerObject.Namespace, timeout) @@ -347,7 +374,7 @@ func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject co } func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObject corev1.ObjectReference, targetPVC *corev1.PersistentVolumeClaim, - operationTimeout time.Duration, label map[string]string, selectedNode string, resources corev1.ResourceRequirements) (*corev1.Pod, error) { + operationTimeout time.Duration, label map[string]string, selectedNode string, resources corev1.ResourceRequirements, nodeType string) (*corev1.Pod, error) { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name @@ -388,7 +415,36 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec args = append(args, podInfo.logFormatArgs...) args = append(args, podInfo.logLevelArgs...) - userID := int64(0) + var securityCtx *corev1.PodSecurityContext + nodeSelector := map[string]string{} + podOS := corev1.PodOS{} + toleration := []corev1.Toleration{} + if nodeType == kube.NodeOSWindows { + userID := "ContainerAdministrator" + securityCtx = &corev1.PodSecurityContext{ + WindowsOptions: &corev1.WindowsSecurityContextOptions{ + RunAsUserName: &userID, + }, + } + + nodeSelector[kube.NodeOSLabel] = kube.NodeOSWindows + podOS.Name = kube.NodeOSWindows + + toleration = append(toleration, corev1.Toleration{ + Key: "os", + Operator: "Equal", + Effect: "NoSchedule", + Value: "windows", + }) + } else { + userID := int64(0) + securityCtx = &corev1.PodSecurityContext{ + RunAsUser: &userID, + } + + nodeSelector[kube.NodeOSLabel] = kube.NodeOSLinux + podOS.Name = kube.NodeOSLinux + } pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -406,6 +462,8 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec Labels: label, }, Spec: corev1.PodSpec{ + NodeSelector: nodeSelector, + OS: &podOS, Containers: []corev1.Container{ { Name: containerName, @@ -429,9 +487,8 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec Volumes: volumes, NodeName: selectedNode, RestartPolicy: corev1.RestartPolicyNever, - SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: &userID, - }, + SecurityContext: securityCtx, + Tolerations: toleration, }, } diff --git a/pkg/exposer/generic_restore_test.go b/pkg/exposer/generic_restore_test.go index d2d56ece7..15f8c1615 100644 --- a/pkg/exposer/generic_restore_test.go +++ b/pkg/exposer/generic_restore_test.go @@ -31,7 +31,6 @@ import ( velerotest "github.com/vmware-tanzu/velero/pkg/test" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" corev1api "k8s.io/api/core/v1" clientTesting "k8s.io/client-go/testing" ) @@ -76,9 +75,9 @@ func TestRestoreExpose(t *testing.T) { APIVersion: appsv1.SchemeGroupVersion.String(), }, Spec: appsv1.DaemonSetSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ + Template: corev1api.PodTemplateSpec{ + Spec: corev1api.PodSpec{ + Containers: []corev1api.Container{ { Image: "fake-image", }, @@ -93,21 +92,21 @@ func TestRestoreExpose(t *testing.T) { kubeClientObj []runtime.Object ownerRestore *velerov1.Restore targetPVCName string - sourceNamespace string + targetNamespace string kubeReactors []reactor err string }{ { name: "wait target pvc consumed fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, err: "error to wait target PVC consumed, fake-ns/fake-target-pvc: error to wait for PVC: error to get pvc fake-ns/fake-target-pvc: persistentvolumeclaims \"fake-target-pvc\" not found", }, { name: "target pvc is already bound", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObjBound, @@ -117,7 +116,7 @@ func TestRestoreExpose(t *testing.T) { { name: "create restore pod fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -137,7 +136,7 @@ func TestRestoreExpose(t *testing.T) { { name: "create restore pvc fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -180,7 +179,12 @@ func TestRestoreExpose(t *testing.T) { } } - err := exposer.Expose(context.Background(), ownerObject, test.targetPVCName, test.sourceNamespace, map[string]string{}, corev1.ResourceRequirements{}, time.Millisecond) + err := exposer.Expose(context.Background(), ownerObject, GenericRestoreExposeParam{ + TargetPVCName: test.targetPVCName, + TargetNamespace: test.targetNamespace, + HostingPodLabels: map[string]string{}, + Resources: corev1api.ResourceRequirements{}, + ExposeTimeout: time.Millisecond}) assert.EqualError(t, err, test.err) }) } @@ -239,21 +243,21 @@ func TestRebindVolume(t *testing.T) { kubeClientObj []runtime.Object ownerRestore *velerov1.Restore targetPVCName string - sourceNamespace string + targetNamespace string kubeReactors []reactor err string }{ { name: "get target pvc fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, err: "error to get target PVC fake-ns/fake-target-pvc: persistentvolumeclaims \"fake-target-pvc\" not found", }, { name: "wait restore pvc bound fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -263,7 +267,7 @@ func TestRebindVolume(t *testing.T) { { name: "retain target pv fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -284,7 +288,7 @@ func TestRebindVolume(t *testing.T) { { name: "delete restore pod fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -306,7 +310,7 @@ func TestRebindVolume(t *testing.T) { { name: "delete restore pvc fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -328,7 +332,7 @@ func TestRebindVolume(t *testing.T) { { name: "rebind target pvc fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -350,7 +354,7 @@ func TestRebindVolume(t *testing.T) { { name: "reset pv binding fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -377,7 +381,7 @@ func TestRebindVolume(t *testing.T) { { name: "wait restore PV bound fail", targetPVCName: "fake-target-pvc", - sourceNamespace: "fake-ns", + targetNamespace: "fake-ns", ownerRestore: restore, kubeClientObj: []runtime.Object{ targetPVCObj, @@ -415,7 +419,7 @@ func TestRebindVolume(t *testing.T) { hookCount = 0 - err := exposer.RebindVolume(context.Background(), ownerObject, test.targetPVCName, test.sourceNamespace, time.Millisecond) + err := exposer.RebindVolume(context.Background(), ownerObject, test.targetPVCName, test.targetNamespace, time.Millisecond) assert.EqualError(t, err, test.err) }) } @@ -521,7 +525,7 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) { }, } - restorePodWithoutNodeName := corev1.Pod{ + restorePodWithoutNodeName := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", @@ -534,19 +538,19 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) { }, }, }, - Status: corev1.PodStatus{ - Phase: corev1.PodPending, - Conditions: []corev1.PodCondition{ + Status: corev1api.PodStatus{ + Phase: corev1api.PodPending, + Conditions: []corev1api.PodCondition{ { - Type: corev1.PodInitialized, - Status: corev1.ConditionTrue, + Type: corev1api.PodInitialized, + Status: corev1api.ConditionTrue, Message: "fake-pod-message", }, }, }, } - restorePodWithNodeName := corev1.Pod{ + restorePodWithNodeName := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", @@ -559,22 +563,22 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) { }, }, }, - Spec: corev1.PodSpec{ + Spec: corev1api.PodSpec{ NodeName: "fake-node", }, - Status: corev1.PodStatus{ - Phase: corev1.PodPending, - Conditions: []corev1.PodCondition{ + Status: corev1api.PodStatus{ + Phase: corev1api.PodPending, + Conditions: []corev1api.PodCondition{ { - Type: corev1.PodInitialized, - Status: corev1.ConditionTrue, + Type: corev1api.PodInitialized, + Status: corev1api.ConditionTrue, Message: "fake-pod-message", }, }, }, } - restorePVCWithoutVolumeName := corev1.PersistentVolumeClaim{ + restorePVCWithoutVolumeName := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", @@ -587,12 +591,12 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) { }, }, }, - Status: corev1.PersistentVolumeClaimStatus{ - Phase: corev1.ClaimPending, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimPending, }, } - restorePVCWithVolumeName := corev1.PersistentVolumeClaim{ + restorePVCWithVolumeName := corev1api.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "fake-restore", @@ -605,35 +609,35 @@ func Test_ReastoreDiagnoseExpose(t *testing.T) { }, }, }, - Spec: corev1.PersistentVolumeClaimSpec{ + Spec: corev1api.PersistentVolumeClaimSpec{ VolumeName: "fake-pv", }, - Status: corev1.PersistentVolumeClaimStatus{ - Phase: corev1.ClaimPending, + Status: corev1api.PersistentVolumeClaimStatus{ + Phase: corev1api.ClaimPending, }, } - restorePV := corev1.PersistentVolume{ + restorePV := corev1api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-pv", }, - Status: corev1.PersistentVolumeStatus{ - Phase: corev1.VolumePending, + Status: corev1api.PersistentVolumeStatus{ + Phase: corev1api.VolumePending, Message: "fake-pv-message", }, } - nodeAgentPod := corev1.Pod{ + nodeAgentPod := corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: velerov1.DefaultNamespace, Name: "node-agent-pod-1", Labels: map[string]string{"role": "node-agent"}, }, - Spec: corev1.PodSpec{ + Spec: corev1api.PodSpec{ NodeName: "fake-node", }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, + Status: corev1api.PodStatus{ + Phase: corev1api.PodRunning, }, } diff --git a/pkg/exposer/mocks/GenericRestoreExposer.go b/pkg/exposer/mocks/GenericRestoreExposer.go index 83a9789af..fb8fc2790 100644 --- a/pkg/exposer/mocks/GenericRestoreExposer.go +++ b/pkg/exposer/mocks/GenericRestoreExposer.go @@ -44,17 +44,17 @@ func (_m *GenericRestoreExposer) DiagnoseExpose(_a0 context.Context, _a1 v1.Obje return r0 } -// Expose provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4, _a5, _a6 -func (_m *GenericRestoreExposer) Expose(_a0 context.Context, _a1 v1.ObjectReference, _a2 string, _a3 string, _a4 map[string]string, _a5 v1.ResourceRequirements, _a6 time.Duration) error { - ret := _m.Called(_a0, _a1, _a2, _a3, _a4, _a5, _a6) +// Expose provides a mock function with given fields: _a0, _a1, _a2 +func (_m *GenericRestoreExposer) Expose(_a0 context.Context, _a1 v1.ObjectReference, _a2 exposer.GenericRestoreExposeParam) error { + ret := _m.Called(_a0, _a1, _a2) if len(ret) == 0 { panic("no return value specified for Expose") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, v1.ObjectReference, string, string, map[string]string, v1.ResourceRequirements, time.Duration) error); ok { - r0 = rf(_a0, _a1, _a2, _a3, _a4, _a5, _a6) + if rf, ok := ret.Get(0).(func(context.Context, v1.ObjectReference, exposer.GenericRestoreExposeParam) error); ok { + r0 = rf(_a0, _a1, _a2) } else { r0 = ret.Error(0) } diff --git a/pkg/exposer/types.go b/pkg/exposer/types.go index d4d8c8730..670c8b661 100644 --- a/pkg/exposer/types.go +++ b/pkg/exposer/types.go @@ -38,4 +38,5 @@ type ExposeByPod struct { HostingPod *corev1.Pod HostingContainer string VolumeName string + NodeOS *string } diff --git a/pkg/label/label.go b/pkg/label/label.go index 411de2cda..98d232f24 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -18,6 +18,7 @@ package label import ( "crypto/sha256" + "encoding/hex" "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,7 +39,7 @@ func GetValidName(label string) string { } sha := sha256.Sum256([]byte(label)) - strSha := fmt.Sprintf("%x", sha) + strSha := hex.EncodeToString(sha[:]) charsFromLabel := validation.DNS1035LabelMaxLength - 6 if charsFromLabel < 0 { // Derive the label name from sha hash in case the DNS1035LabelMaxLength is less than 6 diff --git a/pkg/nodeagent/node_agent.go b/pkg/nodeagent/node_agent.go index b31b1dda5..44d0cebd2 100644 --- a/pkg/nodeagent/node_agent.go +++ b/pkg/nodeagent/node_agent.go @@ -81,6 +81,11 @@ type BackupPVC struct { SPCNoRelabeling bool `json:"spcNoRelabeling,omitempty"` } +type RestorePVC struct { + // IgnoreDelayBinding indicates to ignore delay binding the restorePVC when it is in WaitForFirstConsumer mode + IgnoreDelayBinding bool `json:"ignoreDelayBinding,omitempty"` +} + type Configs struct { // LoadConcurrency is the config for data path load concurrency per node. LoadConcurrency *LoadConcurrency `json:"loadConcurrency,omitempty"` @@ -91,6 +96,9 @@ type Configs struct { // BackupPVCConfig is the config for backupPVC (intermediate PVC) of snapshot data movement BackupPVCConfig map[string]BackupPVC `json:"backupPVC,omitempty"` + // RestoreVCConfig is the config for restorePVC (intermediate PVC) of generic restore + RestorePVCConfig *RestorePVC `json:"restorePVC,omitempty"` + // PodResources is the resource config for various types of pods launched by node-agent, i.e., data mover pods. PodResources *kube.PodResources `json:"podResources,omitempty"` } diff --git a/pkg/persistence/mocks/backup_store.go b/pkg/persistence/mocks/backup_store.go index 8abdd35bb..bd1ef5dbd 100644 --- a/pkg/persistence/mocks/backup_store.go +++ b/pkg/persistence/mocks/backup_store.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -// Code generated by mockery v2.16.0. DO NOT EDIT. +// Code generated by mockery v2.42.2. DO NOT EDIT. package mocks @@ -21,15 +21,17 @@ import ( io "io" mock "github.com/stretchr/testify/mock" - volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" - itemoperation "github.com/vmware-tanzu/velero/pkg/itemoperation" - "github.com/vmware-tanzu/velero/pkg/persistence" + + persistence "github.com/vmware-tanzu/velero/pkg/persistence" + + results "github.com/vmware-tanzu/velero/pkg/util/results" + v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/internal/volume" - "github.com/vmware-tanzu/velero/pkg/util/results" + volume "github.com/vmware-tanzu/velero/internal/volume" + volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" ) // BackupStore is an autogenerated mock type for the BackupStore type @@ -41,14 +43,21 @@ type BackupStore struct { func (_m *BackupStore) BackupExists(bucket string, backupName string) (bool, error) { ret := _m.Called(bucket, backupName) + if len(ret) == 0 { + panic("no return value specified for BackupExists") + } + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(string, string) (bool, error)); ok { + return rf(bucket, backupName) + } if rf, ok := ret.Get(0).(func(string, string) bool); ok { r0 = rf(bucket, backupName) } else { r0 = ret.Get(0).(bool) } - var r1 error if rf, ok := ret.Get(1).(func(string, string) error); ok { r1 = rf(bucket, backupName) } else { @@ -62,6 +71,10 @@ func (_m *BackupStore) BackupExists(bucket string, backupName string) (bool, err func (_m *BackupStore) DeleteBackup(name string) error { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for DeleteBackup") + } + var r0 error if rf, ok := ret.Get(0).(func(string) error); ok { r0 = rf(name) @@ -76,6 +89,10 @@ func (_m *BackupStore) DeleteBackup(name string) error { func (_m *BackupStore) DeleteRestore(name string) error { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for DeleteRestore") + } + var r0 error if rf, ok := ret.Get(0).(func(string) error); ok { r0 = rf(name) @@ -90,7 +107,15 @@ func (_m *BackupStore) DeleteRestore(name string) error { func (_m *BackupStore) GetBackupContents(name string) (io.ReadCloser, error) { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for GetBackupContents") + } + var r0 io.ReadCloser + var r1 error + if rf, ok := ret.Get(0).(func(string) (io.ReadCloser, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) io.ReadCloser); ok { r0 = rf(name) } else { @@ -99,7 +124,6 @@ func (_m *BackupStore) GetBackupContents(name string) (io.ReadCloser, error) { } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -113,7 +137,15 @@ func (_m *BackupStore) GetBackupContents(name string) (io.ReadCloser, error) { func (_m *BackupStore) GetBackupItemOperations(name string) ([]*itemoperation.BackupOperation, error) { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for GetBackupItemOperations") + } + var r0 []*itemoperation.BackupOperation + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*itemoperation.BackupOperation, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) []*itemoperation.BackupOperation); ok { r0 = rf(name) } else { @@ -122,7 +154,6 @@ func (_m *BackupStore) GetBackupItemOperations(name string) ([]*itemoperation.Ba } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -136,7 +167,15 @@ func (_m *BackupStore) GetBackupItemOperations(name string) ([]*itemoperation.Ba func (_m *BackupStore) GetBackupMetadata(name string) (*v1.Backup, error) { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for GetBackupMetadata") + } + var r0 *v1.Backup + var r1 error + if rf, ok := ret.Get(0).(func(string) (*v1.Backup, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) *v1.Backup); ok { r0 = rf(name) } else { @@ -145,7 +184,36 @@ func (_m *BackupStore) GetBackupMetadata(name string) (*v1.Backup, error) { } } + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetBackupVolumeInfos provides a mock function with given fields: name +func (_m *BackupStore) GetBackupVolumeInfos(name string) ([]*volume.BackupVolumeInfo, error) { + ret := _m.Called(name) + + if len(ret) == 0 { + panic("no return value specified for GetBackupVolumeInfos") + } + + var r0 []*volume.BackupVolumeInfo var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*volume.BackupVolumeInfo, error)); ok { + return rf(name) + } + if rf, ok := ret.Get(0).(func(string) []*volume.BackupVolumeInfo); ok { + r0 = rf(name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*volume.BackupVolumeInfo) + } + } + if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -159,7 +227,15 @@ func (_m *BackupStore) GetBackupMetadata(name string) (*v1.Backup, error) { func (_m *BackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error) { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for GetBackupVolumeSnapshots") + } + var r0 []*volume.Snapshot + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*volume.Snapshot, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) []*volume.Snapshot); ok { r0 = rf(name) } else { @@ -168,7 +244,6 @@ func (_m *BackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -182,7 +257,15 @@ func (_m *BackupStore) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot func (_m *BackupStore) GetCSIVolumeSnapshotClasses(name string) ([]*volumesnapshotv1.VolumeSnapshotClass, error) { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for GetCSIVolumeSnapshotClasses") + } + var r0 []*volumesnapshotv1.VolumeSnapshotClass + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*volumesnapshotv1.VolumeSnapshotClass, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshotClass); ok { r0 = rf(name) } else { @@ -191,7 +274,6 @@ func (_m *BackupStore) GetCSIVolumeSnapshotClasses(name string) ([]*volumesnapsh } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -205,7 +287,15 @@ func (_m *BackupStore) GetCSIVolumeSnapshotClasses(name string) ([]*volumesnapsh func (_m *BackupStore) GetCSIVolumeSnapshotContents(name string) ([]*volumesnapshotv1.VolumeSnapshotContent, error) { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for GetCSIVolumeSnapshotContents") + } + var r0 []*volumesnapshotv1.VolumeSnapshotContent + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*volumesnapshotv1.VolumeSnapshotContent, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshotContent); ok { r0 = rf(name) } else { @@ -214,7 +304,6 @@ func (_m *BackupStore) GetCSIVolumeSnapshotContents(name string) ([]*volumesnaps } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -228,7 +317,15 @@ func (_m *BackupStore) GetCSIVolumeSnapshotContents(name string) ([]*volumesnaps func (_m *BackupStore) GetCSIVolumeSnapshots(name string) ([]*volumesnapshotv1.VolumeSnapshot, error) { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for GetCSIVolumeSnapshots") + } + var r0 []*volumesnapshotv1.VolumeSnapshot + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*volumesnapshotv1.VolumeSnapshot, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) []*volumesnapshotv1.VolumeSnapshot); ok { r0 = rf(name) } else { @@ -237,7 +334,6 @@ func (_m *BackupStore) GetCSIVolumeSnapshots(name string) ([]*volumesnapshotv1.V } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -251,14 +347,21 @@ func (_m *BackupStore) GetCSIVolumeSnapshots(name string) ([]*volumesnapshotv1.V func (_m *BackupStore) GetDownloadURL(target v1.DownloadTarget) (string, error) { ret := _m.Called(target) + if len(ret) == 0 { + panic("no return value specified for GetDownloadURL") + } + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(v1.DownloadTarget) (string, error)); ok { + return rf(target) + } if rf, ok := ret.Get(0).(func(v1.DownloadTarget) string); ok { r0 = rf(target) } else { r0 = ret.Get(0).(string) } - var r1 error if rf, ok := ret.Get(1).(func(v1.DownloadTarget) error); ok { r1 = rf(target) } else { @@ -272,7 +375,15 @@ func (_m *BackupStore) GetDownloadURL(target v1.DownloadTarget) (string, error) func (_m *BackupStore) GetPodVolumeBackups(name string) ([]*v1.PodVolumeBackup, error) { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for GetPodVolumeBackups") + } + var r0 []*v1.PodVolumeBackup + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*v1.PodVolumeBackup, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) []*v1.PodVolumeBackup); ok { r0 = rf(name) } else { @@ -281,7 +392,6 @@ func (_m *BackupStore) GetPodVolumeBackups(name string) ([]*v1.PodVolumeBackup, } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -295,7 +405,15 @@ func (_m *BackupStore) GetPodVolumeBackups(name string) ([]*v1.PodVolumeBackup, func (_m *BackupStore) GetRestoreItemOperations(name string) ([]*itemoperation.RestoreOperation, error) { ret := _m.Called(name) + if len(ret) == 0 { + panic("no return value specified for GetRestoreItemOperations") + } + var r0 []*itemoperation.RestoreOperation + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]*itemoperation.RestoreOperation, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) []*itemoperation.RestoreOperation); ok { r0 = rf(name) } else { @@ -304,7 +422,6 @@ func (_m *BackupStore) GetRestoreItemOperations(name string) ([]*itemoperation.R } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -314,48 +431,19 @@ func (_m *BackupStore) GetRestoreItemOperations(name string) ([]*itemoperation.R return r0, r1 } -// GetRestoreItemOperations provides a mock function with given fields: name -func (_m *BackupStore) GetBackupVolumeInfos(name string) ([]*volume.BackupVolumeInfo, error) { - ret := _m.Called(name) - - var r0 []*volume.BackupVolumeInfo - if rf, ok := ret.Get(0).(func(string) []*volume.BackupVolumeInfo); ok { - r0 = rf(name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*volume.BackupVolumeInfo) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(name) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// PutBackupVolumeInfos provides a mock function with given fields: name, volumeInfo -func (_m *BackupStore) PutBackupVolumeInfos(name string, volumeInfo io.Reader) error { - ret := _m.Called(name, volumeInfo) - - var r0 error - if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { - r0 = rf(name, volumeInfo) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // GetRestoreResults provides a mock function with given fields: name func (_m *BackupStore) GetRestoreResults(name string) (map[string]results.Result, error) { ret := _m.Called(name) - r0 := make(map[string]results.Result) + if len(ret) == 0 { + panic("no return value specified for GetRestoreResults") + } + + var r0 map[string]results.Result + var r1 error + if rf, ok := ret.Get(0).(func(string) (map[string]results.Result, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) map[string]results.Result); ok { r0 = rf(name) } else { @@ -364,7 +452,6 @@ func (_m *BackupStore) GetRestoreResults(name string) (map[string]results.Result } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -378,7 +465,15 @@ func (_m *BackupStore) GetRestoreResults(name string) (map[string]results.Result func (_m *BackupStore) GetRestoredResourceList(name string) (map[string][]string, error) { ret := _m.Called(name) - r0 := make(map[string][]string) + if len(ret) == 0 { + panic("no return value specified for GetRestoredResourceList") + } + + var r0 map[string][]string + var r1 error + if rf, ok := ret.Get(0).(func(string) (map[string][]string, error)); ok { + return rf(name) + } if rf, ok := ret.Get(0).(func(string) map[string][]string); ok { r0 = rf(name) } else { @@ -387,7 +482,6 @@ func (_m *BackupStore) GetRestoredResourceList(name string) (map[string][]string } } - var r1 error if rf, ok := ret.Get(1).(func(string) error); ok { r1 = rf(name) } else { @@ -401,6 +495,10 @@ func (_m *BackupStore) GetRestoredResourceList(name string) (map[string][]string func (_m *BackupStore) IsValid() error { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for IsValid") + } + var r0 error if rf, ok := ret.Get(0).(func() error); ok { r0 = rf() @@ -415,7 +513,15 @@ func (_m *BackupStore) IsValid() error { func (_m *BackupStore) ListBackups() ([]string, error) { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for ListBackups") + } + var r0 []string + var r1 error + if rf, ok := ret.Get(0).(func() ([]string, error)); ok { + return rf() + } if rf, ok := ret.Get(0).(func() []string); ok { r0 = rf() } else { @@ -424,7 +530,6 @@ func (_m *BackupStore) ListBackups() ([]string, error) { } } - var r1 error if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { @@ -438,6 +543,10 @@ func (_m *BackupStore) ListBackups() ([]string, error) { func (_m *BackupStore) PutBackup(info persistence.BackupInfo) error { ret := _m.Called(info) + if len(ret) == 0 { + panic("no return value specified for PutBackup") + } + var r0 error if rf, ok := ret.Get(0).(func(persistence.BackupInfo) error); ok { r0 = rf(info) @@ -452,6 +561,10 @@ func (_m *BackupStore) PutBackup(info persistence.BackupInfo) error { func (_m *BackupStore) PutBackupContents(backup string, backupContents io.Reader) error { ret := _m.Called(backup, backupContents) + if len(ret) == 0 { + panic("no return value specified for PutBackupContents") + } + var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { r0 = rf(backup, backupContents) @@ -466,6 +579,10 @@ func (_m *BackupStore) PutBackupContents(backup string, backupContents io.Reader func (_m *BackupStore) PutBackupItemOperations(backup string, backupItemOperations io.Reader) error { ret := _m.Called(backup, backupItemOperations) + if len(ret) == 0 { + panic("no return value specified for PutBackupItemOperations") + } + var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { r0 = rf(backup, backupItemOperations) @@ -480,6 +597,10 @@ func (_m *BackupStore) PutBackupItemOperations(backup string, backupItemOperatio func (_m *BackupStore) PutBackupMetadata(backup string, backupMetadata io.Reader) error { ret := _m.Called(backup, backupMetadata) + if len(ret) == 0 { + panic("no return value specified for PutBackupMetadata") + } + var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { r0 = rf(backup, backupMetadata) @@ -490,10 +611,32 @@ func (_m *BackupStore) PutBackupMetadata(backup string, backupMetadata io.Reader return r0 } +// PutBackupVolumeInfos provides a mock function with given fields: name, volumeInfo +func (_m *BackupStore) PutBackupVolumeInfos(name string, volumeInfo io.Reader) error { + ret := _m.Called(name, volumeInfo) + + if len(ret) == 0 { + panic("no return value specified for PutBackupVolumeInfos") + } + + var r0 error + if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { + r0 = rf(name, volumeInfo) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // PutRestoreItemOperations provides a mock function with given fields: restore, restoreItemOperations func (_m *BackupStore) PutRestoreItemOperations(restore string, restoreItemOperations io.Reader) error { ret := _m.Called(restore, restoreItemOperations) + if len(ret) == 0 { + panic("no return value specified for PutRestoreItemOperations") + } + var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { r0 = rf(restore, restoreItemOperations) @@ -508,6 +651,10 @@ func (_m *BackupStore) PutRestoreItemOperations(restore string, restoreItemOpera func (_m *BackupStore) PutRestoreLog(backup string, restore string, log io.Reader) error { ret := _m.Called(backup, restore, log) + if len(ret) == 0 { + panic("no return value specified for PutRestoreLog") + } + var r0 error if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { r0 = rf(backup, restore, log) @@ -518,13 +665,17 @@ func (_m *BackupStore) PutRestoreLog(backup string, restore string, log io.Reade return r0 } -// PutRestoreResults provides a mock function with given fields: backup, restore, results -func (_m *BackupStore) PutRestoreResults(backup string, restore string, results io.Reader) error { - ret := _m.Called(backup, restore, results) +// PutRestoreResults provides a mock function with given fields: backup, restore, _a2 +func (_m *BackupStore) PutRestoreResults(backup string, restore string, _a2 io.Reader) error { + ret := _m.Called(backup, restore, _a2) + + if len(ret) == 0 { + panic("no return value specified for PutRestoreResults") + } var r0 error if rf, ok := ret.Get(0).(func(string, string, io.Reader) error); ok { - r0 = rf(backup, restore, results) + r0 = rf(backup, restore, _a2) } else { r0 = ret.Error(0) } @@ -532,13 +683,17 @@ func (_m *BackupStore) PutRestoreResults(backup string, restore string, results return r0 } -// PutRestoredResourceList provides a mock function with given fields: restore, results -func (_m *BackupStore) PutRestoredResourceList(restore string, results io.Reader) error { - ret := _m.Called(restore, results) +// PutRestoreVolumeInfo provides a mock function with given fields: restore, volumeInfo +func (_m *BackupStore) PutRestoreVolumeInfo(restore string, volumeInfo io.Reader) error { + ret := _m.Called(restore, volumeInfo) + + if len(ret) == 0 { + panic("no return value specified for PutRestoreVolumeInfo") + } var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { - r0 = rf(restore, results) + r0 = rf(restore, volumeInfo) } else { r0 = ret.Error(0) } @@ -546,26 +701,30 @@ func (_m *BackupStore) PutRestoredResourceList(restore string, results io.Reader return r0 } -// PutRestoreVolumeInfo provides a mock function with given fields: restore, results -func (_m *BackupStore) PutRestoreVolumeInfo(restore string, results io.Reader) error { - ret := _m.Called(restore, results) +// PutRestoredResourceList provides a mock function with given fields: restore, _a1 +func (_m *BackupStore) PutRestoredResourceList(restore string, _a1 io.Reader) error { + ret := _m.Called(restore, _a1) + + if len(ret) == 0 { + panic("no return value specified for PutRestoredResourceList") + } var r0 error if rf, ok := ret.Get(0).(func(string, io.Reader) error); ok { - r0 = rf(restore, results) + r0 = rf(restore, _a1) } else { r0 = ret.Error(0) } return r0 } -type mockConstructorTestingTNewBackupStore interface { - mock.TestingT - Cleanup(func()) -} // NewBackupStore creates a new instance of BackupStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewBackupStore(t mockConstructorTestingTNewBackupStore) *BackupStore { +// The first argument is typically a *testing.T value. +func NewBackupStore(t interface { + mock.TestingT + Cleanup(func()) +}) *BackupStore { mock := &BackupStore{} mock.Mock.Test(t) diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index 29452344e..09ad91db8 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -48,6 +48,8 @@ type Backupper interface { // BackupPodVolumes backs up all specified volumes in a pod. BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api.Pod, volumesToBackup []string, resPolicies *resourcepolicies.Policies, log logrus.FieldLogger) ([]*velerov1api.PodVolumeBackup, *PVCBackupSummary, []error) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velerov1api.PodVolumeBackup + GetPodVolumeBackup(namespace, name string) (*velerov1api.PodVolumeBackup, error) + ListPodVolumeBackupsByPod(podNamespace, podName string) ([]*velerov1api.PodVolumeBackup, error) } type backupper struct { @@ -59,7 +61,10 @@ type backupper struct { pvbInformer ctrlcache.Informer handlerRegistration cache.ResourceEventHandlerRegistration wg sync.WaitGroup - result []*velerov1api.PodVolumeBackup + // pvbIndexer holds all PVBs created by this backuper and is capable to search + // the PVBs based on specific properties quickly because of the embedded indexes. + // The statuses of the PVBs are got updated when Informer receives update events. + pvbIndexer cache.Indexer } type skippedPVC struct { @@ -101,8 +106,22 @@ func (pbs *PVCBackupSummary) addSkipped(volumeName string, reason string) { } } +const indexNamePod = "POD" + +func podIndexFunc(obj interface{}) ([]string, error) { + pvb, ok := obj.(*velerov1api.PodVolumeBackup) + if !ok { + return nil, errors.Errorf("expected PodVolumeBackup, but got %T", obj) + } + if pvb == nil { + return nil, errors.New("PodVolumeBackup is nil") + } + return []string{cache.NewObjectName(pvb.Spec.Pod.Namespace, pvb.Spec.Pod.Name).String()}, nil +} + func newBackupper( ctx context.Context, + log logrus.FieldLogger, repoLocker *repository.RepoLocker, repoEnsurer *repository.Ensurer, pvbInformer ctrlcache.Informer, @@ -118,13 +137,19 @@ func newBackupper( uploaderType: uploaderType, pvbInformer: pvbInformer, wg: sync.WaitGroup{}, - result: []*velerov1api.PodVolumeBackup{}, + pvbIndexer: cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{ + indexNamePod: podIndexFunc, + }), } b.handlerRegistration, _ = pvbInformer.AddEventHandler( cache.ResourceEventHandlerFuncs{ UpdateFunc: func(_, obj interface{}) { - pvb := obj.(*velerov1api.PodVolumeBackup) + pvb, ok := obj.(*velerov1api.PodVolumeBackup) + if !ok { + log.Errorf("expected PodVolumeBackup, but got %T", obj) + return + } if pvb.GetLabels()[velerov1api.BackupUIDLabel] != string(backup.UID) { return @@ -135,7 +160,10 @@ func newBackupper( return } - b.result = append(b.result, pvb) + // the Indexer inserts PVB directly if the PVB to be updated doesn't exist + if err := b.pvbIndexer.Update(pvb); err != nil { + log.WithError(err).Errorf("failed to update PVB %s/%s in indexer", pvb.Namespace, pvb.Name) + } b.wg.Done() }, }, @@ -318,6 +346,12 @@ func (b *backupper) BackupPodVolumes(backup *velerov1api.Backup, pod *corev1api. continue } b.wg.Add(1) + + if err := b.pvbIndexer.Add(volumeBackup); err != nil { + errs = append(errs, errors.Wrapf(err, "failed to add PodVolumeBackup %s/%s to indexer", volumeBackup.Namespace, volumeBackup.Name)) + continue + } + podVolumeBackups = append(podVolumeBackups, volumeBackup) pvcSummary.addBackedup(volumeName) } @@ -343,7 +377,12 @@ func (b *backupper) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velero case <-b.ctx.Done(): log.Error("timed out waiting for all PodVolumeBackups to complete") case <-done: - for _, pvb := range b.result { + for _, obj := range b.pvbIndexer.List() { + pvb, ok := obj.(*velerov1api.PodVolumeBackup) + if !ok { + log.Errorf("expected PodVolumeBackup, but got %T", obj) + continue + } podVolumeBackups = append(podVolumeBackups, pvb) if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed { log.Errorf("pod volume backup failed: %s", pvb.Status.Message) @@ -353,6 +392,37 @@ func (b *backupper) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velero return podVolumeBackups } +func (b *backupper) GetPodVolumeBackup(namespace, name string) (*velerov1api.PodVolumeBackup, error) { + obj, exist, err := b.pvbIndexer.GetByKey(cache.NewObjectName(namespace, name).String()) + if err != nil { + return nil, err + } + if !exist { + return nil, nil + } + pvb, ok := obj.(*velerov1api.PodVolumeBackup) + if !ok { + return nil, errors.Errorf("expected PodVolumeBackup, but got %T", obj) + } + return pvb, nil +} + +func (b *backupper) ListPodVolumeBackupsByPod(podNamespace, podName string) ([]*velerov1api.PodVolumeBackup, error) { + objs, err := b.pvbIndexer.ByIndex(indexNamePod, cache.NewObjectName(podNamespace, podName).String()) + if err != nil { + return nil, err + } + var pvbs []*velerov1api.PodVolumeBackup + for _, obj := range objs { + pvb, ok := obj.(*velerov1api.PodVolumeBackup) + if !ok { + return nil, errors.Errorf("expected PodVolumeBackup, but got %T", obj) + } + pvbs = append(pvbs, pvb) + } + return pvbs, nil +} + func skipAllPodVolumes(pod *corev1api.Pod, volumesToBackup []string, err error, pvcSummary *PVCBackupSummary, log logrus.FieldLogger) { for _, volumeName := range volumesToBackup { log.WithError(err).Warnf("Skip pod volume %s", volumeName) diff --git a/pkg/podvolume/backupper_factory.go b/pkg/podvolume/backupper_factory.go index fb166f110..f75f1d30b 100644 --- a/pkg/podvolume/backupper_factory.go +++ b/pkg/podvolume/backupper_factory.go @@ -32,7 +32,7 @@ import ( // BackupperFactory can construct pod volumes backuppers. type BackupperFactory interface { // NewBackupper returns a pod volumes backupper for use during a single Velero backup. - NewBackupper(context.Context, *velerov1api.Backup, string) (Backupper, error) + NewBackupper(context.Context, logrus.FieldLogger, *velerov1api.Backup, string) (Backupper, error) } func NewBackupperFactory( @@ -59,8 +59,8 @@ type backupperFactory struct { log logrus.FieldLogger } -func (bf *backupperFactory) NewBackupper(ctx context.Context, backup *velerov1api.Backup, uploaderType string) (Backupper, error) { - b := newBackupper(ctx, bf.repoLocker, bf.repoEnsurer, bf.pvbInformer, bf.crClient, uploaderType, backup) +func (bf *backupperFactory) NewBackupper(ctx context.Context, log logrus.FieldLogger, backup *velerov1api.Backup, uploaderType string) (Backupper, error) { + b := newBackupper(ctx, log, bf.repoLocker, bf.repoEnsurer, bf.pvbInformer, bf.crClient, uploaderType, backup) if !cache.WaitForCacheSync(ctx.Done(), bf.pvbInformer.HasSynced) { return nil, errors.New("timed out waiting for caches to sync") diff --git a/pkg/podvolume/backupper_test.go b/pkg/podvolume/backupper_test.go index 941436830..1456bdf11 100644 --- a/pkg/podvolume/backupper_test.go +++ b/pkg/podvolume/backupper_test.go @@ -315,6 +315,7 @@ func TestBackupPodVolumes(t *testing.T) { scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) corev1api.AddToScheme(scheme) + log := logrus.New() tests := []struct { name string @@ -594,7 +595,7 @@ func TestBackupPodVolumes(t *testing.T) { backupObj.Spec.StorageLocation = test.bsl factory := NewBackupperFactory(repository.NewRepoLocker(), ensurer, fakeCtrlClient, pvbInformer, velerotest.NewLogger()) - bp, err := factory.NewBackupper(ctx, backupObj, test.uploaderType) + bp, err := factory.NewBackupper(ctx, log, backupObj, test.uploaderType) require.NoError(t, err) @@ -619,6 +620,91 @@ func TestBackupPodVolumes(t *testing.T) { } } +func TestGetPodVolumeBackup(t *testing.T) { + backupper := &backupper{ + pvbIndexer: cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{ + indexNamePod: podIndexFunc, + }), + } + + obj := &velerov1api.PodVolumeBackup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "pvb", + }, + Spec: velerov1api.PodVolumeBackupSpec{ + Pod: corev1api.ObjectReference{ + Kind: "Pod", + Namespace: "default", + Name: "pod", + }, + }, + } + + err := backupper.pvbIndexer.Add(obj) + require.NoError(t, err) + + // not exist PVB + pvb, err := backupper.GetPodVolumeBackup("invalid-namespace", "invalid-name") + require.NoError(t, err) + assert.Nil(t, pvb) + + // exist PVB + pvb, err = backupper.GetPodVolumeBackup("velero", "pvb") + require.NoError(t, err) + assert.NotNil(t, pvb) +} + +func TestListPodVolumeBackupsByPodp(t *testing.T) { + backupper := &backupper{ + pvbIndexer: cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{ + indexNamePod: podIndexFunc, + }), + } + + obj1 := &velerov1api.PodVolumeBackup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "pvb1", + }, + Spec: velerov1api.PodVolumeBackupSpec{ + Pod: corev1api.ObjectReference{ + Kind: "Pod", + Namespace: "default", + Name: "pod", + }, + }, + } + obj2 := &velerov1api.PodVolumeBackup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "pvb2", + }, + Spec: velerov1api.PodVolumeBackupSpec{ + Pod: corev1api.ObjectReference{ + Kind: "Pod", + Namespace: "default", + Name: "pod", + }, + }, + } + + err := backupper.pvbIndexer.Add(obj1) + require.NoError(t, err) + err = backupper.pvbIndexer.Add(obj2) + require.NoError(t, err) + + // not exist PVBs + pvbs, err := backupper.ListPodVolumeBackupsByPod("invalid-namespace", "invalid-name") + require.NoError(t, err) + assert.Empty(t, pvbs) + + // exist PVBs + pvbs, err = backupper.ListPodVolumeBackupsByPod("default", "pod") + require.NoError(t, err) + assert.Len(t, pvbs, 2) +} + type logHook struct { entry *logrus.Entry } @@ -636,6 +722,7 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) { defer func() { cancelFunc() }() + log := logrus.New() cases := []struct { name string ctx context.Context @@ -691,7 +778,7 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) { logHook := &logHook{} logger.Hooks.Add(logHook) - backuper := newBackupper(c.ctx, nil, nil, informer, nil, "", &velerov1api.Backup{}) + backuper := newBackupper(c.ctx, log, nil, nil, informer, nil, "", &velerov1api.Backup{}) backuper.wg.Add(1) if c.statusToBeUpdated != nil { diff --git a/pkg/repository/udmrepo/repo_options.go b/pkg/repository/udmrepo/repo_options.go index 28eadfdb9..8dbbbee36 100644 --- a/pkg/repository/udmrepo/repo_options.go +++ b/pkg/repository/udmrepo/repo_options.go @@ -200,7 +200,12 @@ func GetRepoDomain() string { func getRepoConfigFile(workPath string, repoID string) string { if workPath == "" { - workPath = filepath.Join(os.Getenv("HOME"), "udmrepo") + home := os.Getenv("HOME") + if home != "" { + workPath = filepath.Join(home, "udmrepo") + } else { + workPath = filepath.Join(os.TempDir(), "udmrepo") + } } name := "repo-" + strings.ToLower(repoID) + ".conf" diff --git a/pkg/restore/actions/csi/pvc_action.go b/pkg/restore/actions/csi/pvc_action.go index 19c687e8b..0462bb74c 100644 --- a/pkg/restore/actions/csi/pvc_action.go +++ b/pkg/restore/actions/csi/pvc_action.go @@ -478,6 +478,7 @@ func newDataDownload( SnapshotID: dataUploadResult.SnapshotID, SourceNamespace: dataUploadResult.SourceNamespace, OperationTimeout: backup.Spec.CSISnapshotTimeout, + NodeOS: dataUploadResult.NodeOS, }, } if restore.Spec.UploaderConfig != nil { diff --git a/pkg/restore/actions/dataupload_retrieve_action.go b/pkg/restore/actions/dataupload_retrieve_action.go index 653f5e340..d5b922d6f 100644 --- a/pkg/restore/actions/dataupload_retrieve_action.go +++ b/pkg/restore/actions/dataupload_retrieve_action.go @@ -78,6 +78,7 @@ func (d *DataUploadRetrieveAction) Execute(input *velero.RestoreItemActionExecut SnapshotID: dataUpload.Status.SnapshotID, SourceNamespace: dataUpload.Spec.SourceNamespace, DataMoverResult: dataUpload.Status.DataMoverResult, + NodeOS: dataUpload.Status.NodeOS, } jsonBytes, err := json.Marshal(dataUploadResult) diff --git a/pkg/restore/restore_test.go b/pkg/restore/restore_test.go index f3da85202..065d23cbe 100644 --- a/pkg/restore/restore_test.go +++ b/pkg/restore/restore_test.go @@ -1709,7 +1709,7 @@ func (a *pluggableAction) AreAdditionalItemsReady(additionalItems []velero.Resou } // TestRestoreActionModifications runs restores with restore item actions that modify resources, and -// verifies that that the modified item is correctly created in the API. Verification is done by looking +// verifies that the modified item is correctly created in the API. Verification is done by looking // at the full object in the API. func TestRestoreActionModifications(t *testing.T) { // modifyingActionGetter is a helper function that returns a *pluggableAction, whose Execute(...) @@ -2006,7 +2006,7 @@ func TestRestoreWithAsyncOperations(t *testing.T) { } // TestRestoreActionAdditionalItems runs restores with restore item actions that return additional items -// to be restored, and verifies that that the correct set of items is created in the API. Verification is +// to be restored, and verifies that the correct set of items is created in the API. Verification is // done by looking at the namespaces/names of the items in the API; contents are not checked. func TestRestoreActionAdditionalItems(t *testing.T) { tests := []struct { diff --git a/pkg/util/csi/volume_snapshot_test.go b/pkg/util/csi/volume_snapshot_test.go index 29a34feed..e5a87c2bd 100644 --- a/pkg/util/csi/volume_snapshot_test.go +++ b/pkg/util/csi/volume_snapshot_test.go @@ -1684,28 +1684,28 @@ func TestWaitUntilVSCHandleIsReady(t *testing.T) { }, }, { - name: "waitDisabled should not find volumesnapshotcontent volumesnapshot status is nil", + name: "waitDisabled should not find volumesnapshotcontent when volumesnapshot status is nil", wait: false, expectError: false, exepctedVSC: nil, volSnap: vsWithNilStatus, }, { - name: "waitDisabled should not find volumesnapshotcontent volumesnapshot status.BoundVolumeSnapshotContentName is nil", + name: "waitDisabled should not find volumesnapshotcontent when volumesnapshot status.BoundVolumeSnapshotContentName is nil", wait: false, expectError: false, exepctedVSC: nil, volSnap: vsWithNilStatusField, }, { - name: "waitDisabled should find volumesnapshotcontent volumesnapshotcontent status is nil", + name: "waitDisabled should find volumesnapshotcontent when volumesnapshotcontent status is nil", wait: false, expectError: false, exepctedVSC: vscWithNilStatus, volSnap: vsForNilStatusVsc, }, { - name: "waitDisabled should find volumesnapshotcontent volumesnapshotcontent status.SnapshotHandle is nil", + name: "waitDisabled should find volumesnapshotcontent when volumesnapshotcontent status.SnapshotHandle is nil", wait: false, expectError: false, exepctedVSC: vscWithNilStatusField, diff --git a/pkg/util/kube/event.go b/pkg/util/kube/event.go index de91d3533..8b5fc9c5b 100644 --- a/pkg/util/kube/event.go +++ b/pkg/util/kube/event.go @@ -16,6 +16,7 @@ limitations under the License. package kube import ( + "math" "sync" "time" @@ -60,6 +61,9 @@ func NewEventRecorder(kubeClient kubernetes.Interface, scheme *runtime.Scheme, e } res.broadcaster = record.NewBroadcasterWithCorrelatorOptions(record.CorrelatorOptions{ + // Bypass the built-in EventCorrelator's rate filtering, otherwise, the event will be abandoned if the rate exceeds. + // The callers (i.e., data mover pods) have controlled the rate and total number outside. E.g., the progress is designed to be updated every 10 seconds and is changeable. + BurstSize: math.MaxInt32, MaxEvents: 1, MessageFunc: func(event *v1.Event) string { return event.Message diff --git a/pkg/util/kube/event_test.go b/pkg/util/kube/event_test.go index 080d95546..2f24b2515 100644 --- a/pkg/util/kube/event_test.go +++ b/pkg/util/kube/event_test.go @@ -43,9 +43,12 @@ func TestEvent(t *testing.T) { } cases := []struct { - name string - events []testEvent - expected int + name string + events []testEvent + generateDiff int + generateSame int + generateEnding bool + expected int }{ { name: "update events, different message", @@ -116,6 +119,18 @@ func TestEvent(t *testing.T) { }, expected: -1, }, + { + name: "auto generate 200", + generateDiff: 200, + generateEnding: true, + expected: 201, + }, + { + name: "auto generate 200, update", + generateSame: 200, + generateEnding: true, + expected: 2, + }, } shutdownTimeout = time.Second * 5 @@ -143,6 +158,28 @@ func TestEvent(t *testing.T) { _, err = client.CoreV1().Pods("fake-ns").Create(context.Background(), pod, metav1.CreateOptions{}) require.NoError(t, err) + for i := 0; i < tc.generateDiff; i++ { + tc.events = append(tc.events, testEvent{ + reason: fmt.Sprintf("fake-reason-%v", i), + message: fmt.Sprintf("fake-message-%v", i), + }) + } + + for i := 0; i < tc.generateSame; i++ { + tc.events = append(tc.events, testEvent{ + reason: "fake-reason", + message: fmt.Sprintf("fake-message-%v", i), + }) + } + + if tc.generateEnding { + tc.events = append(tc.events, testEvent{ + reason: "fake-ending-reason", + message: "fake-ending-message", + ending: true, + }) + } + for _, e := range tc.events { if e.ending { recorder.EndingEvent(pod, e.warning, e.reason, e.message) diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index e91e5dab3..a80405874 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -271,7 +271,7 @@ func SetPVReclaimPolicy(ctx context.Context, pvGetter corev1client.CoreV1Interfa // nothing if the consuming doesn't affect the PV provision. // The latest PVC and the selected node will be returned. func WaitPVCConsumed(ctx context.Context, pvcGetter corev1client.CoreV1Interface, pvc string, namespace string, - storageClient storagev1.StorageV1Interface, timeout time.Duration) (string, *corev1api.PersistentVolumeClaim, error) { + storageClient storagev1.StorageV1Interface, timeout time.Duration, ignoreConsume bool) (string, *corev1api.PersistentVolumeClaim, error) { selectedNode := "" var updated *corev1api.PersistentVolumeClaim var storageClass *storagev1api.StorageClass @@ -282,18 +282,20 @@ func WaitPVCConsumed(ctx context.Context, pvcGetter corev1client.CoreV1Interface return false, errors.Wrapf(err, "error to get pvc %s/%s", namespace, pvc) } - if tmpPVC.Spec.StorageClassName != nil && storageClass == nil { - storageClass, err = storageClient.StorageClasses().Get(ctx, *tmpPVC.Spec.StorageClassName, metav1.GetOptions{}) - if err != nil { - return false, errors.Wrapf(err, "error to get storage class %s", *tmpPVC.Spec.StorageClassName) + if !ignoreConsume { + if tmpPVC.Spec.StorageClassName != nil && storageClass == nil { + storageClass, err = storageClient.StorageClasses().Get(ctx, *tmpPVC.Spec.StorageClassName, metav1.GetOptions{}) + if err != nil { + return false, errors.Wrapf(err, "error to get storage class %s", *tmpPVC.Spec.StorageClassName) + } } - } - if storageClass != nil { - if storageClass.VolumeBindingMode != nil && *storageClass.VolumeBindingMode == storagev1api.VolumeBindingWaitForFirstConsumer { - selectedNode = tmpPVC.Annotations[KubeAnnSelectedNode] - if selectedNode == "" { - return false, nil + if storageClass != nil { + if storageClass.VolumeBindingMode != nil && *storageClass.VolumeBindingMode == storagev1api.VolumeBindingWaitForFirstConsumer { + selectedNode = tmpPVC.Annotations[KubeAnnSelectedNode] + if selectedNode == "" { + return false, nil + } } } } diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index 2a5c2d826..59dfc5788 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -189,14 +189,15 @@ func TestWaitPVCConsumed(t *testing.T) { } tests := []struct { - name string - pvcName string - pvcNamespace string - kubeClientObj []runtime.Object - kubeReactors []reactor - expectedPVC *corev1api.PersistentVolumeClaim - selectedNode string - err string + name string + pvcName string + pvcNamespace string + kubeClientObj []runtime.Object + kubeReactors []reactor + expectedPVC *corev1api.PersistentVolumeClaim + selectedNode string + ignoreWaitForFirstConsumer bool + err string }{ { name: "get pvc error", @@ -213,6 +214,16 @@ func TestWaitPVCConsumed(t *testing.T) { }, expectedPVC: pvcObject, }, + { + name: "success when ignore wait for first consumer", + pvcName: "fake-pvc-2", + pvcNamespace: "fake-namespace", + ignoreWaitForFirstConsumer: true, + kubeClientObj: []runtime.Object{ + pvcObjectWithSC, + }, + expectedPVC: pvcObjectWithSC, + }, { name: "get sc fail", pvcName: "fake-pvc-2", @@ -275,7 +286,7 @@ func TestWaitPVCConsumed(t *testing.T) { var kubeClient kubernetes.Interface = fakeKubeClient - selectedNode, pvc, err := WaitPVCConsumed(context.Background(), kubeClient.CoreV1(), test.pvcName, test.pvcNamespace, kubeClient.StorageV1(), time.Millisecond) + selectedNode, pvc, err := WaitPVCConsumed(context.Background(), kubeClient.CoreV1(), test.pvcName, test.pvcNamespace, kubeClient.StorageV1(), time.Millisecond, test.ignoreWaitForFirstConsumer) if err != nil { assert.EqualError(t, err, test.err) diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index c57ceae4c..d799df9f7 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -786,19 +786,18 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, for _, curLine := range lines { fmt.Printf("%s %s\n", curLine, time.Now().Format("2006-01-02 15:04:05")) comps := strings.Split(curLine, "=") - // SnapshotPhase represents the lifecycle phase of a Snapshot. - // New - No work yet, next phase is InProgress - // InProgress - snapshot being taken - // Snapshotted - local snapshot complete, next phase is Protecting or SnapshotFailed - // SnapshotFailed - end state, snapshot was not able to be taken - // Uploading - snapshot is being moved to durable storage - // Uploaded - end state, snapshot has been protected - // UploadFailed - end state, unable to move to durable storage - // Canceling - when the SanpshotCancel flag is set, if the Snapshot has not already moved into a terminal state, the - // status will move to Canceling. The snapshot ID will be removed from the status status if has been filled in - // and the snapshot ID will not longer be valid for a Clone operation - // Canceled - the operation was canceled, the snapshot ID is not valid - // Canceled - the operation was canceled, the snapshot ID is not valid + // SnapshotPhase represents the lifecycle phase of a Snapshot: + // * New: No work yet, next phase is InProgress + // * InProgress: snapshot being taken + // * Snapshotted: local snapshot complete, next phase is Protecting or SnapshotFailed + // * SnapshotFailed: end state, snapshot was not able to be taken + // * Uploading: snapshot is being moved to durable storage + // * Uploaded: end state, snapshot has been protected + // * UploadFailed: end state, unable to move to durable storage + // * Canceling: when the SanpshotCancel flag is set, if the Snapshot has not already moved into a terminal state, the + // status will move to Canceling. The snapshot ID will be removed from the status if the status has been filled in + // and the snapshot ID will not longer be valid for a Clone operation + // * Canceled: the operation was canceled, the snapshot ID is not valid if len(comps) == 2 { phase := comps[1] actualCount++