diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml index df5fe1fc7..8167e0383 100644 --- a/.github/workflows/stale-issues.yml +++ b/.github/workflows/stale-issues.yml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v3 + - uses: actions/stale@v6.0.1 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: "This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 14 days. If a Velero team member has requested log or more information, please provide the output of the shared commands." diff --git a/changelogs/unreleased/7100-blackpiglet b/changelogs/unreleased/7100-blackpiglet new file mode 100644 index 000000000..1084a29d1 --- /dev/null +++ b/changelogs/unreleased/7100-blackpiglet @@ -0,0 +1 @@ +Generate VolumeInfo for backup. \ No newline at end of file diff --git a/config/crd/v1/bases/velero.io_downloadrequests.yaml b/config/crd/v1/bases/velero.io_downloadrequests.yaml index 2d9e2a981..bf5f5e74a 100644 --- a/config/crd/v1/bases/velero.io_downloadrequests.yaml +++ b/config/crd/v1/bases/velero.io_downloadrequests.yaml @@ -53,6 +53,7 @@ spec: - RestoreItemOperations - CSIBackupVolumeSnapshots - CSIBackupVolumeSnapshotContents + - BackupVolumeInfos type: string name: description: Name is the name of the Kubernetes resource with diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index 7f6639cfd..fff600dc1 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -33,7 +33,7 @@ var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=M\x93\xdb:rw\xff\n\x94r\xf0\xee\xd6H~N.)\u07fcc;Q\xbd\xf7\xec)\xcf<\xef%\x17\x88lIx\x03\x02\\\x00\xd4X\x9b\xca\x7fO\xa1\x01\xf0\x13$AY\xf3\xe2M\x99\x17{(\xa0\xd1\xe8n\xf4\x17\x1a\xe0z\xbd~AK\xf6\x05\x94fR\xbc!\xb4d\xf0Հ\xb0\x7f\xe9\xcd\xe3\xbf\xeb\r\x93\xafN\xaf_<2\x91\xbf!\xb7\x956\xb2\xf8\fZV*\x83w\xb0g\x82\x19&ŋ\x02\fͩ\xa1o^\x10B\x85\x90\x86\xda\xd7\xda\xfeIH&\x85Q\x92sP\xeb\x03\x88\xcdc\xb5\x83]\xc5x\x0e\n\x81\x87\xa1O?m^\xff\xeb\xe6\xa7\x17\x84\bZ\xc0\x1b\xb2\xa3\xd9cU\xea\xcd\t8(\xb9a\xf2\x85.!\xb3 \x0fJV\xe5\x1b\xd2\xfc\xe0\xba\xf8\xe1\x1c\xaa\x7f\xc5\xde\xf8\x823m~n\xbd\xfc\x85i\x83?\x94\xbcR\x94\xd7#\xe1;\xcdġ\xe2T\x85\xb7/\bљ,\xe1\r\xf9h\x87(i\x06\xf9\vB<\xd68\xe4\xda#|z\xed dG(\xa8Å\x10Y\x82x{\xb7\xfd\xf2o\xf7\x9dׄ\xe4\xa03\xc5J\x83sw\x88\x11\xa6\t%_pZDy*\x13s\xa4\x86((\x15h\x10F\x13s\x04\x92\xd1\xd2T\n\x88ܓ\x9f\xab\x1d(\x01\x06t\r\x9a\x90\x8cWڀ\"\xdaP\x03\x84\x1aBI)\x990\x84\tbX\x01\xe4Oo\xef\xb6D\xee~\x87\xcchBEN\xa8\xd62c\xd4@NN\x92W\x05\xb8\xbe\x7f\xde\xd4PK%KP\x86\x05:\xbb\xa7%<\xad\xb7\xbd齴\x14p\xadHn\xa5\x06\xdc4<\x15!\xf7D\xb3\xf31G\xa6\x9b\xe9\xa2\x1cu\x00\x13ۈ\n\x8f\xfc\x86܃\xb2`\x88>ʊ\xe7V\xd8N\xa0,\xc12y\x10\xec\x1f5lM\x8c\xc4A95\xe0\x05\xa0y\x980\xa0\x04\xe5\xe4Dy\x057H\x92\x82\x9e\x89\x02;\n\xa9D\v\x1e6\xd1\x1b\xf2\xabT@\x98\xd8\xcb7\xe4hL\xa9\u07fczu`&,\x9aL\x16E%\x989\xbfB\xf9g\xbb\xcaH\xa5_\xe5p\x02\xfeJ\xb3Ú\xaa\xec\xc8\fd\x96\x91\xafh\xc9ֈ\xba\xc0\x85\xb3)\xf2\x7f\t\x02\xa0_vp5g+\x8c\xda(&\x0e\xad\x1fP\xea'8`\x17\x80\x93/\xd7\xd5͢!\xb4}e\xa9\xf3\xf9\xfd\xfdC[\xf6\x98\xeeS\x1f\xe9\xde\x12Ȇ\x05\x96`L\xecA9&\xee\x95,\x10&\x88\xdcI\x1f\x8a.g \xfa\xe4\xd7ծ`\xc6\xf2\xfd\xef\x15h+\xe4rCnQ\x93\x90\x1d\x90\xaa̭dn\xc8V\x90[Z\x00\xbf\xa5\x1a\x9e\x9d\x01\x96\xd2zm\t\x9bƂ\xb6\x12\xec7vTk\xfd\x10t\xd9\b\xbf\x9cB\xb8/!\xeb,\x18ۋ\xedY\x86˂\xec\xa5j\xf4\x85SW\x9b\x0e\xc8\xf8\x92\xb5O\xa6ٽ\xa0\xa5>J\xf3\xc0\n\x90\x95\xe9\xb7\xe8!t{\xbf\xedu\b\xc8x\xd4P\xadT\x1ar\xbbΞ(3\x16\xbd\x01Lb\x01\x91/\xa8a\x02<\xd44\x95&\xa6R\x02W\xe9g\xa0\xf9\xf9A\xfe\xa6\x81\xe4\x15\nk\xa6\x00\xa7|Cv\xb0\x97\n\"p\x15\xd8\xfe\xb61(e\t\xa3\x11%Y\x99\ry8\x82%#\xad\xb8\xf1r\xcf4y\xfd\x13)\x98\xa8\fl\x06\xd0F\x18\x8cD\xa1\x86\x16\xf2\x04j\x86^﨡\xbf\xdav=2\xd9\xfe\x04\x01ؙ\xee<\xc9vg\xfbcdV\x9e\xabd\xbboAd\x9a\xacVD*\xb2r&pu\x83\xa0\xadQ5k&ZcD >1\xceø\xcbf\xee\b\xe8x\xa7\x1f\xe4\a\xed\x84t\x8e\x10#\xddZty:\x829\x82\"\xa5\f\xc6'\x82\xf7\x9eq \xfa\xac\r\x14\x9e*A\xe5\a\"\xe2r\xe0܃Ж\xa8\x1e\xe7\xe1\t\x927\x98\xc8\xdb\x1e\xb2\xed\xa1\xbdS\x9e:\r\xef\xfa\xd4\xf1\x8d\xdbF\xb8!\x94<\xc2\xd9y,T\x10\xcb\x1cj\a\x1a\x89t\x86\xc4\xc1\xfd\f\x14\xb2G8#\x18\xbfA1\xdb;U\x14\xdc\xf3\b\xe7\x94f=\x02Z\x9c\x98\xf6\x1b/\x96\x92\xf6\x05\x12\x02\xf3\xd9\xe9\xc4#\xb8\xd9\x14t\xd1\xfc\xe4H\xba\"\tO\xa0\xfd\x05Ӭ\xd9\xd6ڨCƾԎEv\x15\x1cY\x998Qk\xe60\x95 \xf7\xf5v\xd3\x17\xcaY^\x0f\xe4\xe4~+ƽ\xe1\xee\xf3Q\x9a\xad\xb8!\xef\xbf2\xedw\xfc\xdeI\xd0\x1f\xa5\xc17\xcfBN\x87\xf8\x05\xc4t\x1dqy\t\xa7\xb6-\x1d\xda\xfbV\t\xc2\ud7ad\x8b\xf0j\xf60M\xb6\xc2\xc6-\x9e\x1e\xb8\v醛\xb6\x0fݧ\xa84nL\t)\xd6.\xf5\x12\x1b\xc9\x11;\x11\xa4T\x1d\x8e\fQ\xab\a\x1d\xc9\xf5ğ\akI\\\x7f\xb7\xaf\xcai\x06y\xd8W\xc1\xdd@j\xe0\xc02R\x80:L\x19\x8e\xf6SZ\xfd\x9e\x86B\xa2\xd6u\xcfB\tK3\xed\xe1\xf1\xaa;\x9a\xfc\xee>k\xbbr\x13Z\x05f\xcf6\x1d\xd9\x04\x1co:?#4\xb1\xe8\x7f\xccR\x97\xe69\x96iP~\xb7@\xe3/\xe0\xc5\xd0\xf6;Ĝ\x85,(nN\xfc\xb75s(\xd0\xffCJ\xcaT\xc2\x1a~\x8b\xe5\x18\x1c:}}\x16\xab=\x8c\x1d\x81ib\xf9{\xa2|\xb8\xbd\x1c\x99\x9c\xb4\xba\x05\xb83\xe4r?\xf0Xn\xc8\xd3QjgSqSd\x16$\xd3d\xf5\bg\xbf\x19\xd7\xd6\x03\xab\xadX9\x03\xbfX\xdd\xd4ނ\x14\xfcLV\xd8w\xf5-NP\xa2$&6\xfb\xba~\xac\xcbO\xd6\x05-\xd7^z\x8d,X6\xda\x0f\xcbeR]l\x1b\x83\x06\x0f\xc2v\xackD\xac{<5\xdb$\xf9-\xa5\x8e\xec|\x8f\xa0r'\xb5q\x19Ɏ;\xbb$\xfbE\x9c\xec\xf9\xac\x17\xa1{W\xa5#U\xa8\xbf\xb0겗\xa8\xb5\xdc\xd6Ӛ\xd9\xed\x19\xf8L\x9a\x03j\x03\xb2U\xb3\xf2\x9d\x1e^\xb9=\v\x1c\x84f\xe8\x94\xcc\xc2-\x95\xcc@Gw\x8b\x9b'A\xcb\xcf$\x17\xeb\xc4\"u\x81\x8f+n\x98Nf\x86'ݑ\xb5DZ\x18\x02\xbc\xff\xda\xcazZ\xa5a\xff\x9e\x13\xbe\xa5x\x11\\\xebEA\xfbU\v\xd9f\xc6J\xc6bO\x87m\xa1v\xac]\xd4VЯ\xac\xa8\nB\vK\xfa\xd4pi\xef*\xce:\x1c\xaf\xeb\xce\x10.\x9a\x11#\xed\xa2*9\x98\xd4\x15\xe9*\xcc\xec2\xd1,\x87\xda0{)\x90\x82P\xb2\xa7\x8c\x8f\x94\xbb\f\x9fE\xb4]\x12\xa3xeq\xbd\xe0#m\xf05\x92\"!\x81\x9b\xe8dNk\xebR\xa5\xbb\x8aw\n\xd2ܳ\xb9dvp\xcfJ\xc5$\xd6\xe9]\xd9C\xf3\"F\xc5\xf9\x87\x8b6x~\xb8h3\xcf\x0f\x17m\xf4\xf9\xe1\xa2\xcd??\\4\xff\xfcp\xd1\xc2\xf3\xc3E\xfb\xe1\xa2M5\x9b\xd2\xd6s\x18\xb9\xd3q#?\xceb\x91\xb0\xad=\x85\xe2\x04|_\x85\xe1\xeb\xbcS+3\xb7\xf1^\x91:\xfe\xe4\xdap\xdd2%u\xa9\xa6] A\xbc\xdda\x9f\x99\xe2\xcdo\xa8\x97\x0f\x83^T/\xbf\x9d\xec|\xa5zy\x8fa\xdf\xeb\xbeR\xb5|\x98\xff\xb2j\xf9\x1b_\xaaQ\x00\r\xe9y\xb7\x17\x9f\x8f\r\xd9\x1bm\x00\xf8\x0f\xae\xbf\x1dԇ]\xc6\xf8g\xaf\xb6\x1fa~ba\xfc\xea/\xab\xef\x8fҋi;J\xcd\x01\x99\"\x93\n\xc7|m\\\xd9.\xee\xea\x16\xd2}\x9f¹T\x1aS+\xe6\xa7\xe85\xd42-\x82}\xaf\x8b\xd9@\xf1\xa9\xf4\xb6\"\xedD\xe76\xd2e\xeeLgd>\x98\x04\xd0g\x91\x1d\x95\x14\xb2\xd2>o`\xa1\xbf\xc5\xf4\x85\xdf\n\xc52\xb0D\x05\xfb\x9a\x1ce\x15\xa9؞\xa0\xddL\xfd\xdex՞ߣ\x06CO\xaf7\xdd_\x8c\xf45|䉙c\x04ϧ#\b\xdc]\x17\x87vA~Xp\xfe\xc0y_\x90\x88TD0>f\xb0\xeaS\xf9\x1d\xd3\xf4\xa9tI\xa2Ŗ\x7f:\xc1\x91V\xe5wqm_\xb7vo\xc4\t\\\xba\x99\x9d~\x84!\xbdzo\xba\xdcnI\xcd^\xbf\"o\x14\xe8|\xa5^Jnj\xa6*\xef\x82Z\xbc\xc4:\xeco\xdezO\xa9\xb6\xbb\xa8\xc6n\xb6T9\xb1\xb2\xae[37\rrA=]\x12q\xe6k\xe7\x16W\xcc\xf9\n\xb5\xc9y$\xd7\xc9E*\xe0&\x01\x8fV\xc7Mս\xcd佇5q\xe9\xd5n\x93\xa0\xb1\x12n\xbe\xc6\xedz\x95\xec\u05c8\xb2\xc7U\xcdl\x9d\xdal\x14>\x8d\xdfl%ڒ\xfa\xb3Y\x8a]XkVג\x8d\x8c\xbb\xb4¬[A6\x024\xa5\xael\xa4nl\x04\xe2d5Yj\xb5\xd8\b\xec\x19\xb3;)%\x93?.\xa9\x12\x8b_\xa2Bf\xad!\xff\xa3\xe4\xefR2H\xd5q.\xe7\x02\x9aO\xbd\xe6\x96\xf3\xc1ǚvVc~*3\xc7\xe5\xcejQq\xc3J\x8eۋ'\x96Gcvs\x84s}1\xc4\xef\x12\x8fk\xba\xcbLȧϵ0oz.7\xd5\xe4\t8'4&\x8a\x83\x99g\xee\x1e\xa0L\xae\xc1Z\b\xbb<\xfd\x95\x17\xfe\xba\xa0\x1b'\xefx\"5\xb6\x03c\x8ePX(\xe3מ\x8c\xaa\xf2iw\xd2y\xbe\xf8\xee\xef\x15\xa83\xc1\xebYj\xffb\xe6<\x94[\x96\xda\xc6BAQxm\xe3n\x9f\xea\xb9\xd9\xcd\xf2$o\x853xQ\xb0=\x1c\x11\x8e\xd5\x10\xbc\xe6\xb5U\x866j\x18i\x1aO\xc4ʺw\xe4\xf79O5\xf50\xd1\xf3\x06\x1a\xcbC\x8dY#\xff,\xe1\xc6\xe5\x01\xc7\x04\xc8\xd4\xc3Ai\x1bⳇ\x81\x9e+\xf0\x98\v=\x92}\xae\xb4\xc3>\xcfq\xc8g\xc1\xe1\x9e\x05!Ȳ $\x99L)\x87x\x9e%\x14y\xc6`\xe49\u0091\xcb\x02\x92\x19\x90\xbd\xc39)\xc7n\x92\x8a=\x92\xf7;S\x8a5\xe6\xb7$\xa7\x8f\xd3$\x1c\xa3Iج\x9c\xc34\xe1\xb8̲c2\t4|\xa6P噂\x95\xe7\bW\x9e7`\x99\rYf%g\xe6\xe7e\xc7[.N\xdeK\x95\x83\x9a\xdc\xebH\x15\xcdI\xa1\xec\xc5\x17\xdd1{\x99\xffp\xa7\x9cm\xd5qec\t\xeb\xfa\xd4{F~f\xc2\xef\xa3Z!l\xd9\xfd\xce\x06L\xe3\x88\xc4\xf3\xff\x8d\x97\xe7o\x1bu\xbb6\x1aJ\xaap\x87uwv\xa5\x15zC\xde\xd3\xec\u0603~\x8c\xc6\x15{\xa9\njȪ\xde\xf2z\xe5\x80ۿW\x1bB>\xc8zӾ}\x93\x8cfE\xc9\xcf6n\x88\xc0\\\xb5A\\&\x10Q\xe1\v\xe3\xdfIβ\x88\xa7\x15\xbd\\\xc85\x1e\\\t\x81W\x1ee\xed\xad\xef\xd26\x8c;Z\xe8\x94u\xafW\xdcK\xce\xe5\xd3\xc2p\x9c\x96\xec?\xf0\x96\xe6\xf9\x1c\xceۻ-6\r\x92\x82\xb7;\xd7\x15B5\xd2;\xb0\x16\xb3\x99\xce؊\xdf\xee;\x10#\x95v\xf5\x9f(\xad\xb5\xc5fc\xb7.\xb9\xaa?\xabi\xee\xb6\x0e\xbb\r\n\v\x15g\"\xb1\xd6\xc3\x1c\x99\xca\xd7%U\xe6\xec\xea\nnj\x1c\xc6\xf38\xc1nNe[F\xcd\xcb\xf0\xba\xdf(mí\xbf\xb8\x99w.\xbb[\xa1}\x8a^\x82\xc7\xf8Q\xbe\xd9C|W\xc4c\xdc\x05Y#\xa5\"\xaf\xa3EIW\xcbbi\x7f\xb5\xed\xaf\xf2\x04\xef\xa2٬\x0ey\xee{\xcd#\xe5D\x01\xa2\xbb\xdcu\xeavP\xbc\x9c\xf32]\x14\xaf\x0f\nC\xfb\xeb;\x13\xe7\xe2[G\xa6\x12n.\rpu\x01\xfe\x95wݭ\x1b\xc39\xf0\x0f\x8c\x83vh%(ѻa\xafZ\xa7V\xc5\xce9j{\xfbc=\xc0\x88\xf5q\xd3\u009co\t\xca:F.;\\\xe9 \x96\xe3\x13o8\u0084\x81C$\xdf<\xa1EO\x9d\x8b\xbf\x83Hϩ\x94/\xf1^\xad|`kQ9W1\xaaQ\xc6ാ}\x80\x99r<^~\xdd[2ǜ\xff\xb1\xdb\xe1\xf1F\xf4\xf9\xfb\xe1\xdd\xc5\xe9\xfek\x10^\x90+\x85W\\\xfaK\xd5\xf1Jȋ\xae\x88\xdf\xd5uP\xc6/\xa2\x83\xeb8B\x047\xb7Q\xbb\x97\xc4f_\x84\v\"\x0f\x8bw`\xba탧D\x96\xd1\xc1\xb3\xc0\xd7\xc6iC\x8b\xb9\x9b\xeeo\x87=\xf03$*oU\xd3\xd5\u05f5?Qݰ9\xa6f\x1bp\xae'\x86\f\x16\x1a\xe4\x04N \x88\xb5S\x8e\xbe\xe1S9\xfd>\x11\xa8m(\xfe\x18\x85\xd3\xf5A\xf3\x87\xc8\xdf\x7f^\xe5\x01\xdd-u\x02\xf5RO\xc0\xac/\xe0\x8f\x10a(\x99.\x14\x7fc}YXG\x81&\xb9jQ]\x9bi\xd6\xd5\xf3\xc9J\xeb\xf6~;\xd6sT\x82C\x83\x18\xff\x06\x1f\xba\xf8F%5\x9cY\xaa\x8a\x1a\xcelNAu\xd4Qdr\x8d\x82\xba\xfa4q\xad\xceސ\x8d\x8d\x9c\a\x80\xc7\xc1\xc2\xe7\x0f\xdcy\xb0\x02\xb4\xa6\x87p5\xf6\x93u\x98\x0f \x00\x93G\x91\xd9\xf8t|s~\xa8{1\xb4\xdb7\xa4\x99\xa9\xa8\x1f Tg\xb6Z\xbd\x8c)`.\x0f\xee\xc3\x15,|\xb0(D\x12\vi\xf2\xb5d*%\xf2x_7\xb4\xb4A\xa7\x0e\x19\xd1|`\n8;0\xeb\xb6[&\x1d\xa8\xda\xd1\x03\xac3\xc99\xa0\xae\x1d\xe2\xf5\x9c\x8b՟\xd2\xfa\fT\xcfN\xedC\xbb\xad\xdf_r\xdcv\xf7*RW\xa8\x8c\xdf\x1b2LA\xf3\x01\xaf\x01B\x12\a^\x14i8*D?u5Ĵ\xdd6,0\xafW}\x16\xd2\x7f\xf9\xea\xc6Ǯ\xf1\xe4JA\x7f\x97\xea\x86\x14L\xd8\x7f\xa8\xc8\xdd\x06P\xe8\xbc\b\x7f\xbc\xf1|\x06\xef;ۦ>,\xdb\xf2#!,\x88\xb1\xc8:~@rM>\xc20\x10tg\x1e!\xc7-\xcf\xd8\xf7\xbdl\x93\xad\xb8S\xf2\xa0@\x0fW՚\xfc\x8d2\xc3\xc4\xe1\x83Tw\xbc:0\xd1\xf8\x1b\x8b\x1a\xdfQe\x18\xe5\xfc\xec\xf0\x89!\xca\x04\xe5\xec\x1f1\xee\xb4\x7f\x9c\aT\xab\xdb\xc8o\th\x8c\xfd\xf0\x0e\xac\xa9\x1d\r7\xe2\x82\xe0\xe9:'\v\xbeY\xb3EÄ\x93]<_\xb9\x93\x95\xe9(\xbfFy\xc6\xe2]\x0flC>J\x03a\xe7\x9fuaZs\x01ڬa\xbf\x97ʸ\x1d\xa1\xf5\x9a\xb0\xbd\x0f_b)}\xca8F\xb1\xee\x03a\x84\x99\xa6ԳYo\x98IR\xa86\xf0Fゞ]\xbe\x97f\x99\r\xf8\xe1\x956\x94G4\xf27\xc5\xf8\x18'\xda\xf5\x02\xf9o)i\xf0m\xbb\xfd0\xaeGp\x8erx\x16\xdaY\xa3\xa8m&\x98\a\x04A\x9e\x143\xc6Z\x80vi\x171V\xe7sN\xb4Ղ\x17\x05\xf8\xc4y\v\xdb\xf1\xad\xe6n\x02\xa9n<\xe6l\xf8\xc9\xe1\xf7\xb0vH\x82\xd1ą\xdfR\xf7}-+\xb3#\x15\a+TJV\x87c\x90\xcb\x11[>\x027\xaf\x00\x93!\xa8!t(\xab1\x95\x12\xad\xad\xb7\xfa|_\x83.\xcd\x1eG1\xf5\xa5\x03\xe1#\x95\xaf\xfcE\xf7뽒\xc5\xda\xf3\x02\xaban\xfcv\x98b\xd2\x06\xec\xe6\x18%9q\xdf\xc0\xf27J\xa3\x18\x94%\bB\xb5\xc7'\xe1\"\x90\x8b\xf36\xdaPeR\xe3\xa0\xfbN\xe3\x99\x10\b!\xc7\xf1\xbd\xf7\xdb}\xeeB\x94[\xff\t\xb8\x1a\xf0\r\xd1L\x84\xefc\xba\xcdD'\n\xdaFF\n0\x19\x1a-t\x1a\xc44\x9d\b\xa6\x8b\xfe\x1f\x1b\xbc\x9cj\x9b\xf8>\xc5\v\xfe\xd2k\xde;\xf4\x86\x1fC\xab\x9bx\xcf5B\x8f?\xb1\xbd\xab\xbd\xca,\xd6\x7f\xfe??\xccvJ\xf2\xb2^N:X\xe8;՞\xd2̧\xcf\xee8X\xcfG\x03t}\xb7\x97\x8b\x9c\xf4\xd3ea\xe75c\xce\xf0\xe9\xd6\xebDb\xa7ˢ\xcdg\v5\xaf;\xbb'\x8a\x9f\x8b\x9c[c\x7f\xf3\xcd\"\xb1\xa6\x87\x10\x896#Ө\xe3\xcf\xd9h\xb3\x15l\x06\x1cG\xbe\xee\xd4\v@\xaf\x14nF\xed\xc0\xe0%*м\xb5\xb6\xfdH\xfe\xcd\xff\x06\x00\x00\xff\xffw\xe0\xabZYz\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\xe3\xb8\x11\xbe\xebWt\xed\x1e|YQ3\x9bKJ\x97\x94FN\xaa\xa6\xe2\x89]#ǹ\xe4\xb0\x10Д\xb0\x06\x01\x06\x0fi\x94T\xfe{\xaa\x01\xf0!\x92\xb2\xe4\xa9$\x8b\x8bM\xb2\xd1\xe8\xfe\xfa\r\xcd\xe7\xf3\x19\xab\xe5\vZ'\x8d^\x02\xab%~\xf3\xa8\xe9\xc9\x15\xaf\xbfw\x854\x8b\xc3\xc7٫\xd4b\t\xeb༩\xbe\xa23\xc1r\xbc\xc7Rj\xe9\xa5ѳ\n=\x13̳\xe5\f\x80im<\xa3\u05ce\x1e\x01\xb8\xd1\xde\x1a\xa5\xd0\xcew\xa8\x8bװ\xc5m\x90J\xa0\x8d̛\xa3\x0f\x1f\x8a\x8f?\x17\x1ff\x00\x9aU\xb8\x84-㯡v\xdeX\xb6CexbY\x1cP\xa15\x8543W#\xa7\x13vքz\t݇\xc4!\x9f\x9e$\xff\x14\x99m\x12\xb3\x87\xcc,~W\xd2\xf9?_\xa6y\x90\xceG\xbaZ\x05\xcb\xd4%\xb1\"\x89\xdb\x1b\xeb\xff\xd2\x1d=\x87\xadS\xe9\x8bԻ\xa0\x98\xbd\xb0}\x06ษq\tqw\xcd8\x8a\x19@\x86&r\x9b\x03\x13\"\x82\xcdԓ\x95ڣ]\x1b\x15*ݞ%\xd0q+k\x1f\xc1L\xba@V\x06\x1am\xc0y\xe6\x83\x03\x17\xf8\x1e\x98\x83ՁIŶ\n\x17\x7fլ\xf9?\xf2\x03\xf8\xd5\x19\xfd\xc4\xfc~\tE\xdaU\xd4{暯\xc9FO\xbd7\xfeD\n8o\xa5\xdeM\x89\xf4\xc0\x9c\x7faJ\x8a(ɳ\xac\x10\xa4\x03\xbfGP\xccy\xf0\xf4\x82\x9e\x12B@\x10!4\b\xc1\x91\xb9|\x0e\xc0!q\x89\x18MK\xaaFg\x9d\x89M\xa2\xc0ˀK\x92\x9f\xded\xe9{l\x1b\xff.\xb8Ŗ\xa5\xf3\xac\xaa\xcf\xf8\xaevx\x89\xd9\x19\x14\xf7X\xb2\xa0|_U\xb2\x92\xea\xfb\xe5\xb9Z5\xf2B\xa4]g'ޟ\xbdK\xa7n\x8dQ\xc8\x12\x97Du\xf8\x98\xbc\x90\xef\xb1b\xcbLljԫ\xa7\xcf/\xbfۜ\xbd\x86)G\x1a\x04\x05\x19\x8e\xf5l\xb3G\x8b\xf0\x12\xe3/\xd9\xcde\xd5Z\x9e\x00f\xfb+r\xdf\x19\xb1\xb6\xa6F\xebe\x13,i\xf5rQ\xef\xed@\xa6;\x12;Q\x81\xa0$\x84ɏr\xbc\xa0Ț\x82)\xc1\xef\xa5\x03\x8b\xb5E\x87\xda\xf7\xe1m\x05+\x81\xe9,^\x01\x1b\xb4Ćb9(A\xb9\xeb\x80փEnvZ\xfe\xb3\xe5\xed\xc0\x9b\xec\xbc\x1e\x9d\x1f\xf0\x8c\xf1\xa9\x99\"W\r\xf8\x130-\xa0b'\xb0H\xa7@\xd0=~\x91\xc4\x15\xf0\x85\xfc]\xea\xd2,a\xef}햋\xc5N\xfa&\asSUAK\x7fZ\xc4t*\xb7\xc1\x1b\xeb\x16\x02\x0f\xa8\x16N\xee\xe6\xcc\xf2\xbd\xf4\xc8}\xb0\xb8`\xb5\x9cG\xd1uJ\x9a\x95\xf8\xd1\xe6\xac\xed\xee\xced\x1dEmZ1k\xbea\x01ʘ\xc9\v\xd2֤E\a4\xbd\"t\xbe\xfeq\xf3\f\xcd\xd1\xd1\x18C\xf4#\xee\xddFי\x80\x00\x93\xbaD\x9b\x8cXZSE\x9e\xa8Em\xa4\xf6\xf1\x81+\x89z\b\xbf\v\xdbJz\xb2\xfb?\x02:O\xb6*`\x1d\v\x13l\x11B\x1d㾀\xcf\x1a֬B\xb5f\x0e\xff\xe7\x06 \xa4ݜ\x80\xbd\xcd\x04\xfd\x9a:$N\xa8\xf5>4\xb5\xf0\x82\xbd&\xa3xS#?\x8b\x1f\x81NZ\xf2p\xcf<Ƹ\x18\xe0\x9aC\xfcr1m\xd6tp\xd3b\x9c\xa3s_\x8c\xc0ᗁȫ\x96\xf0L\xc6\x1am%],\x8bP\x1a;\xac\x18\xac\xcd\xc0\xfd\xd5d\xaab\xf4\ru\xa8Ƃ\xcc\xe1+2\xf1\xa8\xd5\xe9§\xbfY\xe9\xc7\a]0$\xad$\xe2\xe6\xa4\xf9\x13Zi\xc4\x15\xe5?\r\xc8[\b\xf6\xe6\betk\xedՉr\x90;i>ζ\xcdZ=}n2o\n\xa0\x1co\x19\xab\x02V9rM\t\x1f@HG\r\x80\x8bL\xc7`\xe9\xa0b\x83\xb0\x04oû\xd4\xe7F\x97r7V\xba\xdf\xd3\\\xf2\x98+\xac\aȭ\xe3I\x94\x9a\xc8;jk\x0eR\xa0\x9dS|\xc8R\xf2,I\xb0\xa9r\x95\x12\x95pcM/DYTŢ\xa0\xa8f\xea\x8a\r\xd7-a쀙\xd4Ƀ;\x061\xd9\xd8*\x97T\xedQ\x8b\xb6\x1b9\x93\xc6Ĭ\xe5P\xc0Q\xfa}J\x87j*\xee\xe0\xcdأ\xf5\x8a\xa7\xa9\xd7\x03ٟ\xf7H\x94\xa9\x80\"8\xe4\x16}\xf46T\xe4>\xe4J\x05\xc0\x97\xe0bB\x1d\xe6\x89f\xc5F\xad\xd9\xfd\x8a\xa71\xd0p\u0378\xb9\x85\xb9.\xf2\x1d\xb5\u038d\xc0\x16K\xb4\xa8\xfddR\xa7\x01\xc4j\xf4\x18\xf3\xba0\xdcQJ\xe7X{\xb70\a\xb4\a\x89\xc7\xc5\xd1\xd8W\xa9ws\x02|\x9e#h\x11NJŏ\xf1\xcf\x05\x95\x9f\x1f\xef\x1f\x97\xb0\x12\x02\x8cߣ%\xab\x95A5\x8e\xd6\xebo~\x8a5\xf6'\bR\xfc\xe1\xee{p1u\x8a\x9c\x1b\xb0\xd9D\xef?Q\xa3\x16\x85\"\x886\xc9*\xc6\x02UJ2v\x95\xad\x99r͔#Nu\x98\xfdE\x89\x89*\xc8TF}\xc5q2}#\xcc\x00\xbe\xcd;C\xcd+V\xcf\x135\xf3\xa6\x92|6\xd46\xb6\xc1W\"\xb2i\xbb\xa5\x16\x92S\xdbv\x1eI\xcd8\"κ\xf3\t\x18\x86\xfd\xfa\xa5\xfc1\rSR7W\xcf+\x12?\xf6i\xbb!.%\xb3\\\x11\x1dzj\xb7\x1ch\xa4\x8a\xc9\xec\x18\xe7\x98B\xb8њb\xd7\x1b`mb\xbcsÊ\xf0\xce|\xb2\r\xfc\x15'\x80\x1f\xa9\xf2)\x126\x18\xa7m$Kp\x18S\xf551\xe0zDp\xb6F{\x8b,\xeb\x15\x11\xb6E\x95\xc1z\x05۠\x85\xc2F\xa2\xe3\x1e5\xcd\x13\xb2\x14?\xfcf3\x93b\xce\xd3\b\x84\xe2+\x1e\xe4\xf8Nh\x8c\xee\xc3hG\x13\xf8m8\xd0\xc3/\xcdh\xbd\xb0\x99\xec\x97\t0J\xa9\xa8s\x9c\xc8\x13]\xc70\xbe\xbd\xfc\xb4y\xb8s\xb1\xe1G\xed\xa7\x9a\xc4#Z\x8c\xf3\x15\n\xea\xf9M\xbe\xc5\bΣ\x9dp\x80\xd6z\xd1栌\xde\r\x02'\xad|\xa7A\xfd\\r(cA\xa0\xa7Ҥw\xc0\xf7Lﰻ\xb3\xca\xf2\xbf-)\xb9\xcf\xc0g:\x0f\x91\xfa\x92{\xdcd\xd1g9\xd5ԏ\xee\x8b;\xe2\xe9\xbb\xe2F\xfaƲ\x17\x87\xa2+\xb8\x8f\xe8\x9b*M\xa0\xce}w\x7fܭ\xef\x1f\x86Ǘ\xd37 \xf1ޛ\xf37nA\xe0\xc8\\w\x87\xfe\xdb\xe1PQ\xb7z\xb5\x05\xfe\x92\xa8\xd2ec\xde\x02lk\x82\x7f+2\xef\xa6\x1c:\xff8\xf0\x1e\x19\xe3O\x1eך\f\xa2i,\u0083\xa5\xc1\xb3\xbbC\x8bIa\xaa\xb6\xdc~\x19\xb5\x1a\xfc2\xd3\xff6\xfe\xdd\xe6\x06\xbd&k\xed\xe8e\xaa\x97=\xbbf\x90\xfbo¶\xbdW^¿\xfe=\xfbO\x00\x00\x00\xff\xff\x80.\x12\xd3P\x1c\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4\x96M\x93\xdb6\x0f\xc7\xef\xfe\x14\x98y\x0e\xb9<\x92\xb3\xed\xa5\xa3[\xb3\xc9a\xa7mƳ\x9bɝ&a\x8bY\x8ad\x01\xd0[\xb7\xd3\xef\xde!)\xf9E\xb67\xdbCy\x13\t\x02\x7f\xfe@\x80j\x9af\xa1\xa2\xfd\x8a\xc46\xf8\x0eT\xb4\xf8\x87\xa0\xcf_\xdc>\xffĭ\r\xcb\xdd\xdd\xe2\xd9z\xd3\xc1}b\t\xc3#rH\xa4\xf1#n\xac\xb7b\x83_\f(\xca(Q\xdd\x02@y\x1fD\xe5iΟ\x00:x\xa1\xe0\x1cR\xb3E\xdf>\xa75\xae\x93u\x06\xa98\x9fB\xef\u07b7w?\xb4\xef\x17\x00^\r\u0601A\x87\x82k\xa5\x9fS$\xfc=!\v\xb7;tH\xa1\xb5a\xc1\x11u\xf6\xbf\xa5\x90b\aDž\xba\x7f\x8c]u\x7f,\xae>\x14W\x8f\xd5UYu\x96\xe5\x97[\x16\xbf\xda\xd1*\xbaD\xca]\x17T\f\xd8\xfamr\x8a\xae\x9a,\x00X\x87\x88\x1d|β\xa2\xd2h\x16\x00㱋\xcc\x06\x941\x05\xa4r+\xb2^\x90\xee\x83K\xc3\x04\xb0\x01\x83\xac\xc9F)\xa0\xbe\xf4X\x8e\ba\x03\xd2#\xd4p \x01\xd68*0e\x1f\xc07\x0e~\xa5\xa4\xef\xa0ͼ\xdaj\x9a\x85\x8c\x06\x15\xf5\x87\xf9\xb4\xec\xb3`\x16\xb2~{K\x02\x8b\x92ē\x88\x12\xd7\x06\x0ft\xc2\xf7\\@\xb1oc\xaf\xf8<\xfaSY\xb8\x15\xb9\xda\xec\xee*i\xdd㠺\xd16D\xf4?\xaf\x1e\xbe\xfe\xf8t6\r\xe7Z\xaf\xa4\x16,\x83\x9a\x94fp\x95\x1a\x04\x8f\x10\b\x86@\x13Un\x0fN#\x85\x88$v\xbaZu\x9c\x14\xcf\xc9\xecL»\xac\xb2Z\x81\xc9U\x83\\\xa0\x8d\x97\x00\xcdx\xb0\n\xd32\x10FBF_\xeb\xe8\xcc1d#\xe5!\xac\xbf\xa1\x96\x16\x9e\x90\xb2\x1b\xe0>$gr\xb1\xed\x90\x04\bu\xd8z\xfb\xe7\xc17\xe7s\xe6\xa0N\xc91?\xd3(\x97\xce+\a;\xe5\x12\xfe\x1f\x9470\xa8=\x10\xe6(\x90\xfc\x89\xbfb\xc2-\xfc\x961Y\xbf\t\x1d\xf4\"\x91\xbb\xe5rkej\x1a:\fC\xf2V\xf6\xcbR\xffv\x9d$\x10/\r\xee\xd0-\xd9n\x1bE\xba\xb7\x82Z\x12\xe1RE\xdb\x14\xe9\xbe4\x8ev0\xff\xa3\xb1\xcd\xf0\xbb3\xad\x17\x17\xa4\x8eR\xe8\xafd \x97yM{\xddZOq\x04\x9d\xa72\x9d\xc7OO_`\n]\x921\xa7_\xb8\x1f7\xf21\x05\x19\x98\xf5\x1b\xa4\x9a\xc4\r\x85\xa1\xf8Dob\xb0^ʇv\x16\xfd\x1c?\xa7\xf5`\x85\xa7+\x99s\xd5\xc2}餹\xa8S4Jд\xf0\xe0\xe1^\r\xe8\xee\x15\xe3\x7f\x9e\x80L\x9a\x9b\f\xf6m)8}\x04\xe6ƕ\xda\xc9\xc2Ծo\xe4\xebJ\xd1>E\xd49\x83\x19b\xdem7V\x97\xf2\x80M x\xe9\xad\ue9e2\x9d\xd1=\x14x{\xb6p\xbd\xa0\xf38\xb6\xc9\xf9\xca\xcd\xc3Cɝ%\x9c\xdd\xc2\x06.z\xee\xeb\\J3\xfc\x97dj'\x1e\xd9\xe8D\x84^N\xfa\xb3\xba\xb6\xe9\xad,\x90(\xd0\xc5\xecLԧbT^ze=\x83\xf2\xfbq#H\xaf\x04^\x90r\x19\xe8\x90r\x9fA\x03&]\xf0\x1b\xb1\x9c\xbe%\x91\x82F\xe6\xf6\xc2\xce\n\x0eW4\xbd\x92\x9d<|rN\xad\x1dv \x94\xf0Ff\x15\x91\xda\xcf\xd6ʛ\xf5\x1d\x04\xabls-\a\x87w\xfa\xbbI(\xb8}\x1a.#5\xf0\x19_\xae\xcc>\xf8\x15\x85-!ϯ|^\\Uz\x87\x9f\x817P\xbaz)/&9\xf7;sB\x91%\x90ڞr\xe5\xb4>\xf4\xef\x0e\xfe\xfa{\xf1O\x00\x00\x00\xff\xff\x045\f\xc6i\n\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VM\x93\xdb6\f\xbd\xfbW`\xa6\x87\xb43\x91\x9c\xb4\x97\x8eo\xad\x93\xc3N\xd24\xb3N\xf7NS\xb0\xc4.E\xb2\x04\xe8\xcd\xf6\xd7w@J\xfe\x94\xbd\xdeCu\x13\t\x82\x8f\x0f\x0f\x8f\xac\xaaj\xa6\x82y\xc0Hƻ\x05\xa8`\xf0;\xa3\x93?\xaa\x1f\x7f\xa5\xda\xf8\xf9\xf6\xfd\xecѸf\x01\xcbD\xec\xfb{$\x9f\xa2\xc6\x0f\xb81ΰ\xf1n\xd6#\xabF\xb1Z\xcc\x00\x94s\x9e\x95\f\x93\xfc\x02h\xef8zk1V-\xba\xfa1\xadq\x9d\x8cm0\xe6\xe4\xe3\xd6\xdbw\xf5\xfb\x9f\xebw3\x00\xa7z\\@㟜\xf5\xaa\x89\xf8OBb\xaa\xb7h1\xfa\xda\xf8\x19\x05Ԓ\xbb\x8d>\x85\x05\xec'\xca\xdaa߂\xf9Ð澤\xc93\xd6\x10\x7f\x9a\x9a\xfdl\x86\x88`ST\xf6\x1cD\x9e$\xe3\xdadU<\x9b\x9e\x01\x90\xf6\x01\x17\xf0E`\x04\xa5\xb1\x99\x01\fG̰\xaa\xe1t\xdb\xf7%\x95\xee\xb0W\x05/\x80\x0f\xe8~\xfbz\xf7\xf0\xcb\xeah\x18\xa0A\xd2\xd1\x04\xceD\x9d`\x06C\xa0`@\x00\xecw\xa0@9P\x91\xcdFi\x86M\xf4=\xac\x95~La\x97\x15\xc0\xaf\xffF\xcd@\xec\xa3j\xf1-P\xd2\x1d(\xc9WB\xc1\xfa\x166\xc6b\xbd[\x14\xa2\x0f\x18ٌ,\x97\xef@C\a\xa3'\xc0\xdf\xc8\xd9J\x144\"\x1e$\xe0\x0eG~\xb0\x19\xe8\x00\xbf\x01\xee\fA\xc4\x10\x91\xd0\x159\x1d%\x06\tRn8A\r+\x8c\x92\x06\xa8\xf3\xc96\xa2\xb9-F\x86\x88ڷ\xce\xfc\xbb\xcbM\u0090lj\x15\x8fr\xd8\x7f\xc61F\xa7,l\x95M\xf8\x16\x94k\xa0W\xcf\x101\xf3\x94\xdcA\xbe\x1cB5\xfc\xe1#\x82q\x1b\xbf\x80\x8e9\xd0b>o\r\x8f\xbd\xa3}\xdf'g\xf8y\x9e\xdb\xc0\xac\x13\xfbH\xf3\x06\xb7h\xe7d\xdaJE\xdd\x19F\xcd)\xe2\\\x05Se\xe8.\xf7O\xdd7?ġ\xdb\xe8\xcd\x11V~\x16\x99\x11G\xe3ڃ\x89\xac\xf9+\x15\x10\xd5\x17\xc1\x94\xa5\xe5\x14{\xa2eHع\xff\xb8\xfa\x06\xe3ֹ\x18\xa7\xec\x17\xe5\xec\x16Ҿ\x04B\x98q\x1b\x8c\xa5\x88Yy\x92\x13]\x13\xbcq\x9c\x7f\xb45\xe8N駴\xee\r\xd3(f\xa9U\r\xcbl(\xb0FH\xa1Q\x8cM\rw\x0e\x96\xaaG\xbbT\x84\xff{\x01\x84i\xaa\x84\xd8\xdbJp腧\xc1\x85\xb5\x83\x89\xd1\xc9.\xd4\xeb\xa4\xd5W\x01\xb5TO\b\x94\x95fctn\r\xd8\xf8\bj\xdf\xf9\x03\x81\xf5Q\xe6\xe9\xce\xcd\xe0Tl\x91OGO\xb0|\xcbA\xb2\xfdS\xa7\x8e\x8d\xe6G\xac\xdbZ\xbc\x82\x06 \xc5=~\xaa\xcf2^\xc6\x00\x93\xea\x9dD2\x8aXh\x10^\xc5\nĤ\x0e1\x9do-\x1f\xba\xd4OoP\xc1\xef\x19\xf3g\xdf^\x9d_z\xc7\"\xf7\xabA\x0fަ\x1eWN\x05\xea\xfc\v\xb1w\x8c\xfd\x9f\x01c\xb91\xaf\x86\x8e\x17\xef\ue5ba\x12\x98\xec\xc5}\xefQ\xfc\x1e/\x9ft\b\xb8)\xcb\r\x98\x86ț\x0e\xba\\ݽ\x86\xc2\v\xe1W\x8bt\xa1m\xc7/_\xcf/kP.\xf8Q\x83\xb2\xa4\xdcY\b\x9f\xd2\x1a\xa3CF\xda\xdb\xe7\x93\xe1n2#\xc0Sgt\x97\x17f\x01\x8b3\x13ym\xb2Ͻ\x1e\xbe\xf4\xbd\x898\xd1DUn\xae\x89a\x01\x7f6|\xc1\xad.mP\r\x0er\x93\xe3\xb1\xe2D\xaf\xf0\xbc\x1c?R\xadS\x8c\xe8xȒ\xdf\x00\xa7\vn5\xbd\xd1)\xfe\xba\xff\xfc\x82\xf3}\xd8G\xe6Ǭ2\xae\xa0\t\x11+2\xad\xbc\\dN\xbc/{\xd29\x19\xe5;~I\x1d\x135YQ\xfc\x1eLi\x98\x17 ~\xdc\x05\x16\x83FW.\xdfӷbN\x88\x94\x1f6Z\x9d>\xa9\xe4[#4h\x91\xb1\x81\xf5s\xb9i\x9e\x89\xb1?ǽ\xf1\xb1W\xbc\x00\xb9\x94+6\x132r\xc9Z\xb5\xb6\xb8\x00\x8e\xe9\x92\xca&\x0f\x1e:E\x13mxt\xe6\xaf\x123%\x8c]3^U\x06\\\xbc\x0f*\xf8\x82O\x13\xa3_\xa3\xd7H\x84\xe7mt\xf1$\x93Mp6H\xf2rj\x0eX\x1a\x1e\xe4\xc3\xc8\x7f\x01\x00\x00\xff\xff\xa7\x94\xfb\xf9\xa5\r\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VM\x93\xdb6\f\xbd\xfbW`\xa6\x87\xb43\x91\x9c\xb4\x97\x8eo\xad\x93\xc3N\xd24\xb3N\xf7NS\xb0\xc4.E\xb2\x04\xe8\xcd\xf6\xd7w@J\xfe\x94\xbd\xdeCu\x13\t\x82\x8f\x0f\x0f\x8f\xac\xaaj\xa6\x82y\xc0Hƻ\x05\xa8`\xf0;\xa3\x93?\xaa\x1f\x7f\xa5\xda\xf8\xf9\xf6\xfd\xecѸf\x01\xcbD\xec\xfb{$\x9f\xa2\xc6\x0f\xb81ΰ\xf1n\xd6#\xabF\xb1Z\xcc\x00\x94s\x9e\x95\f\x93\xfc\x02h\xef8zk1V-\xba\xfa1\xadq\x9d\x8cm0\xe6\xe4\xe3\xd6\xdbw\xf5\xfb\x9f\xebw3\x00\xa7z\\@㟜\xf5\xaa\x89\xf8OBb\xaa\xb7h1\xfa\xda\xf8\x19\x05Ԓ\xbb\x8d>\x85\x05\xec'\xca\xdaa߂\xf9Ð澤\xc93\xd6\x10\x7f\x9a\x9a\xfdl\x86\x88`ST\xf6\x1cD\x9e$\xe3\xdadU<\x9b\x9e\x01\x90\xf6\x01\x17\xf0E`\x04\xa5\xb1\x99\x01\fG̰\xaa\xe1t\xdb\xf7%\x95\xee\xb0W\x05/\x80\x0f\xe8~\xfbz\xf7\xf0\xcb\xeah\x18\xa0A\xd2\xd1\x04\xceD\x9d`\x06C\xa0`@\x00\xecw\xa0@9P\x91\xcdFi\x86M\xf4=\xac\x95~La\x97\x15\xc0\xaf\xffF\xcd@\xec\xa3j\xf1-P\xd2\x1d(\xc9WB\xc1\xfa\x166\xc6b\xbd[\x14\xa2\x0f\x18ٌ,\x97\xef@C\a\xa3'\xc0\xdf\xc8\xd9J\x144\"\x1e$\xe0\x0eG~\xb0\x19\xe8\x00\xbf\x01\xee\fA\xc4\x10\x91\xd0\x159\x1d%\x06\tRn8A\r+\x8c\x92\x06\xa8\xf3\xc96\xa2\xb9-F\x86\x88ڷ\xce\xfc\xbb\xcbM\u0090lj\x15\x8fr\xd8\x7f\xc61F\xa7,l\x95M\xf8\x16\x94k\xa0W\xcf\x101\xf3\x94\xdcA\xbe\x1cB5\xfc\xe1#\x82q\x1b\xbf\x80\x8e9\xd0b>o\r\x8f\xbd\xa3}\xdf'g\xf8y\x9e\xdb\xc0\xac\x13\xfbH\xf3\x06\xb7h\xe7d\xdaJE\xdd\x19F\xcd)\xe2\\\x05Se\xe8.\xf7O\xdd7?ġ\xdb\xe8\xcd\x11V~\x16\x99\x11G\xe3ڃ\x89\xac\xf9+\x15\x10\xd5\x17\xc1\x94\xa5\xe5\x14{\xa2eHع\xff\xb8\xfa\x06\xe3ֹ\x18\xa7\xec\x17\xe5\xec\x16Ҿ\x04B\x98q\x1b\x8c\xa5\x88Yy\x92\x13]\x13\xbcq\x9c\x7f\xb45\xe8N駴\xee\r\xd3(f\xa9U\r\xcbl(\xb0FH\xa1Q\x8cM\rw\x0e\x96\xaaG\xbbT\x84\xff{\x01\x84i\xaa\x84\xd8\xdbJp腧\xc1\x85\xb5\x83\x89\xd1\xc9.\xd4\xeb\xa4\xd5W\x01\xb5TO\b\x94\x95fctn\r\xd8\xf8\bj\xdf\xf9\x03\x81\xf5Q\xe6\xe9\xce\xcd\xe0Tl\x91OGO\xb0|\xcbA\xb2\xfdS\xa7\x8e\x8d\xe6G\xac\xdbZ\xbc\x82\x06 \xc5=~\xaa\xcf2^\xc6\x00\x93\xea\x9dD2\x8aXh\x10^\xc5\nĤ\x0e1\x9do-\x1f\xba\xd4OoP\xc1\xef\x19\xf3g\xdf^\x9d_z\xc7\"\xf7\xabA\x0fަ\x1eWN\x05\xea\xfc\v\xb1w\x8c\xfd\x9f\x01c\xb91\xaf\x86\x8e\x17\xef\ue5ba\x12\x98\xec\xc5}\xefQ\xfc\x1e/\x9ft\b\xb8)\xcb\r\x98\x86ț\x0e\xba\\ݽ\x86\xc2\v\xe1\xaf(ҝ\xdb\xf8\xe9\xb8\v\xed=~\xf9\x1a\x7fY\xab\xf2\x10\x18\xb5*K\xca݆\xf0)\xad1:d\xa4\xbd\xcd>\x19\xee&3\x02}S\xe6\x84H\xf9\x01\xa4\xd5\xe9\xd3K\xbe5B\x83\x16\x19\x1bX?\x97\x1b\xe9\x99\x18\xfbs\xdc\x1b\x1f{\xc5\v\x90˻b3!#\x97\xacUk\x8b\v\xe0\x98.\xa9l\xf2\xe0\xa1S4цGg\xfe*1S\xc2\xd85\xe3Ue\xc0\xc5{\xa3\x82/\xf841\xfa5z\x8dDx\xdeF\x17O2\xd9\x04g\x83$/\xac急\xe1\xe1>\x8c\xfc\x17\x00\x00\xff\xff\t\x15i;\xcd\r\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKs\xdc6\xf2\xbf\xebSt9\a\xfdSerb\xff\xb7\xb6\xb6\xe6f\xcb\xeb-\xed&\xb2ʒ}I\xe5\x80!\x9a$\"\x12@\x00p\xa4\xd9T\xbe\xfbV\x03\x04\x87\x0f̌F\xb5\xce\xf2\"\r\x1e\xdd?4\xfa\x8d,\xcb.\x98\x16_\xd1X\xa1\xe4\x1a\x98\x16\xf8\xe4P\xd2/\x9b?\xfc\xcd\xe6B\xad\xb6o.\x1e\x84\xe4k\xb8\xea\xacS\xedg\xb4\xaa3\x05~\xc0RHᄒ\x17-:ƙc\xeb\v\x00&\xa5r\x8c\x86-\xfd\x04(\x94tF5\r\x9a\xacB\x99?t\x1b\xdct\xa2\xe1h<\xf1\xc8z\xfbC\xfe\xe6m\xfe\xc3\x05\x80d-\xaeA+\xbeUM\xd7\xe2\x86\x15\x0f\x9d\xb6\xf9\x16\x1b4*\x17\xea\xc2j,\x88veT\xa7װ\x9f\b{{\xbe\x01\xf3\xad\xe2_=\x99\xf7\x9e\x8c\x9fi\x84u\xffJ\xcd\xfe(\xac\xf3+t\xd3\x19\xd6,A\xf8I+d\xd55\xcc,\xa6/\x00l\xa14\xae\xe1\x86`hV \xbf\x00\xe8\x8f\xe8ae\xc08\xf7Bcͭ\x11ҡ\xb9\"\nQX\x19p\xb4\x85\x11\xday\xa1\xdc*\x0e\x01 \x04\x84`\x1ds\x9d\x05\xdb\x1550\v7\xf8\xb8\xba\x96\xb7FU\x06m\x80\a\xf0\xabU\xf2\x96\xb9z\ryX\x9e\xeb\x9aY\xecg\x83x\xef\xfcD?\xe4v\x04\xda:#d\x95\x82q/Z\x84\xc7\x1a%\xb8ZX\b\xa7\x85Gf\t\x8eq\xfe\x94i\xc6~\x9e\xb6[\xc7Z=Ape\x90\xed\xb7\x06\b\x9c9L\x01\x18\xe4\t\xaa\x04W#I\xde+\x16\x13R\xc8\xca\x0f\x85\x9b\x00\xa7`\x83\x1e\"r\xe8t\x02\x99\xc6\"\u05ca\xe72\x12\x9d\xc0\xba\x99\x8d\x9e\x92\r\xad\xffo\xa3\x9a\x00\xbaU\xfc\x05P\xce\xe2\x1b\x16O\xb8~\x1d\x0f\x9dԏ\x1a\xfd\x9aȼӍb\x1c\r\xb1\xaf\x99\xe4\r\xd2\xcd2p\x86I[\xa29\x00#n\xbb\xdf\xe9)\x98/\x91\xdeh\xe6\x1ca\xf4\xb6s\xe7\x94a\x15\u008f\xaa\xf0\x0e\x8aT\xda\xe0D\xa7m\xad\xba\x86\xc3&r\x01\xb0N\x99\xa4\x82\x13Ⱛ\xa7\x1b\xc9\xce\xecl\xca\xf30\xfa\x11\xed\xe8O\xf3\x82lD(\x99\xb6\xa0w\x15\xa6\xad'Lo\xdf\x04wU\xd4زu\xbfRi\x94\xefn\xaf\xbf\xfe\xff\xddd\x18@\x1b\xa5\xd18\x11\xddg\xf8F\xc1a4\nSQ_\x12\xc1\xb0\n8E\x05\xb4A\a\xc3\x18\xf2\x1eC\xb8\x0ea\xc1\xa06hQ\xba\xb1H\xe2\xa7J`\x12\xd4\xe6W,\\\x0ewh\x88L\xbc\x98B\xc9-\x1a\a\x06\vUI\xf1\uf076%]#\xa6\rs\xd8{\xf1\xfd\xe7\x1d\xadd\rlY\xd3\xe1k`\x92C\xcbv`\x90\xb8@'G\xf4\xfc\x12\x9b\xc3O\xca \bY\xaa5\xd4\xcei\xbb^\xad*\xe1bP,T\xdbvR\xb8\xdd\xca\xc77\xb1\xe9\x9c2v\xc5q\x8b\xcdʊ*c\xa6\xa8\x85\xc3\xc2u\x06WL\x8b\xccC\x97>0\xe6-\xff\xce\xf4a\xd4^N\xb0.\x14#|>\x98\x1d\xb9\x01\ng ,\xb0~k8\xc5^\xd0\xd1\x1d}\xfe\xfb\xdd=D\xd6\xfe2\xe6\xd2\xf7r\xdfo\xb4\xfb+ \x81\tY\x92Y\xd3%\x96F\xb5\x9e&J\xae\x95\x90\xce\xff(\x1a\x81r.~\xdbmZ\xe1\xe8\xde\x7f\xeb\xd0:\xba\xab\x1c\xae|\xa6@\xee\xa9Ӥ\xb9<\x87k\tW\xac\xc5\xe6\x8aY\xfc\xe6\x17@\x92\xb6\x19\t\xf6yW0Nr拃\xd4F\x131E9p_\xb3\xbc\xe3NcA\xb7G\x02\xa4\x9d\xa2\x14\xbd\x87*\x95\x016_\x9eO\b\xa7\r\x97\xbe\xa4w\x9a/\x9a!{\x9f\xda\x13\xb1ɑO\x8d\x0e3\xac\\\x10\x05h\xe6^v\xd8cP++\x9c2;\"\x1c\x1cl\xbe\xa0p\xe0\x1a蓊\xe3\x89s\xdc(\x8e)ش\x15\\͂\xb6R~E\xfe\xa8\x93rɅ>%\xcf\x02\xa6\x15?\x81\xab\xe7\xc8\xc0`\x89\x06e\x81\xd1q\x1dK\x1e\x12\xc8\xc6a}\x89\xf1\xb0R\xc0\x11\xaf\x9eD\xfc\xee\xf6:z\xf2(\xc4\x1e\xbb[\xf2=!\x1f\xfaJ\x81\r\xf7\x81\xee4\xef\xcb\xeb20\xf3>\xcd)`\xa0\x05\x864p\b\x12 \xa4u\xc88\xa82I\x91j\x12 \xc37\xd8\xefx\x1d\xad\x032R\xa8\xca{b\x8a\x9a\xa5\xa0l\x86Ҙ\x10\x8b\xbd6.\x82y\xfcl\x17\xd4\xc7)(j&+\f\xe7E(;\x8a\x8e\xf9\xe5K\xecx\x99\x92\xc4/\x91\x9a\xcc\x1d\xc7\xff,\xb8?\xf3p>\x83~\xc6\xe1\xc6U\xc6\xd1\xc3=t\x1b4\x12\x1d\xfa\xf3qUX:Z\x81\xdaٕڢ\xd9\n|\\=*\xf3 d\x95\x91jfA\a\xecʗ\xa9\xab\xef\xfc\x9f\x17\x9f\xc5W\xb4\xcf=Ф\xd2\xfe\x96\xa7\">v\xf5\xa2C\xc5\x1c\xf6\xf9q\xec\xf2\xaeϬ\xe6{\xc9,\x1ekQԱ8\xe9}\xec\x01c\x12\x94\t\xf3\xe0\x9a\x99\xdc}sU&\x81v\x86\x10\xed\xb2\xbe\x97\x961\xc9\xe9\x7f+\xac\xa3\xf1\x17I\xb0\x13\xcf2\xdf/\xd7\x1f\xfe\x1c\x05\xefċl\xf5@\x02\x1e\xbe\xa7l\x0f+k\x99\xce\xc2j\xe6T+\x8a\xd9j\xcaJ\xaf9\t\xbe\x14hN\xa4q\x9f'\x8bc\xa2\x99\xc8o\x875g呎U\x89\xc4m\xdc:<\x96\xde\x1d\x95״q\xc3*\v\xcc 0h\x99\xa6{~\xc0]\x16\x12\x02\xcd\x04Es\n\xd8CW\x04\x98֍H\x06\xee>\xec\xf7)k/\t*\xcbYe\x0f\x9d=yk\xb1\vt\xa5d)\xaa\x13\xf7\xf0e\xb2x\xc8\tlL;JQuf_J\x8d\x9bS\xe7&͚\x19\xd64\xd8|\x14\r\xda\xc0\xf7\x19\xf6s\xbb\xdc5\x94%]\xbbACB/ir`p\xc0\xef\x04\xdc>\xbf\xd2hJe\xdaВ\xedl,\x17\x0e\x9fl/r!\x1dVC\a\xee\xac\v\xb9\xa7%ϻ\x0eZ\x1a\x8fy\xa21\xe8ꔣ\x9d\xb4\v\x97\xe7AٵK(\x19<(-Xbܠu\v\x83\xa7\x89W\xcbD\xf1\x88\xf5\x04\xe5>!\x83\xbe_\x9f\xa8={\xdb\b\x89\xfaP\x7f\xfa\x16m\xaa\xc0;\\\xdd\x1d\x84h\xf0\xb7\x8eʎ)\xc4,\xdd\t\x98\xad\xa1\xcax6\xa4\x15\xbf\x98\vr\xec\xf7f\x93\x936\xf2\x18\xe9\xb2=\xe2_\a\xceh\x90\x84W\x8f^\xa6!\x1c\xba\xf8\x16B\xb5\xe0K[$\x85\xa22k\xd2b=q\xbdW\xcb\x1d\xbe\x1bix\xaf\xee\xa2%w:z#\xe9y\xa4z\x1c0\"\x17v\xfa\x04\x84\xa8!\xf75\x10\x95h%\x13\rr\x88\x0fa\xf3=\t\xaac*\x1b,)Z\aӋ\xae\xa2\x877\xd4\x195\x82\xf5m\xbeK{\x84fg\x91{?\x9a\x10²\xf6 \x17\xc5\\hKgI\xa2\xb2k\x1a\xb6ip\r\xcet\xcb\xe9#\x96آ\xb5\xac:e\x8a?\x85U\xa1\xe1\xd2o\x01\xb6Q\x9d\x1b:.\x93xui{\x9d:\xaf\xe9\x93\xece̢\x00՚\xb6\xaf\xb9\x9a\xc6\xef\x19;\x82\xfd#\xa9G\xb5\xc1t\xce\xf6\x12\x9f\x00\xe0_\xffN!\xa45)\x03\x1b\xbc\xd7Q\v\x83#N\xf9\x06\x1f\x13\xa3\x8bW\xcb\xf1\xe4U4\x99\xc4\xdcGo\rg\x9d\xbfgtJ\x04\xfd2\xa8U\x13\x8dY9\u058c\x02\xf5f\xe7\xd0N\xddy\xaa\xbd\xe6\xcb\xf2\xbd\x18G\xfb\xe3\xfd\x05J}\xa7\xa1`ҷ\xc1ɺ\x9c\x02.\xacn\xd8.\x95\x82D\x84\x94z\x93q\x91\v\xd8\xebs4j\x8d!\xe597\xc3\xf1\x98>(y\xa0P\x8c\xf6,\xa4\xfb\xeb_^\x90b@\x10\xe7{\xe2\xf2m8\x1cIb\xacd\xda\xd6\xca]\x7f8\xa1\x05w\xc3\xc2h\r\xfb\x1c~\xc8!#\xb5^\x15RW5\xf8\x96\xb3Lu\xfa^~\n\xead\xf1\x89(Կԧb\xd0\x1dR\xea\xe90<\xe9\\\xcd\xdf\x1c_\x83\x15\xbeOM\xa5@\xa8\rB\x17\xc9Rp\xa2\xd4J\x19L\xb8LX\x86\x95I\x10\x99\xc2\xff3\xe3GRO\x16\x83\x1e9\x1f\xd1\xee\xdf:\xc6#\xddfx\xc7[\xc3\xef\x7f\\\xfc'\x00\x00\xff\xffo\x05\xb2\xa4Q#\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Y_\x93۶\x11\x7fק\xd8q\x1e\xae\x991\xa9\xd8\xedt:z\xb3\xef\x9aε\xc9Yc\x9d\xfd\x92\xc9\x03D\xacHD$\x80\x02\xa0tj&߽\xb3\x00A\xf1\x9f\xa4\xd3M.\xe1\x8b}\xc0b\xf1\xc3\x0f\xfb\x0f\xab$IfL\x8b\xafh\xacPr\x01L\v|r(\xe9/\x9bn\xffaS\xa1\xe6\xbbw\xb3\xad\x90|\x01\xb7\xb5u\xaa\xfa\x8cV\xd5&\xc3;\xdc\b)\x9cPrV\xa1c\x9c9\xb6\x98\x010)\x95c4l\xe9O\x80LIgTY\xa2Ir\x94\xe9\xb6^\xe3\xba\x16%G\xe3\x95ǭwߥ\xefާ\xdf\xcd\x00$\xabp\x01Z\xf1\x9d*\xeb\n\rZ\xa7\f\xdat\x87%\x1a\x95\n5\xb3\x1a3R\x9e\x1bU\xeb\x05\x1c'\xc2\xe2f\xe3\x00z\xa9\xf8W\xaf\xe7s\xd0\xe3\xa7Ja\xdd\x7f&\xa7\x7f\x10\xd6y\x11]ֆ\x95\x138\xfc\xac\x152\xafKf\xc6\xf33\x00\x9b)\x8d\vx (\x9ae\xc8g\x00\xcd9=\xb4\x04\x18\xe7\x9e9V.\x8d\x90\x0e\xcd-\xa9\x88\x8c%\xc0\xd1fFh\xe7\x99i\xf5\x80ڀ+\x90\xb6\xf4\xac2!\x85\xcc\xfdP\x80\x00N\xc1\x1a\xa1A½2\x80_\xac\x92K\xe6\x8a\x05\xa4D\\\xaa\x15Oe\xd4\xd9\xc8\x04\xce\x1f\x06\xa3\xee@\xe7\xb0\xce\b\x99\x9fB\xf6;\x83\xea\xe1Y*\xfeL$\x8f\x05z\x99\x88\xa6֥b\x1c\rm^0\xc9K\x042Pp\x86I\xbbAs\x02E\\\xf6x\xd0}$_\xa2\xbe\xce\xcc5\xec\\CE\x90\xedm\xff\xb5;tiߥ\xe2\xcd\x02h\x8c\x1a\xacc\xae\xb6`\xeb\xac\x00f\xe1\x01\xf7\xf3{\xb94*7h\xed\x04\f/\x9e\xea\x82\xd9>\x8e\x95\x9fx]\x1c\x1be*\xe6\x16 \xa4\xfb\xfb\xdfNck\x16\xa5N9V~<8\xb4=\xa4\x8f\xc3ဖ\x9c-o\xae\xffO\x81\xbb&HwJ\xf6y\xfd8\x18\x9d\x02\xdbQ\x1a\xe3m\x9a\x19\xf4\xa1\xf6QTh\x1d\xabtO뇼\xaf\x8f3\x17\x06\xc2\xf4\xee]\beY\x81\x15[4\x92J\xa3\xfc\xb0\xbc\xff\xfa\xd7Uo\x18@\x1b\xa5\xd18\x11\xa3k\xf8:ɣ3\n}foHa\x90\x02NY\x03mp\x8a0\x86\xbc\xc1\x10\x9cEX0\xa8\rZ\x94!\x8f\xf4\x14\x03\t1\tj\xfd\vf.\x85\x15\x1aR\x03\xb6Pu\xe9#\xd0\x0e\x8d\x03\x83\x99ʥ\xf8_\xabے\xefѦ%s\u0604\xf8\xe3\xe7c\xb0d%\xecXY\xe3[`\x92C\xc5\x0e`\x90v\x81Zv\xf4y\x11\x9b\u008fd!Bn\xd4\x02\n\xe7\xb4]\xcc\xe7\xb9p1if\xaa\xaaj)\xdca\xee\xf3\x9fX\xd7N\x19;\xe7\xb8\xc3rnE\x9e0\x93\x15\xc2a\xe6j\x83s\xa6E\xe2\xa1K\x9f8ӊ\x7fc\x9a4kozXGN\x17>\x9f\xeb\xce\xdc\x00%;\x10\x16X\xb34\x9c\xe2Ht\fٟ\xff\xb9z\x84\xb8\xb5\xbf\x8c!\xfb\x9e\xf7\xe3B{\xbc\x02\"L\xc8\r\x05]\xbačQ\x95\u05c9\x92k%\xa4\xf3\x7fd\xa5@9\xa4\xdf\xd6\xebJ8\xba\xf7\xff\xd6h\x1d\xddU\n\xb7\xbe\x92\xa0xYk\xb2\\\x9e½\x84[Vay\xcb,\xbe\xfa\x05\x10\xd36!b\x9fw\x05\xdd\"h(\x1cX\xebL\xc4\n\xe6\xc4}\r\xab\x92\x95ƌ\xae\x8f\x18\xa4\xa5b#2\xef\x1b\x14~\x80\x8d\xe4Ӟ\xeaiץoͲm\xadWN\x19\x96\xe3\x0f*\xe8\x1c\n\r\xb0}\x9cZ\x13\xc1\xc9N\xce\v\xca\xc1\x06ɑR\x802.\xde\x17h\xb0\xbbƠVV8e\x0e\xa48d\xcbt\xa4\xe1\xc4E\xf8#+~\xe1\x18\x14\xee\xbdC\x18ܠA\x99a\x8c\x10\xe7*\x99\x89St\x12\xfa\x18\xe2i\xea\xe1L\xf4\x9c\x04\xfcay\x1f#fd\xb8\x81\xee\xc6\xfb^\xa0\x87\xbe\x8d\xc0\x92\xfb\x84ry\xef\x9b\xfbM\xd8\xcc\xc7\x0e\xa7\x80\x81\x16\x18*\xd26\x18\x83\x90\xd6!\xe3\xa06\x93\x1a\xe9m\x00\xe4`\x06\x9b\x15oC\xa4hB\xd21\x84\x13\xf5\xc0(F\t\x0e\xff^}z\x98\xffk\x8a\xf9\xf6\x14\xc0\xb2\f\xad\xf5\xf9\x1a+\x94\xeem\x9b\xb39Za\x90S\xe1\x82iŤؠui\xb3\a\x1a\xfb\xd3\xfb\x9f\xa7\xd9\x03\xf8^\x19\xc0'V\xe9\x12߂\b\x8c\xb7\xe1/ڌ\xb0\x81\x8eV#\xec\x85+\xc40i\xb5\f\x90u5\xc7\xde\xfb\xe3:\xb6EP\xcdqk\x84Rlq\x01o|%x\x84\xf9+9\xd6ooNh\xfdKp\xa07$\xf4&\x80k\xf3]\xd7#\x8f ]\xc1\x1c8#\xf2\x1c\x8f\x85\xe8\xf0\xf3\xc1\x9bBⷠ\f1 UG\x85WL\xb7\x17\xe2\x11\xf2\x11\xe8\x9f\xde\xff|\x12q\x9f/\x10\x92\xe3\x13\xbc\a!\x037Z\xf1oSx\xf4\xd6q\x90\x8e=\xd1NY\xa1,\x9ebV\xc9\xf2\x10\xaa\xfd\x1d\x82U\x15\xc2\x1e\xcb2\t\xf5\x06\x87=;\x10\v\xf1\xe2\xc8\xde\x18hf\xdcYk\x8dU\xc6㧻O\x8b\x80\x8c\f*\xf7\xf1\x8e\xb2\xd3FP\xd5@\xe5B\xc8y\xde\x1aGI3~\xb6\x0e\xe6\xe3\x14d\x05\x939\x86\xf3\"lj\xcaB\xe9\xcdK\xfcx\x9c\xfa\xe37Q\x02\f\x03ǟ\x96D\x9fy8_\xa9>\xe3pݷ\xd6\xd9\xc3m\xeb5\x1a\x89\x0e\xfd\xf9\xb8\xca,\x1d-C\xed\xec\\\xed\xd0\xec\x04\xee\xe7{e\xb6B\xe6\t\x99f\x12l\xc0\xce\xfd\x93y\xfe\x8d\xff\xe7\xc5g\xf1\xaf\xeb\xe7\x1e\xa8\xf7\xe8\x7f\xcdS\xd1>v\xfe\xa2C\xc5Z\xf1\xf9y\xecf\xd5\x140õ\xe4\x16\xfbBdE|\x0441\xf6\x843\t\xaa8y\b\xcdL\x1e^ݔ\x89\xd0\xda\x10\xa2C\xd2\xf4\xb4\x12&9\xfd\xdf\n\xebh\xfcE\f\xd6\xe2Y\xee\xfb\xe5\xfe\xee\x8f1\xf0Z\xbc\xc8WO\x14\xba\xe1{J\x8e\xb0\x92\x8a\xe9$H3\xa7*\x91\r\xa4\xa9\xf6\xbb\xe7D\xfcF\xa0\xb9P\xc5}\xee\t\xc7*t\xa2\x8ale\xae*#\xadd\xda\x16\xca\xdd\xdf]\xc0\xb1j\x05#\x86\xe3u5\xc5c\xd45h\x02]\x87\xc7\xfb\xcb\xc3\xe9@\xd2\a\u0557\x8eȔ\x11\xb9O[\xad\xef\xfbW\x84d\x15\xeb6\xff\xba_Ŵ\x162\xbf\nk\xb7\x97v\x01藎hDy\xa1\x9b\xe7\x8a)\x9c\xbd\x1e\xdf\x18-ʺ\x1aCI`\xab\xb4`\x13\xe3tG#\xfb\xa4\x897\xe3\xba\xe6\f\x13\xc1\x00.pд\x9e&\xdeQ\x8d\xfd\x84\xbaҏ\xd0\xdb\xc5[\xd1t@\xbe֮\xe8\xd9MEr\x1fa2\xfd:\x1c\xc8h\xc5gCҺ.9\x98<:\xd4p\xa2o\xab\x83\xd9^K\xb4{\x9a\xf1\xc3\xda\xf7ۮyZ\x87\x1e_\xc3{\x88\xf0.v\xfe\xe8y\xf3\xe2\xc7u\xa6\xe8\xe9\xd0k\xcf]\xb0\x81\xdb\xf1\n\xdf\xc92\xbc\xf1\tQ\xa1\x7f\xb1\x86\xf6\xe4\x9eٸ\xc9\xd4}CG_X\xea\xb3*\xa9C\xee\v{zwl\x98(\x91C\xfb+\x8bo\xa5[\xdfҹ\x99\xaac\xa3\xa2\xda\"\xf7qc\x02\xf4x]\xec\x92r\xe60!\x15#\tY\x97%[\x97\xb8\x00g\xea\xf1\xf4\x19\xf7\xaa\xd0Z\x96_\xf2\xaf\x1f\x83Tx\xf37K\x80\xadU\xed\xdaG\x7f\xe3h\r\x157\xb6\xb1\x82\xeb\x1a\x0f\x05\xb3\x97\xa0,If\xca\xe2Z\x97?orp&\x94=\xe0~btԵ\xeeN\xdeF\x13\x9a\x98\xfb\xde[\xc7U\x044\x1b]\xe2\xa0\x11\x83B\x95Ѻ\x95\xa3\xa4TWk4D\x84o\x95GFb\xe0\x98\xea\xa2\xf8\xd7בɣ\x86\x18\v\x83\xaa\xe6=\x991雊d\xbfN\x01\x17V\x97\xec0\xa17\x9e\xc4\x17Xd\xbe\xe4GG\x8b\x89^H\xee\xef\xe7\xae\xed\xfe\xb4?\x05L\x97\x7fS?,L\xddB\xf7W\x82\xc1|\xfb\x1b\xc8\xeb\xecp\xa6䳎\x19\xf7ܰ\xb7\xea\t_\x8ax^\xf5t\xbc놮q\xa0\xeao\xf3GƨI\xa2F\x83\x1e9\xef\xe8n:\xa7ݑz\xdd\xfe.\xb0\x80_\x7f\x9b\xfd?\x00\x00\xff\xffg\b\x17r\xc1\x1f\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc]O\x93\xdb:r\xbfϧ@9\a'U#y\x9d\\Rs\x9b\xf8\xd9Ye\xf7\xd9Sc\x97\xdf\x19\"[\x12vH\x80\x0f\x005VR\xf9\xee\xa9n\x80\xe0\x1f\x81$\xa8\x19\xbd\xf5\x06\xb7\xa1\x80\x06\xd0\xddht7~\xc0\xacV\xab\x1b^\x89\uf80dP\xf2\x8e\xf1J\xc0\x0f\v\x12\xff2\xeb\xa7\x7f7k\xa1\xde\x1d\xdf\xdf<\t\x99߱\x0f\xb5\xb1\xaa|\x04\xa3j\x9d\xc1/\xb0\x13RX\xa1\xe4M\t\x96\xe7\xdc\xf2\xbb\x1bƸ\x94\xcar\xfcl\xf0O\xc62%\xadVE\x01z\xb5\a\xb9~\xaa\xb7\xb0\xadE\x91\x83&\xe2M\xd7\xc7?\xad\xdf\xff\xeb\xfaO7\x8cI^\xc2\x1d\xd3`\xac\xd2`\xd6G(@\xab\xb5P7\xa6\x82\fi\ued6a\xab;\xd6\xfe\xe0\xda\xf8\xfe\xdcX\x1f]s\xfaR\bc\xff\xd2\xfd\xfaWa,\xfdR\x15\xb5\xe6E\xdb\x19}4B\xee\xeb\x82\xeb\xf0\xf9\x861\x93\xa9\n\xee\xd8g\xec\xa6\xe2\x19\xe47\x8c\xf9\xa1S\xb7+?\xea\xe3{G\";@\xc9\xddx\x18S\x15\xc8\xfb\x87\xcd\xf7\x7f\xfb\xda\xfb\xccX\x0e&Ӣ\xb2\xc4\x00?6&\f\xe3\xec;\xcd\r\a@\xbcf\xf6\xc0-\xd3Pi0 \xada\xf6\x00\x8cWU!2bu\xa0ȘڅV\x86\xed\xb4*[j[\x9e=\xd5\x15\xb3\x8aqf\xb9ރe\x7f\xa9\xb7\xa0%X0,+jcA\xaf\x03\xadJ\xab\n\xb4\x15\rc]\xe9\xa8K\xe7\xeb`.oq\xba\xae\x16\xcbQO\xc0\rٳ\fr\xcf!\x1c\xad=\b\xd3Nm8\x1d?%.\x99\xda\xfe\r2\xbbf_A#\x19f\x0e\xaa.rT\xaf#hdN\xa6\xf6R\xfcw\xa0mp\xa2\xd8i\xc1-xy\xb7EH\vZ\xf2\x82\x1dyQ\xc3-\xe32g%?1\r\xd8\v\xabe\x87\x1eU1k\xf6+\x89G\xee\xd4\x1d;X[\x99\xbbw\xef\xf6\xc26\xcb$SeYKaO\xefH\xe3Ŷ\xb6J\x9bw9\x1c\xa1xg\xc4~\xc5uv\x10\x162[kx\xc7+\xb1\xa2\xa1KZ*\xeb2\xff\xa7 \xb6\xb7\xbd\xb1\xda\x13j\x9e\xb1Z\xc8}\xe7\aR\xf3\t\t\xa0\xc2;]rM\xdd,ZF\xe3'\xe4\xce\xe3ǯߺz&̐\xfb\xc4\xf7\x8e\xf2\xb5\"@\x86\t\xb9\x03\xed\x84Hچ4A\xe6\x95\x12\xd2\xd2\x1fY!@\x0e\xd9o\xeam),\xca\xfd\xf7\x1a\f*\xb4Z\xb3\x0fd;\xd8\x16X]\xe5\xdcB\xbef\x1b\xc9>\xf0\x12\x8a\x0f\xdc\xc0\xd5\x05\x80\x9c6+dl\x9a\b\xbafoX\xd9q\xad\xf3Cc\xbcF\xe4\xe5W\xff\xd7\n\xb2ފ\xc1fb\xe7\x979\xdb)\xdd3\x0e\xd8d\xdd#\x1a_\xb4X\xdc\xeaG\v6\xfce0\x94\xff\b\x15Q\x7fp\x10\xb5\x14\xbf\xd7@&έX83)g$Y3>R\x8b\xf5\xd9\xef#<\xc5\x02?\xb2\xa2\xce!\x0f\xd6\xf6l.\x83\x11\x7f\xe7d\n?]I\xb1\x95#|\x8cX;\xfa\xb7\x13\x9b \xc9P͟\x0f\";8'\x00u\x93\xe8\xb0\\\x81!Á\x8e\xeail\x92lN\xf6\xbe\x93)\xd3і\x9955\xa4\x173'mI0\xb7m\x991\xbcg\x86\xc5\x7f\x8f\xee\x9cm\xf9\xff\xc9\xd8f'\xb9@i7gM_Wi)\xa8Bg\x7f\xb3cPV\xf6t˄m\xbe\xceQ\xe4E\xd1\xe9\xff\x1fX0\xcb5~3l\xf9\xaa\x1a?)\x959\x8a(\x95\xd0\xfd?\xa0Ph\xb3\xf8\xea\xf7\x8ad\x81\xfc\xb5\xdbꖉ]\x10H~\xcbv\xa2\xb0\xa0\a\x92y\xd1zy\rf\xa4\xecwXJn\xb3\xc3\xc7\x1f\xe8٘6ϔȗac\xe7\x1271B\x7fc\x9e\xa1\xcb(|\x15\x1aJ\x17\x16\x7f#n\xb6_(\x9e\xb8\xff\xfc\v\xe4S\xecai\x9aw6\x91\xfb\xc1`\xbb]{??u\x1a\xde\xf5\t1\x93Kx\xdc2Ξ\xe0\xe4<\x16.\x19\n\x87[\xf2w\xa3\xd1\xd39s(\xf3BJ\xf6\x04'\"\xe3S)\xb3\xadSU\xc1\x95'\x88\xb8\xfb\xb1\xd2c \x8e\xc9\a\xb8\x8e\x93\xf8\x81\x18A\x81w:\xf3\x18\xa5\xc5\x1a[4?9\x96nH\x9a\xd2\xf0\xfe\x82i\x06\xb1u҇$طƉ\bW\xc1AT\x89\x13\xa5\xec\xa1\x01Z-Mb\xec;/D\x1e:rz\xbf\x91\xe3\xdep\xbf|Vv#o]DfHK~Q`>+K_\xae\xc2N7\xf0\v\x98\xe9\x1a\xd2\xf2\x92\xcel#\x1f\xba\x19\xb6\x04\xe5ve\xe3\x12)A<°\x8d\xc4\xc0\xc5\xf3\x83\U000a5bbb\xe9\xfd\xa1_\xca\xdaP\nM*\xb9\xa2\xadr\x1d\xeb\xc91;\x91\xa4\xd2=\x89\x9c\x0f-t\xea:L$\xfb\rw\x12\xd7\xdee\x80\v\x9eA\xdeD\x9b\x94\xb7\xe4\x16\xf6\"c%\xe8\xfd\xd4\xc6\xd1-\x15\xda\xf7\xb4!$Z]W\x16jX\xda\xd6\xde\x14o\xba\xf3\xf9\xc1\xacp\xe5&\xd4j\x84=[u$]9^u~F\xb4Œ\xff1\xcb]\x9e\xe7t\x84ċ\x87\x05\x16\x7f\x81,\xce\xf7~70\xb7C\x96\xbc\xc2\xf5\xfb?\xb8͑B\xff/\xab\xb8\xd0\tk\xf8\x9e\x8e\x89\n\xe8\xb5\xf5\x89\xb1n7\u06030\f\xe5{\xe4\xc5y\"<29\x85\xb6\x05\n\xb7\x91\xabݙ\xc7r˞\x0fʸ=u'\xa0\x88\xa5l\xfaE\x18\xf6\xe6\tNon\xcf\xec\xc0\x9b\x8d|\xe36\xf8\xc5\xe6&x\vJ\x16'\xf6\x86ھy\x89\x13\x94\xa8\x89\x89\xd5~\xac\x9eBJnU\xf2j\xe5\xb5תRd\xa3\xedd4=ޖ\x9e:uS\xe4mnܻ\xc7S\xb3M\xd2\xdfJ\x19\xfb\xe7x\xa2od<\x0fM\x8b\xbeO\x1bɗ\xcd\xfa\xfa>\xf7\x15\x8c1z\x80;\v\xda'\xff\x9c\x81n\"\x87\x17\xc6Tsɽ\x90\xd8\xe3!!\x8b\f\x9e\xd1&wT\x922\xc4%\xde&\xf2e\xa1\x9f\xfe\xf1G'7\x89+\x1b\xff\xeeN䵽\xe1L\x95%\x1f\x1e\x0e&\r\xf5\x83k\xd9\xe8\xb4'䤯\xf75\xad\xe7t7\xb1\xd1!:\x16|\x16\xf6 $\xe3\x8d\xd9\x00\xed\x15\x8a\xb3J\xcd[0W\x0eܰ-\x80\f9\xf5\x9fa\x9f/\x85\xdcP\a\xec\xfd\xab\xfb\x05\xace\xd7E\xe2lX\x1d\x04\x1a>\xd0N\x95\xeaR\xa9\x9c=\x1f@CO+\xce\x13\xe5\xe8i&\x92\x94\xcav\xf3\x11H\xb7R\xf9[\xc3vB\x1b\xdb\x1dh\xaa\xc2\xd5&U\x1d\x16J\x18g\xf7M\x94\xa0j{\x81\f>\xb6\xad{\xe7\xba%\xff!ʺd\xbcTu\x82S\xe0\n\xee/\xa2\f\x87\xaf^\x02\xcf\\\xd8p\x0eE\x99\x19\xabPJU\x016U\xc4[ء9ʔ4\"\a݀\x03\x9cd\x85\u0085\xbb㢨c\xc7>\xb1\xb24\xbc\x95\x1f\xb5\xbe(\xba\xfd\xe2Zv\xb2\x8d\a\xf5\xdcgP2\v\x0e\xfc\bL옰\fd\x86r\x01\xedL6u\xe1\x99A\xacIV\xcb4\x03\x8f\x05d]\xa61`E+[\xc8\xc9dZ\xb7\xfa'.\x8ak\x88\r5\xef\x93ҏ\xc0\xf3K\x120\xbfu\x9a3\x90\xa6\xd6tp\xef\xcc˳(\xd2ƌ\x92c\x05\xafev\x00\xb2S\xb2g>\x98#/\xa4\xb1\xc0Su\x01\xbd\xa6ZJ!\xf7i\xb2KNq\xb6űz\xabT\x01|\bx\x8a\x15\xe4\xf5\xe5f跶\xf5\x1fb\x86\x82\x04\xd2݅-xQy[ĭ\x85\xb2r\xebM1]\xcb\xee\xees\x05+\xb4$\x06\xf7\xa3x\xcd\xe0ZH\x91 \xd8\xc1\x99\x8b\xb0]\xcf\x12I\\ճ\xc4\x0e\x82SqI\xfal\xd3#\x80\xab\xb3\tRh\xecAk\x16x\x99[`<\xcf!w\x89ItU|\xcc\xe2\xe0e#P\x85\xe8얻\x89I\x92mJ/\"\xa5T\xac>ª\x96OR=\xcb\x15E\xf2f\xb1\x01IO\r\xbej\xf7\xf6bK\xf4GZ\xa1\xbe\xbe\xa6\xebT\xe3<]\xc1\xca$\xeb͢lȔ\x16\xcc\xd95\a]\x1e\xf9qv\x14S\xfdO4\xf6\a\xcd\x1f\x1c\xe68\x15϶\x89\xb7\xea8\x7f\xcf\a\xb0\a\xd0\r\x98yE\xb8혝nϣ\xdb8&\x00\xdcP\x7f\x1aW\xd8\x01/\a\x90\xb7x\xa0\x83^\xc0-*6\xaf\v\xeb\xe0Ǻ\x8e(Q\x12\xf0+\xee\x19\xa4 '\xe6\xf0\x12}\f`\xc0+4 @\xd5t\x12\x99\xa1\x93\xa5C\xfav\x0f\xe3\xfb\xc0\aJ\xf95#\xfd\xbb\xc3\x03\x130\r3H\x86i\xd0\xe4\x14\xbf\xceզ˱V\a}=\x8f\xa6\xfd\xb9\xd8g\xa1\xfcR\xf9u0\xea\x80\xf69\x18i2\x80\x83\x90\xe5Ɛ\x9d\x80\x05\\Č\v\xaeB\x9f\x0eD\x8a\xf7\x19\xadD\xd5\x106\x94j\xf6\xabͣۅa\xef\xd9A\xd5\x11H\xdd\x04wf\x00\x16\xe3\xb0\n\x7f\x88\x00\x96\x1f߯\xfb\xbfX\xe5A\x16\x94\xf9\x8a̎\x02\x956\x9b*d.\x8e\"\xafy\xd1[d\x1d\xb5h\xb5\x87)ͤ(b竨VM\xfb\x9e\x1a\xb1/\x95;gYl\x8e\xa6]\xc44,\xc6\xc5\b\x8c>\xc2bd\x93Zz\xe4\x90\x0e5M\xc7XL\x83\"\x96 +\x86\xb8\x89Q\xa2\xf3x\x8a\x14\xef~\x06;q\x01b\"\x11-\xf7\xe2\x03\x92\x14L\xc4EH\x88Y@Y\"\xfe\xa1\x8fl\x98&\xb9\x00\xf5\x90Ĝy\x84\xc3b\\\x83\xc7\x11L\xce#\x19\xcd\x10\xc1)L\x12\x1e\xc50L\xa1\x13\xa6Y\x1eA.\xa4c\x12&I\x13^a\x1e\x89\xf0zx\xc3\u05c8\x02\xc6M\xcd,\x9a\xe0EQB\x02^`\tJ`\x96c\x17\"\x02\u0089\xffH\xbfKq\x00\xfds\xfe\x11\xa2)\xa7\xff#\xa7\xfb#\x14'\xcf\xfcS\xcf\xf4Gh\xcfl\xbb\x93Z2\xf9㒳\xfc\x10\x86\xfcʫJ\xc8\xfd\xb9\x9e\xa4jӤ&\x9d\x01\x01\xba}\xf6T\xa9\x1b-\xf4\xe2\xacX\x97\xeeVn$&k\xd2zBZ\xb5f\xf7\xf2tF\x97\xae\x04Dc\x90\xfe\xb5-\x1cֳ(\x8a\xee\xdd$\"\xdb%\xe5o\xf9\x99xf\x00+\x8ey\xd8Q\x11*\xdd\xf3\x8e\xe7B\xb0/\x83\xea\xddDᴷ\x1ds\xb4\x85=\\\xe8m\x97uaE\x15]\xf2\x95VGAi\xc7\x03\x9c\x02?\xff\xa6\xe8V\xd0\xf6D\x94\xbe<\x86ո\x1e\x04\x0e<\xb6\x86\x9e\xa1(\x187\xe7\xd3\xcf\xdc\xc5\xd8L\xad\xe8\xae\x1bJ\xb2\xd1\a\x7f\x81\xf6\x96Vl,b\x97͕\xcd\x12\xc9\xd0\xe5Z\x13Ɉ\x8c\xeeE\xd3\xfe\xb0s\xdd\xe9\xdb\xef5\xe8\x13SG:\xd3\xf7\x0e\xd2\f\xec\xde\xd9\x15\x83\xf1[c鼹tױ\aqBk_ؽt;v\x94\xec`\x8cD\aM\\\x1b\x1b\xa15ǰg\xa4j\x94\xaaT\xa1u\\\x1f&7\xa6T\xcc\xfau#\xa5\xe5\xb1Ҭ\x97r\x95x\xe9\xf2\x88i\x82d*\x06=\xedLd\x16s~\xad\xc8i.vJv\x1a\xd30\xe5\xd7\xc0\x92/\xc0\x90/\x88\xa1\x96EQ\xc9lJ\xc1\x8a_%\x96\xbab4u\x8dx겈j\x86\xe4\x00\x03\x9e\x82\xeeN:\xc6K>\xb3I9e\x9b?9\x9eFm'\xa0\xb5\x13N\x83\xe6F\x9a\x80\xca^\x86\xc6N\xe0\xe1\x95b\xad+E[\u05c8\xb7\xae\x1bq\xcd\xc6\\\xb3\x9a3\xf3\xf32\x14\xf5Ň\f\xcdq\xf4g\x95Ã\xd2v.@x\x18֏\x1c\x01v\x82&U\xe4L6Uc'\r\xe8\xfb{\xbf\xff\xb2I\xc5O\xeb\x1a\xf7\xf7W\x95\xe3\xd8\xe6\xce\x16\x1e\a\xd5Ϯ\xd0\xee@\x83t\x0fK\xfc\xd7\xd7/\x9f\x03\xfd\x98?\xea\x9d\xde\xc1\x9b\x06\xce\xc1\xc8=s\xfc\xe9\x93\a\xdc8n\xd1\x1e\xfeʇ\x04\xbc\x12\xffIov\xcd'd\xee\x1f6T\xb5\xf1\x96譯p\xa0\x1f\xce\u07b6\x80\xbbG\xe0Ȩ\xf6ov=\x8a\x11\xd8i\xf8\x93ыI\xcd\xee%\xc60Y\x0e\x84\x84\xab\xeea\xe3F\xb7f\x9f\xd0u\x93'\xa6\x9c\xe2\x1d\x84\xceW\x15\xd7\xf6D\xdaan\xc3\x18Ɠ2\xcd\x1e2\x95:\x195\xb5\xe7oAEy\xdb<\tE\ap\xa7\xaa\x7f\x9a9\xe4\xe8%\xe3\x18\xbf=1{o\xe2\x15\xc71\xbe\x1d\xaf\x88S\x91\xcfQ\x04ī\xa5\xa4\xbc\x19z\xf8>g\xd6\x1eC\xc5i{\x86\x91l\x93։\xf0\aۓI3\x92W\xe6\x10\xc9\n\xbd̦\xd1CU\x96\xdb:q>\xaenoJ\";t\f\xd034&Jw\x9e\xed\x1b\x8c\tת#D\xdb0\xb9\xd0R\x14\xb7\x9d\xc0\xfc\x8f9\xf2L|\x16\xe4\xe2\aA\x1c{FL\x05e\x9aЌ5\xba\xd0\xf2\xe5\x82\xc3\xceY\x17.\x01\xd8:\xedw&\xbe(q\xf1[\x12\xf3̊0j\xec\x19\x89\x94\xa7\"\xfe\xae\xfc\x9c0I&;@^\x17\x90\xf0\xc0\xdb\xd7N\xd5\xf9'\xde\x1a±5\xa9\xfa\x8f\xbc!_;\xdb+:\xbc\xfd\xc7\xe4<\xd3=\xe5\x11\x88w\x97\xa4\xf3\xecݫS\x19\xfa\xf1\xa6\xce20fW\x17\rZ&\xd3\xc0-\xe4M\xf5(2\xbf\x99\xc3\x02XH|\x17Yu\x9eѻI\x90\x8c\x89\x98\xc9\t\x13\x99\xf1\xca\x12\n\x9e\xbc\x8cZk\x9a\xb2\xfbM\xedΞ\xfe\xeb\x91\x1d7Z\x1e\xce\xe8\xc18\xc6\xf22\xe2\x89\r/\x82\r[\xd0\x03\x9b:\xef\xc0w\xba/\xa4\x05TN,\xad\xcdM@T\xe6\xeb\x0emG\x86\x9c\x1f$\r9\x83#H\xa6$\xdd5\x81|\n\xbe\xfb\x8d\xd2f\xfa\b\xfa\xad\tt\bP\x84\xbe\xe2W˵\rC?\u05c8\x9d\xd2%\xb7w,\xe7\x16V\xd8\xfa\xb2\x1d2\xfez\xa1\xd6\xf3'\x1ctiŇ\xc1tӄ\xc4[\x14\xfe\xaaI\t\xc6\xf0}\xe3\xbe?\x83\x06\xb6\a\x89,\x9ez\xa0\xad\xbd\xad\xe3Wp\xc0\x9d!\xb7xfk\xee;p;e8\xfb\x89\x9d\x1b\xb8W?)\"؏\xae\x1b!-\xec\xcfN]\xfcM\xa1G\xe0f\xf8J\xec\x19#>u\xeb\xfa\x9c\x99\xe3\x81{\x92\x84;\x90\x18=*jE\x88RF\xac\x11\xf6\xbc\b\xfaU\x1d\xb8\x993\x97\x0fX'\\\xa1\xeb,\xca`)\x1fG\xc6\x14\xbfҳb\x9f\xe19\xf2\xf5\x13)=\xe5A\xe3Ki\xc56\xf2A\xab\xbd\x06s\xae\xd2+\xba\xe3!\xe4\xfe\x93\xd2\x0fE\xbd\x172@\xf0\x96U~\xe0\xda\n^\x14'7\x9eH\xdb\x0f\xcdb\x8e\xfc6\xdfz\xe4\x87)!\xf99\xcf&\x05\\\xb56\xa7\"\xa4[\xe8t\x81m\xabj\xdb]\x15oM\xbb`b\x01\xb4\xa7\xb6f\x9f\x95\x85&W/\xfaD\x05\x06\xcfƮ`\xb7Sں\x1c\xcej\xc5\xc4\xce\x1b\xeaX\xae\x81\x8b\x82|\r\xf7\xc6-: \x01]\x12v>\x1fOjZ\x15䤔\xfc\xe4\xc2R\x9ee5ځw\xc6\xf2؆\xf6\"ז\x9c\x1b\xaf\xcd)\x11\xe5\xa6[?\x84tu\xb9\x05M\x97:\xf0g\xc7:\xba[\xe7LP\xf4\x9c\x92\xd1=\xae\xce\xd5^fp9\xc7\xd3jSƇ~W\x96\x17\x9bqG\xad\x7f!!T\x0e\xb11~9\x9fF\xef5\xcfq,\xa20MS\x94Yv\xe0r\x8f\xea\xa3U\xbd?4*8f\xa9\xc7Ҩ5\xe5|*Z\xa9\xa69\U000f2d56\x9d\x94\xad?\x05\xcb\xdb\xe1N\x11\x9dfᄟ\xa9[Dnk3\xee\xdd]\xad\x98\xceļ\x9d\x91\xc6#\xfc\x8f%\x94B\x13nN2\x9b\x86\t\xbb䑘\xb8\r4Ō\xe8|\x83\x05\xbcd\xbe\xa1q\xfa|[\xaf\xb78\xb5\xbeԒɏ\xfbٯ\xc0\x0eg\xd2/\xe1\x85k9\xb6\xf0h~\x91\x91/\x12\xb7\xcf6\x80D\a\x93\xc0 \xd1\xfb\x96\xe4t,\xe3\x85\xe9y\x99sAW\xaf\xf2˼i\xea\x18}\xe9\x9f\xd7\v>\x067\xe6c\x8a?\xfc}P}p\xe7\x02=㖢\xf7a#\xcc\xf9g\xb1k\xfe-¶\x80\x7f9\xab\xf1\aߝx\xe6Z\n\xb9\x9f\x9b\xfco\xbeZ$\x1c\xf0\x14\"\x01Ad\x12!DX\x14\x104\x83\x1cy\xf9;\x04\t/\b\t\xa2\xdb\xc9\xd9GR\xe4\xbc\xc3dߓ\xff\xf2\x7f\x01\x00\x00\xff\xffT\xf5\x7f\x80\xacd\x00\x00"), diff --git a/design/pv_backup_info.md b/design/pv_backup_info.md index 771f8255c..107305fe5 100644 --- a/design/pv_backup_info.md +++ b/design/pv_backup_info.md @@ -61,7 +61,7 @@ type VolumeInfo struct { // CSISnapshotInfo is used for displaying the CSI snapshot status type CSISnapshotInfo struct { SnapshotHandle string // It's the storage provider's snapshot ID for CSI. - Size int64 // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. + Size int64 // The snapshot corresponding volume size. Driver string // The name of the CSI driver. VSCName string // The name of the VolumeSnapshotContent. @@ -70,7 +70,7 @@ type CSISnapshotInfo struct { // SnapshotDataMovementInfo is used for displaying the snapshot data mover status. type SnapshotDataMovementInfo struct { DataMover string // The data mover used by the backup. The valid values are `velero` and ``(equals to `velero`). - UploaderType string // The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover. + UploaderType string // The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`. RetainedSnapshot string // The name or ID of the snapshot associated object(SAO). SAO is used to support local snapshots for the snapshot data mover, e.g. it could be a VolumeSnapshot for CSI snapshot data moign/pv_backup_info. SnapshotHandle string // It's the filesystem repository's snapshot ID. @@ -79,7 +79,6 @@ type SnapshotDataMovementInfo struct { // VeleroNativeSnapshotInfo is used for displaying the Velero native snapshot status. type VeleroNativeSnapshotInfo struct { SnapshotHandle string // It's the storage provider's snapshot ID for the Velero-native snapshot. - Size int64 // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. VolumeType string // The cloud provider snapshot volume type. VolumeAZ string // The cloud provider snapshot volume's availability zones. @@ -89,11 +88,12 @@ type VeleroNativeSnapshotInfo struct { // PodVolumeBackupInfo is used for displaying the PodVolumeBackup snapshot status. type PodVolumeBackupInfo struct { SnapshotHandle string // It's the file-system uploader's snapshot ID for PodVolumeBackup. - Size int64 // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. + Size int64 // The snapshot corresponding volume size. - UploaderType string // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover. + UploaderType string // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. VolumeName string // The PVC's corresponding volume name used by Pod: https://github.com/kubernetes/kubernetes/blob/e4b74dd12fa8cb63c174091d5536a10b8ec19d34/pkg/apis/core/types.go#L48 - PodName string // The Pod name mounting this PVC. The format should be /. + PodName string // The Pod name mounting this PVC. + PodNamespace string // The Pod namespace. NodeName string // The PVB-taken k8s node's name. } diff --git a/internal/resourcemodifiers/json_patch.go b/internal/resourcemodifiers/json_patch.go index b5af7c362..d137d40be 100644 --- a/internal/resourcemodifiers/json_patch.go +++ b/internal/resourcemodifiers/json_patch.go @@ -19,30 +19,37 @@ type JSONPatch struct { } func (p *JSONPatch) ToString() string { - if addQuotes(p.Value) { + if addQuotes(&p.Value) { return fmt.Sprintf(`{"op": "%s", "from": "%s", "path": "%s", "value": "%s"}`, p.Operation, p.From, p.Path, p.Value) } return fmt.Sprintf(`{"op": "%s", "from": "%s", "path": "%s", "value": %s}`, p.Operation, p.From, p.Path, p.Value) } -func addQuotes(value string) bool { - if value == "" { +func addQuotes(value *string) bool { + if *value == "" { + return true + } + // if value is escaped, remove escape and add quotes + // this is useful for scenarios where boolean, null and numbers are required to be set as string. + if strings.HasPrefix(*value, "\"") && strings.HasSuffix(*value, "\"") { + *value = strings.TrimPrefix(*value, "\"") + *value = strings.TrimSuffix(*value, "\"") return true } // if value is null, then don't add quotes - if value == "null" { + if *value == "null" { return false } // if value is a boolean, then don't add quotes - if _, err := strconv.ParseBool(value); err == nil { + if strings.ToLower(*value) == "true" || strings.ToLower(*value) == "false" { return false } // if value is a json object or array, then don't add quotes. - if strings.HasPrefix(value, "{") || strings.HasPrefix(value, "[") { + if strings.HasPrefix(*value, "{") || strings.HasPrefix(*value, "[") { return false } // if value is a number, then don't add quotes - if _, err := strconv.ParseFloat(value, 64); err == nil { + if _, err := strconv.ParseFloat(*value, 64); err == nil { return false } return true diff --git a/internal/resourcemodifiers/resource_modifiers_test.go b/internal/resourcemodifiers/resource_modifiers_test.go index b09b473d9..648d827be 100644 --- a/internal/resourcemodifiers/resource_modifiers_test.go +++ b/internal/resourcemodifiers/resource_modifiers_test.go @@ -256,6 +256,64 @@ func TestGetResourceModifiersFromConfig(t *testing.T) { }, }, } + cm9 := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: deployments.apps\n resourceNameRegex: \"^test-.*$\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: replace\n path: \"/value/bool\"\n value: \"\\\"true\\\"\"\n\n\n", + }, + } + + rules9 := &ResourceModifiers{ + Version: "v1", + ResourceModifierRules: []ResourceModifierRule{ + { + Conditions: Conditions{ + GroupResource: "deployments.apps", + ResourceNameRegex: "^test-.*$", + Namespaces: []string{"bar", "foo"}, + }, + Patches: []JSONPatch{ + { + Operation: "replace", + Path: "/value/bool", + Value: `"true"`, + }, + }, + }, + }, + } + cm10 := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "sub.yml": "version: v1\nresourceModifierRules:\n- conditions:\n groupResource: deployments.apps\n resourceNameRegex: \"^test-.*$\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: replace\n path: \"/value/bool\"\n value: \"true\"\n\n\n", + }, + } + + rules10 := &ResourceModifiers{ + Version: "v1", + ResourceModifierRules: []ResourceModifierRule{ + { + Conditions: Conditions{ + GroupResource: "deployments.apps", + ResourceNameRegex: "^test-.*$", + Namespaces: []string{"bar", "foo"}, + }, + Patches: []JSONPatch{ + { + Operation: "replace", + Path: "/value/bool", + Value: "true", + }, + }, + }, + }, + } type args struct { cm *v1.ConfigMap @@ -338,6 +396,22 @@ func TestGetResourceModifiersFromConfig(t *testing.T) { want: rules8, wantErr: false, }, + { + name: "bool value as string", + args: args{ + cm: cm9, + }, + want: rules9, + wantErr: false, + }, + { + name: "bool value as bool", + args: args{ + cm: cm10, + }, + want: rules10, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -480,7 +554,24 @@ func TestResourceModifiers_ApplyResourceModifierRules(t *testing.T) { }, }, } - + cmTrue := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "data": map[string]interface{}{ + "test": "true", + }, + }, + } + cmFalse := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "data": map[string]interface{}{ + "test": "false", + }, + }, + } type fields struct { Version string ResourceModifierRules []ResourceModifierRule @@ -496,6 +587,33 @@ func TestResourceModifiers_ApplyResourceModifierRules(t *testing.T) { wantErr bool wantObj *unstructured.Unstructured }{ + { + name: "configmap true false string", + fields: fields{ + Version: "v1", + ResourceModifierRules: []ResourceModifierRule{ + { + Conditions: Conditions{ + GroupResource: "configmaps", + ResourceNameRegex: ".*", + }, + Patches: []JSONPatch{ + { + Operation: "replace", + Path: "/data/test", + Value: `"false"`, + }, + }, + }, + }, + }, + args: args{ + obj: cmTrue.DeepCopy(), + groupResource: "configmaps", + }, + wantErr: false, + wantObj: cmFalse.DeepCopy(), + }, { name: "Invalid Regex throws error", fields: fields{ diff --git a/pkg/apis/velero/v1/download_request_types.go b/pkg/apis/velero/v1/download_request_types.go index 7c7be5337..07f44b38d 100644 --- a/pkg/apis/velero/v1/download_request_types.go +++ b/pkg/apis/velero/v1/download_request_types.go @@ -25,7 +25,7 @@ type DownloadRequestSpec struct { } // DownloadTargetKind represents what type of file to download. -// +kubebuilder:validation:Enum=BackupLog;BackupContents;BackupVolumeSnapshots;BackupItemOperations;BackupResourceList;BackupResults;RestoreLog;RestoreResults;RestoreResourceList;RestoreItemOperations;CSIBackupVolumeSnapshots;CSIBackupVolumeSnapshotContents +// +kubebuilder:validation:Enum=BackupLog;BackupContents;BackupVolumeSnapshots;BackupItemOperations;BackupResourceList;BackupResults;RestoreLog;RestoreResults;RestoreResourceList;RestoreItemOperations;CSIBackupVolumeSnapshots;CSIBackupVolumeSnapshotContents;BackupVolumeInfos type DownloadTargetKind string const ( @@ -41,6 +41,7 @@ const ( DownloadTargetKindRestoreItemOperations DownloadTargetKind = "RestoreItemOperations" DownloadTargetKindCSIBackupVolumeSnapshots DownloadTargetKind = "CSIBackupVolumeSnapshots" DownloadTargetKindCSIBackupVolumeSnapshotContents DownloadTargetKind = "CSIBackupVolumeSnapshotContents" + DownloadTargetKindBackupVolumeInfos DownloadTargetKind = "BackupVolumeInfos" ) // DownloadTarget is the specification for what kind of file to download, and the name of the diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index e9c15dbc4..6760d627b 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -71,6 +71,7 @@ func TestBackedUpItemsMatchesTarballContents(t *testing.T) { req := &Request{ Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), + PVMap: map[string]PvcPvInfo{}, } backupFile := bytes.NewBuffer([]byte{}) @@ -84,8 +85,8 @@ func TestBackedUpItemsMatchesTarballContents(t *testing.T) { builder.ForDeployment("zoo", "raz").Result(), ), test.PVs( - builder.ForPersistentVolume("bar").Result(), - builder.ForPersistentVolume("baz").Result(), + builder.ForPersistentVolume("bar").ClaimRef("foo", "pvc1").Result(), + builder.ForPersistentVolume("baz").ClaimRef("bar", "pvc2").Result(), ), } for _, resource := range apiResources { diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 61f6834d6..ae8074521 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -250,6 +250,10 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti namespace = metadata.GetNamespace() if groupResource == kuberesource.PersistentVolumes { + if err := ib.addVolumeInfo(obj, log); err != nil { + backupErrs = append(backupErrs, err) + } + if err := ib.takePVSnapshot(obj, log); err != nil { backupErrs = append(backupErrs, err) } @@ -685,6 +689,39 @@ func (ib *itemBackupper) unTrackSkippedPV(obj runtime.Unstructured, groupResourc } } +func (ib *itemBackupper) addVolumeInfo(obj runtime.Unstructured, log logrus.FieldLogger) error { + pv := new(corev1api.PersistentVolume) + err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), pv) + if err != nil { + log.WithError(err).Warnf("Fail to convert PV") + return err + } + + if ib.backupRequest.PVMap == nil { + ib.backupRequest.PVMap = make(map[string]PvcPvInfo) + } + + pvcName := "" + pvcNamespace := "" + if pv.Spec.ClaimRef != nil { + pvcName = pv.Spec.ClaimRef.Name + pvcNamespace = pv.Spec.ClaimRef.Namespace + + ib.backupRequest.PVMap[pvcNamespace+"/"+pvcName] = PvcPvInfo{ + PVCName: pvcName, + PVCNamespace: pvcNamespace, + PV: *pv, + } + } + + ib.backupRequest.PVMap[pv.Name] = PvcPvInfo{ + PVCName: pvcName, + PVCNamespace: pvcNamespace, + PV: *pv, + } + return nil +} + // convert the input object to PV/PVC and get the PV name func getPVName(obj runtime.Unstructured, groupResource schema.GroupResource) (string, error) { if groupResource == kuberesource.PersistentVolumes { diff --git a/pkg/backup/item_backupper_test.go b/pkg/backup/item_backupper_test.go index 7bd7548bc..481092a0c 100644 --- a/pkg/backup/item_backupper_test.go +++ b/pkg/backup/item_backupper_test.go @@ -19,10 +19,12 @@ package backup import ( "testing" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/volume" "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" @@ -237,3 +239,55 @@ func TestRandom(t *testing.T) { err2 := runtime.DefaultUnstructuredConverter.FromUnstructured(o, pvc) t.Logf("err1: %v, err2: %v", err1, err2) } + +func TestAddVolumeInfo(t *testing.T) { + tests := []struct { + name string + pv *corev1api.PersistentVolume + expectedVolumeInfo map[string]PvcPvInfo + }{ + { + name: "PV has ClaimRef", + pv: builder.ForPersistentVolume("testPV").ClaimRef("testNS", "testPVC").Result(), + expectedVolumeInfo: map[string]PvcPvInfo{ + "testPV": { + PVCName: "testPVC", + PVCNamespace: "testNS", + PV: *builder.ForPersistentVolume("testPV").ClaimRef("testNS", "testPVC").Result(), + }, + "testNS/testPVC": { + PVCName: "testPVC", + PVCNamespace: "testNS", + PV: *builder.ForPersistentVolume("testPV").ClaimRef("testNS", "testPVC").Result(), + }, + }, + }, + { + name: "PV has no ClaimRef", + pv: builder.ForPersistentVolume("testPV").Result(), + expectedVolumeInfo: map[string]PvcPvInfo{ + "testPV": { + PVCName: "", + PVCNamespace: "", + PV: *builder.ForPersistentVolume("testPV").Result(), + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ib := itemBackupper{} + ib.backupRequest = new(Request) + ib.backupRequest.VolumeInfos.VolumeInfos = make([]volume.VolumeInfo, 0) + + pvObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pv) + require.NoError(t, err) + logger := logrus.StandardLogger() + + err = ib.addVolumeInfo(&unstructured.Unstructured{Object: pvObj}, logger) + require.NoError(t, err) + require.Equal(t, tc.expectedVolumeInfo, ib.backupRequest.PVMap) + }) + } +} diff --git a/pkg/backup/pv_skip_tracker.go b/pkg/backup/pv_skip_tracker.go index 859456a37..64241a240 100644 --- a/pkg/backup/pv_skip_tracker.go +++ b/pkg/backup/pv_skip_tracker.go @@ -10,6 +10,14 @@ type SkippedPV struct { Reasons []PVSkipReason `json:"reasons"` } +func (s *SkippedPV) SerializeSkipReasons() string { + ret := "" + for _, reason := range s.Reasons { + ret = ret + reason.Approach + ": " + reason.Reason + ";" + } + return ret +} + type PVSkipReason struct { Approach string `json:"approach"` Reason string `json:"reason"` diff --git a/pkg/backup/pv_skip_tracker_test.go b/pkg/backup/pv_skip_tracker_test.go index 9fdcb034f..16de8f555 100644 --- a/pkg/backup/pv_skip_tracker_test.go +++ b/pkg/backup/pv_skip_tracker_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSummary(t *testing.T) { @@ -41,3 +42,14 @@ func TestSummary(t *testing.T) { } assert.Equal(t, expected, tracker.Summary()) } + +func TestSerializeSkipReasons(t *testing.T) { + tracker := NewSkipPVTracker() + //tracker.Track("pv5", "", "skipped due to policy") + tracker.Track("pv3", podVolumeApproach, "it's set to opt-out") + tracker.Track("pv3", csiSnapshotApproach, "not applicable for CSI ") + + for _, skippedPV := range tracker.Summary() { + require.Equal(t, "csiSnapshot: not applicable for CSI ;podvolume: it's set to opt-out;", skippedPV.SerializeSkipReasons()) + } +} diff --git a/pkg/backup/request.go b/pkg/backup/request.go index 44bc5578f..6735c23a2 100644 --- a/pkg/backup/request.go +++ b/pkg/backup/request.go @@ -20,6 +20,8 @@ import ( "fmt" "sort" + corev1api "k8s.io/api/core/v1" + "github.com/vmware-tanzu/velero/internal/hook" "github.com/vmware-tanzu/velero/internal/resourcepolicies" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -52,6 +54,16 @@ type Request struct { itemOperationsList *[]*itemoperation.BackupOperation ResPolicies *resourcepolicies.Policies SkippedPVTracker *skipPVTracker + // A map contains the backup-included PV detail content. + // The key is PV name or PVC name(The format is PVC-namespace/PVC-name) + PVMap map[string]PvcPvInfo + VolumeInfos volume.VolumeInfos +} + +type PvcPvInfo struct { + PVCName string + PVCNamespace string + PV corev1api.PersistentVolume } // GetItemOperationsList returns ItemOperationsList, initializing it if necessary diff --git a/pkg/backup/snapshots.go b/pkg/backup/snapshots.go index a5c659705..e9724b9e3 100644 --- a/pkg/backup/snapshots.go +++ b/pkg/backup/snapshots.go @@ -4,7 +4,6 @@ import ( "context" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" - snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -17,27 +16,25 @@ import ( // Common function to update the status of CSI snapshots // returns VolumeSnapshot, VolumeSnapshotContent, VolumeSnapshotClasses referenced -func UpdateBackupCSISnapshotsStatus(client kbclient.Client, volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, backup *velerov1api.Backup, backupLog logrus.FieldLogger) (volumeSnapshots []snapshotv1api.VolumeSnapshot, volumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, volumeSnapshotClasses []snapshotv1api.VolumeSnapshotClass) { +func UpdateBackupCSISnapshotsStatus(client kbclient.Client, globalCRClient kbclient.Client, backup *velerov1api.Backup, backupLog logrus.FieldLogger) (volumeSnapshots []snapshotv1api.VolumeSnapshot, volumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, volumeSnapshotClasses []snapshotv1api.VolumeSnapshotClass) { if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { backupLog.Info("backup SnapshotMoveData is set to true, skip VolumeSnapshot resource persistence.") } else if features.IsEnabled(velerov1api.CSIFeatureFlag) { selector := label.NewSelectorForBackup(backup.Name) vscList := &snapshotv1api.VolumeSnapshotContentList{} - if volumeSnapshotLister != nil { - tmpVSs, err := volumeSnapshotLister.List(label.NewSelectorForBackup(backup.Name)) - if err != nil { - backupLog.Error(err) - } - for _, vs := range tmpVSs { - volumeSnapshots = append(volumeSnapshots, *vs) - } - } - - err := client.List(context.Background(), vscList, &kbclient.ListOptions{LabelSelector: selector}) + vsList := new(snapshotv1api.VolumeSnapshotList) + err := globalCRClient.List(context.TODO(), vsList, &kbclient.ListOptions{ + LabelSelector: label.NewSelectorForBackup(backup.Name), + }) if err != nil { backupLog.Error(err) } + volumeSnapshots = append(volumeSnapshots, vsList.Items...) + + if err := client.List(context.Background(), vscList, &kbclient.ListOptions{LabelSelector: selector}); err != nil { + backupLog.Error(err) + } if len(vscList.Items) >= 0 { volumeSnapshotContents = vscList.Items } diff --git a/pkg/builder/volume_snapshot_builder.go b/pkg/builder/volume_snapshot_builder.go index bbaedd16e..0abc48d2a 100644 --- a/pkg/builder/volume_snapshot_builder.go +++ b/pkg/builder/volume_snapshot_builder.go @@ -18,6 +18,7 @@ package builder import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -68,7 +69,21 @@ func (v *VolumeSnapshotBuilder) BoundVolumeSnapshotContentName(vscName string) * return v } +// SourcePVC set the built VolumeSnapshot's spec.Source.PersistentVolumeClaimName. func (v *VolumeSnapshotBuilder) SourcePVC(name string) *VolumeSnapshotBuilder { v.object.Spec.Source.PersistentVolumeClaimName = &name return v } + +// RestoreSize set the built VolumeSnapshot's status.RestoreSize. +func (v *VolumeSnapshotBuilder) RestoreSize(size string) *VolumeSnapshotBuilder { + resourceSize := resource.MustParse(size) + v.object.Status.RestoreSize = &resourceSize + return v +} + +// VolumeSnapshotClass set the built VolumeSnapshot's spec.VolumeSnapshotClassName value. +func (v *VolumeSnapshotBuilder) VolumeSnapshotClass(name string) *VolumeSnapshotBuilder { + v.object.Spec.VolumeSnapshotClassName = &name + return v +} diff --git a/pkg/builder/volume_snapshot_content_builder.go b/pkg/builder/volume_snapshot_content_builder.go index 734eeedf3..bbfbe5477 100644 --- a/pkg/builder/volume_snapshot_content_builder.go +++ b/pkg/builder/volume_snapshot_content_builder.go @@ -59,6 +59,7 @@ func (v *VolumeSnapshotContentBuilder) DeletionPolicy(policy snapshotv1api.Delet return v } +// VolumeSnapshotRef sets the built VolumeSnapshotContent's spec.VolumeSnapshotRef value. func (v *VolumeSnapshotContentBuilder) VolumeSnapshotRef(namespace, name string) *VolumeSnapshotContentBuilder { v.object.Spec.VolumeSnapshotRef = v1.ObjectReference{ APIVersion: "snapshot.storage.k8s.io/v1", @@ -68,3 +69,18 @@ func (v *VolumeSnapshotContentBuilder) VolumeSnapshotRef(namespace, name string) } return v } + +// VolumeSnapshotClassName sets the built VolumeSnapshotContent's spec.VolumeSnapshotClassName value. +func (v *VolumeSnapshotContentBuilder) VolumeSnapshotClassName(name string) *VolumeSnapshotContentBuilder { + v.object.Spec.VolumeSnapshotClassName = &name + return v +} + +// ObjectMeta applies functional options to the VolumeSnapshotContent's ObjectMeta. +func (v *VolumeSnapshotContentBuilder) ObjectMeta(opts ...ObjectMetaOpt) *VolumeSnapshotContentBuilder { + for _, opt := range opts { + opt(v.object) + } + + return v +} diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 9ff2040c6..9fcb097fb 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -24,6 +24,7 @@ import ( k8scheme "k8s.io/client-go/kubernetes/scheme" kbclient "sigs.k8s.io/controller-runtime/pkg/client" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" "github.com/pkg/errors" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" @@ -158,6 +159,9 @@ func (f *factory) KubebuilderClient() (kbclient.Client, error) { if err := apiextv1.AddToScheme(scheme); err != nil { return nil, err } + if err := snapshotv1api.AddToScheme(scheme); err != nil { + return nil, err + } kubebuilderClient, err := kbclient.New(clientConfig, kbclient.Options{ Scheme: scheme, }) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 6333dd401..884246762 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -23,21 +23,16 @@ import ( "net/http" "net/http/pprof" "os" - "reflect" "strings" "time" logrusr "github.com/bombsimon/logrusr/v3" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" - snapshotv1client "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" - snapshotv1informers "github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions" - snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -244,15 +239,17 @@ func NewCommand(f client.Factory) *cobra.Command { } type server struct { - namespace string - metricsAddress string - kubeClientConfig *rest.Config - kubeClient kubernetes.Interface - discoveryClient discovery.DiscoveryInterface - discoveryHelper velerodiscovery.Helper - dynamicClient dynamic.Interface - csiSnapshotClient *snapshotv1client.Clientset - csiSnapshotLister snapshotv1listers.VolumeSnapshotLister + namespace string + metricsAddress string + kubeClientConfig *rest.Config + kubeClient kubernetes.Interface + discoveryClient discovery.DiscoveryInterface + discoveryHelper velerodiscovery.Helper + dynamicClient dynamic.Interface + // controller-runtime client. the difference from the controller-manager's client + // is that the the controller-manager's client is limited to list namespaced-scoped + // resources in the namespace where Velero is installed, or the cluster-scoped + // resources. The crClient doesn't have the limitation. crClient ctrlclient.Client ctx context.Context cancelFunc context.CancelFunc @@ -399,23 +396,6 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s featureVerifier: featureVerifier, } - // Setup CSI snapshot client and lister - var csiSnapClient *snapshotv1client.Clientset - if features.IsEnabled(velerov1api.CSIFeatureFlag) { - csiSnapClient, err = snapshotv1client.NewForConfig(clientConfig) - if err != nil { - cancelFunc() - return nil, err - } - s.csiSnapshotClient = csiSnapClient - - s.csiSnapshotLister, err = s.getCSIVolumeSnapshotListers() - if err != nil { - cancelFunc() - return nil, err - } - } - return s, nil } @@ -615,40 +595,6 @@ func (s *server) initRepoManager() error { return nil } -func (s *server) getCSIVolumeSnapshotListers() (vsLister snapshotv1listers.VolumeSnapshotLister, err error) { - _, err = s.discoveryClient.ServerResourcesForGroupVersion(snapshotv1api.SchemeGroupVersion.String()) - switch { - case apierrors.IsNotFound(err): - // CSI is enabled, but the required CRDs aren't installed, so halt. - s.logger.Warnf("The '%s' feature flag was specified, but CSI API group [%s] was not found.", velerov1api.CSIFeatureFlag, snapshotv1api.SchemeGroupVersion.String()) - err = nil - case err == nil: - wrapper := NewCSIInformerFactoryWrapper(s.csiSnapshotClient) - - s.logger.Debug("Creating CSI listers") - // Access the wrapped factory directly here since we've already done the feature flag check above to know it's safe. - vsLister = wrapper.factory.Snapshot().V1().VolumeSnapshots().Lister() - - // start the informers & and wait for the caches to sync - wrapper.Start(s.ctx.Done()) - s.logger.Info("Waiting for informer caches to sync") - csiCacheSyncResults := wrapper.WaitForCacheSync(s.ctx.Done()) - s.logger.Info("Done waiting for informer caches to sync") - - for informer, synced := range csiCacheSyncResults { - if !synced { - err = errors.Errorf("cache was not synced for informer %v", informer) - return - } - s.logger.WithField("informer", informer).Info("Informer cache synced") - } - case err != nil: - s.logger.Errorf("fail to find snapshot v1 schema: %s", err) - } - - return vsLister, err -} - func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string) error { s.logger.Info("Starting controllers") @@ -775,10 +721,10 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.metrics, backupStoreGetter, s.config.formatFlag.Parse(), - s.csiSnapshotLister, s.credentialFileStore, s.config.maxConcurrentK8SConnections, s.config.defaultSnapshotMoveData, + s.crClient, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", controller.Backup) } @@ -837,7 +783,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string cmd.CheckError(err) r := controller.NewBackupFinalizerReconciler( s.mgr.GetClient(), - s.csiSnapshotLister, + s.crClient, clock.RealClock{}, backupper, newPluginManager, @@ -1027,37 +973,6 @@ func (s *server) runProfiler() { } } -// CSIInformerFactoryWrapper is a proxy around the CSI SharedInformerFactory that checks the CSI feature flag before performing operations. -type CSIInformerFactoryWrapper struct { - factory snapshotv1informers.SharedInformerFactory -} - -func NewCSIInformerFactoryWrapper(c snapshotv1client.Interface) *CSIInformerFactoryWrapper { - // If no namespace is specified, all namespaces are watched. - // This is desirable for VolumeSnapshots, as we want to query for all VolumeSnapshots across all namespaces using this informer - w := &CSIInformerFactoryWrapper{} - - if features.IsEnabled(velerov1api.CSIFeatureFlag) { - w.factory = snapshotv1informers.NewSharedInformerFactoryWithOptions(c, 0) - } - return w -} - -// Start proxies the Start call to the CSI SharedInformerFactory. -func (w *CSIInformerFactoryWrapper) Start(stopCh <-chan struct{}) { - if features.IsEnabled(velerov1api.CSIFeatureFlag) { - w.factory.Start(stopCh) - } -} - -// WaitForCacheSync proxies the WaitForCacheSync call to the CSI SharedInformerFactory. -func (w *CSIInformerFactoryWrapper) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { - if features.IsEnabled(velerov1api.CSIFeatureFlag) { - return w.factory.WaitForCacheSync(stopCh) - } - return nil -} - // if there is a restarting during the reconciling of backups/restores/etc, these CRs may be stuck in progress status // markInProgressCRsFailed tries to mark the in progress CRs as failed when starting the server to avoid the issue func markInProgressCRsFailed(ctx context.Context, cfg *rest.Config, scheme *runtime.Scheme, namespace string, log logrus.FieldLogger) { diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 746c9d789..e8a8c2eec 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -21,12 +21,11 @@ import ( "context" "fmt" "os" + "strconv" "strings" "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" - snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" - snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" @@ -43,14 +42,18 @@ import ( "github.com/vmware-tanzu/velero/internal/resourcepolicies" "github.com/vmware-tanzu/velero/internal/storage" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" + "github.com/vmware-tanzu/velero/pkg/itemoperation" + "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/encode" @@ -84,11 +87,10 @@ type backupReconciler struct { metrics *metrics.ServerMetrics backupStoreGetter persistence.ObjectBackupStoreGetter formatFlag logging.Format - volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister - volumeSnapshotClient snapshotterClientSet.Interface credentialFileStore credentials.FileStore maxConcurrentK8SConnections int defaultSnapshotMoveData bool + globalCRClient kbclient.Client } func NewBackupReconciler( @@ -110,10 +112,10 @@ func NewBackupReconciler( metrics *metrics.ServerMetrics, backupStoreGetter persistence.ObjectBackupStoreGetter, formatFlag logging.Format, - volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, credentialStore credentials.FileStore, maxConcurrentK8SConnections int, defaultSnapshotMoveData bool, + globalCRClient kbclient.Client, ) *backupReconciler { b := &backupReconciler{ ctx: ctx, @@ -135,10 +137,10 @@ func NewBackupReconciler( metrics: metrics, backupStoreGetter: backupStoreGetter, formatFlag: formatFlag, - volumeSnapshotLister: volumeSnapshotLister, credentialFileStore: credentialStore, maxConcurrentK8SConnections: maxConcurrentK8SConnections, defaultSnapshotMoveData: defaultSnapshotMoveData, + globalCRClient: globalCRClient, } b.updateTotalBackupMetric() return b @@ -317,6 +319,7 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg request := &pkgbackup.Request{ Backup: backup.DeepCopy(), // don't modify items in the cache SkippedPVTracker: pkgbackup.NewSkipPVTracker(), + PVMap: map[string]pkgbackup.PvcPvInfo{}, } // set backup major version - deprecated, use Status.FormatVersion @@ -665,7 +668,7 @@ func (b *backupReconciler) runBackup(backup *pkgbackup.Request) error { backup.Status.VolumeSnapshotsCompleted++ } } - volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses := pkgbackup.UpdateBackupCSISnapshotsStatus(b.kbClient, b.volumeSnapshotLister, backup.Backup, backupLog) + volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses := pkgbackup.UpdateBackupCSISnapshotsStatus(b.kbClient, b.globalCRClient, backup.Backup, backupLog) // Iterate over backup item operations and update progress. // Any errors on operations at this point should be added to backup errors. @@ -734,6 +737,8 @@ func (b *backupReconciler) runBackup(backup *pkgbackup.Request) error { if logFile, err := backupLog.GetPersistFile(); err != nil { fatalErrs = append(fatalErrs, errors.Wrap(err, "error getting backup log file")) } else { + backup.VolumeInfos.VolumeInfos = generateVolumeInfo(backup, volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses, b.globalCRClient, backupLog) + if errs := persistBackup(backup, backupFile, logFile, backupStore, volumeSnapshots, volumeSnapshotContents, volumeSnapshotClasses, results); len(errs) > 0 { fatalErrs = append(fatalErrs, errs...) } @@ -796,7 +801,6 @@ func persistBackup(backup *pkgbackup.Request, ) []error { persistErrs := []error{} backupJSON := new(bytes.Buffer) - volumeInfos := make([]volume.VolumeInfo, 0) if err := encode.To(backup.Backup, "json", backupJSON); err != nil { persistErrs = append(persistErrs, errors.Wrap(err, "error encoding backup")) @@ -843,7 +847,7 @@ func persistBackup(backup *pkgbackup.Request, persistErrs = append(persistErrs, errs...) } - volumeInfoJSON, errs := encode.ToJSONGzip(volumeInfos, "backup volumes information") + volumeInfoJSON, errs := encode.ToJSONGzip(backup.VolumeInfos, "backup volumes information") if errs != nil { persistErrs = append(persistErrs, errs...) } @@ -908,3 +912,328 @@ func oldAndNewFilterParametersUsedTogether(backupSpec velerov1api.BackupSpec) bo return haveOldResourceFilterParameters && haveNewResourceFilterParameters } + +func generateVolumeInfo(backup *pkgbackup.Request, csiVolumeSnapshots []snapshotv1api.VolumeSnapshot, + csiVolumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, csiVolumesnapshotClasses []snapshotv1api.VolumeSnapshotClass, + crClient kbclient.Client, logger logrus.FieldLogger) []volume.VolumeInfo { + volumeInfos := make([]volume.VolumeInfo, 0) + + skippedVolumeInfos := generateVolumeInfoForSkippedPV(backup, logger) + volumeInfos = append(volumeInfos, skippedVolumeInfos...) + + nativeSnapshotVolumeInfos := generateVolumeInfoForVeleroNativeSnapshot(backup, logger) + volumeInfos = append(volumeInfos, nativeSnapshotVolumeInfos...) + + csiVolumeInfos := generateVolumeInfoForCSIVolumeSnapshot(backup, csiVolumeSnapshots, csiVolumeSnapshotContents, csiVolumesnapshotClasses, logger) + volumeInfos = append(volumeInfos, csiVolumeInfos...) + + pvbVolumeInfos := generateVolumeInfoFromPVB(backup, crClient, logger) + volumeInfos = append(volumeInfos, pvbVolumeInfos...) + + dataUploadVolumeInfos := generateVolumeInfoFromDataUpload(backup, crClient, logger) + volumeInfos = append(volumeInfos, dataUploadVolumeInfos...) + + return volumeInfos +} + +// generateVolumeInfoForSkippedPV generate VolumeInfos for SkippedPV. +func generateVolumeInfoForSkippedPV(backup *pkgbackup.Request, logger logrus.FieldLogger) []volume.VolumeInfo { + tmpVolumeInfos := make([]volume.VolumeInfo, 0) + + for _, skippedPV := range backup.SkippedPVTracker.Summary() { + if pvcPVInfo, ok := backup.PVMap[skippedPV.Name]; ok { + volumeInfo := volume.VolumeInfo{ + PVCName: pvcPVInfo.PVCName, + PVCNamespace: pvcPVInfo.PVCNamespace, + PVName: skippedPV.Name, + SnapshotDataMoved: false, + Skipped: true, + SkippedReason: skippedPV.SerializeSkipReasons(), + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), + Labels: pvcPVInfo.PV.Labels, + }, + } + tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) + } else { + logger.Warnf("Cannot find info for PV %s", skippedPV.Name) + continue + } + } + + return tmpVolumeInfos +} + +// generateVolumeInfoForVeleroNativeSnapshot generate VolumeInfos for Velero native snapshot +func generateVolumeInfoForVeleroNativeSnapshot(backup *pkgbackup.Request, logger logrus.FieldLogger) []volume.VolumeInfo { + tmpVolumeInfos := make([]volume.VolumeInfo, 0) + + for _, nativeSnapshot := range backup.VolumeSnapshots { + var iops int64 + if nativeSnapshot.Spec.VolumeIOPS != nil { + iops = *nativeSnapshot.Spec.VolumeIOPS + } + + if pvcPVInfo, ok := backup.PVMap[nativeSnapshot.Spec.PersistentVolumeName]; ok { + volumeInfo := volume.VolumeInfo{ + BackupMethod: volume.NativeSnapshot, + PVCName: pvcPVInfo.PVCName, + PVCNamespace: pvcPVInfo.PVCNamespace, + PVName: pvcPVInfo.PV.Name, + SnapshotDataMoved: false, + Skipped: false, + NativeSnapshotInfo: volume.NativeSnapshotInfo{ + SnapshotHandle: nativeSnapshot.Status.ProviderSnapshotID, + VolumeType: nativeSnapshot.Spec.VolumeType, + VolumeAZ: nativeSnapshot.Spec.VolumeAZ, + IOPS: strconv.FormatInt(iops, 10), + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), + Labels: pvcPVInfo.PV.Labels, + }, + } + + tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) + } else { + logger.Warnf("cannot find info for PV %s", nativeSnapshot.Spec.PersistentVolumeName) + continue + } + } + + return tmpVolumeInfos +} + +// generateVolumeInfoForCSIVolumeSnapshot generate VolumeInfos for CSI VolumeSnapshot +func generateVolumeInfoForCSIVolumeSnapshot(backup *pkgbackup.Request, csiVolumeSnapshots []snapshotv1api.VolumeSnapshot, + csiVolumeSnapshotContents []snapshotv1api.VolumeSnapshotContent, csiVolumesnapshotClasses []snapshotv1api.VolumeSnapshotClass, + logger logrus.FieldLogger) []volume.VolumeInfo { + tmpVolumeInfos := make([]volume.VolumeInfo, 0) + + for _, volumeSnapshot := range csiVolumeSnapshots { + var volumeSnapshotClass *snapshotv1api.VolumeSnapshotClass + var volumeSnapshotContent *snapshotv1api.VolumeSnapshotContent + + // This is protective logic. The passed-in VS should be all related + // to this backup. + if volumeSnapshot.Labels[velerov1api.BackupNameLabel] != backup.Name { + continue + } + + if volumeSnapshot.Spec.VolumeSnapshotClassName == nil { + logger.Warnf("Cannot find VolumeSnapshotClass for VolumeSnapshot %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name) + continue + } + + if volumeSnapshot.Status == nil || volumeSnapshot.Status.BoundVolumeSnapshotContentName == nil { + logger.Warnf("Cannot fine VolumeSnapshotContent for VolumeSnapshot %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name) + continue + } + + if volumeSnapshot.Spec.Source.PersistentVolumeClaimName == nil { + logger.Warnf("VolumeSnapshot %s/%s doesn't have a source PVC", volumeSnapshot.Namespace, volumeSnapshot.Name) + continue + } + + for index := range csiVolumesnapshotClasses { + if *volumeSnapshot.Spec.VolumeSnapshotClassName == csiVolumesnapshotClasses[index].Name { + volumeSnapshotClass = &csiVolumesnapshotClasses[index] + } + } + + for index := range csiVolumeSnapshotContents { + if *volumeSnapshot.Status.BoundVolumeSnapshotContentName == csiVolumeSnapshotContents[index].Name { + volumeSnapshotContent = &csiVolumeSnapshotContents[index] + } + } + + if volumeSnapshotClass == nil || volumeSnapshotContent == nil { + logger.Warnf("fail to get VolumeSnapshotContent or VolumeSnapshotClass for VolumeSnapshot: %s/%s", + volumeSnapshot.Namespace, volumeSnapshot.Name) + continue + } + + var operation itemoperation.BackupOperation + for _, op := range *backup.GetItemOperationsList() { + if op.Spec.ResourceIdentifier.GroupResource.String() == kuberesource.VolumeSnapshots.String() && + op.Spec.ResourceIdentifier.Name == volumeSnapshot.Name && + op.Spec.ResourceIdentifier.Namespace == volumeSnapshot.Namespace { + operation = *op + } + } + + var size int64 + if volumeSnapshot.Status.RestoreSize != nil { + size = volumeSnapshot.Status.RestoreSize.Value() + } + snapshotHandle := "" + if volumeSnapshotContent.Status.SnapshotHandle != nil { + snapshotHandle = *volumeSnapshotContent.Status.SnapshotHandle + } + if pvcPVInfo, ok := backup.PVMap[volumeSnapshot.Namespace+"/"+*volumeSnapshot.Spec.Source.PersistentVolumeClaimName]; ok { + volumeInfo := volume.VolumeInfo{ + BackupMethod: volume.CSISnapshot, + PVCName: pvcPVInfo.PVCName, + PVCNamespace: pvcPVInfo.PVCNamespace, + PVName: pvcPVInfo.PV.Name, + Skipped: false, + SnapshotDataMoved: false, + PreserveLocalSnapshot: true, + OperationID: operation.Spec.OperationID, + StartTimestamp: &volumeSnapshot.CreationTimestamp, + CSISnapshotInfo: volume.CSISnapshotInfo{ + VSCName: *volumeSnapshot.Status.BoundVolumeSnapshotContentName, + Size: size, + Driver: volumeSnapshotClass.Driver, + SnapshotHandle: snapshotHandle, + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), + Labels: pvcPVInfo.PV.Labels, + }, + } + + tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) + } else { + logger.Warnf("cannot find info for PVC %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Spec.Source.PersistentVolumeClaimName) + continue + } + } + + return tmpVolumeInfos +} + +// generateVolumeInfoFromPVB generate VolumeInfo for PVB. +func generateVolumeInfoFromPVB(backup *pkgbackup.Request, crClient kbclient.Client, logger logrus.FieldLogger) []volume.VolumeInfo { + tmpVolumeInfos := make([]volume.VolumeInfo, 0) + + for _, pvb := range backup.PodVolumeBackups { + volumeInfo := volume.VolumeInfo{ + BackupMethod: volume.PodVolumeBackup, + SnapshotDataMoved: false, + Skipped: false, + StartTimestamp: pvb.Status.StartTimestamp, + PVBInfo: volume.PodVolumeBackupInfo{ + SnapshotHandle: pvb.Status.SnapshotID, + Size: pvb.Status.Progress.TotalBytes, + UploaderType: pvb.Spec.UploaderType, + VolumeName: pvb.Spec.Volume, + PodName: pvb.Spec.Pod.Name, + PodNamespace: pvb.Spec.Pod.Namespace, + NodeName: pvb.Spec.Node, + }, + } + + pod := new(corev1api.Pod) + pvcName := "" + err := crClient.Get(context.TODO(), kbclient.ObjectKey{Namespace: pvb.Spec.Pod.Namespace, Name: pvb.Spec.Pod.Name}, pod) + if err != nil { + logger.WithError(err).Warn("Fail to get pod for PodVolumeBackup: ", pvb.Name) + continue + } + for _, volume := range pod.Spec.Volumes { + if volume.Name == pvb.Spec.Volume && volume.PersistentVolumeClaim != nil { + pvcName = volume.PersistentVolumeClaim.ClaimName + } + } + + if pvcName != "" { + if pvcPVInfo, ok := backup.PVMap[pod.Namespace+"/"+pvcName]; ok { + volumeInfo.PVCName = pvcPVInfo.PVCName + volumeInfo.PVCNamespace = pvcPVInfo.PVCNamespace + volumeInfo.PVName = pvcPVInfo.PV.Name + volumeInfo.PVInfo = volume.PVInfo{ + ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), + Labels: pvcPVInfo.PV.Labels, + } + } else { + logger.Warnf("Cannot find info for PVC %s/%s", pod.Namespace, pvcName) + continue + } + } else { + logger.Debug("The PVB %s doesn't have a corresponding PVC", pvb.Name) + } + + tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) + } + + return tmpVolumeInfos +} + +// generateVolumeInfoFromDataUpload generate VolumeInfo for DataUpload. +func generateVolumeInfoFromDataUpload(backup *pkgbackup.Request, crClient kbclient.Client, logger logrus.FieldLogger) []volume.VolumeInfo { + tmpVolumeInfos := make([]volume.VolumeInfo, 0) + vsClassList := new(snapshotv1api.VolumeSnapshotClassList) + if err := crClient.List(context.TODO(), vsClassList); err != nil { + logger.WithError(err).Errorf("cannot list VolumeSnapshotClass %s", err.Error()) + return tmpVolumeInfos + } + + for _, operation := range *backup.GetItemOperationsList() { + if operation.Spec.ResourceIdentifier.GroupResource.String() == kuberesource.PersistentVolumeClaims.String() { + var duIdentifier velero.ResourceIdentifier + + for _, identifier := range operation.Spec.PostOperationItems { + if identifier.GroupResource.String() == "datauploads.velero.io" { + duIdentifier = identifier + } + } + if duIdentifier.Empty() { + logger.Warnf("cannot find DataUpload for PVC %s/%s backup async operation", + operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name) + continue + } + + dataUpload := new(velerov2alpha1.DataUpload) + err := crClient.Get( + context.TODO(), + kbclient.ObjectKey{ + Namespace: duIdentifier.Namespace, + Name: duIdentifier.Name}, + dataUpload, + ) + if err != nil { + logger.Warnf("fail to get DataUpload for operation %s: %s", operation.Spec.OperationID, err.Error()) + continue + } + + driverUsedByVSClass := "" + for index := range vsClassList.Items { + if vsClassList.Items[index].Name == dataUpload.Spec.CSISnapshot.SnapshotClass { + driverUsedByVSClass = vsClassList.Items[index].Driver + } + } + + if pvcPVInfo, ok := backup.PVMap[operation.Spec.ResourceIdentifier.Namespace+"/"+operation.Spec.ResourceIdentifier.Name]; ok { + volumeInfo := volume.VolumeInfo{ + BackupMethod: volume.CSISnapshot, + PVCName: pvcPVInfo.PVCName, + PVCNamespace: pvcPVInfo.PVCNamespace, + PVName: pvcPVInfo.PV.Name, + SnapshotDataMoved: true, + Skipped: false, + OperationID: operation.Spec.OperationID, + StartTimestamp: operation.Status.Created, + CSISnapshotInfo: volume.CSISnapshotInfo{ + Driver: driverUsedByVSClass, + }, + SnapshotDataMovementInfo: volume.SnapshotDataMovementInfo{ + DataMover: dataUpload.Spec.DataMover, + UploaderType: "kopia", + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(pvcPVInfo.PV.Spec.PersistentVolumeReclaimPolicy), + Labels: pvcPVInfo.PV.Labels, + }, + } + + tmpVolumeInfos = append(tmpVolumeInfos, volumeInfo) + } else { + logger.Warnf("Cannot find info for PVC %s/%s", operation.Spec.ResourceIdentifier.Namespace, operation.Spec.ResourceIdentifier.Name) + continue + } + } + } + + return tmpVolumeInfos +} diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index df2e22a22..736209e4a 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -29,15 +29,16 @@ import ( "github.com/google/go-cmp/cmp" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" - snapshotfake "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned/fake" - snapshotinformers "github.com/kubernetes-csi/external-snapshotter/client/v4/informers/externalversions" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/version" "k8s.io/utils/clock" @@ -45,11 +46,14 @@ import ( ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/vmware-tanzu/velero/pkg/backup" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" + "github.com/vmware-tanzu/velero/pkg/volume" fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/discovery" @@ -61,6 +65,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" "github.com/vmware-tanzu/velero/pkg/plugin/framework" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/boolptr" @@ -1062,12 +1067,11 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { - name: "backup with snapshot data movement set to false when CSI feature is enabled", - backup: defaultBackup().SnapshotMoveData(false).Result(), - //backup: defaultBackup().Result(), + name: "backup with snapshot data movement set to false when CSI feature is enabled", + backup: defaultBackup().SnapshotMoveData(false).Result(), backupLocation: defaultBackupLocation, defaultVolumesToFsBackup: false, expectedResult: &velerov1api.Backup{ @@ -1103,7 +1107,7 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement not set when CSI feature is enabled", @@ -1143,7 +1147,7 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement set to true and defaultSnapshotMoveData set to false", @@ -1184,7 +1188,7 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement set to false and defaultSnapshotMoveData set to true", @@ -1225,7 +1229,7 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, { name: "backup with snapshot data movement not set and defaultSnapshotMoveData set to true", @@ -1266,35 +1270,43 @@ func TestProcessBackupCompletions(t *testing.T) { CSIVolumeSnapshotsCompleted: 0, }, }, - volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), + volumeSnapshot: builder.ForVolumeSnapshot("velero", "testVS").VolumeSnapshotClass("testClass").Status().BoundVolumeSnapshotContentName("testVSC").RestoreSize("10G").SourcePVC("testPVC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).Result(), }, } + snapshotHandle := "testSnapshotID" + for _, test := range tests { t.Run(test.name, func(t *testing.T) { formatFlag := logging.FormatText var ( - logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) - pluginManager = new(pluginmocks.Manager) - backupStore = new(persistencemocks.BackupStore) - backupper = new(fakeBackupper) - snapshotClient = snapshotfake.NewSimpleClientset() - sharedInformer = snapshotinformers.NewSharedInformerFactory(snapshotClient, 0) - snapshotLister = sharedInformer.Snapshot().V1().VolumeSnapshots().Lister() + logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) + pluginManager = new(pluginmocks.Manager) + backupStore = new(persistencemocks.BackupStore) + backupper = new(fakeBackupper) + fakeGlobalClient = velerotest.NewFakeControllerRuntimeClient(t) ) var fakeClient kbclient.Client // add the test's backup storage location if it's different than the default if test.backupLocation != nil && test.backupLocation != defaultBackupLocation { - fakeClient = velerotest.NewFakeControllerRuntimeClient(t, test.backupLocation) + fakeClient = velerotest.NewFakeControllerRuntimeClient(t, test.backupLocation, + builder.ForVolumeSnapshotClass("testClass").Driver("testDriver").Result(), + builder.ForVolumeSnapshotContent("testVSC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).VolumeSnapshotClassName("testClass").Status(&snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: &snapshotHandle, + }).Result(), + ) } else { - fakeClient = velerotest.NewFakeControllerRuntimeClient(t) + fakeClient = velerotest.NewFakeControllerRuntimeClient(t, + builder.ForVolumeSnapshotClass("testClass").Driver("testDriver").Result(), + builder.ForVolumeSnapshotContent("testVSC").ObjectMeta(builder.WithLabels(velerov1api.BackupNameLabel, "backup-1")).VolumeSnapshotClassName("testClass").Status(&snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: &snapshotHandle, + }).Result(), + ) } if test.volumeSnapshot != nil { - snapshotClient.SnapshotV1().VolumeSnapshots(test.volumeSnapshot.Namespace).Create(context.Background(), test.volumeSnapshot, metav1.CreateOptions{}) - sharedInformer.Snapshot().V1().VolumeSnapshots().Informer().GetStore().Add(test.volumeSnapshot) - sharedInformer.WaitForCacheSync(make(chan struct{})) + require.NoError(t, fakeGlobalClient.Create(context.TODO(), test.volumeSnapshot)) } apiServer := velerotest.NewAPIServer(t) @@ -1328,8 +1340,7 @@ func TestProcessBackupCompletions(t *testing.T) { backupStoreGetter: NewFakeSingleObjectBackupStoreGetter(backupStore), backupper: backupper, formatFlag: formatFlag, - volumeSnapshotClient: snapshotClient, - volumeSnapshotLister: snapshotLister, + globalCRClient: fakeGlobalClient, } pluginManager.On("GetBackupItemActionsV2").Return(nil, nil) @@ -1731,3 +1742,749 @@ func TestPatchResourceWorksWithStatus(t *testing.T) { } } +func TestGenerateVolumeInfoForSkippedPV(t *testing.T) { + tests := []struct { + name string + skippedPVName string + pvMap map[string]backup.PvcPvInfo + expectedVolumeInfos []volume.VolumeInfo + }{ + { + name: "Cannot find info for PV", + skippedPVName: "testPV", + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Normal Skipped PV info", + skippedPVName: "testPV", + pvMap: map[string]backup.PvcPvInfo{ + "velero/testPVC": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + "testPV": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + Skipped: true, + SkippedReason: "CSI: skipped for PodVolumeBackup;", + PVInfo: volume.PVInfo{ + ReclaimPolicy: "Delete", + Labels: map[string]string{ + "a": "b", + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + request := new(backup.Request) + request.SkippedPVTracker = backup.NewSkipPVTracker() + if tc.skippedPVName != "" { + request.SkippedPVTracker.Track(tc.skippedPVName, "CSI", "skipped for PodVolumeBackup") + } + if tc.pvMap != nil { + request.PVMap = tc.pvMap + } + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) + + volumeInfos := generateVolumeInfoForSkippedPV(request, logger) + require.Equal(t, tc.expectedVolumeInfos, volumeInfos) + }) + } +} + +func TestGenerateVolumeInfoForCSIVolumeSnapshot(t *testing.T) { + resourceQuantity := resource.MustParse("100Gi") + now := metav1.Now() + tests := []struct { + name string + volumeSnapshot snapshotv1api.VolumeSnapshot + volumeSnapshotContent snapshotv1api.VolumeSnapshotContent + volumeSnapshotClass snapshotv1api.VolumeSnapshotClass + pvMap map[string]backup.PvcPvInfo + operation *itemoperation.BackupOperation + expectedVolumeInfos []volume.VolumeInfo + }{ + { + name: "VS doesn't have VolumeSnapshotClass name", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{}, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "VS doesn't have status", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + VolumeSnapshotClassName: stringPtr("testClass"), + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "VS doesn't have PVC", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + VolumeSnapshotClassName: stringPtr("testClass"), + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: stringPtr("testContent"), + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Cannot find VSC for VS", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + VolumeSnapshotClassName: stringPtr("testClass"), + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: stringPtr("testPVC"), + }, + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: stringPtr("testContent"), + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Cannot find VolumeInfo for PVC", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + VolumeSnapshotClassName: stringPtr("testClass"), + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: stringPtr("testPVC"), + }, + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: stringPtr("testContent"), + }, + }, + volumeSnapshotClass: *builder.ForVolumeSnapshotClass("testClass").Driver("pd.csi.storage.gke.io").Result(), + volumeSnapshotContent: *builder.ForVolumeSnapshotContent("testContent").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: stringPtr("testSnapshotHandle")}).Result(), + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Normal VolumeSnapshot case", + volumeSnapshot: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testVS", + Namespace: "velero", + CreationTimestamp: now, + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + VolumeSnapshotClassName: stringPtr("testClass"), + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: stringPtr("testPVC"), + }, + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: stringPtr("testContent"), + RestoreSize: &resourceQuantity, + }, + }, + volumeSnapshotClass: *builder.ForVolumeSnapshotClass("testClass").Driver("pd.csi.storage.gke.io").Result(), + volumeSnapshotContent: *builder.ForVolumeSnapshotContent("testContent").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: stringPtr("testSnapshotHandle")}).Result(), + pvMap: map[string]backup.PvcPvInfo{ + "velero/testPVC": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + OperationID: "testID", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "snapshot.storage.k8s.io", + Resource: "volumesnapshots", + }, + Namespace: "velero", + Name: "testVS", + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + BackupMethod: volume.CSISnapshot, + OperationID: "testID", + StartTimestamp: &now, + PreserveLocalSnapshot: true, + CSISnapshotInfo: volume.CSISnapshotInfo{ + Driver: "pd.csi.storage.gke.io", + SnapshotHandle: "testSnapshotHandle", + Size: 107374182400, + VSCName: "testContent", + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: "Delete", + Labels: map[string]string{ + "a": "b", + }, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + request := new(backup.Request) + request.Backup = new(velerov1api.Backup) + if tc.pvMap != nil { + request.PVMap = tc.pvMap + } + operationList := request.GetItemOperationsList() + if tc.operation != nil { + *operationList = append(*operationList, tc.operation) + } + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) + + volumeInfos := generateVolumeInfoForCSIVolumeSnapshot(request, []snapshotv1api.VolumeSnapshot{tc.volumeSnapshot}, []snapshotv1api.VolumeSnapshotContent{tc.volumeSnapshotContent}, []snapshotv1api.VolumeSnapshotClass{tc.volumeSnapshotClass}, logger) + require.Equal(t, tc.expectedVolumeInfos, volumeInfos) + }) + } + +} + +func TestGenerateVolumeInfoForVeleroNativeSnapshot(t *testing.T) { + tests := []struct { + name string + nativeSnapshot volume.Snapshot + pvMap map[string]backup.PvcPvInfo + expectedVolumeInfos []volume.VolumeInfo + }{ + { + name: "Native snapshot's IPOS pointer is nil", + nativeSnapshot: volume.Snapshot{ + Spec: volume.SnapshotSpec{ + PersistentVolumeName: "testPV", + VolumeIOPS: nil, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Cannot find info for the PV", + nativeSnapshot: volume.Snapshot{ + Spec: volume.SnapshotSpec{ + PersistentVolumeName: "testPV", + VolumeIOPS: int64Ptr(100), + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Normal native snapshot", + pvMap: map[string]backup.PvcPvInfo{ + "testPV": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + nativeSnapshot: volume.Snapshot{ + Spec: volume.SnapshotSpec{ + PersistentVolumeName: "testPV", + VolumeIOPS: int64Ptr(100), + VolumeType: "ssd", + VolumeAZ: "us-central1-a", + }, + Status: volume.SnapshotStatus{ + ProviderSnapshotID: "pvc-b31e3386-4bbb-4937-95d-7934cd62-b0a1-494b-95d7-0687440e8d0c", + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + BackupMethod: volume.NativeSnapshot, + PVInfo: volume.PVInfo{ + ReclaimPolicy: "Delete", + Labels: map[string]string{ + "a": "b", + }, + }, + NativeSnapshotInfo: volume.NativeSnapshotInfo{ + SnapshotHandle: "pvc-b31e3386-4bbb-4937-95d-7934cd62-b0a1-494b-95d7-0687440e8d0c", + VolumeType: "ssd", + VolumeAZ: "us-central1-a", + IOPS: "100", + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + request := new(backup.Request) + request.VolumeSnapshots = append(request.VolumeSnapshots, &tc.nativeSnapshot) + if tc.pvMap != nil { + request.PVMap = tc.pvMap + } + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) + + volumeInfos := generateVolumeInfoForVeleroNativeSnapshot(request, logger) + require.Equal(t, tc.expectedVolumeInfos, volumeInfos) + }) + } +} + +func TestGenerateVolumeInfoFromPVB(t *testing.T) { + tests := []struct { + name string + pvb *velerov1api.PodVolumeBackup + pod *corev1api.Pod + pvMap map[string]backup.PvcPvInfo + expectedVolumeInfos []volume.VolumeInfo + }{ + { + name: "cannot find PVB's pod, should fail", + pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(), + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "PVB doesn't have a related PVC", + pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(), + pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{ + Name: "test", + VolumeMounts: []corev1api.VolumeMount{ + { + Name: "testVolume", + MountPath: "/data", + }, + }, + }).Volumes( + &corev1api.Volume{ + Name: "", + VolumeSource: corev1api.VolumeSource{ + HostPath: &corev1api.HostPathVolumeSource{}, + }, + }, + ).Result(), + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "", + PVCNamespace: "", + PVName: "", + BackupMethod: volume.PodVolumeBackup, + PVBInfo: volume.PodVolumeBackupInfo{ + PodName: "testPod", + PodNamespace: "velero", + }, + }, + }, + }, + { + name: "Backup doesn't have information for PVC", + pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(), + pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{ + Name: "test", + VolumeMounts: []corev1api.VolumeMount{ + { + Name: "testVolume", + MountPath: "/data", + }, + }, + }).Volumes( + &corev1api.Volume{ + Name: "", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "testPVC", + }, + }, + }, + ).Result(), + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "PVB's volume has a PVC", + pvMap: map[string]backup.PvcPvInfo{ + "velero/testPVC": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + pvb: builder.ForPodVolumeBackup("velero", "testPVB").PodName("testPod").PodNamespace("velero").Result(), + pod: builder.ForPod("velero", "testPod").Containers(&corev1api.Container{ + Name: "test", + VolumeMounts: []corev1api.VolumeMount{ + { + Name: "testVolume", + MountPath: "/data", + }, + }, + }).Volumes( + &corev1api.Volume{ + Name: "", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: "testPVC", + }, + }, + }, + ).Result(), + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + BackupMethod: volume.PodVolumeBackup, + PVBInfo: volume.PodVolumeBackupInfo{ + PodName: "testPod", + PodNamespace: "velero", + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), + Labels: map[string]string{"a": "b"}, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + crClient := velerotest.NewFakeControllerRuntimeClient(t) + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) + request := new(pkgbackup.Request) + request.PodVolumeBackups = append(request.PodVolumeBackups, tc.pvb) + if tc.pvMap != nil { + request.PVMap = tc.pvMap + } + if tc.pod != nil { + require.NoError(t, crClient.Create(context.TODO(), tc.pod)) + } + + volumeInfos := generateVolumeInfoFromPVB(request, crClient, logger) + require.Equal(t, tc.expectedVolumeInfos, volumeInfos) + }) + } +} + +func TestGenerateVolumeInfoFromDataUpload(t *testing.T) { + now := metav1.Now() + tests := []struct { + name string + volumeSnapshotClass *snapshotv1api.VolumeSnapshotClass + dataUpload *velerov2alpha1.DataUpload + operation *itemoperation.BackupOperation + pvMap map[string]backup.PvcPvInfo + expectedVolumeInfos []volume.VolumeInfo + }{ + { + name: "Operation is not for PVC", + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "configmaps", + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "Operation doesn't have DataUpload PostItemOperation", + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "persistentvolumeclaims", + }, + Namespace: "velero", + Name: "testPVC", + }, + PostOperationItems: []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{ + Group: "", + Resource: "configmaps", + }, + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "DataUpload cannot be found for operation", + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + OperationID: "testOperation", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "persistentvolumeclaims", + }, + Namespace: "velero", + Name: "testPVC", + }, + PostOperationItems: []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{ + Group: "velero.io", + Resource: "datauploads", + }, + Namespace: "velero", + Name: "testDU", + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{}, + }, + { + name: "VolumeSnapshotClass cannot be found for operation", + dataUpload: builder.ForDataUpload("velero", "testDU").DataMover("velero").CSISnapshot(&velerov2alpha1.CSISnapshotSpec{ + VolumeSnapshot: "testVS", + }).SnapshotID("testSnapshotHandle").Result(), + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + OperationID: "testOperation", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "persistentvolumeclaims", + }, + Namespace: "velero", + Name: "testPVC", + }, + PostOperationItems: []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{ + Group: "velero.io", + Resource: "datauploads", + }, + Namespace: "velero", + Name: "testDU", + }, + }, + }, + }, + pvMap: map[string]backup.PvcPvInfo{ + "velero/testPVC": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + BackupMethod: volume.CSISnapshot, + SnapshotDataMoved: true, + OperationID: "testOperation", + SnapshotDataMovementInfo: volume.SnapshotDataMovementInfo{ + DataMover: "velero", + UploaderType: "kopia", + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), + Labels: map[string]string{"a": "b"}, + }, + }, + }, + }, + { + name: "Normal DataUpload case", + dataUpload: builder.ForDataUpload("velero", "testDU").DataMover("velero").CSISnapshot(&velerov2alpha1.CSISnapshotSpec{ + VolumeSnapshot: "testVS", + SnapshotClass: "testClass", + }).SnapshotID("testSnapshotHandle").Result(), + volumeSnapshotClass: builder.ForVolumeSnapshotClass("testClass").Driver("pd.csi.storage.gke.io").Result(), + operation: &itemoperation.BackupOperation{ + Spec: itemoperation.BackupOperationSpec{ + OperationID: "testOperation", + ResourceIdentifier: velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{ + Group: "", + Resource: "persistentvolumeclaims", + }, + Namespace: "velero", + Name: "testPVC", + }, + PostOperationItems: []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{ + Group: "velero.io", + Resource: "datauploads", + }, + Namespace: "velero", + Name: "testDU", + }, + }, + }, + Status: itemoperation.OperationStatus{ + Created: &now, + }, + }, + pvMap: map[string]backup.PvcPvInfo{ + "velero/testPVC": { + PVCName: "testPVC", + PVCNamespace: "velero", + PV: corev1api.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testPV", + Labels: map[string]string{"a": "b"}, + }, + Spec: corev1api.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1api.PersistentVolumeReclaimDelete, + }, + }, + }, + }, + expectedVolumeInfos: []volume.VolumeInfo{ + { + PVCName: "testPVC", + PVCNamespace: "velero", + PVName: "testPV", + BackupMethod: volume.CSISnapshot, + SnapshotDataMoved: true, + OperationID: "testOperation", + StartTimestamp: &now, + CSISnapshotInfo: volume.CSISnapshotInfo{ + Driver: "pd.csi.storage.gke.io", + }, + SnapshotDataMovementInfo: volume.SnapshotDataMovementInfo{ + DataMover: "velero", + UploaderType: "kopia", + }, + PVInfo: volume.PVInfo{ + ReclaimPolicy: string(corev1api.PersistentVolumeReclaimDelete), + Labels: map[string]string{"a": "b"}, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + request := new(backup.Request) + operationList := request.GetItemOperationsList() + if tc.operation != nil { + *operationList = append(*operationList, tc.operation) + } + if tc.pvMap != nil { + request.PVMap = tc.pvMap + } + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatJSON) + + crClient := velerotest.NewFakeControllerRuntimeClient(t) + if tc.dataUpload != nil { + crClient.Create(context.TODO(), tc.dataUpload) + } + + if tc.volumeSnapshotClass != nil { + crClient.Create(context.TODO(), tc.volumeSnapshotClass) + } + + volumeInfos := generateVolumeInfoFromDataUpload(request, crClient, logger) + require.Equal(t, tc.expectedVolumeInfos, volumeInfos) + }) + } +} + +func int64Ptr(val int) *int64 { + i := int64(val) + return &i +} + +func stringPtr(str string) *string { + return &str +} diff --git a/pkg/controller/backup_finalizer_controller.go b/pkg/controller/backup_finalizer_controller.go index eb99f6ee5..ea9c0364b 100644 --- a/pkg/controller/backup_finalizer_controller.go +++ b/pkg/controller/backup_finalizer_controller.go @@ -29,8 +29,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" kbclient "sigs.k8s.io/controller-runtime/pkg/client" - snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1" - velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/metrics" @@ -42,21 +40,21 @@ import ( // backupFinalizerReconciler reconciles a Backup object type backupFinalizerReconciler struct { - client kbclient.Client - volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister - clock clocks.WithTickerAndDelayedExecution - backupper pkgbackup.Backupper - newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - backupTracker BackupTracker - metrics *metrics.ServerMetrics - backupStoreGetter persistence.ObjectBackupStoreGetter - log logrus.FieldLogger + client kbclient.Client + globalCRClient kbclient.Client + clock clocks.WithTickerAndDelayedExecution + backupper pkgbackup.Backupper + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager + backupTracker BackupTracker + metrics *metrics.ServerMetrics + backupStoreGetter persistence.ObjectBackupStoreGetter + log logrus.FieldLogger } // NewBackupFinalizerReconciler initializes and returns backupFinalizerReconciler struct. func NewBackupFinalizerReconciler( client kbclient.Client, - volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, + globalCRClient kbclient.Client, clock clocks.WithTickerAndDelayedExecution, backupper pkgbackup.Backupper, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, @@ -67,6 +65,7 @@ func NewBackupFinalizerReconciler( ) *backupFinalizerReconciler { return &backupFinalizerReconciler{ client: client, + globalCRClient: globalCRClient, clock: clock, backupper: backupper, newPluginManager: newPluginManager, @@ -191,7 +190,7 @@ func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Requ backup.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} recordBackupMetrics(log, backup, outBackupFile, r.metrics, true) - pkgbackup.UpdateBackupCSISnapshotsStatus(r.client, r.volumeSnapshotLister, backup, log) + pkgbackup.UpdateBackupCSISnapshotsStatus(r.client, r.globalCRClient, backup, log) // update backup metadata in object store backupJSON := new(bytes.Buffer) if err := encode.To(backup, "json", backupJSON); err != nil { diff --git a/pkg/controller/backup_finalizer_controller_test.go b/pkg/controller/backup_finalizer_controller_test.go index f759d0318..74f6da57c 100644 --- a/pkg/controller/backup_finalizer_controller_test.go +++ b/pkg/controller/backup_finalizer_controller_test.go @@ -23,7 +23,6 @@ import ( "testing" "time" - snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -44,14 +43,13 @@ import ( "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" - velerotestmocks "github.com/vmware-tanzu/velero/pkg/test/mocks" ) -func mockBackupFinalizerReconciler(fakeClient kbclient.Client, fakeVolumeSnapshotLister snapshotv1listers.VolumeSnapshotLister, fakeClock *testclocks.FakeClock) (*backupFinalizerReconciler, *fakeBackupper) { +func mockBackupFinalizerReconciler(fakeClient kbclient.Client, fakeGlobalClient kbclient.Client, fakeClock *testclocks.FakeClock) (*backupFinalizerReconciler, *fakeBackupper) { backupper := new(fakeBackupper) return NewBackupFinalizerReconciler( fakeClient, - fakeVolumeSnapshotLister, + fakeGlobalClient, fakeClock, backupper, func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, @@ -164,9 +162,9 @@ func TestBackupFinalizerReconcile(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) - fakeVolumeSnapshotLister := velerotestmocks.NewVolumeSnapshotLister(t) + fakeGlobalClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) - reconciler, backupper := mockBackupFinalizerReconciler(fakeClient, fakeVolumeSnapshotLister, fakeClock) + reconciler, backupper := mockBackupFinalizerReconciler(fakeClient, fakeGlobalClient, fakeClock) pluginManager.On("CleanupClients").Return(nil) backupStore.On("GetBackupItemOperations", test.backup.Name).Return(test.backupOperations, nil) backupStore.On("GetBackupContents", mock.Anything).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil) diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 26bddc565..48a48dbf1 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -608,6 +608,8 @@ func (s *objectBackupStore) GetDownloadURL(target velerov1api.DownloadTarget) (s return s.objectStore.CreateSignedURL(s.bucket, s.layout.getCSIVolumeSnapshotContentsKey(target.Name), DownloadURLTTL) case velerov1api.DownloadTargetKindBackupResults: return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupResultsKey(target.Name), DownloadURLTTL) + case velerov1api.DownloadTargetKindBackupVolumeInfos: + return s.objectStore.CreateSignedURL(s.bucket, s.layout.getBackupVolumeInfoKey(target.Name), DownloadURLTTL) default: return "", errors.Errorf("unsupported download target kind %q", target.Kind) } diff --git a/pkg/persistence/object_store_test.go b/pkg/persistence/object_store_test.go index 6c149f188..ba6a7bcb8 100644 --- a/pkg/persistence/object_store_test.go +++ b/pkg/persistence/object_store_test.go @@ -768,6 +768,13 @@ func TestGetDownloadURL(t *testing.T) { velerov1api.DownloadTargetKindRestoreResourceList: "restores/b-cool-20170913154901-20170913154902/restore-b-cool-20170913154901-20170913154902-resource-list.json.gz", }, }, + { + name: "", + targetName: "my-backup", + expectedKeyByKind: map[velerov1api.DownloadTargetKind]string{ + velerov1api.DownloadTargetKindBackupVolumeInfos: "backups/my-backup/my-backup-volumeinfos.json.gz", + }, + }, } for _, test := range tests { diff --git a/pkg/volume/volume_info_common.go b/pkg/volume/volume_info_common.go index cfca31df9..14ede0c6b 100644 --- a/pkg/volume/volume_info_common.go +++ b/pkg/volume/volume_info_common.go @@ -51,6 +51,8 @@ type VolumeInfo struct { SnapshotDataMoved bool `json:"snapshotDataMoved"` // Whether the local snapshot is preserved after snapshot is moved. + // The local snapshot may be a result of CSI snapshot backup(no data movement) + // or a CSI snapshot data movement plus preserve local snapshot. PreserveLocalSnapshot bool `json:"preserveLocalSnapshot"` // Whether the Volume is skipped in this backup. @@ -69,6 +71,7 @@ type VolumeInfo struct { SnapshotDataMovementInfo SnapshotDataMovementInfo `json:"snapshotDataMovementInfo,omitempty"` NativeSnapshotInfo NativeSnapshotInfo `json:"nativeSnapshotInfo,omitempty"` PVBInfo PodVolumeBackupInfo `json:"pvbInfo,omitempty"` + PVInfo PVInfo `json:"pvInfo,omitempty"` } // CSISnapshotInfo is used for displaying the CSI snapshot status @@ -76,7 +79,7 @@ type CSISnapshotInfo struct { // It's the storage provider's snapshot ID for CSI. SnapshotHandle string `json:"snapshotHandle"` - // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. + // The snapshot corresponding volume size. Size int64 `json:"size"` // The name of the CSI driver. @@ -91,7 +94,7 @@ type SnapshotDataMovementInfo struct { // The data mover used by the backup. The valid values are `velero` and ``(equals to `velero`). DataMover string `json:"dataMover"` - // The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover. + // The type of the uploader that uploads the snapshot data. The valid values are `kopia` and `restic`. UploaderType string `json:"uploaderType"` // The name or ID of the snapshot associated object(SAO). @@ -111,9 +114,6 @@ type NativeSnapshotInfo struct { // It's the storage provider's snapshot ID for the Velero-native snapshot. SnapshotHandle string `json:"snapshotHandle"` - // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. - Size int64 `json:"size"` - // The cloud provider snapshot volume type. VolumeType string `json:"volumeType"` @@ -129,19 +129,32 @@ type PodVolumeBackupInfo struct { // It's the file-system uploader's snapshot ID for PodVolumeBackup. SnapshotHandle string `json:"snapshotHandle"` - // The snapshot corresponding volume size. Some of the volume backup methods cannot retrieve the data by current design, for example, the Velero native snapshot. + // The snapshot corresponding volume size. Size int64 `json:"size"` - // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. It's useful for file-system backup and snapshot data mover. + // The type of the uploader that uploads the data. The valid values are `kopia` and `restic`. UploaderType string `json:"uploaderType"` // The PVC's corresponding volume name used by Pod // https://github.com/kubernetes/kubernetes/blob/e4b74dd12fa8cb63c174091d5536a10b8ec19d34/pkg/apis/core/types.go#L48 VolumeName string `json:"volumeName"` - // The Pod name mounting this PVC. The format should be /. + // The Pod name mounting this PVC. PodName string `json:"podName"` + // The Pod namespace + PodNamespace string `json:"podNamespace"` + // The PVB-taken k8s node's name. NodeName string `json:"nodeName"` } + +// PVInfo is used to store some PV information modified after creation. +// Those information are lost after PV recreation. +type PVInfo struct { + // ReclaimPolicy of PV. It could be different from the referenced StorageClass. + ReclaimPolicy string `json:"reclaimPolicy"` + + // The PV's labels should be kept after recreation. + Labels map[string]string `json:"labels"` +}