From dbd0aa49155ff35b3c2fb09edac89cb41689c21b Mon Sep 17 00:00:00 2001 From: Carlisia Campos Date: Tue, 14 Jul 2020 14:47:00 -0700 Subject: [PATCH] Add a BSL controller to handle validation + update BSL status phase (#2674) * Add BSL controller Signed-off-by: Carlisia * Add changelog Signed-off-by: Carlisia * Make update Signed-off-by: Carlisia * Update docs Signed-off-by: Carlisia * Add kubebuilder dependency Signed-off-by: Carlisia * Add export Signed-off-by: Carlisia * add kubebuilder binaries into velero builder image Signed-off-by: Ashish Amarnath * Reset velero dockerfile Signed-off-by: Carlisia * Consolidate all logic Signed-off-by: Carlisia * Add copyright header Signed-off-by: Carlisia * Clean up + add "last validated" column Signed-off-by: Carlisia * Better tests Signed-off-by: Carlisia * Add more tests Signed-off-by: Carlisia * Better logging Signed-off-by: Carlisia * Format Signed-off-by: Carlisia * Code reviews Signed-off-by: Carlisia * Address code review Signed-off-by: Carlisia * Remove redundant logic Signed-off-by: Carlisia Co-authored-by: Ashish Amarnath --- changelogs/unreleased/2674-carlisia | 1 + .../velero.io_backupstoragelocations.yaml | 16 ++ config/crd/crds/crds.go | 2 +- go.mod | 11 +- go.sum | 57 +++- hack/build-image/Dockerfile | 7 + internal/velero/storagelocation.go | 132 +++++++++ internal/velero/storagelocation_test.go | 261 ++++++++++++++++++ .../velero/v1/backupstoragelocation_types.go | 15 + pkg/apis/velero/v1/zz_generated.deepcopy.go | 9 + .../backup_storage_location_builder.go | 20 ++ pkg/cmd/cli/backuplocation/create.go | 32 ++- pkg/cmd/server/server.go | 96 +++---- .../output/backup_storage_location_printer.go | 8 + pkg/controller/backup_sync_controller.go | 11 +- .../backupstoragelocation_controller.go | 142 ++++++++++ .../backupstoragelocation_controller_test.go | 185 +++++++++++++ pkg/controller/suite_test.go | 101 ++++++- .../master/api-types/backupstoragelocation.md | 1 + 19 files changed, 1015 insertions(+), 92 deletions(-) create mode 100644 changelogs/unreleased/2674-carlisia create mode 100644 internal/velero/storagelocation.go create mode 100644 internal/velero/storagelocation_test.go create mode 100644 pkg/controller/backupstoragelocation_controller.go create mode 100644 pkg/controller/backupstoragelocation_controller_test.go diff --git a/changelogs/unreleased/2674-carlisia b/changelogs/unreleased/2674-carlisia new file mode 100644 index 000000000..b09446515 --- /dev/null +++ b/changelogs/unreleased/2674-carlisia @@ -0,0 +1 @@ +Add a BSL controller to handle validation + update BSL status phase (validation removed from the server and no longer blocks when there's any invalid BSL) \ No newline at end of file diff --git a/config/crd/bases/velero.io_backupstoragelocations.yaml b/config/crd/bases/velero.io_backupstoragelocations.yaml index 0a4eb4fb1..f7a335912 100644 --- a/config/crd/bases/velero.io_backupstoragelocations.yaml +++ b/config/crd/bases/velero.io_backupstoragelocations.yaml @@ -13,6 +13,11 @@ spec: description: Backup Storage Location status such as Available/Unavailable name: Phase type: string + - JSONPath: .status.lastValidationTime + description: LastValidationTime is the last time the backup store location was + validated + name: Last Validated + type: date - JSONPath: .metadata.creationTimestamp name: Age type: date @@ -86,6 +91,11 @@ spec: provider: description: Provider is the provider of the backup storage. type: string + validationFrequency: + description: ValidationFrequency defines how frequently to validate + the corresponding object storage. A value of 0 disables validation. + nullable: true + type: string required: - objectStorage - provider @@ -114,6 +124,12 @@ spec: format: date-time nullable: true type: string + lastValidationTime: + description: LastValidationTime is the last time the backup store location + was validated the cluster. + format: date-time + nullable: true + type: string phase: description: Phase is the current state of the BackupStorageLocation. enum: diff --git a/config/crd/crds/crds.go b/config/crd/crds/crds.go index ea51c0d1f..add6ff3c3 100644 --- a/config/crd/crds/crds.go +++ b/config/crd/crds/crds.go @@ -30,7 +30,7 @@ import ( var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xecwà\x18C\xe7\xf2(\xf3J(\x90\xday\xa1\xb3@\x87\xa8q:\xa5\x03\xc6\xc5\xd9\xc36\xb8\xb5\x843\xf1\xbe\xe3\xe2\x8cF0\x16\n\xf2\xe6\xfd\xa9n\x10>\x8c\x92\xbb\x15\xe4kLPC[)t\xf1E9{\xceƮ\xafG\x00\xd7R\b{\xbf\x12[T\xe0Pa\xe6\x8d\x1dbôPØ\xf7Q#\xbc\x1b\xf0V\x8d\xff%\x12ێʌ\xc2\x04x=\xc8\xec\x10\xb6e\xd2\x17\x86\x02\xb9A\xc7\xf6+\xcaR\xbd\x0f\x13\aӒ\x0ec\u0084\x9b1i̧\xb0\xfaf\u074cY?\u05cc\x19\x8f\xd7\xe5e-\xfa\xdf\x0f+\x93\xe3>[17\xbd\x85\x1f\xa9\x98\xc4DI\xa1\xf5f\aX\x94\xfe\xfd\x1a\xa4OO)\x92\x10\x9c\x18\x8e\xb2\xa7~\xf7oN\x10\xe7\xea\xf4\xe6t\xdd\a\xea\xf4/\x94B\xfd\xeaߌ\x10\xd8\xd9?F_\xbfP\x00?\xb5\xd7\\\x83\xdc\xd5\x02ȯa'\x95G{\"\x89)rʹ$~)\v\xe6w*\x1a\x85\xf0\xd9\xe1\xfe\x8d\xa2#\xd7\xd4s\x16q\xe3ti\x88)ST\xdd\xddL'\xa1\x02'\x83\xd2b\x11R\xcc'\xe6`\xf3\x84#\x9f\xcf_\xbf`>\xce\x14X\xa2a=\x12>\x9f\xa0\xd9~m\f\x91\x97\x11\x10\x83\x94:\xbb\b\xe5\x82k\x10\xf0\x82\xef!\xba\x10\x1aH \x82^C\x93g!Z\xe4\x9a\x05+\xd4\v\xbe3\x90X\x86\x98Y\xbbL\xf4a\xbc\xe0\xfb\xfc\xa4\x13\xb6\x116\xd2Ų\n\xf1\x8f\x1e0\x038\x87]\xca2\xe0\"R\xf20sD\xc1R\x17\x91F\xe2\xf6\xd9\xe4\xd5bj\xea\x1eA\x90W.\b\x85\xb4\xfd \xcbE\x04\x92\xeb\x04\x87l\x13\xa9\x88\xf4,\x94\xcc\xeb\xd7\x04\xfd\xde\xe8k\xf8j\xfcF\x8f\x05\xab\xddq\xff&]\xac\xdd}1\xe8\xbe\x1a\xcfO>\x9c\x89\x01\xe5\xb3Y\x18\x96\xb1\t\xe9\xe0\x86\x89\xfev-jV\x89\xc3\u0604\f\xab\x16\x89t\xb0єC\x04^\x85jbxٔ\xb7\uf3a2r\\l\xd2F\xafx\xb3[\x0f\xbd'\xb2x\xa1\"\xb7\xa5\xd0G\xab~ex\xdd\"\x88O\xb4/\x84ա2\xaaD\x869\xe4\x153\x91+{\xc2\xe3^fP\xa0ݏo\x04\xedQ\x92\xcf^\xf2\xfaE\xbe4\x8c\xb3\xf4i\xc9֜Ft\xc6\xf9\x1c\x1a+\xb2\xcd\xd99I\xb43\x13\aKy\xe3\x13\xe7\xe8\xe0M\x92\xe3\x86\x19n\x8a<\xe7\x83\x16\xa1\x1e\x16{\xefŜ\xef\xef\xdb\x01\xa5\xb0\xc7\x15\x82\vt\xffK[\x15+\xed\xffA)\xa4\x9d\xb5\xd0\xcf|b\xa2\xb0\xb32V\x85\xda/!\xf8\xd2\x01I\xf3(\xd4iAx\x80,C^\x03U؆ͮ\x17i\\\xc3\xeb\xc1\xb8\xb0+\xee$\xaa\x1c\xe4T\xa4E\xe3\xf2\x05\xdf/\xaf{6~\xb9їa{\xeeYl\xda\xcbg\x00\x1b\xad\xde\xe1\x92W^~{\xe8\xb2H\xeb\x16L\xe2\xe3\xb3e\xc1,esi\x17\xa7e\xf5\x19\f\x85\xa2\xe3\xd8.й\xd28\xbf\x10\x89\a\xe3|\xa8\xd0u\x82ǁ\xda\xd0tN\x13kB v\xe1\xdc\xcb\xd8t\xc2A\x8e\xec\xa4TIRr8X\xe0\xecA\xcc#H\xa1\x14\\66\x1a\xfc\xe3e8\xf6\xe0W\x88\x8cÂ\t\x88\xa4\n\xa55\x19:7\xa5\x0e\xb3\x9ew\xa6\xe0V\x17\xdbDH*\xc2!\xc2Tq/\x8d\xa5a#\xb1\xe6\xac0\xfb\xfe\xadU\x03$Ӧ\xff\xa7\xd5\xec<\x8c\x80Ϡ\x8bB\xe8\xd9͢\x87\xdc]X\x97L!\x82\t!\xbb\xddWl\xc6K#\xbd\xa84\xbf\xee\x06[H\xbda\xe0\xf0\xe9C\xb7cH.\x11\xcf\x0f\xa9\xef\xd2ʆ\xcd\xf5\x83`\x9b\xa5\xe9\x97܇\xc6\xeb\x01-v$կ\fs8\xa7\x8do\xa5\xe7\xcb\x18\x1d\xf0\xb8r\xb0\x93\xd6\xf96\x92\x8e\x0f\xb6>>G\xd1\xf7\xd6~C\x8a\xf2簮U\x00:\x98\xd7tR8r874\xf8\x18\x04A\xee@z@\x9d\x99Js\x11\x83\x8c\x94_\x10X\x1a\x9c\xe9\xec&\x1b\xc6\x12æ\x81\xba*\x96\x10\xbeb\xed\x91z\xa2\xd6ў\xfc\xa3\x90S\x95\xaa4\xce\x12\x93\x97\x05\x9ajbSkFGLOa]爷\x10o\xb2\xa8\n\x10\x051{\x11Gig\x96\x05v\xe5\v\xafBz\xf6\xee\x04\x95]\xbd7d\x14\xa5B\xbf,\x1b\xd8\xe2\xceX\xb6E's\xac\xb7\xcc(s\xa3A\xc0NHU\xd9E\x1e\xed\f\x8e.\x8f죑\u007fLо\xe4\xb5+&\u007f\xb6L\xb9(T\x9b\xf2\xaa\xa5]\x1a\xa8=X\xfc\xc8\x10\xa9\xb4\x92t\xc6|l\x94\x14UI\xe8\xf7\xefaR\x8b7\xdfä\xde\xf8\x1e&u\xc6\xf70\xe9{\x9849\xbe\x87I\xdfä\xdfk\x984\x8dɊ\xebV\x83?ͼ}\xf6\bu\x1c\xb1Q\xc8\xf1T\xff.\xf4^/\xeb\xcb\xdb\f\xaf\x19軌-\xdd+\xee8\xef˹9\xfao\xdc|ݨGʟ\x9474\x96N\xb6\xee-h\xc4\x1b\xea͜o/\x99k*\xe9\xf6$֍\x1d\xa9)ѤW\xf4\xa8O\x9d\xec\x14f\xb6;\x18\x84R\xed\xde\x14a\x1b\xa6\xfcJ\xfd\x8a\xb3\xad\x1f3\r\x1f\xd3m\x9b\xe3\x1c:\t\xed\xbb,\xb2\x9d\x16\xc3_\x99C\x93}\x19\xe3\xdd\x18\xf1$\x03\xbd8~Zw\u007f\xf1&\xf6f\xc0\xab\xf4\x87\x1e\x01\xdc4I)\x8b\u07b7\x9b#\x93N\xc5\xeb\x03\xa7\x9c\x03cAKu=\xd8\x17S߬h\xb3\x13\xfe\\\x86\xa4\xe8,{\x9b\n\xed\x97\xf4n|s\xc7F\xb7'c\xd0ɞwر\xb4\x85tyOF\xb7\xe7bd\x93YЉqv\xa7\xc5|\xbe5\xd9U\xf1\r\xbd\x14\xa9ObjÝ\xe8\xa0X\x10s\xccwK|S\x8f\x04\x1f\xe6M`}VgD\xab\xeba\x02\xe4\xb2~\x88\x05,\x99\xeb}8\xbb\xe3\xe1\xb4\xcb`\x82\x88\xb9>\x87\xf1\x1e\x86\t\xa0\x83\xdd\rK:\x17&`\xd6=\r\x1fد0ӥ\xf01\x9d\x84\xbf4\xf6\x1c\xeb9\x98\xe94\x98\x89L\xa7\xb0\x9a\xe9%X\xdeA0ßo\xec\x16\xa8\xfb\x01\x06\xdfyn\x8f@\xb7\v`\x10\xe4\xc2\u0380\x91\xb3\xffA\x90\v\xfa\x01fN\xfc\a\xc1Nn\x8c\x13\x1a1\xfa\x93Ӣt\a\x93.lM\xc6I\x8fݹ\x03\xc9E\xba\xae\x95)S\xe55\xec>)|\x05\xf3\x1d\x1e\x9e\xd9\xc9\xf3U\x98\xac\xb9\b\x14]y\n~N\xef\t\xfd\xf0\x91Ɇ\xf3Ɗ=\xfed\xb2\xd6M\xdb1\xfa\xbbs;\xd7\"\xa3PSJ\x9f\xfa D\xba\x9f\xd7]:\x14;\xc6*[\xbc\xe7\xd6d_\x84a_ޣ\x96罚$\xe2\xe9駀\xb8\x97\x05\xae\xbfT!\x91[\x95\xc2:$\xfe%\x82¢-\xfdy0\xaf=\x84\x95\x89\x94\xfep\x8a\xafE\xae\xe1q\xb6\xb8\x18\xebp\x97/)XbӴ:>\x0f\xafiŢ-\xa1\x84\xc4\xc6\xec\xc6V\xf5\bl]d\xa6h?t\xb4|\xd4\x15\xb5a\xe7<|\xf9\xd3\v_\xb9\xb9\xeb\x9f<)]\xe6\x8e\x15\xdf\xca\xf2\r\xb3\x00 (\xe3\xd97@cy\xabs\xc3~J&w\xfd\xf9|\x95\xda\xe6\x01).\xab\u05579_\x85\xab\vh\x03\x1e\xad\x01\x16\xd6q0@\xb00\a<\xa2\x06\xa3\xb9^\xc67\xb8\xc2=\xff\xd35\xfd\xfc\xb5\x05#\x96\xe3\xaaR\x19\x91'\xcbM\xf7L\xe3\xf5\xf0'\xf6G\xf6\x88\xf6ʍB䫪;c\x87\xc8?լ\x9d\xb1\x85\xf0\xb7\x90\v\x8f\xab\x01\x80\v\xfc\u0600Jq\xf1x\xe6\xea&O\t\xd6\xc1u\xe7t\x976\x14\x9e\vtN\xecӝ\xcdWrG{Դ\xc9\rT\x89b(\xd6\x14.\xbb\xf7\x17CF'2O\xf9o@-\xa6\xb0\xadYW}\x9bSfO\x196O\x8c7ƣ\u007f\x1ev$R{\xdcc7<·R\xday_~_O#\x8ep\xea\xce\x16\xde|@\x01\x95\xdcKr\x88$ؽ\xb0[\xb1\xc7Uf\x14\xe5Q\xd2\xe8S\x8c\xfe6r\rP\a\xbe\x8e\xd0#\xe8\xc7\xf6̔\bFe\x0eP\xd2\xc7\x12\xae\xe3\x8eJ\x12,\xc4_\x8d\xed\x1f\xd6\x14R\x1b\x1b\xc2\x17\x0e\xa1\xd3\xd2\xc5\xfe\x9c\xaf\xb5N\xe2\xfb@3\xeaө\x96\xaf¤L\xc3\xfb\xfc\xd0)\xc6\n\xbe\xe2\xe9\x16\x15\x0e&0\u007f\xae\xbf\xa2ћ\xb0\xd1\x0f\xd6\xec)\xe2\xeb\xfdt\x97\xbcR\xef\x97\aa\xbd\x14J\xbd\a\xf0#o\xed=\xfe\x82\xe4\x17F6\x82!\x06F̦y\x18'5!\xa5\xd4A\xd6|\x98\xb05\x95\xef\x18\\c\xb0=\x89\xa7\xf7\xad)O\xc4T7\x90]\x88\xb4\x03\xa2\xf3+\xdc\xed\x8c\xf5!~]\xad@\xee\xe2\xc6҃Jޙ+_\xe1k\f }\x93\xc55\xbaɱ\xa0E\xe1X7=\u007f\x11\x82\xcb\xce\"\xcb(>\xc1\x1b\xe7\x85\xea\xf9\x80o.w\xf1~Mڅ\xf9_z\xdbY\x8fɛ\xf6\xec\xbaϵ*\xb6hIS\x19X\xe0\x17\x1f\xed\x05\xaf\xa7\x86S\xba-\xa2\x86W+\xbd'\u007f\xd3.\b\x82'\x0f\xa3\x148\x03;1xgx\xdc\xe7\xf1\xaf\xc6\v\xb5\x19Kh\xbb!`=5\x91Ë\xfbD\x19\x12ÖI\x1f$'4wH\x97V\x92ಃ\xd0{R k\xaa\xfd!i\xe0\xc8N1\\ī\b!(U\xb5'\x95\x8e\x855_Y\xdd\xcaLc\xa9-o\xa1*\xb2\x17\xa8\xca\xe1\x93\xe7\xf0\xad\x98\xf8\xa9\x9f\x9bx\x17x\xb5\xb3\xa6XE\xfes\xcd\xec:V\xec\xac4\x142qN\x13\xaf㍀e\xb1\x97%j\x10.\xe22\xdbw2%\xc8\xf1D\xcd\v\xeb\x97\x05a\x8f\x9d\xa93\xf1\x17\xc3\xc5|\r\x8fX\n2\xb6\xbe\x11[S\xc0\xdd釖\xae)GO_\x15\n\x1f\x85\t\xa2\xa7\xbc\x96\xbf\xeea,\x86;x=\x88\x9d\x80\xaa\x13@uQ\xff\xfb\xc4N\xcdw\x96\xee磨\xe7\x93\xc9'\a)d\xc1\r\xbc\x14\xfb\xfcA\xee\xfa\xf9EY*\x99\x11\xb6\u007f\xfc\x95\x0eH\x8e\v\xa2\x8a\xabɀ\x82\xa3\x87:6\x80/XZ\xcc\xc8*\xfb\xc8?(\xa4\xfd\xde!v#\x95\xabŁ]7Et\x9f\xbdǢ\x1cx\xd7D\x8e\xd8,\x1as|\"M\xe8\x11\x90\xbef\x95@\xc5N\x80Ѥp1!u\xa8q\x0e!\xf5\xa21B\\\x95\x91\x03\xdaUC[Q\x9ds} U\xaf\xc2R\xa2=m=\xff\x11'\rd!q\xfd\xc7\xe6!\xad4$\xe1\xf7wJD\x06\xfc\xf8ɣ\xe6cv\x9f\x9a\xff\x98}\xab\xf8\xf1\xbac\xe8'co\x99\xb7L;\xa2\x12\x9f4\x05\x02\x91eH\xba\xfb\xf5\xf4;v\x97\x97\xfcO\xfaT\x1d\xff\x9b\x19\x1d\xf6Rw\v\xff\xf9_\x17\x10\xebL\xcf\t\x0fz\xf8\xff\x01\x00\x00\xff\xff\xff\x8cC\xfd\xf8O\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcXM\x8fۼ\x11\xbe\xfbW\f\xb6\x87\xbd\xd4r\x92\xf6P\xe8\xe68-\x90v\x935\xd6\xc9\xf6\xd0\x16\bM\x8elv)R\xe5\x90\u07b8E\xff\xfb\x8b!%ْ\xb5\x1f\x01\u07bc<\x99\xe4\x98|\xe6\x99Oj6\x9f\xcfg\xa2\xd1\xf7\xe8I;[\x82h4~\x0fhyF\xc5ß\xa8\xd0nqx\xbb\xc5 \xde\xce\x1e\xb4U%\xac\"\x05W\xdf!\xb9\xe8%~\xc0J[\x1d\xb4\xb3\xb3\x1a\x83P\"\x88r\x06 \xacuA\xf02\xf1\x14@:\x1b\xbc3\x06\xfd|\x87\xb6x\x88[\xdcFm\x14\xfatCw\xff\xe1M\xf1\xae\xf8\xe3\f@zL\u007f\xff\xa2k\xa4 \xea\xa6\x04\x1b\x8d\x99\x01XQc\t[!\x1fbC\xc1y\xb1C\xe3d\xbe\xab8\xa0A\xef\n\xedfԠLH\x94J\xf0\x84Y{m\x03\xfa\x953\xb1ΰ\xe6\xf0\xd7\xcd\xed\xe7\xb5\b\xfb\x12\n\n\"D*\x9a\xbd L\x90\x15\x92\xf4\xba\t\t\xd8\xfbt\x1fl\xf2\x85p\xd3\xde\b\xf9_@Q\xeeA\x10,\x0fB\x1b\xb15\xb8\xf8jE\xf7;\x9d\x96a\xaf\xfb\xd3ñ\xc1\x12(xmwc(\x1d\x93\xc5\x05\vgG-w\xe7\a)\x11x\xba\xf3.6%\x9ch\xc8ҭ\x11\xb2\x01\xb3*\xad&\x9d\"i\xdfh\n\u007f{Z\xe6FSHr\x8d\x89^\x98\xa7\x8c\x90DH\xdb]4\xc2?!4\x03h<\x12\xfa\x03~\xb5\x0f\xd6=ڿh4\x8aJ\xa8\x84I\f\x91t\xac\xd7gF\xdf\b\x89\x8a\xd7\xe2ַ~\xd7j\x94\xe9/\xe1\u007f\xff\x9f\x01\x1c\x84\xd1*\x1d\x9f7]\x83v\xb9\xfex\xff\x87\x8d\xdcc-\xf2\xe2\xa4]G\x8a\x82&\x10\xd0a\x85\xc7=z\x84\xfb\xc4)\xb0\"H\xadV\xed\x89\x00n\xfbo\x94\x81\xday\xe3]\x83>\xe8\x0e%\x8f\xb3(\xeb\xd7FX\xae\x19l\x96\x01\xc5q\x85\x04a\x8f\xd0F\a*\xa0\xa4\b\xb8\n\xc2^\x13xL$\xdap2a\x0f\xa8\x02a[X\x05l\x98hO@{\x17\x8d\xe2`<\xa0\x0f\xe0Q\xba\x9d\xd5\xff\xedO&\b.]iD\xc0\xd6\xd8\xddH\xc1c\x85a\x9a#\xfe\x1e\x84UP\x8b#x\xe4; ڳӒ\b\x15\xf0\xc9y\x04m+W\xc2>\x84\x86\xca\xc5b\xa7C\x97W\xa4\xab\xebhu8.Rv\xd0\xdb\x18\x9c\xa7\x85\xc2\x03\x9a\x05\xe9\xdd\\x\xb9\xd7\x01e\x88\x1e\x17\xa2\xd1\xf3\x04\xdc\xe6P\xaf\xd5\xefzg\xb8>C:\n\xac<\x92\xe7?\xc9;\xfb|\xb6y\xfe[\xc6\u007f\xa2\x97\x97\x98\x95\xbb?o\xbe@wi2\xc1\x90\xf3\xc4\xf6\xe9ot\"\x9e\x89ҶB\x9f\rWyW\xa7\x13Ѫ\xc6i\x1b\xd2D\x1a\x8dvH:\xc5m\xad\x03[\xfa?\x11)\xb0}\nX\xa5\xec\n[\x84\xd8pܫ\x02>ZX\x89\x1a\xcdJ\x10\xfetڙa\x9a3\xa5/\x13\u007f^\x14\x86\x82\x99\xad~\xb9\xcbד\x16\x9a\x8c\xd2M\x83r\x10'\nI{\xf6\xe5 \x02\xa6\bh\x83v@\xe9\xd3\xe9\xef\xe9\xe0M\x01,%\x12}r\n\x87\xeb#\xa8\xcb^l\x80\xadA_kJe\x15*\xe7\xd3Z\xce!Ц\xc6ѡ\xd0\xe7\x9fb\xb4\x836\xd6c\bs\xb8C\xa1n\xad9Nn\xfc\xdd\xeb0\xbe`\xd2\\<2\xac\xcd\xd1\xca5z\xedԳ\xea\xbe\x1f\t\xf7J\xef\xdd#T\xc9mm0G\xce+t\xb4r\x9c7\xbb\xb1\\\u007f\xecrh\x0e\x8e6\x96Zn\nX\xb61\xe9*x\x03J\x13\xd7UJG\x8e\xe9\xe16\x81wK\b>\xbeZi\xe9l\xa5wcUϛ\x87i\xafx\xf6\xd0\x11W\xabt\a'\x1a\xf6\x80ƻ\x83V\xe8\xe7\xec\xf9\xbaҲ\xc5\x10}\xae:U*\x88c\xed&c\a\xfa\xe4Ӻ\xf5\xb3&\xbb=\x97<52\x19E뮄\x81\xb3\x1e\x81Evg\xe1\xc7~\x05lQ\xe9\xace+\x05\a\xa2\xd7\xe7\x9a\xc6\xc6\x1b\xfd\xf5\xa9\x00㱍\xf2\x01\xc3\xe5\xfa\xd8\xeb\x92\x183\x99\xe2(ς\x83H\x98\xb8}\x1e\xc0\v6\x03\x90b\x85\xfee\x14\xab%\x8b\xf5\x1e/`\xb5\x84m\xb4\xca`\x87\xe5q\x8f\x96˷\xae\x8e\\C\xbe\xdcl&΄\x8eǔ\x1c\xda\x02ܱ9\x85\xbdr\xbe\x16\xa1\x84\xed\xf1\"\xa8_T\xad\xf1X\xe9\xef/\xaa\xb6Nb\x1d\xc1\x8d\b{Ж\xb4B\x10\x13tOd\xd9n\xf4\x01|\xdb\xe4@\xfaAcp\x06\xe1\xa4~\x99\xf02\x8c׆G\xc7糑\xb1n\x85z\xbd\xbbyj\xb8\xc6\t{:4/\xb4\x98\xd2`>\f\xd7\xc1Nw\xe9\x8b\x053\xb7\xbe?V2\xf3k\xe5\xbc0\xb9m\xea\xc2Ϫ\xe6O.\x90Wg\x15\x92{.\v\xd1FB\x95\x13^\x01\xff\xb4\xf0\x81;(ɝM\xc9\x18\xb9\x99\xa1\v\xb7\xb1\xee\x91\xff|vZ:\x00\x9cMz\xa5\xee\x80{\xd4\xdcp\xa5\xadGm\f\xb7M\x1ekwH/\x8a\xe1\xe0\x1eǣ9\xf2K\xceUpxW\xbc)\xae~\xe3\xeak\x04\x05.\xa7\xa8\xee\xf0\xa0\xc7\xef\x85K6o.\xe4;\xef\xed\v&O\xbeu\x8d\xd8·b\xdf.ԯ\xb4\xe1\xb6q\xc2\xd5Oo\xa1\xfc8\xa0\x00Aטf\xef77ה\xde\xf7\xdc\xf2^\x1c\xfa\xc8\xe6\xa3\x04\x90\x9f\x10\xae\xedt#\x05\xf4\x13\xc6\xeem\xa5\t\xac\x03\xe3\xecn\x10\ny\xb4}/8\x0f\xd9u\x9c\a\x85ܲr\x9a\x95{awxz˴\xd8\xcfP\xb2c\\\"\x1dz\xc7\xc9\x1b\xb4\x9dv\x85Wؐ\x1f\xee\xcf\xda\xeff ڙn\xc8p\x8f\xba\xb5\xa5\xb9\x8c\xc9Wp=\x92\xee\x8a\b\x139\xe7\xab~\x956*}=y>DzD\xa7\xa7\x8cޣ\r\xa7ܓ\x1cj*\xff\xbc\xae\t^\x0e>\xb9\x9c\xef\x8c?Ǽ\xa8\xcbD\xce\x1d-\x9d>Z\xbd=\xcd\xda\xefJ\xf9\x13M\xda\x00\xc8\t\xf6\x8c\xc86\xaaڕS\"\xe7\f\xda\x04T\x9fǟl\xae\xae\x06\xdf]\xd2T:\x9b\x9bS*\xe1\x1f\xff\x9a\xe5SQ\xddw8x\xf1\x97\x00\x00\x00\xff\xff\xbd\xf1$\xc1\xe0\x13\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcXMsۼ\x11\xbe\xebW\xec\xb8\a_**y\xdbC\x877Eo3\x93։5\x96\xe3\x1e\xda\xce\x04\x02\x97\x12j\x10`\xf1!G\xed\xf4\xbfw\x16\x00?DR\x92\x93y\x13\x9cD`\xb5x\xf6\xd9\x0f,0\x9b\xcf\xe73V\x8b'4Vh\x95\x03\xab\x05~u\xa8\xe8\xcbf\xcf\u007f\xb2\x99Ћ\xc3\xdb-:\xf6v\xf6,T\x91\xc3\xca[\xa7\xab\a\xb4\xda\x1b\x8e\xbfb)\x94pB\xabY\x85\x8e\x15̱|\x06\xc0\x94ҎѴ\xa5O\x00\xae\x953ZJ4\xf3\x1d\xaa\xec\xd9oq\xeb\x85,Є\x1d\x9a\xfd\x0fo\xb2_\xb2?\xce\x00\xb8\xc1\xf0\xf7GQ\xa1u\xac\xaasP^\xca\x19\x80b\x15\xe6\xb0e\xfc\xd9\xd7\xd6i\xc3v(5\x8f{e\a\x94ht&\xf4\xcc\xd6\xc8\x03\x92\xa2\b\xf0\x98\\\x1b\xa1\x1c\x9a\x95\x96\xbe\x8a\xb0\xe6\xf0\x97\xcd\xfd\xa75s\xfb\x1c2\xeb\x98\xf36\xab\xf7\xccb\x80\\\xa0\xe5F\xd4.\x00{\x17\xf6\x83M\xdc\x10\xeeҎ\x10\xff\x05\xd6\xf3=0\v\xcb\x03\x13\x92m%.>+\xd6\xfc\x0e\xda\"\xecu\xab\xdd\x1dk\xcc\xc1:#\xd4\xee\f\x14ɬ{bR\x14-\x13c\\w#\x19\x10\x16\xdc\x1e\x81\xfe\r\x8e&\xe8+\xf2\x05D\x18B\xc3\x17\xbc0\x1bT\x02\x1c\xa2\x0e,z`I7<\x9d,D\xd4\xf4=\xc4\xdcx?\x1by\xae\xa7q\xb9ñ\x9a\x9dѾΡs]\x94N\x81\x13\x83.ҟ\xd8o\xc8\x0f\xebRX\xf7\xd7\xf32wº WKo\x98<\x178A\xc4\n\xb5\xf3\x92\x993B3\x80ڠEs\xc0\xcf\xeaY\xe9\x17\xf5^\xa0,l\x0e%\x93\xc1\xab\x96k\xb2\xeb\x13\xa1\xaf\x19\x0f\x9cY\xbf5)W\x92Eѻ9\xfc\xf7\u007f\xb3\x96w\xf2eX\xd45\xaa\xe5\xfa\xc3\xd3\x1f6|\x8f\x15˓{&bq`(\xb9\x9d\xf5<\xbbG\x83\xf0\x148\x8d^\xb7ɪ\xa4\x11@o\xff\x85\xdc5\x01P\x1b]\xa3q\xa2AI\xa3W\x19ڹ\x01\x96[\x02\x1be\xa0\xa0Z\x801\xfaRFc\x016\x18\x02\xba\x04\xb7\x17\x16\f\x06\x12\x95\xeb\\\xd8\x02*\x81\xa9\x04+\x83\r\x11m,ؽ\xf6\xb2\xa0\x02r@\xe3\xc0 \xd7;%\xfe\xd3j\xb6\xe0t\nx\x87\xc9\xd9\xcd\b\t\xaf\x98$\x9a=\xfe\x1e\x98*\xa0bG0H{\x80W=mA\xc4f\xf0\x912D\xa8R\xe7\xb0w\xae\xb6\xf9b\xb1\x13\xae\xa9\x85\\W\x95W\xc2\x1d\x17\xa1\xa2\x89\xadw\xda\xd8E\x81\a\x94\v+vsf\xf8^8\xe4\xce\x1b\\\xb0Z\xcc\x03p\x15\xcbSU\xfc\xae\r\x86\xdb\x1e\xd2A1\x88#D\xfeY\xde)\xe6\xa3\xcf\xe3\xdf\"\xfe\x8e^\x9a\"V\x1e\xfe\xbcy\x84f\xd3\xe0\x82S\xce\x03\xdb\xdd\xdflG<\x11%T\x89&:\xae4\xba\n\x1aQ\x15\xb5\x16ʅ\x0f.\x05\xaaSҭ\xdfV\u0091\xa7\xff\xed\xd1:\xf2O\x06\xabp\"\xc0\x16\xc1ס\x9cd\xf0A\xc1\x8aU(W\xcc\xe2\x0f\xa7\x9d\x18\xb6s\xa2\xf4:\xf1\xfd\x83\xecT0\xb2\xd5N7g̤\x87&\xb3tS#?ɓ\x02\xad0\x14ˎ9\f\x19\x90\x92\xf6\x84\xd2\xf3\xe5\xef|\xf2\x86\x04\xe6\x1c\xad\xfd\xa8\v<\x9d\x1f@]\xb6b'\xd8j4\x95\xb0\xa1\x15\x80R\x9b\xe19\xc2R1\uf3e6\xfed\x83\x15T\xbe\x1aB\x98\xc3\x03\xb2\xe2^\xc9\xe3\xe4\xc2ߌp\xc3\r&\xddE#\xc2\xda\x1c\x15_\xa3\x11\xba\xb8h\ueec1pk\xf4^\xbf@\x19\xc2V9y\xa4\xbab\x8f\x8a\x0f\xebf3\x96\xeb\x0fM\r\x8dɑr)q\x93\xc12\xe5\xa4.\xe1\r\x14\xc2R/`\x83\xca!=\xd4\xda\xd0j\x0e\xce\xf8W\x1b͵*\xc5nhj\xbfᙎ\x8a\x8bJ\a\\\xad\xc2\x1eTh(\x02j\xa3\x0f\xa2@3\xa7\xc8\x17\xa5\xe0\t\x837\xf1\xd4)Á8\xb4n2w\xa0->)\xac/\xba\xec\xbe/\xd95_\x11E\nW\x8b\x8e\xaa\x9e\x05\x85\x14\xce\xcc\f\xe3\nȣ\\+E^r\x1aXkϭ\x1d:o\xf0\xd7s\tFc\xeb\xf93\xba\xf1\xfc0\xea\x82Xӝ\xc5?\x11\no1p{\x19\xc0\x15\x9f\x01p\xb6Bs\x1d\xc5jIbm\xc43X-a\xebU!\xb1\xc1\xf2\xb2GEǷ(\x8ft\x86<\xdem&tB\xc3c(\x0e\xe9\x00n\u061c\xc2^jS1\x97\xc3\xf68Jꫦ\xd5\x06K\xf1\xf5\xaai\xeb \xd6\x10\\3\xb7\a\xa1\xac(\x10\xd8\x04\xdd\x13U\xb6\x19m\x02\xdf\xd71\x91\xbe\xd1\x19TA\xa8\xa8\x8f\v^\x84\xf1\xda\xf4h\xf8\xbc\x98\x19\xeb$\xd4\xda\xdd|\x87\x86kX\xb0\xa7Ss\u008a\xae/}\x1f\v\"?^\x84\xf14\x96\xbfPV\x9b\xdb\xc68A\xa9\xa5\xd0Ơ\xad\xb5*(\xfe^WT;\xb8\xbfEi\x9dr\xe0\xfc\xb4Z\x9d\xac4\x9c_\xed\x17b\xe7\xffm\x1dC\xbc`\xf6\xcfe\xbd\r\x97\x90^\xd3\xf0\x83\xfb\x83\x9b^\x83@-\xa7\x02\xaf\xbc\xc5\"\xd6\xfb\f\xfe\xa1\xe0Wj 95v9a\xa4^Ύ\xbc\xab\xf4\v\xfd\xb9\xa7-(\x00\xad\x82]\xa19\xa2\x16=\xf6\x9ba\xe9EHI]\xa3\xc1J\x1f\xd2%\xb4?\xa8\xc53(\x8ft\xf9\xd6%\x1c~\xc9\xded7?\xb9\xf9\xa0\x9b6u\x13X<\xe0A\f\xafKc6\xefF\xf2M\xf2\xb6\xa1M\x1f_\x9a>ta\x92ؗ\x91\xf9\xa5\x90\xd45Odzw\x15\x1c?\x06\xbc\xdb\xdc\xdd\xda\xf0$C\x1d\xffH\xe9\v\xb9\xcf\x06\x80t\x83ҩ\xd1\xf7֡\x99pv\xeb+aAi\x90Z\xedNR!\x8e\xd4\xf6\x836\x10CG\x1b(\x90:v\xcar\xbegj\x87\xddU.a\uf864\xc0\x18#=\x8d\x8e.\x1a\x84\x9a\x0e\x85W\xf8\xf0QT\x97\xb3\xe1\xeeDt\xfa\xb9\xa5E\x9d|)\xc79\xf9\n\xae\a\xd2\xcd\x19JD\xce]\xf3\x1cԍ\xef\xeb\"ǯLW\xad\xff\xee\a\xa7\xb1\xf9\xccvOO?\xdf\xf6\xf0\xd8w\xf9x%\x89\xc6B\xee\x8dA庺\x1b\x92i\xaa\xf6\xbe\xee\xfe\xb3TS?US\xb7\xa3\xa9\xba\xebI\xf4\x97\xd74~\xa5I+\xf9\xcc\xc6/\aT\x15\x84\xc2.{Ë*\r@b\x14\xe4=\xfe\x1e\x1eB|\f?\x13z'\x1dl\x8d\x17l\x00\xc4Ƅ\x1d\\\x97\xa8\x93\xb1\xe8\x1a\x80\xbd\xf1\xe4*\x9e1\x8f\x980\xfcxsu\xffݝ\xedq0\xa3\x10\xc0\xa1X\xa6T\xf5\x96r\x00\x1200E\x02\x1a\xa7\x00!\x06\x84\xc80DF\x18\xa3\x95v2\x998&d\xa5\x99`\xf9\x8e\xfa\xe7Yv\xe6\xfc}\x89n\xd4\x01W:\x06\x05\xb4G\x98\xea\x8e\x0e\xa4F\x0eq\vړ\x00c\xc5\x12\xc6\x1e:2\vE\xc5\x04\x88\x9b\xbf\xd0j\vw\x05\x1d\vH\x1f\xb3w\xa5\xcd\xf6\xc8\n\x8c6\xee\x02\xfd\xfblYJ~ť7:\x17x\xfe((r0\xbep\xcd\xf85\x98\xe0`0O\xc0X|@\x0eG֪\x8a\xb4\xf0[\x81Ca\x1b;\xe8U\x93t\xeb\xf5\x8et\x9e\x18\x1b\x87!\aҧu\xed{\xdad\x8d,k\x87{\xf4k\xa1\xddʰ\xedI\xd1jf\\\x9bD\xab\x1ax\xa8\x03\xd3\x0e\xee+\x9e\xc6K\xde\x1fE\xaaO\xa5\x13D\x99\xc2\xeeY\\{\xf8U\xee\xa5\u007f\xc72\x8f\xc7\xc6\xf8\x0fx\x8b\xa8P\xb9\xfdx\xf7\tf\xa7\xb5\x04\xa7\xcc+\xed\xc319\x80/\xa0(l\x91\xc7\xc2m9\x0e\xd5\"\x06\x97\"\x05\xad\v\xeb\t\xc3)tɛ\x81T\xe6\xf6+\xf5i\xe1\xb2\xde\x1b\xb0A\xc8\xc9\x19E\xd7\xc2U\x80K3\xa0\xbf4\x82_\x1c{!,\xab\x82\xf4m\xf0\xc7\xd7ݩ\xe2H\xebY<\xdfE\x8b\x15Z\x18˻\x84\xb6Ԭ\x80+giK\xb6\x8e\x01l#\xc3cO\xb6\x9f\xc7\xf2\x84\xe8\xf3\x00\xb7G⥁-\xdfh\xa0\xdc*\xa7\xf2W\x92\x85Z'b<\xe9\xb5Ց\x997)\xa8\xd1,\xff\x8bC=1\x93\xb0\x99\x19\x83Nv\xea-\xb0t\xe8srG\xe6\xc8r\x9e\xf7I8\x1f\xabJ\xfdk\x19\n\x02&t\xb1\xb1\x16\x93\xa2\xbb>\u007fN\xbc{w\xf2.\xa8K\x1b\x83\xa3\xf15\x04\u007f\xfcٌV\xd1\xdd\xcfq\x14\xe1\u007f\x01\x00\x00\xff\xff\xcb0\x9b\f\x8c\t\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WOoܶ\x13\xbd\xebS\f\xf2;\xe4W \xd26m\x0f\x85n\xad\x93\x02A\xd3 \xb0\x13_\x8a\x1e\xb8\xe4\xacĚ\"Y\xcep\x1d\xf7\xd3\x17CI\xfbG+\xdb\xe9\xa1{\xd3p8|||oȭ꺮T\xb4\xb7\x98\xc8\x06߂\x8a\x16\xbf0z\xf9\xa2\xe6\xeeGjl\xd8\xec_o\x91\xd5\xeb\xea\xcez\xd3\xc2U&\x0e\xc35R\xc8I\xe3\x1b\xdcYo\xd9\x06_\r\xc8\xca(Vm\x05\xa0\xbc\x0f\xac$L\xf2\t\xa0\x83\xe7\x14\x9c\xc3Tw蛻\xbc\xc5m\xb6\xce`*+\xcc\xeb\xef\xbfm\xbek~\xa8\x00t\xc22\xfd\x93\x1d\x90X\r\xb1\x05\x9f\x9d\xab\x00\xbc\x1a\xb0\x05\x13\xee\xbd\v\xca$\xfc+#15{t\x98BcCE\x11\xb5,ڥ\x90c\vǁq\xee\x04h\xdc̛\xa9\xcc\xf5X\xa6\x8c8K\xfc\xeb\xda\xe8{;eD\x97\x93r\x97 \xca Y\xdfe\xa7\xd2\xc5p\x05\x10\x13\x12\xa6=~\xf6w>\xdc\xfb_,:C-\xec\x94#\xac\x00H\x87\x88-|\x10\x94Qi4\x15\xc0^9k\n\x15#\xee\x10\xd1\xff\xf4\xf1\xdd\xed\xf77\xba\xc7A\x8dA\x00\x83\xa4\x93\x8d%o\x89\x1b,\x81\x82\t\x05p8\x00\x03\xe5A%\xb6;\xa5\x19v)\f\xb0U\xfa.ǩ&@\xd8\xfe\x89\x9a\x818$\xd5\xe1+\xa0\xac{PRmL\x04\x17:\xd8Y\x87\xcd4%\xa6\x101\xb1\x9dY\x96߉\xbe\x0e\xb1\x05\xe0\x97\xb2\xa31\a\x8c(\n\t\xb8G\x98t\x81\x06\xa8\xec\x16\xc2\x0e\xb8\xb7\x04\t\v\x95~\xd4\xd8IY\x90\x14\xe5'\xe4\r\xdc\b݉\x80\xfa\x90\x9d\x11\x19\xee11$ԡ\xf3\xf6\xefCe\x12^dI\xa7x\x16\xc2\xfc\xb3\x9e1y\xe5\xe4,2\xbe\x02\xe5\r\f\xea\x01\x12\x16v\xb2?\xa9VR\xa8\x81\xdfBB\xb0~\x17Z\xe8\x99#\xb5\x9bMgyv\x94\x0eÐ\xbd\xe5\x87M\xf1\x85\xddf\x0e\x896\x06\xf7\xe86d\xbbZ%\xdd[F\xcd9\xe1FE[\x17\xe0\xbe\x18\xaa\x19\xcc\xff\xd2d?zy\x82\x94\x1fD=\xc4\xc9\xfa\xee\x10.:\u007f\x94w\xd1\xf9(\x8fqڈ\xffH\xaf\x84\x84\x95\xeb\xb77\x9f`^\xb4\x1c\xc19\xe7\xa3N\x0e\xd3\xe8H\xbc\x10e\xfd\x0e\xd3xpEeR\x11\xbd\x89\xc1z.\x1f\xdaY\xf4\xe7\xa4S\xde\x0e\x96i\x96\xad\x9cO\x03W\xa5\xaf\xc0\x16!G\xa3\x18M\x03\xef<\\\xa9\x01ݕ\"\xfc\xcfi\x17\x86\xa9\x16J\x9f'\xfe\xb4\x1d\x9e'\x8el\x1d\xc2s\xbfZ=\xa1\x85\x95o\"j9/!M\xe6ٝ\xd5\xc5\x02\xb0\v\t\xd4\xd1\xd9\x13m\xcdI\xdd5o\x16P*u\xc8\xe7\xb1\x05\x8aO%E\x16\xbe\xef\xd5y\v\xf9?6]#}\x80&\bcg\xf8\xa6Y\xd4{l\xf55\x8d\xaeb\x98\xa5*[\x17\x1e\xc5\xe8\xd2zN\xd1,\x17\x95\x1f\xfa<\xac\x15\xaf\xe1\xe7\x82\xf4}\xe8\x9e\x18\xbd\n\x9eE\xd0O\xa4\xdc\x06\x97\a\xbc\xf1*R\x1f\x9e̜/\xcd\xc3E\xb2L\xbbFi\xb5\xf8\x18\xa4i\xf8\x1a)\xbbՅV\x858\xff\xca\xc5\xf9\x1c\xcbr\xf7\xcc,˄\xb1\xe3\"ȅ\x9d<2ұ\r\xdc[\xeeᾷ\xba_\xa9\neZ9 \xe9/DA\xdb\xe2\xd8\u007f\a[tl\x13^ȣ.\xa2\xb9\b\n\xe4j\xad\xf8\xc2s\xeb\x85\xeb\xc9\v\xcf:\x96\x15g\xfajϖ\xec\x99T\x9dSB\xcfS\x8dr[-'|\x8dig\xc5\u007f\xbe~\xff\xa4s\xdf\x1c\xf3\xca\x1bLY?\xe2\x88\tk\xb2\x9dܭ2&\xde-\xceZ\x120\xfeN\xef\xf8gO\r\xbfD\x9bN\x9e,\x8f@{{H\x1b\x1b\v\xfa\xf1\x8aX\xbe^J9\xa4r\xedj\xe5/\xb0m\x11\f:d4\xb0}\x18;\xe3\x031\x0eK\xbc\xbb\x90\x06\xc5-\xc8\xc5Q\xb3\xbd\x10\x8a\xbc/\xd5\xd6a\v\x9c\xf2\xba\x8aV6\x1b{E\x17\xb6:\xdb\xe7G\xc9X;\xfe\x83\xb9\x9e8\u007fx\xa4\x83\xd5\xf0\x01\xef/b\x1fS\xd0H\x84Kc<\x82~E܋\xd0\xf1a\xfe\xfa\xf8U\xa4XO\x0f\xf12\x00P\x9e\xb5愺\xe9\xcd8E\x8e\x8eQZcd4\x1f\x96O\xf1\x17/\xce\xde\xd6\xe5S\ao\xec\xf8/\x02~\xff\xa3\x1a\xab\xa2\xb9\x9dqH\xf0\x9f\x00\x00\x00\xff\xff\xbbظ3\xc4\f\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Y_s۸\x11\u007fק\xd8\xc9=\xb87\x13RMz\xd3\xe9\xe8\xedb7\x1d\xb7w\x8e'\xf2\xe5%\x93\x87\x15\xb1\x12Q\x83\x00\x8a\x05\xa5\xa8\x9d~\xf7\xce\x02\xa4$J\xb4,_{)_l\x82\x8b\xc5\xfe\xdf\xdfB\x93\xa2(&\xe8\xf5'\n\xac\x9d\x9d\x01zM_#Yy\xe3\xf2\xf1O\\j7]\xbfYP\xc47\x93Gm\xd5\f\xae[\x8e\xae\xf9H\xec\xdaP\xd1\r-\xb5\xd5Q;;i(\xa2\u0088\xb3\t\x00Z\xeb\"\xca2\xcb+@\xe5l\f\xce\x18\nŊl\xf9\xd8.h\xd1j\xa3(\xa4\x13\xfa\xf3\u05ff/ߖ?L\x00\xaa@i\xfb\x83n\x88#6~\x06\xb65f\x02`\xb1\xa1\x19x\xa7\xd6δ\r-\xb0zl=\x97k2\x14\\\xa9݄=Ur\xe8*\xb8\xd6\xcf`\xff!\xef\xed\x04\xca\xca\xdc;\xf5)\xb1y\x97ؤ/Fs\xfc\xdb\xd8ן4\xc7D\xe1M\x1bМ\n\x91>\xb2\xb6\xab\xd6`8\xf9<\x01\xf0\x81\x98\u009a~\xb1\x8f\xd6m\xec{MF\xf1\f\x96h\x98&\x00\\9O3\xb8\x13)=V\xa4&\x00k4Z%Sd\xb9\x9d'\xfb\xe3\xfd\xed\xa7?̫\x9a\x1a̋\xc2\xd9y\nQ\xf7\xea\xc9s\xe0\xd8\xdd\x1a\x80\"\xae\x82\xf6\x89#\\\t\xabL\x03J\\I\f\xb1&\xe8\x1cB\n8\x1d\x03n\t\xb1\xd6\f\x81\x92\x0e6;\xf7\x80-\b\tZp\x8b\xbfS\x15K\x98\x8b\x9e\x81\x81k\xd7\x1a%\xfe_S\x88\x10\xa8r+\xab\xff\xb9\xe3\xcc\x10]:\xd2`\xa4ξ\xfd\xa3m\xa4`ш\x11Zz\rh\x154\xb8\x85@r\x06\xb4\xf6\x80[\"\xe1\x12~v\x81@ۥ\x9bA\x1d\xa3\xe7\xd9t\xbaұ\x0f\xe5\xca5Mku\xdcNS@\xeaE\x1b]\u0a625\x99)\xebU\x81\xa1\xaau\xa4*\xb6\x81\xa6\xe8u\x91\x04\xb7)\x92\xcbF}\x17\xba\xb8\xe7\xab\x03I\xe3V\xdc\xc61h\xbb\xda-\xa7\x00{\xd2\xee\x12`\xa0\x19\xb0ۖ\xe5ߛW\x96\xc4*\x1f\xff<\u007f\x80\xfe\xd0䂡͓\xb5\xf7\xdbxox1\x94\xb6K\n\xd9q\xcb\xe0\x9ađ\xac\xf2Nۘ^*\xa3\xc9\x0e\x8d\xce\xed\xa2\xd1Q<\xfd\x8f\x968\x8a\u007fJ\xb8N\t\r\v\x82\xd6+\x8c\xa4J\xb8\xb5p\x8d\r\x99kd\xfa\xcd\xcd.\x16\xe6BL\xfa\xbc\xe1\x0f\xebА0[k\xb7\xdc\x17\x8aQ\x0f\x1d\xe5\xfe\xdcS%\xfe\x12\xa3\xc9>\xbd\xd4UJ\x01X\xba\x00xL^\x1e\xb0\x1dKMyrU\x98G\x17pE?\xb9\xea ɟ\x90\xe9\xdd؎^*\xa9m9M\xa9c\r\x9c)\x8fX\x02\x98~릦@iG \x8e\xba\x92@r\xac\xa3\v[a+\xfbI\x95G\xfbG\x8d.\x8fu\x8a\xce\xca\u007f\xe7\x14\x8d\x89+\x1b!֘c\xf2ޥ\xcc\b\xad\xb5\x92\x05\xce^,\x80w\xea\xec\xf9\x1dg\x84@K\nd%\xa3r\xf1\xf1.\x95\xa8\x88\xda\xf6\x99\x97K7Dwb\xbeE60)\x18:\xfa\x9c\xb3\xe1\xc9z<*\xe9\x8f\xf7\xb7}\r\xee\x8d\xd4\xc9\x1c\x8fO|\xb8\xf90\xcbRI\b\xadR%\x95.\xb7Ԃ9\x04l\xe4\xce)1\x99\xcc\xd1\xe6\xe0\x88\x0e\xaa\x1a\xedHa\x85\x04Z\x92u\x97\xad\xf4\xb2\xf2\xea\xa5\xd9z\f\x1b\xfag\x04>\x1c\x17\x86\xffS\x13\xbeH\xad\x84ڟU\xeb\xee \x9eϪ%\xf3C\xb0\x14)i\xa6\\ŢTE>\xf2ԭ)\xac5m\xa6\x1b\x17\x1e\xb5]\x15\x12\x88E\x8e\x04\x9e\xa6\x11`\xfa]\xfa\xf3\xab\xb4H\xc8\xfc2U\x12\xe9\xb7\xd0G\xce\xe1\xe9\x8b\xd5\xe9q\xe5\xa5]\xe9j\xde!\x9f㝒\x12\x9bZWu?$\xec\xab\xe7h\x8e4\xa8r\xc9E\xbb\xfd\xcd\xc3V\f\xd9\x06\x91g[tch\x81V\xc9\xff\xac9\xca\xfa\x8b-\xd7\xea\v\x92\xf4\x97ۛo\x13̭~qF\x8e\x02\xe2\x1c\x13\xde\xdd*1\xdfRS8\v\xa7>\x0eH{`7\x82$w4\x17#\xb9\x88\xab\x13\x00\x85J\xa5\x8b\x064\xf7g@\xd6\x19\x9d\a\xc2?\xe0\x8a\x01\x03\x01B\x83^\xfc\xf4H\xdb\"7i\x8fZz\xac\xb4\xd1\x0e\xaf,\b\xd0{\xa3G\xdai\u05ca;\xb8\xd8!o\x19kq\xc5\xe3\xfa\x8eX=\xef>k\xed<^\x8c\xc1\xe7\xee\xe8\x8cKv\x10:\xba=P=\x8d\xdf\x13\xe0\xfa\x84\xddd\n\x14tu(Z1>\xba\f(\x04\xd2\x0f\x16\xbcS\x83\xf7a\x9c\r>e}\x9e\x9d\xde\"Ɩ/\x9e\xdf\x12uo\xbd\\\x0fb\xc7#a\x85_3\xc1UN\xb0\xe3\xf0\x9a\xea\x9c\v\xafO\xe9ӅHPY\xac\xa8\x1b\x89\xc7.\x866\xc8\xfd\t\xa7C\x18\x1c0\xcb\xfbR\xdd\x15^\xa4\x12\xb4\x13ԹDmHA\u007fGv\xbc\xe7\x84\xe7!\x8f\x05-\xa5T\xb5\xde8T\xfdPԉ\xd6_\xf2<\xc84\x9c\xee\x1b\xae\xf8I\x8e-\x93JS\xf2\x88\xfa\xc7\xeda\xe9B\x83q\x06\n#\x15#\fmk\f.\f\xcd \x86\xf6\xf8㓩\xdf\x103\xaeΧ\xd7ϙ&χ\xdd\x06\xc0\x85k\xe3n@\x1c\xa4\xf8\x15w\xd1s\xf9t:2\x82\rC\x16\x050s\a\x1f\x8dI;\x0e\xd3z\u007f\x89\x9a\xe4Y\x90\xb8\xe5\xbf\xcdp\x00_#\x9f7νP\x8c%Ϯ\x06\x9d\xc9\x1eH\x13a\xdb\x1c\x9fP\xc0\x1dmN\xd6n\xed}p\xab@|\x1c\x1aE\x1f?'\xca\x16\xf0>\xc5\xf9\xc5\xfav\a\x9cW\xb9#\x82ڙ>=]D\x03\xb6m\x16\x14D\xef\xc56\x12\x0f\x8b\xf0\xe9̟\xa6\x88\xbd\xd1\x0ev\xf7W\b\x99O7\x14Uh\xd3-\x9b\xe4Lt\xa04{\x83\xa7SQ\xafBB\x12\x922\x92\xd2\xfbh\xed\xd3\xd4SH\x9f^rK\x91\xa4\xb9qv\x14\xe3\xf6\xf9\xa9m\xfc\xe3\x0fO\"\x0em#\xad\x06E\xbd\xfb*\x06|'\xfc\xff\u05fc\x9fl\xacl\xd1s\xed\xe2\xed\xcdYo\xcfwd}\x94\xefAK\xaa]\xe9ޯ#\xea]>li\xf9\xc9apq\xeaq\xc4\x10/k\x1e\xf3\x01\xe93}#\xf1%U\u009c<\x06\x8c\xa7\x81\x99\ue0ef\x8f\u007fey\r\xacӵ\x98`\x9f\f\x86\xf2\xa8\xcb\xd2N\x04ڹ\x90c\xf5\x94\xe3\xa0\x11\f\n\xffP\xf4oQ\xf3G\xe2\xe1hi\xff\x93ӛ\xfd[\x8aˢ\xfb\x89)}\xe8\xd4R\a\x87w\xb7\xaa\xdd\xca\x1e\x86`%\x90\x9d\xd4\xdd\xf1\x8fL\xaf\xf2UI\xff\xabQz\xad\x9c\xcdh\x96g\xf0\xf9\xcb\x04\xba\xbb\xd6O\xbd\x1c\xb2\xf8\x9f\x00\x00\x00\xff\xff\x80\xb6\xf7)\x9e\x1b\x00\x00"), diff --git a/go.mod b/go.mod index 45c6dbd2c..57696720d 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,14 @@ require ( github.com/go-ini/ini v1.28.2 // indirect github.com/gobwas/glob v0.2.3 github.com/gofrs/uuid v3.2.0+incompatible - github.com/golang/protobuf v1.3.2 + github.com/golang/protobuf v1.4.2 github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect github.com/joho/godotenv v1.3.0 github.com/kubernetes-csi/external-snapshotter/v2 v2.1.0 + github.com/onsi/ginkgo v1.13.0 + github.com/onsi/gomega v1.10.1 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.0.0 github.com/robfig/cron v0.0.0-20170309132418-df38d32658d8 @@ -30,8 +32,11 @@ require ( github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 - golang.org/x/net v0.0.0-20200602114024-627f9648deb9 - google.golang.org/grpc v1.26.0 + golang.org/x/net v0.0.0-20200625001655-4c5254603344 + golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect + golang.org/x/text v0.3.3 // indirect + google.golang.org/grpc v1.27.0 + google.golang.org/protobuf v1.25.0 // indirect k8s.io/api v0.17.4 k8s.io/apiextensions-apiserver v0.17.4 k8s.io/apimachinery v0.17.4 diff --git a/go.sum b/go.sum index 4794e33fc..fa62da030 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -197,12 +199,23 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -307,6 +320,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -317,11 +332,17 @@ github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0 h1:M76yO2HkZASFjXL0HSoZJ1AYEmQxNJmY41Jx1zNUq1Y= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -350,6 +371,7 @@ github.com/robfig/cron v0.0.0-20170309132418-df38d32658d8/go.mod h1:JGuDeoQd7Z6y github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= @@ -390,6 +412,7 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -416,6 +439,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -456,8 +481,9 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -487,16 +513,24 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220220014-0732a990476f h1:72l8qCJ1nGxMGH26QVBVIxKd/D34cfGt0OvrPtpemyY= golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= @@ -526,6 +560,8 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= @@ -550,6 +586,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191220175831-5c49e3ecc1c1 h1:PlscBL5CvF+v1mNR82G+i4kACGq2JQvKDnNq7LSS65o= google.golang.org/genproto v0.0.0-20191220175831-5c49e3ecc1c1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -558,6 +596,19 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -582,6 +633,8 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hack/build-image/Dockerfile b/hack/build-image/Dockerfile index db0baa970..90cb4cd0d 100644 --- a/hack/build-image/Dockerfile +++ b/hack/build-image/Dockerfile @@ -24,6 +24,13 @@ WORKDIR /go/src/k8s.io RUN git config --global advice.detachedHead false RUN git clone -b kubernetes-1.17.0 https://github.com/kubernetes/code-generator +RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/kubebuilder_2.3.1_linux_amd64.tar.gz && \ + tar -zxvf kubebuilder_2.3.1_linux_amd64.tar.gz && \ + mv kubebuilder_2.3.1_linux_amd64 /usr/local/kubebuilder && \ + chmod +x /usr/local/kubebuilder && \ + export PATH=$PATH:/usr/local/kubebuilder/bin && \ + rm kubebuilder_2.3.1_linux_amd64.tar.gz + # get controller-tools RUN go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.4 diff --git a/internal/velero/storagelocation.go b/internal/velero/storagelocation.go new file mode 100644 index 000000000..d020d2c29 --- /dev/null +++ b/internal/velero/storagelocation.go @@ -0,0 +1,132 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package velero + +import ( + "context" + "errors" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/persistence" + + "github.com/sirupsen/logrus" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" +) + +// StorageLocation holds information for connecting with storage +// for a backup storage location. +type StorageLocation struct { + Client client.Client + Ctx context.Context + + DefaultStorageLocation string + DefaultStoreValidationFrequency time.Duration + + // use variables to refer to these functions so they can be + // replaced with fakes for testing. + NewPluginManager func(logrus.FieldLogger) clientmgmt.Manager + NewBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) +} + +// IsReadyToValidate calculates if a given backup storage location is ready to be validated. +// +// Rules: +// Users can choose a validation frequency per location. This will overrite the server's default value +// To skip/stop validation, set the frequency to zero +// This will always return "true" for the first attempt at validating a location, regardless of its validation frequency setting +// Otherwise, it returns "ready" only when NOW is equal to or after the next validation time +// (next validation time: last validation time + validation frequency) +func (p *StorageLocation) IsReadyToValidate(location *velerov1api.BackupStorageLocation, log logrus.FieldLogger) bool { + validationFrequency := p.DefaultStoreValidationFrequency + // If the bsl validation frequency is not specifically set, skip this block and continue, using the server's default + if location.Spec.ValidationFrequency != nil { + validationFrequency = location.Spec.ValidationFrequency.Duration + } + + if validationFrequency == 0 { + log.Debug("Validation period for this backup location is set to 0, skipping validation") + return false + } + + if validationFrequency < 0 { + log.Debugf("Validation period must be non-negative, changing from %d to %d", validationFrequency, p.DefaultStoreValidationFrequency) + validationFrequency = p.DefaultStoreValidationFrequency + } + + lastValidation := location.Status.LastValidationTime + if lastValidation != nil { // always ready to validate the first time around, so only even do this check if this has happened before + nextValidation := lastValidation.Add(validationFrequency) // next validation time: last validation time + validation frequency + if time.Now().UTC().Before(nextValidation) { // ready only when NOW is equal to or after the next validation time + return false + } + } + + return true +} + +// IsValidFor verifies if a storage is valid for a given backup storage location. +func (p *StorageLocation) IsValid(location *velerov1api.BackupStorageLocation, log logrus.FieldLogger) error { + pluginManager := p.NewPluginManager(log) + defer pluginManager.CleanupClients() + + backupStore, err := p.NewBackupStore(location, pluginManager, log) + if err != nil { + return err + } + + if err := backupStore.IsValid(); err != nil { + return err + } + + return nil +} + +// PatchStatus patches the status.phase field as well as the status.lastValidationTime to the current time +func (p *StorageLocation) PatchStatus(location *velerov1api.BackupStorageLocation, phase velerov1api.BackupStorageLocationPhase) error { + statusPatch := client.MergeFrom(location.DeepCopyObject()) + location.Status.Phase = phase + location.Status.LastValidationTime = &metav1.Time{Time: time.Now().UTC()} + if err := p.Client.Status().Patch(p.Ctx, location, statusPatch); err != nil { + return err + } + + return nil +} + +// ListBackupStorageLocations verifies if there are any backup storage locations. +// For all purposes, if either there is an error while attempting to fetch items or +// if there are no items an error would be returned since the functioning of the system +// would be haulted. +func ListBackupStorageLocations(kbClient client.Client, ctx context.Context, namespace string) (velerov1api.BackupStorageLocationList, error) { + var locations velerov1api.BackupStorageLocationList + if err := kbClient.List(ctx, &locations, &client.ListOptions{ + Namespace: namespace, + }); err != nil { + return velerov1api.BackupStorageLocationList{}, err + } + + if len(locations.Items) == 0 { + return velerov1api.BackupStorageLocationList{}, errors.New("no backup storage locations found") + } + + return locations, nil +} diff --git a/internal/velero/storagelocation_test.go b/internal/velero/storagelocation_test.go new file mode 100644 index 000000000..f0a53d000 --- /dev/null +++ b/internal/velero/storagelocation_test.go @@ -0,0 +1,261 @@ +/*Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package velero + +import ( + "context" + "errors" + "testing" + "time" + + . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/scheme" + "github.com/vmware-tanzu/velero/pkg/persistence" + persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" + "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +func TestIsReadyToValidate(t *testing.T) { + tests := []struct { + name string + serverDefaultValidationFrequency time.Duration + backupLocation *velerov1api.BackupStorageLocation + ready bool + }{ + { + name: "don't validate, since frequency is set to zero", + serverDefaultValidationFrequency: 0, + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").ValidationFrequency(0).Result(), + ready: false, + }, + { + name: "validate as per location setting, as that takes precedence, and always if it has never been validated before regardless of the frequency setting", + serverDefaultValidationFrequency: 0, + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").ValidationFrequency(1 * time.Hour).Result(), + ready: true, + }, + { + name: "don't validate as per location setting, as it is set to zero and that takes precedence", + serverDefaultValidationFrequency: 1, + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").ValidationFrequency(0).Result(), + ready: false, + }, + { + name: "validate as per default setting when location setting is not set", + serverDefaultValidationFrequency: 1, + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").Result(), + ready: true, + }, + { + name: "don't validate when default setting is set to zero and the location setting is not set", + serverDefaultValidationFrequency: 0, + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").Result(), + ready: false, + }, + { + name: "don't validate when now is before the NEXT validation time (validation frequency + last validation time)", + serverDefaultValidationFrequency: 0, + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").ValidationFrequency(1 * time.Second).LastValidationTime(time.Now()).Result(), + ready: false, + }, + { + name: "validate when now is equal to the NEXT validation time (validation frequency + last validation time)", + serverDefaultValidationFrequency: 0, + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").ValidationFrequency(1 * time.Second).LastValidationTime(time.Now().Add(-1 * time.Second)).Result(), + ready: true, + }, + { + name: "validate when now is after the NEXT validation time (validation frequency + last validation time)", + serverDefaultValidationFrequency: 0, + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").ValidationFrequency(1 * time.Second).LastValidationTime(time.Now().Add(-2 * time.Second)).Result(), + ready: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + storageLocationInfo := StorageLocation{ + DefaultStoreValidationFrequency: tt.serverDefaultValidationFrequency, + } + + g.Expect(storageLocationInfo.IsReadyToValidate(tt.backupLocation, velerotest.NewLogger())).To(BeIdenticalTo(tt.ready)) + }) + } +} + +func TestIsValid(t *testing.T) { + tests := []struct { + name string + backupLocation *velerov1api.BackupStorageLocation + isValidError error + expectError bool + }{ + { + name: "do not expect an error when store is valid", + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").Result(), + isValidError: nil, + expectError: false, + }, + { + name: "expect an error when store is not valid", + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").Result(), + isValidError: errors.New("an error"), + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + var ( + pluginManager = &pluginmocks.Manager{} + backupStores = make(map[string]*persistencemocks.BackupStore) + ) + pluginManager.On("CleanupClients").Return(nil) + + location := tt.backupLocation + backupStores[location.Name] = &persistencemocks.BackupStore{} + backupStore := backupStores[location.Name] + backupStore.On("IsValid").Return(tt.isValidError) + + storageLocationInfo := StorageLocation{ + NewPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + NewBackupStore: func(loc *velerov1api.BackupStorageLocation, _ persistence.ObjectStoreGetter, _ logrus.FieldLogger) (persistence.BackupStore, error) { + return backupStores[loc.Name], nil + }, + } + + actual := storageLocationInfo.IsValid(tt.backupLocation, velerotest.NewLogger()) + if tt.expectError { + g.Expect(actual).NotTo(BeNil()) + } else { + g.Expect(actual).To(BeNil()) + } + }) + } +} + +func TestPatchStatus(t *testing.T) { + tests := []struct { + name string + backupLocation *velerov1api.BackupStorageLocation + newPhase velerov1api.BackupStorageLocationPhase + expectError bool + }{ + { + name: "an update to the same phase should succeed", + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(), + newPhase: velerov1api.BackupStorageLocationPhaseAvailable, + expectError: false, + }, + { + name: "an update to a different phase should succeed", + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(), + newPhase: velerov1api.BackupStorageLocationPhaseUnavailable, + expectError: false, + }, + { + name: "an update to a location that doesn't exist should fail (see actual test)", + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result(), + newPhase: velerov1api.BackupStorageLocationPhaseUnavailable, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + storageLocationInfo := StorageLocation{ + Client: fake.NewFakeClientWithScheme(scheme.Scheme, tt.backupLocation), + } + + if tt.expectError { + backupLocation := builder.ForBackupStorageLocation("ns-1", "location-2").Phase(velerov1api.BackupStorageLocationPhaseAvailable).Result() + // an update to a location that was never created will fail: + g.Expect(storageLocationInfo.PatchStatus(backupLocation, tt.newPhase)).NotTo(BeNil()) + } else { + g.Expect(storageLocationInfo.PatchStatus(tt.backupLocation, tt.newPhase)).To(BeNil()) + + key := client.ObjectKey{Name: tt.backupLocation.Name, Namespace: tt.backupLocation.Namespace} + instance := &velerov1api.BackupStorageLocation{} + err := storageLocationInfo.Client.Get(context.Background(), key, instance) + g.Expect(err).To(BeNil()) + g.Expect(instance.Status.Phase).To(BeIdenticalTo(tt.newPhase)) + } + }) + } +} + +func TestListBackupStorageLocations(t *testing.T) { + tests := []struct { + name string + backupLocations *velerov1api.BackupStorageLocationList + expectError bool + }{ + { + name: "1 existing location", + backupLocations: &velerov1api.BackupStorageLocationList{ + Items: []velerov1api.BackupStorageLocation{ + *builder.ForBackupStorageLocation("ns-1", "location-1").Result(), + }, + }, + expectError: false, + }, + { + name: "multiple existing location", + backupLocations: &velerov1api.BackupStorageLocationList{ + Items: []velerov1api.BackupStorageLocation{ + *builder.ForBackupStorageLocation("ns-1", "location-1").Result(), + *builder.ForBackupStorageLocation("ns-1", "location-2").Result(), + *builder.ForBackupStorageLocation("ns-1", "location-3").Result(), + }, + }, + expectError: false, + }, + { + name: "no existing locations", + backupLocations: &velerov1api.BackupStorageLocationList{}, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + client := fake.NewFakeClientWithScheme(scheme.Scheme, tt.backupLocations) + if tt.expectError { + _, err := ListBackupStorageLocations(client, context.Background(), "ns-1") + g.Expect(err).NotTo(BeNil()) + } else { + _, err := ListBackupStorageLocations(client, context.Background(), "ns-1") + g.Expect(err).To(BeNil()) + } + }) + } +} diff --git a/pkg/apis/velero/v1/backupstoragelocation_types.go b/pkg/apis/velero/v1/backupstoragelocation_types.go index 675c1852d..e932d5752 100644 --- a/pkg/apis/velero/v1/backupstoragelocation_types.go +++ b/pkg/apis/velero/v1/backupstoragelocation_types.go @@ -40,6 +40,11 @@ type BackupStorageLocationSpec struct { // +optional // +nullable BackupSyncPeriod *metav1.Duration `json:"backupSyncPeriod,omitempty"` + + // ValidationFrequency defines how frequently to validate the corresponding object storage. A value of 0 disables validation. + // +optional + // +nullable + ValidationFrequency *metav1.Duration `json:"validationFrequency,omitempty"` } // BackupStorageLocationStatus defines the observed state of BackupStorageLocation @@ -54,6 +59,12 @@ type BackupStorageLocationStatus struct { // +nullable LastSyncedTime *metav1.Time `json:"lastSyncedTime,omitempty"` + // LastValidationTime is the last time the backup store location was validated + // the cluster. + // +optional + // +nullable + LastValidationTime *metav1.Time `json:"lastValidationTime,omitempty"` + // LastSyncedRevision is the value of the `metadata/revision` file in the backup // storage location the last time the BSL's contents were synced into the cluster. // @@ -79,6 +90,7 @@ type BackupStorageLocationStatus struct { // +kubebuilder:storageversion // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="Backup Storage Location status such as Available/Unavailable" +// +kubebuilder:printcolumn:name="Last Validated",type="date",JSONPath=".status.lastValidationTime",description="LastValidationTime is the last time the backup store location was validated" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // BackupStorageLocation is a location where Velero stores backup objects @@ -94,6 +106,8 @@ type BackupStorageLocation struct { // the k8s:deepcopy marker will no longer be needed and should be removed. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:object:root=true +// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations/status,verbs=get;update;patch // BackupStorageLocationList contains a list of BackupStorageLocation type BackupStorageLocationList struct { @@ -124,6 +138,7 @@ type ObjectStorageLocation struct { // BackupStorageLocationPhase is the lifecycle phase of a Velero BackupStorageLocation. // +kubebuilder:validation:Enum=Available;Unavailable +// +kubebuilder:default=Unavailable type BackupStorageLocationPhase string const ( diff --git a/pkg/apis/velero/v1/zz_generated.deepcopy.go b/pkg/apis/velero/v1/zz_generated.deepcopy.go index 7e5e8ad39..7bf9796b9 100644 --- a/pkg/apis/velero/v1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v1/zz_generated.deepcopy.go @@ -379,6 +379,11 @@ func (in *BackupStorageLocationSpec) DeepCopyInto(out *BackupStorageLocationSpec *out = new(metav1.Duration) **out = **in } + if in.ValidationFrequency != nil { + in, out := &in.ValidationFrequency, &out.ValidationFrequency + *out = new(metav1.Duration) + **out = **in + } return } @@ -399,6 +404,10 @@ func (in *BackupStorageLocationStatus) DeepCopyInto(out *BackupStorageLocationSt in, out := &in.LastSyncedTime, &out.LastSyncedTime *out = (*in).DeepCopy() } + if in.LastValidationTime != nil { + in, out := &in.LastValidationTime, &out.LastValidationTime + *out = (*in).DeepCopy() + } return } diff --git a/pkg/builder/backup_storage_location_builder.go b/pkg/builder/backup_storage_location_builder.go index 9b0fa6638..a74f598be 100644 --- a/pkg/builder/backup_storage_location_builder.go +++ b/pkg/builder/backup_storage_location_builder.go @@ -17,6 +17,8 @@ limitations under the License. package builder import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -86,3 +88,21 @@ func (b *BackupStorageLocationBuilder) AccessMode(accessMode velerov1api.BackupS b.object.Spec.AccessMode = accessMode return b } + +// ValidationFrequency sets the BackupStorageLocation's validation frequency. +func (b *BackupStorageLocationBuilder) ValidationFrequency(frequency time.Duration) *BackupStorageLocationBuilder { + b.object.Spec.ValidationFrequency = &metav1.Duration{Duration: frequency} + return b +} + +// LastValidationTime sets the BackupStorageLocation's last validated time. +func (b *BackupStorageLocationBuilder) LastValidationTime(lastValidated time.Time) *BackupStorageLocationBuilder { + b.object.Status.LastValidationTime = &metav1.Time{Time: lastValidated} + return b +} + +// Phase sets the BackupStorageLocation's status phase. +func (b *BackupStorageLocationBuilder) Phase(phase velerov1api.BackupStorageLocationPhase) *BackupStorageLocationBuilder { + b.object.Status.Phase = phase + return b +} diff --git a/pkg/cmd/cli/backuplocation/create.go b/pkg/cmd/cli/backuplocation/create.go index 633fd5593..ce874974f 100644 --- a/pkg/cmd/cli/backuplocation/create.go +++ b/pkg/cmd/cli/backuplocation/create.go @@ -58,14 +58,14 @@ func NewCreateCommand(f client.Factory, use string) *cobra.Command { } type CreateOptions struct { - Name string - Provider string - Bucket string - Prefix string - BackupSyncPeriod time.Duration - Config flag.Map - Labels flag.Map - AccessMode *flag.Enum + Name string + Provider string + Bucket string + Prefix string + BackupSyncPeriod, ValidationFrequency time.Duration + Config flag.Map + Labels flag.Map + AccessMode *flag.Enum } func NewCreateOptions() *CreateOptions { @@ -83,7 +83,8 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.Provider, "provider", o.Provider, "name of the backup storage provider (e.g. aws, azure, gcp)") flags.StringVar(&o.Bucket, "bucket", o.Bucket, "name of the object storage bucket where backups should be stored") flags.StringVar(&o.Prefix, "prefix", o.Prefix, "prefix under which all Velero data should be stored within the bucket. Optional.") - flags.DurationVar(&o.BackupSyncPeriod, "backup-sync-period", o.BackupSyncPeriod, "how often to ensure all Velero backups in object storage exist as Backup API objects in the cluster. Optional. Set this to `0s` to disable sync") + flags.DurationVar(&o.BackupSyncPeriod, "backup-sync-period", o.BackupSyncPeriod, "how often to ensure all Velero backups in object storage exist as Backup API objects in the cluster. Optional. Set this to `0s` to disable sync. Default: 1 minute.") + flags.DurationVar(&o.ValidationFrequency, "validation-frequency", o.ValidationFrequency, "how often to verify if the backup storage location is valid. Optional. Set this to `0s` to disable sync. Default 1 minute.") flags.Var(&o.Config, "config", "configuration key-value pairs") flags.Var(&o.Labels, "labels", "labels to apply to the backup storage location") flags.Var( @@ -119,12 +120,16 @@ func (o *CreateOptions) Complete(args []string, f client.Factory) error { } func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { - var backupSyncPeriod *metav1.Duration + var backupSyncPeriod, validationFrequency *metav1.Duration if c.Flags().Changed("backup-sync-period") { backupSyncPeriod = &metav1.Duration{Duration: o.BackupSyncPeriod} } + if c.Flags().Changed("validation-frequency") { + validationFrequency = &metav1.Duration{Duration: o.ValidationFrequency} + } + backupStorageLocation := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace(), @@ -139,9 +144,10 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { Prefix: o.Prefix, }, }, - Config: o.Config.Data(), - AccessMode: velerov1api.BackupStorageLocationAccessMode(o.AccessMode.String()), - BackupSyncPeriod: backupSyncPeriod, + Config: o.Config.Data(), + AccessMode: velerov1api.BackupStorageLocationAccessMode(o.AccessMode.String()), + BackupSyncPeriod: backupSyncPeriod, + ValidationFrequency: validationFrequency, }, } diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 844cbd852..87c06856f 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -49,7 +49,6 @@ import ( snapshotv1beta1informers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions" snapshotv1beta1listers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" - api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" @@ -72,10 +71,10 @@ import ( ctrl "sigs.k8s.io/controller-runtime" - kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/vmware-tanzu/velero/internal/util/managercontroller" + "github.com/vmware-tanzu/velero/internal/velero" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) @@ -84,6 +83,7 @@ const ( defaultMetricsAddress = ":8085" defaultBackupSyncPeriod = time.Minute + defaultStoreValidationFrequency = time.Minute defaultPodVolumeOperationTimeout = 240 * time.Minute defaultResourceTerminatingTimeout = 10 * time.Minute @@ -125,7 +125,7 @@ var disableControllerList = []string{ type serverConfig struct { pluginDir, metricsAddress, defaultBackupLocation string backupSyncPeriod, podVolumeOperationTimeout, resourceTerminatingTimeout time.Duration - defaultBackupTTL time.Duration + defaultBackupTTL, storeValidationFrequency time.Duration restoreResourcePriorities []string defaultVolumeSnapshotLocations map[string]string restoreOnly bool @@ -154,6 +154,7 @@ func NewCommand(f client.Factory) *cobra.Command { defaultVolumeSnapshotLocations: make(map[string]string), backupSyncPeriod: defaultBackupSyncPeriod, defaultBackupTTL: defaultBackupTTL, + storeValidationFrequency: defaultStoreValidationFrequency, podVolumeOperationTimeout: defaultPodVolumeOperationTimeout, restoreResourcePriorities: defaultRestorePriorities, clientQPS: defaultClientQPS, @@ -217,6 +218,7 @@ func NewCommand(f client.Factory) *cobra.Command { command.Flags().StringSliceVar(&config.disabledControllers, "disable-controllers", config.disabledControllers, fmt.Sprintf("list of controllers to disable on startup. Valid values are %s", strings.Join(disableControllerList, ","))) command.Flags().StringSliceVar(&config.restoreResourcePriorities, "restore-resource-priorities", config.restoreResourcePriorities, "desired order of resource restores; any resource not in the list will be restored alphabetically after the prioritized resources") command.Flags().StringVar(&config.defaultBackupLocation, "default-backup-storage-location", config.defaultBackupLocation, "name of the default backup storage location") + command.Flags().DurationVar(&config.storeValidationFrequency, "store-validation-frequency", config.storeValidationFrequency, "how often to verify if the storage is valid. Optional. Set this to `0s` to disable sync. Default 1 minute.") command.Flags().Var(&volumeSnapshotLocations, "default-volume-snapshot-locations", "list of unique volume providers and default volume snapshot location (provider1:location-01,provider2:location-02,...)") command.Flags().Float32Var(&config.clientQPS, "client-qps", config.clientQPS, "maximum number of requests per second by the server to the Kubernetes API once the burst limit has been reached") command.Flags().IntVar(&config.clientBurst, "client-burst", config.clientBurst, "maximum number of requests by the server to the Kubernetes API in a short period of time") @@ -296,7 +298,7 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s } var csiSnapClient *snapshotv1beta1client.Clientset - if features.IsEnabled(api.CSIFeatureFlag) { + if features.IsEnabled(velerov1api.CSIFeatureFlag) { csiSnapClient, err = snapshotv1beta1client.NewForConfig(clientConfig) if err != nil { cancelFunc() @@ -359,21 +361,6 @@ func (s *server) run() error { return err } - if err := s.validateBackupStorageLocations(); err != nil { - return err - } - - // Fetching from the server directly since at this point - // the cache has not yet started - bsl := &velerov1api.BackupStorageLocation{} - if err := s.mgr.GetAPIReader().Get(context.Background(), kbclient.ObjectKey{ - Namespace: s.namespace, - Name: s.config.defaultBackupLocation, - }, bsl); err != nil { - s.logger.WithError(errors.WithStack(err)). - Warnf("A backup storage location named %s has been specified for the server to use by default, but no corresponding backup storage location exists. Backups with a location not matching the default will need to explicitly specify an existing location", s.config.defaultBackupLocation) - } - if err := s.initRestic(); err != nil { return err } @@ -427,14 +414,14 @@ func (s *server) veleroResourcesExist() error { var veleroGroupVersion *metav1.APIResourceList for _, gv := range s.discoveryHelper.Resources() { - if gv.GroupVersion == api.SchemeGroupVersion.String() { + if gv.GroupVersion == velerov1api.SchemeGroupVersion.String() { veleroGroupVersion = gv break } } if veleroGroupVersion == nil { - return errors.Errorf("Velero API group %s not found. Apply examples/common/00-prereqs.yaml to create it.", api.SchemeGroupVersion) + return errors.Errorf("Velero API group %s not found. Apply examples/common/00-prereqs.yaml to create it.", velerov1api.SchemeGroupVersion) } foundResources := sets.NewString() @@ -443,13 +430,13 @@ func (s *server) veleroResourcesExist() error { } var errs []error - for kind := range api.CustomResources() { + for kind := range velerov1api.CustomResources() { if foundResources.Has(kind) { s.logger.WithField("kind", kind).Debug("Found custom resource") continue } - errs = append(errs, errors.Errorf("custom resource %s not found in Velero API group %s", kind, api.SchemeGroupVersion)) + errs = append(errs, errors.Errorf("custom resource %s not found in Velero API group %s", kind, velerov1api.SchemeGroupVersion)) } if len(errs) > 0 { @@ -461,43 +448,6 @@ func (s *server) veleroResourcesExist() error { return nil } -// validateBackupStorageLocations checks to ensure all backup storage locations exist -// and have a compatible layout, and returns an error if not. -func (s *server) validateBackupStorageLocations() error { - s.logger.Info("Checking that all backup storage locations are valid") - - pluginManager := clientmgmt.NewManager(s.logger, s.logLevel, s.pluginRegistry) - defer pluginManager.CleanupClients() - - // Fetching from the server directly since at this point - // the cache has not yet started - locations := &velerov1api.BackupStorageLocationList{} - if err := s.mgr.GetAPIReader().List(context.Background(), locations, &kbclient.ListOptions{ - Namespace: s.namespace, - }); err != nil { - return errors.WithStack(err) - } - - var invalid []string - for _, location := range locations.Items { - backupStore, err := persistence.NewObjectBackupStore(&location, pluginManager, s.logger) - if err != nil { - invalid = append(invalid, errors.Wrapf(err, "error getting backup store for location %q", location.Name).Error()) - continue - } - - if err := backupStore.IsValid(); err != nil { - invalid = append(invalid, errors.Wrapf(err, "backup store for location %q is invalid", location.Name).Error()) - } - } - - if len(invalid) > 0 { - return errors.Errorf("some backup storage locations are invalid: %s", strings.Join(invalid, "; ")) - } - - return nil -} - // - Custom Resource Definitions come before Custom Resource so that they can be // restored with their corresponding CRD. // - Namespaces go second because all namespaced resources depend on them. @@ -595,12 +545,12 @@ func (s *server) getCSISnapshotListers() (snapshotv1beta1listers.VolumeSnapshotL // If CSI is enabled, check for the CSI groups and generate the listers // If CSI isn't enabled, return empty listers. - if features.IsEnabled(api.CSIFeatureFlag) { + if features.IsEnabled(velerov1api.CSIFeatureFlag) { _, err = s.discoveryClient.ServerResourcesForGroupVersion(snapshotv1beta1api.SchemeGroupVersion.String()) switch { case apierrors.IsNotFound(err): // CSI is enabled, but the required CRDs aren't installed, so halt. - s.logger.Fatalf("The '%s' feature flag was specified, but CSI API group [%s] was not found.", api.CSIFeatureFlag, snapshotv1beta1api.SchemeGroupVersion.String()) + s.logger.Fatalf("The '%s' feature flag was specified, but CSI API group [%s] was not found.", velerov1api.CSIFeatureFlag, snapshotv1beta1api.SchemeGroupVersion.String()) case err == nil: // CSI is enabled, and the resources were found. // Instantiate the listers fully @@ -901,6 +851,22 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.logger.WithField("informer", informer).Info("Informer cache synced") } + storageLocationInfo := velero.StorageLocation{ + Client: s.mgr.GetClient(), + Ctx: s.ctx, + DefaultStorageLocation: s.config.defaultBackupLocation, + DefaultStoreValidationFrequency: s.config.storeValidationFrequency, + NewPluginManager: newPluginManager, + NewBackupStore: persistence.NewObjectBackupStore, + } + if err := (&controller.BackupStorageLocationReconciler{ + Scheme: s.mgr.GetScheme(), + StorageLocation: storageLocationInfo, + Log: s.logger, + }).SetupWithManager(s.mgr); err != nil { + s.logger.Fatal(err, "unable to create controller", "controller", "BackupStorageLocation") + } + // TODO(2.0): presuming all controllers and resources are converted to runtime-controller // by v2.0, the block from this line and including the `s.mgr.Start() will be // deprecated, since the manager auto-starts all the caches. Until then, we need to start the @@ -944,7 +910,7 @@ func NewCSIInformerFactoryWrapper(c snapshotv1beta1client.Interface) *CSIInforme // This is desirable for VolumeSnapshots, as we want to query for all VolumeSnapshots across all namespaces using this informer w := &CSIInformerFactoryWrapper{} - if features.IsEnabled(api.CSIFeatureFlag) { + if features.IsEnabled(velerov1api.CSIFeatureFlag) { w.factory = snapshotv1beta1informers.NewSharedInformerFactoryWithOptions(c, 0) } return w @@ -952,14 +918,14 @@ func NewCSIInformerFactoryWrapper(c snapshotv1beta1client.Interface) *CSIInforme // Start proxies the Start call to the CSI SharedInformerFactory. func (w *CSIInformerFactoryWrapper) Start(stopCh <-chan struct{}) { - if features.IsEnabled(api.CSIFeatureFlag) { + if features.IsEnabled(velerov1api.CSIFeatureFlag) { w.factory.Start(stopCh) } } // WaitForCacheSync proxies the WaitForCacheSync call to the CSI SharedInformerFactory. func (w *CSIInformerFactoryWrapper) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { - if features.IsEnabled(api.CSIFeatureFlag) { + if features.IsEnabled(velerov1api.CSIFeatureFlag) { return w.factory.WaitForCacheSync(stopCh) } return nil diff --git a/pkg/cmd/util/output/backup_storage_location_printer.go b/pkg/cmd/util/output/backup_storage_location_printer.go index 8eed52f05..00874a0db 100644 --- a/pkg/cmd/util/output/backup_storage_location_printer.go +++ b/pkg/cmd/util/output/backup_storage_location_printer.go @@ -31,6 +31,7 @@ var ( {Name: "Provider"}, {Name: "Bucket/Prefix"}, {Name: "Phase"}, + {Name: "Last Validated"}, {Name: "Access Mode"}, } ) @@ -64,11 +65,18 @@ func printBackupStorageLocation(location *velerov1api.BackupStorageLocation) []m status = "Unknown" } + lastValidated := location.Status.LastValidationTime + LastValidatedStr := "Unknown" + if lastValidated != nil { + LastValidatedStr = lastValidated.String() + } + row.Cells = append(row.Cells, location.Name, location.Spec.Provider, bucketAndPrefix, status, + LastValidatedStr, accessMode, ) diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index de05f2145..fcf43f9a6 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" + "github.com/vmware-tanzu/velero/internal/velero" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/features" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" @@ -122,16 +123,14 @@ func orderedBackupLocations(locationList *velerov1api.BackupStorageLocationList, func (c *backupSyncController) run() { c.logger.Debug("Checking for existing backup storage locations to sync into cluster") - locationList := &velerov1api.BackupStorageLocationList{} - if err := c.kbClient.List(context.Background(), locationList, &client.ListOptions{ - Namespace: c.namespace, - }); err != nil { - c.logger.WithError(errors.WithStack(err)).Error("Error getting backup storage locations from lister") + locationList, err := velero.ListBackupStorageLocations(c.kbClient, context.Background(), c.namespace) + if err != nil { + c.logger.WithError(err).Error("No backup storage locations found, at least one is required") return } // sync the default location first, if it exists - locations := orderedBackupLocations(locationList, c.defaultBackupLocation) + locations := orderedBackupLocations(&locationList, c.defaultBackupLocation) pluginManager := c.newPluginManager(c.logger) defer pluginManager.CleanupClients() diff --git a/pkg/controller/backupstoragelocation_controller.go b/pkg/controller/backupstoragelocation_controller.go new file mode 100644 index 000000000..317fd5059 --- /dev/null +++ b/pkg/controller/backupstoragelocation_controller.go @@ -0,0 +1,142 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "strings" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/vmware-tanzu/velero/internal/velero" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" +) + +// BackupStorageLocationReconciler reconciles a BackupStorageLocation object +type BackupStorageLocationReconciler struct { + Scheme *runtime.Scheme + StorageLocation velero.StorageLocation + + Log logrus.FieldLogger +} + +// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=velero.io,resources=backupstoragelocations/status,verbs=get;update;patch +func (r *BackupStorageLocationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithField("controller", "backupstoragelocation") + + log.Info("Checking for existing backup locations ready to be verified; there needs to be at least 1 backup location available") + + locationList, err := velero.ListBackupStorageLocations(r.StorageLocation.Client, r.StorageLocation.Ctx, req.Namespace) + if err != nil { + log.WithError(err).Error("No backup storage locations found, at least one is required") + } + + var defaultFound bool + var unavailableErrors []string + var anyVerified bool + for i := range locationList.Items { + location := &locationList.Items[i] + log := r.Log.WithField("controller", "backupstoragelocation").WithField("backupstoragelocation", location.Name) + + if location.Name == r.StorageLocation.DefaultStorageLocation { + defaultFound = true + } + + if !r.StorageLocation.IsReadyToValidate(location, log) { + continue + } + + anyVerified = true + + log.Debug("Verifying backup storage location") + + if err := r.StorageLocation.IsValid(location, log); err != nil { + log.Debug("Backup location verified, not valid") + unavailableErrors = append(unavailableErrors, errors.Wrapf(err, "Backup location %q is unavailable", location.Name).Error()) + + if location.Name == r.StorageLocation.DefaultStorageLocation { + log.Warnf("The specified default backup location named %q is unavailable; for convenience, be sure to configure it properly or make another backup location that is available the default", r.StorageLocation.DefaultStorageLocation) + } + + if err2 := r.StorageLocation.PatchStatus(location, velerov1api.BackupStorageLocationPhaseUnavailable); err2 != nil { + log.WithError(err).Errorf("Error updating backup location phase to %s", velerov1api.BackupStorageLocationPhaseUnavailable) + continue + } + } else { + log.Debug("Backup location verified and it is valid") + if err := r.StorageLocation.PatchStatus(location, velerov1api.BackupStorageLocationPhaseAvailable); err != nil { + log.WithError(err).Errorf("Error updating backup location phase to %s", velerov1api.BackupStorageLocationPhaseAvailable) + continue + } + } + } + + if !anyVerified { + log.Info("No backup locations were ready to be verified") + } + + r.logReconciledPhase(defaultFound, locationList, unavailableErrors) + + return ctrl.Result{Requeue: true}, nil +} + +func (r *BackupStorageLocationReconciler) logReconciledPhase(defaultFound bool, locationList velerov1api.BackupStorageLocationList, errs []string) { + var availableBSLs []*velerov1api.BackupStorageLocation + var unAvailableBSLs []*velerov1api.BackupStorageLocation + var unknownBSLs []*velerov1api.BackupStorageLocation + log := r.Log.WithField("controller", "backupstoragelocation") + + for i, location := range locationList.Items { + phase := location.Status.Phase + switch phase { + case velerov1api.BackupStorageLocationPhaseAvailable: + availableBSLs = append(availableBSLs, &locationList.Items[i]) + case velerov1api.BackupStorageLocationPhaseUnavailable: + unAvailableBSLs = append(unAvailableBSLs, &locationList.Items[i]) + default: + unknownBSLs = append(unknownBSLs, &locationList.Items[i]) + } + } + + numAvailable := len(availableBSLs) + numUnavailable := len(unAvailableBSLs) + numUnknown := len(unknownBSLs) + + if numUnavailable+numUnknown == len(locationList.Items) { // no available BSL + if len(errs) > 0 { + log.Errorf("Current backup storage locations available/unavailable/unknown: %v/%v/%v, %s)", numAvailable, numUnavailable, numUnknown, strings.Join(errs, "; ")) + } else { + log.Errorf("Current backup storage locations available/unavailable/unknown: %v/%v/%v)", numAvailable, numUnavailable, numUnknown) + } + } else if numUnavailable > 0 { // some but not all BSL unavailable + log.Warnf("Invalid backup locations detected: available/unavailable/unknown: %v/%v/%v, %s)", numAvailable, numUnavailable, numUnknown, strings.Join(errs, "; ")) + } + + if !defaultFound { + log.Warnf("The specified default backup location named %q was not found; for convenience, be sure to create one or make another backup location that is available the default", r.StorageLocation.DefaultStorageLocation) + } +} + +func (r *BackupStorageLocationReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&velerov1api.BackupStorageLocation{}). + Complete(r) +} diff --git a/pkg/controller/backupstoragelocation_controller_test.go b/pkg/controller/backupstoragelocation_controller_test.go new file mode 100644 index 000000000..07f39bbaf --- /dev/null +++ b/pkg/controller/backupstoragelocation_controller_test.go @@ -0,0 +1,185 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + + "github.com/vmware-tanzu/velero/internal/velero" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/persistence" + persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" + "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +var _ = Describe("Backup Storage Location Reconciler", func() { + BeforeEach(func() {}) + AfterEach(func() {}) + + It("Should successfully patch a backup storage location object status phase according to whether its storage is valid or not", func() { + tests := []struct { + backupLocation *velerov1api.BackupStorageLocation + isValidError error + expectedPhase velerov1api.BackupStorageLocationPhase + }{ + { + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").ValidationFrequency(1 * time.Second).Result(), + isValidError: nil, + expectedPhase: velerov1api.BackupStorageLocationPhaseAvailable, + }, + { + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").ValidationFrequency(1 * time.Second).Result(), + isValidError: errors.New("an error"), + expectedPhase: velerov1api.BackupStorageLocationPhaseUnavailable, + }, + } + + // Setup + var ( + pluginManager = &pluginmocks.Manager{} + backupStores = make(map[string]*persistencemocks.BackupStore) + ) + pluginManager.On("CleanupClients").Return(nil) + + locations := new(velerov1api.BackupStorageLocationList) + for i, test := range tests { + location := test.backupLocation + locations.Items = append(locations.Items, *location) + backupStores[location.Name] = &persistencemocks.BackupStore{} + backupStore := backupStores[location.Name] + backupStore.On("IsValid").Return(tests[i].isValidError) + } + + // Setup reconciler + Expect(velerov1api.AddToScheme(scheme.Scheme)).To(Succeed()) + storageLocationInfo := velero.StorageLocation{ + Client: fake.NewFakeClientWithScheme(scheme.Scheme, locations), + DefaultStorageLocation: "default", + DefaultStoreValidationFrequency: 0, + NewPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + NewBackupStore: func(loc *velerov1api.BackupStorageLocation, _ persistence.ObjectStoreGetter, _ logrus.FieldLogger) (persistence.BackupStore, error) { + return backupStores[loc.Name], nil + }, + } + + r := &BackupStorageLocationReconciler{ + StorageLocation: storageLocationInfo, + Log: velerotest.NewLogger(), + } + + actualResult, err := r.Reconcile(ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "ns-1"}, + }) + + Expect(actualResult).To(BeEquivalentTo(ctrl.Result{Requeue: true})) + Expect(err).To(BeNil()) + + // Assertions + for i, location := range locations.Items { + key := client.ObjectKey{Name: location.Name, Namespace: location.Namespace} + instance := &velerov1api.BackupStorageLocation{} + err := r.StorageLocation.Client.Get(ctx, key, instance) + Expect(err).To(BeNil()) + Expect(instance.Status.Phase).To(BeIdenticalTo(tests[i].expectedPhase)) + } + }) + + It("Should not patch a backup storage location object status phase if the location's validation frequency is specifically set to zero", func() { + tests := []struct { + backupLocation *velerov1api.BackupStorageLocation + isValidError error + expectedPhase velerov1api.BackupStorageLocationPhase + wantErr bool + }{ + { + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-1").ValidationFrequency(0).Result(), + isValidError: nil, + expectedPhase: "", + wantErr: false, + }, + { + backupLocation: builder.ForBackupStorageLocation("ns-1", "location-2").ValidationFrequency(0).Result(), + isValidError: nil, + expectedPhase: "", + wantErr: false, + }, + } + + // Setup + var ( + pluginManager = &pluginmocks.Manager{} + backupStores = make(map[string]*persistencemocks.BackupStore) + ) + pluginManager.On("CleanupClients").Return(nil) + + locations := new(velerov1api.BackupStorageLocationList) + for i, test := range tests { + location := test.backupLocation + locations.Items = append(locations.Items, *location) + backupStores[location.Name] = &persistencemocks.BackupStore{} + backupStores[location.Name].On("IsValid").Return(tests[i].isValidError) + } + + // Setup reconciler + Expect(velerov1api.AddToScheme(scheme.Scheme)).To(Succeed()) + storageLocationInfo := velero.StorageLocation{ + Client: fake.NewFakeClientWithScheme(scheme.Scheme, locations), + DefaultStorageLocation: "default", + DefaultStoreValidationFrequency: 0, + NewPluginManager: func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, + NewBackupStore: func(loc *velerov1api.BackupStorageLocation, _ persistence.ObjectStoreGetter, _ logrus.FieldLogger) (persistence.BackupStore, error) { + // this gets populated just below, prior to exercising the method under test + return backupStores[loc.Name], nil + }, + } + + r := &BackupStorageLocationReconciler{ + StorageLocation: storageLocationInfo, + Log: velerotest.NewLogger(), + } + + actualResult, err := r.Reconcile(ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "ns-1"}, + }) + + Expect(actualResult).To(BeEquivalentTo(ctrl.Result{Requeue: true})) + Expect(err).To(BeNil()) + + // Assertions + for i, location := range locations.Items { + key := client.ObjectKey{Name: location.Name, Namespace: location.Namespace} + instance := &velerov1api.BackupStorageLocation{} + err := r.StorageLocation.Client.Get(ctx, key, instance) + Expect(err).To(BeNil()) + Expect(instance.Status.Phase).To(BeIdenticalTo(tests[i].expectedPhase)) + } + }) +}) diff --git a/pkg/controller/suite_test.go b/pkg/controller/suite_test.go index 8fe4412c4..6b3293820 100644 --- a/pkg/controller/suite_test.go +++ b/pkg/controller/suite_test.go @@ -17,19 +17,116 @@ limitations under the License. package controller import ( + "context" + "path/filepath" "testing" - - "github.com/stretchr/testify/require" + "time" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/manager" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/stretchr/testify/require" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + // +kubebuilder:scaffold:imports ) +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +const ( + timeout = time.Second * 30 +) + +var ( + env *envtest.Environment + testEnv *testEnvironment + ctx = context.Background() +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func(done Done) { + By("bootstrapping test environment") + testEnv = newTestEnvironment() + + By("starting the manager") + go func() { + defer GinkgoRecover() + Expect(testEnv.startManager()).To(Succeed()) + }() + + close(done) +}, 60) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.stop() + Expect(err).ToNot(HaveOccurred()) +}) + +// testEnvironment encapsulates a Kubernetes local test environment. +type testEnvironment struct { + manager.Manager + client.Client + Config *rest.Config + + doneMgr chan struct{} +} + +// newTestEnvironment creates a new environment spinning up a local api-server. +// +// This function should be called only once for each package you're running tests within, +// usually the environment is initialized in a suite_test.go file within a `BeforeSuite` ginkgo block. +func newTestEnvironment() *testEnvironment { + err := velerov1api.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + env = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + } + + if _, err := env.Start(); err != nil { + panic(err) + } + + mgr, err := manager.New(env.Config, manager.Options{ + Scheme: scheme.Scheme, + }) + if err != nil { + klog.Fatalf("Failed to start testenv manager: %v", err) + } + + return &testEnvironment{ + Manager: mgr, + Client: mgr.GetClient(), + Config: mgr.GetConfig(), + doneMgr: make(chan struct{}), + } +} + +func (t *testEnvironment) startManager() error { + return t.Manager.Start(t.doneMgr) +} + +func (t *testEnvironment) stop() error { + t.doneMgr <- struct{}{} + return env.Stop() +} + func newFakeClient(t *testing.T, initObjs ...runtime.Object) client.Client { err := velerov1api.AddToScheme(scheme.Scheme) require.NoError(t, err) diff --git a/site/docs/master/api-types/backupstoragelocation.md b/site/docs/master/api-types/backupstoragelocation.md index 8fb3a0974..f77e538b1 100644 --- a/site/docs/master/api-types/backupstoragelocation.md +++ b/site/docs/master/api-types/backupstoragelocation.md @@ -40,6 +40,7 @@ The configurable parameters are as follows: | `config` | map[string]string | None (Optional) | Provider-specific configuration keys/values to be passed to the object store plugin. See [your object storage provider's plugin documentation][0] for details. | | `accessMode` | String | `ReadWrite` | How Velero can access the backup storage location. Valid values are `ReadWrite`, `ReadOnly`. | | `backupSyncPeriod` | metav1.Duration | Optional Field | How frequently Velero should synchronize backups in object storage. Default is Velero's server backup sync period. Set this to `0s` to disable sync. | +| `validationFrequency` | metav1.Duration | Optional Field | How frequently Velero should validate the object storage . Default is Velero's server validation frequency. Set this to `0s` to disable validation. Default 1 minute. | [0]: ../supported-providers.md