diff --git a/changelogs/unreleased/9307-sseago b/changelogs/unreleased/9307-sseago new file mode 100644 index 000000000..c69891510 --- /dev/null +++ b/changelogs/unreleased/9307-sseago @@ -0,0 +1 @@ +Concurrent backup processing diff --git a/config/crd/v1/bases/velero.io_backups.yaml b/config/crd/v1/bases/velero.io_backups.yaml index 47cbc3734..794c342c8 100644 --- a/config/crd/v1/bases/velero.io_backups.yaml +++ b/config/crd/v1/bases/velero.io_backups.yaml @@ -594,6 +594,8 @@ spec: description: Phase is the current state of the Backup. enum: - New + - Queued + - ReadyToStart - FailedValidation - InProgress - WaitingForPluginOperations @@ -625,6 +627,11 @@ spec: filters that happen as items are processed. type: integer type: object + queuePosition: + description: |- + QueuePosition is the position of the backup in the queue. + Only relevant when Phase is "Queued" + type: integer startTimestamp: description: |- StartTimestamp records the time a backup was started. diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index 130cae45e..045f529da 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -30,7 +30,7 @@ import ( var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccXK\x8f۶\x13\xbf\xfbS\f\xf6\x7f\xfd\xcbnP\xb4(|K\xdc\x06\b\x9a\x04\v{\x91;-\x8dlf)\x92%\x87N\xdd\xc7w/\x86\x94lY\xa2\xad\xf5\x1e\x8a\xf2&r\u07bfy\xd9EQ̄\x95_\xd0yi\xf4\x12\x84\x95\xf8;\xa1\xe6/?\x7f\xfe\xc9ϥY\x1c\xde̞\xa5\xae\x96\xb0\n\x9eL\xb3Fo\x82+\xf1g\xac\xa5\x96$\x8d\x9e5H\xa2\x12$\x963\x00\xa1\xb5!\xc1מ?\x01J\xa3\xc9\x19\xa5\xd0\x15;\xd4\xf3\xe7\xb0\xc5m\x90\xaaB\x17\x85w\xaa\x0f\xdf\xcd\xdf\xfc8\xffa\x06\xa0E\x83K؊\xf29X\x87\xd6xI\xc6I\xf4\xf3\x03*tf.\xcd\xcc[,Y\xfaΙ`\x97p~Hܭ\xe6d\xf5\xbb(h\xdd\t:\xc6'%=\xfd\x9a}\xfe(=E\x12\xab\x82\x13*gH|\xf6R\xef\x82\x12nD\xc0\n|i,.\xe13\xdbbE\x89\xd5\f\xa0\xf54\xdaV\x80\xa8\xaa\x18;\xa1\x1e\x9dԄneTh\xba\x98\x15\xf0\xd5\x1b\xfd(h\xbf\x84y\x17\xddy\xe90\x06\xf6I6\xe8I46\xd2v\x01{\xbb\xc3\xf6\x9b\x8e\xac\xbc\x12\x84ca\x1c\xb9\xf9\xd9֧\xa3\xc5\v)\xe7@@\xef-I\xf4\xe4\xa4\xde\xcd\xceć7)\x14\xe5\x1e\x1b\xb1li\x8dE\xfd\xf6\xf1×\xef7\x17\xd7\x00\xd6\x19\x8b\x8ed\aO:\xbd\xf4\xeb\xdd\x02T\xe8K'-\xc5\xe4\xf8\xab\xb8x\x03`\x05\x89\v*\xceC\xf4@{\xecb\x8cUk\x13\x98\x1ah/=8\xb4\x0e=ꔙ|-4\x98\xedW,i>\x10\xbdA\xc7b\xc0\xefMP\x15\xa7\xef\x01\x1d\x81\xc3\xd2\xec\xb4\xfc\xe3$\xdb\x03\x99\xa8T\tBO\x10Q\xd4B\xc1A\xa8\x80\xff\a\xa1\xab\x81\xe4F\x1c\xc1!넠{\xf2\"\x83\x1f\xda\xf1\xc98\x04\xa9k\xb3\x84=\x91\xf5\xcb\xc5b'\xa9+\xca\xd24MВ\x8e\x8bX_r\x1b\xc88\xbf\xa8\xf0\x80j\xe1\xe5\xae\x10\xae\xdcK\u0092\x82Å\xb0\xb2\x88\x8e\xe8X\x98\xf3\xa6\xfa\x9fk\xcb\xd8_\xa8\x1d\x01\x9dN\xac\xa4;\xe0\xe1\xd2\x02\xe9A\xb4\xa2\x92\x8bg\x14\xf8\x8aC\xb7\xfee\xf3\x04\x9d%\t\xa9\x04ʙt\x14\x97\x0e\x1f\x8e\xa6\xd45\xba\xc4W;\xd3D\x99\xa8+k\xa4\xa6\xf8Q*\x89\x9a\xc0\x87m#\x89\xd3ව\x9e\x18\xba\xa1\xd8Ul\\\xb0E\b\x96K\xa7\x1a\x12|а\x12\r\xaa\x95\xf0\xf8/cŨ\xf8\x82Ax\x11Z\xfdv<$N\xe1\xed=t\xad\xf4\n\xb4\xc3\xf6\xb8\xb1X2\xb2\x1c\\f\x95\xb5,SM\xd5Ɓ\x18\xd1_F*\xdf\x02\xf8\xa4&\xba!\xe3\xc4\x0e?\x9a$sH4\x95v|\xde\xe5\x04u\x16s\xdbJ=\x01\xf3\x84\x19\x81\xb4\x17\xd4k\x06$\xa4>\xf5\x94\xac\x937\x90\x89\xe8\b\xee\x14Z\xe8\x12\xdf\xc7|\xd4\xe5q\xc2\xd1O\x19\x16vio\xbe\x81\xa9\tu_hkkƓ-\x82\v\xfa.c\xcf>\xae\x8c\xae\xe5nlh\x7f\x90]\x03wB\xc9\xc0\xdb\xf5@'{\xca\xc9u\xb6\xa5\xe82\x8f\x01\xa9\xe5.\xb8k\xe0\xd5\x12U5j!\x00:(%\xb6\n\x97@.\xe0\x95\x88\x8cj\xe52\"<\x1f'\x80[_\x10\x83\xd4\x15WK;\xacXI\x97\x8c\x9c\xfe\xa8+p\x97kJ\xff\xa0\x0e\xcdX]\x01\xcf\xc6J\x91\xb9w\xe8I\x96\x99\x87\x87\x87\xfb2\x80\xc5|\xa8\xb8\x1d\xd5\x12\xddkjr=\x90ѕc\x1d\x94j\x15\x14\xa5i\xac \xb9U\xd8\xcd\f\xc6\\&\x9ec.i`T\x86\xf0\x14'\x01c\xce*\x8cVG\b\x1e+\xf8\xb6G=\x02\xc3\xc3C\xd2\xfdpWI\x1cxQ\xc3\xd3j\xf7\x9ax|\xb9\x14\xd1\xefN\xe9\":\x96ZbϿ\xae\xfd\xf8\x8cHk\xaaֲ\x96/\xd6\xcc\x1d\x8eq_\x91\x0e\ac\xbe\xc87\xe6\x01M\xae\xa5\rH\x06Q{\xd1d\"A\xc1\xdf3\x9b\"C\x17\xcd28\x17g\x7f\xba\xe5\x95\xef\xd5\xd3I\tO\xbd&\xcc\v\xf8\x04\xee\x1f\xc7\x1c\x9da,\f\x88/\x18\xda~\xf02\xb8\xfaP\x96\x88\xd5x\x1d\x01Ʒ\x11\x94\x16\xfd\x82彮\xcb\xe5\x87\x14z/vSN~JTi\xd3kY@lM\xa0+\b\xd0>\xe7\xe3mT&,\xb5{\xe1\xa7\xec|d\x9a\\^\f\x96\x81[&\\k\xbf\x9f\xf1[\xe6v\x8d\xa2\x1a\xb7\xf0\x02>\x1b\xca?\xdd\xec\xc0%\xea~2M\x0e\x9d\x01={~\x81A+r\x94\x7fc\xaf%a\x93\x1d\xe7\xd7k%\x1dn\xe7\n\tO\xbfU\xf3d\x03\xd3WC\xae\x13h\xe9\x81W\xb9X9Ws\xa9\vٔc\xe9L\x97P:\x13\x85\x94\xce\xcd\r\an\x15U&\x12\xf7\x96\xd6\xd5P$\xb8_\x16\x8eI\x0f\x1c\xfa\xa0\xe8E\x0e\xac#i\x87_b<\xa7\xdf\xcb\xec\xc9\xd7\\:\x05l\xba\xd6x\x95⽐\xea\xea\U000e4cde\x84\xa3\xfb\xf2ws\xc1r\xfa\x9dķ\xfd\xbc\xfdO\xe6獝\xb7{\x14Ή\xe3\xf4\xe8\x1e]z\xfe\xc9^\xf5\x8c\xf3i\x9d\xe8߄\xed\xe9\x1f\x89%\xfc\xf9\xf7\xec\x9f\x00\x00\x00\xff\xff\x18\xd9g\x90\x9b\x14\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec}_s\x1c)\x92\xf8\xbb?\x05\xa1\xdf\xc3\xecntK\xeb\xf8\xdd]\\\xe8\xcd+\xdb;\x1d3c+,\x8d\xf6\x99\xae\xca\xeefDA-P-\xf7\xde\xddw\xbf \x81\xfa\xd3EUQ\xad\x96ƻg^luAB\xfe!3I\x12X.\x97oh\xc9\x1e@i&\xc55\xa1%\x83\xaf\x06\x84\xfdK_>\xfe\xa7\xbed\xf2j\xff\xf6\xcd#\x13\xf95\xb9\xa9\xb4\x91\xc5\x17вR\x19\xbc\x87\r\x13\xcc0)\xde\x14`hN\r\xbd~C\b\x15B\x1aj\x7f\xd6\xf6OB2)\x8c\x92\x9c\x83ZnA\\>VkXW\x8c\xe7\xa0\x10x\xe8z\xff\xe7˷\xffq\xf9\xefo\b\x11\xb4\x80k\xb2\xa6\xd9cU\xea\xcb=pP\xf2\x92\xc97\xba\x84̂\xdc*Y\x95פ\xf9\xe0\x9a\xf8\xee\xdcP\xff\x82\xad\xf1\aδ\xf9\xa9\xf5\xe3\xcfL\x1b\xfcP\xf2JQ^\xf7\x84\xbfi&\xb6\x15\xa7*\xfc\xfa\x86\x10\x9d\xc9\x12\xae\xc9'\xdbEI3\xc8\xdf\x10\xe2G\x8d].\xfd\x80\xf7o\x1d\x84l\a\x05uc!D\x96 \xdeݮ\x1e\xfe\xff]\xe7gBrЙb\xa5A\xdc\xff{Y\xffN\xfc(\tӄ\x92\ađ(Orbv\xd4\x10\x05\xa5\x02\r\xc2hbv@2Z\x9aJ\x01\x91\x1b\xf2S\xb5\x06%\xc0\x80n\xc1\xcbx\xa5\r(\xa2\r5@\xa8!\x94\x94\x92\tC\x98 \x86\x15@\xfe\xf0\xeevE\xe4\xfa7Ȍ&T\xe4\x84j-3F\r\xe4d/yU\x80k\xfb\xc7\xcb\x1aj\xa9d\tʰ@tWZ\x92\xd4\xfau\fW[,y\\+\x92[\x91\x02\x87\x96'1䞢\x16?\xb3c\xbaA\x1f\x85\xcc\xfeL\x85\x1f\xfe\xe5\x11\xe8;P\x16\f\xd1;Y\xf1\xdcJ\xe2\x1e\x94%`&\xb7\x82\xfd\xa3\x86\xad\x89\x91\xd8)\xa7\x06\xb4\xa5\x8c\x01%('{\xca+XX\xa2\x1cA.\xe8\x81(\xb0}\x92J\xb4\xe0a\x03}<\x8e_\xa4\x02\xc2\xc4F^\x93\x9d1\xa5\xbe\xbe\xba\xda2\x13\xe6W&\x8b\xa2\x12\xcc\x1c\xaep\xaa\xb0ue\xa4\xd2W9\xec\x81_i\xb6]R\x95혁̲\xf9\x8a\x96l\x89\x88\b\x9cc\x97E\xfe\xff\x82x\xe8N\xb7\xe6`\xc5V\x1b\xc5Ķ\xf5\x01\xe7\xc7\f\xf6ة\xe3\x84сr(6\\\xb0?Y\xd2}\xf9pw\xdf\x16T\xa6=SZ\xf2:\xc4\x1fKM&6\xa0\\\xbb\x8d\x92\x05\xc2\x04\x91;QE9\xe7\f\x84!\xbaZ\x17\xccX1\xf8{\x05\xda\xce\x01y\f\xf6\x06u\x10Y\x03\xa9\xca܊\xf1q\x85\x95 7\xb4\x00~C5\xbc2\xaf,W\xf4\xd22!\x89[m\xcdz\\ّ\xb7\xf5!(\xc8\x01\xd6:\xc5rWB֙h\xb6\x15۰\xccM\xa7\x8dT\x8d\xdeq:\xb0K\xa1\xf8Է%\xd3\xecN\xd0R魯g\x05\xc8\xca\x1cט\x925d\xde\xdd\xea\bJ\x18\xa1\x1f/\xea\xacJCn'\xed\x13e\x06\xc7|s\xb7\"\x0f\xa8\xacBkTZ\x95&\xa6R\xc2JI\xa4\xaf/@\xf3ý\xfcU\x03\xc9+\x14\xeeL\x01\xd2aAְ\xb1\x92\xa0\xc0\xb6\xb7\x9f@)K\x1b\x8d\x03\x90UO\xd9\xd8r\xbf\x03K[Zq\xe3\xe7\t\xd3\xe4\xed\x9fI\xc1Dez\xa26\xc8u\xa4\x145\xb4\x90{P\xa7\x10\xf1=5\xf4\x17\xdb\xf8\x88v\x16(A\xa8\x96xkO\xc7\xf5\x01?Ƹ\xed\xcajӂ\xc84\xb9\xb8 R\x91\vg\x81/\x16\xaeuŸY2\xd1\xee\xe3\x89q\x1ez\x99\x87\xbc\xa3\xa1c\xa8\xbe\x97\x1f\xb5\x13ޓh1\x00\xabE\x9a\xa7\x1d\x98\x1d(R\xca\xda\xe2m\x18\a\xa2\x0f\xda@\xe1\t\x13\xac\x88\xc7'\xd2\x13\xce\x1d\xce=\bm\xe9\xea\x11\xe9#/*\xce\xe9\x9a\xc351\xaa\x82\x01ڬ\xa5\xe4@\xc5\x04q\xbe\x806,;\ai\x1c\xa4\ba\x94\xffС\x00\x1aM\xfa\b\x84F@{\x9aY\xeb\xccy\x8b\xb0]\xaaD\xc7T*Ȭ־\xf6ր\x01G\v$$\xe1RlA\xb9ޭ\xa7\x12\x04L\x81\x15\xb8\x9cXE\xab\x80[kB6\x95\xd5\xc1\x97\xc4\xce\xeeA\x19`B\x1b\xa0\x11\xe1|\x06\x7f\xe0kƫ\x1c\xf2\x1b\xe7x\xddY\xff1\x0f^sOk\xa6\xf0\xe9\xc3(Do\x9d9\xcb\xd0\t\xf4\xfe\xde\x12\xfd֘\x986F\xfaP\x82s\x9d-+\xfd\xb0\x1b\xeb;\xaa\x0f4\x18\xdb\xe8\xe2O\x17\v\xe4p\xb7\xd7n\x1f\x9aP\x055Y\x92\xf5&\x14\xa59\xf4k3\x03E\x84\x8a\xa3\xfa$\x91\x9fT)z\x18\xe0f\xed\xff\x9f\x91\x9fC0\x8f8*B\xb5W\xe6\xe9q\xbf\xff\xca\\=\x0f\x1f5\xaev)\x13\x96\x7fv\xe1\xd9a\x9fv\xeb7K6!M\x04\x1e\x13\x0e\x1e.\xcdF\xb8\xf5;\x11\xeb,2?$\xe4\xb5ly\xe1\xfd\xa7\xa4\xd4N\xca\xc7)\xea\xfch\xeb4\x8b\"\x92aT\x85\xacaG\xf7L*\x8fzcj\xe1+d\x95\x89\xcezjH\xce6\x1bP\x16N\xb9\xa3\x1a\xb4[&\x0f\x13d\xd8}'-5\x12\xfdx\x84G\xc3H\xcb&\xc4|h\xe8֏8\xb6\x92\xa1\u0601Z\xf7\x1a\x8dq\xce\xf6,\xaf(G\xbbLE\xe6\xf0\xa1\xf5\xb8bZf\x84ɽ1G%\xd3\x15\xe7\x10\x04\xa4,\x93:+%)\xc0\xfa\xbc\x85]\x13\xf4\xab\x0ec\xbe\xa6\xd6W\x91C\xd8\x13d\x96\xaa8h\xdfU\x8end\xa33\x16\rS0\x10A8]\x03'\x1a8dF\xaa8E\xa6\xf8\xecJ\x8a\x12\x1c dD\xf3uW\x1a\r\x02# \t.\xe1v,\xdb9W\xcf\n\x11\xc2!\xb9\x04\xeb\xf0\x19B˒G\xccESF\x99\xef;\x19\x9b\xebM\x99\x98\xf5\xc7\xf0b\xf3\xbf)\t:\xb3)Q\xd26\xf3\xabK\xd9Z\x1c\xe2kڦ\xfck\x126h\xfe\x13\x84vd\xf6\x13\x8c\n%\xcb\xf4\xa0\xdcZ\xaa2З֝BOgA\x98\t\xbfN̈́\x8e\xcf\xd5\v\x96u\x88\xf0m\xf3f\xbe\xd0'\xb2&eN\xbc\x10c\xea.\xfe\t\xf9\x82&\xe3\xce[\x8cd\x9e\xfc\xdcn\xb5 lS\x13=_\x90\r\xe3\x06\xd4\x11\xf5OR\xf5\x813\xe7 F\x8a\xd5#\x18\xbe7\xd9\xee\xc3W\xeb\x82\xe9f\xa7*\x91.Ǎ\x9d#\x1b\xbc\xfd\xaey\x9e\x80K0\x8c\xcd\x14\x14\x18\x1e\xc7\x15S\xfb\x17t\xad\xde}z\x1f__\xb5K\x82\xe4\xf5\x10\x99\x98t\xae\xbc;¨=>\xef\u0087/\xe8\x03\xd5\v \xb7\x15\xb2 \x94<\xc2\xc1\xb9.T\x10\xcb\x1f\x1a*'t\xaf\x00\xf7dP\xce\x1e\xe1\x80`\xe2\x9b,\xfd\x92*\r\xae<\xc2!\xa5\xda\x11\r혘\xf6\x9bG\x96N\xf6\a$\x04\xc6\xd6S\xc5\xc0\x15?\x15\"[\x1a\xf1\x92\xa8KB\t\xb4?\x01\xcd$Qi\xf7\xd1ޥD\t\xf8A;^\xda\x19\xb3c%\xaaU\x8c8\xc8M2C]y\xa0\x9c\xe5uGn\x8e\xacĂ|\x92\xc6\xfe\xf3\xe1+\xd3~#\xf3\xbd\x04\xfdI\x1a\xfc\xe5E(\xea\x06\xfe\x92\xf4t=\xe0D\x13N\xcb[\x82\xb5\xb7\xe2\x9cM\xb3\xd2VӞi\xb2\x12v\xb9\xe2H\x92\xd8\x15\ueeba\xee\\GE\xa5q\x17MH\xb1ta\x9bXO\x9e\xdeRu\xc8\xfd\xecN}\x87\xf7\xd6X\xb8/n\xef\x97\xd3\f\xf2\xb0]\x83\x9b\x92\xd4\xc0\x96e\x89\xfd\x15\xa0\xb6@J\xab\xc2\xd3$\"Q\xb1zl\xe6\x89O\x9a\xf5n\x97\xaf\xcb\xc7z\x8f\x7fiM\xce\xd2C0\xb2H\xa0\x81\xd7\xdd\xf94>K;g\x13j\x05I\x98\xac:\xb0g9\\5\x85(\xcf \aZqtq&\xb9K\xf3\x1c\xf3\\(\xbf\x9daQf\xc8\xc2\\\xd5\xd0\x1a\xbb3\xc1\x05ŭ\x96\xff\xb2\x96\x16g\xd3\xff\x90\x922\xa5/\xc9;Li\xe1\xd0\xf9\xe6\x83f-0\t]bJ\x8a\x95\x9f=\xe5\xd6\xf6[\x05.\bp\xe7\t\xc8M\xcf/Z\x90\xa7\x9d\xd4\xcelכ8\x17\x8fpp;\x86\x93]\xb6\x95\xcc\xc5J\\8\x1f\xa2\xa70j\x87C\n~ \x17\xf8\xed\xe29\xaeT\xa2\xa4&V\xeb\x88hA\xcb4\tŔ\xa2TG\xdd.X\x83\x13b\x1b֩2\xd6\xc9\x1e\xc36IDK\xa9#\x1b\xf9\x03C\x99\x10\xde[\xa9\x8d\x8b\x97u|\xe6h@M\x86 \x1a\xa1\x1b\x97\xbf$UH6\xb1Jy*\xf4\xdb.\xf7;\xd0\xe0\xf7+|`\xce\x01\xb5+\xbb\x8bf~;m\x7f\xe1\xf6K\xb0\x13\x9a\xa1ǂmK%3\xd0ѽ\xec\xa6$؋HVF\x1b\xf7:\xe6H\xdd*ɥd\x8c\x87@CIwy-!f\xae\x17>|m\x05D\xedܷ\x7fO\xc9\xd8\xdcq\x11L\x19,\nz\x9c\xa6\x944\xc4\x1b\xd72\xcc\x06\x0f\xc8->ԶBM\x90j\xcbk\x01\xfc\x16\x1c\x85\x82\x89\x15v@\u07be\x80c\xe1uh,\xd9$VNseoB'\rw\xea\x1f\xdcT.%n\x15(\xe80\xaf\x1fUG?TH\xd3\nH\xccp7K\x99\xff\xa0Ɇ)m\xdaC\xd0\x03i*Q03\x17^\xe2\x83R'\xad\xbb>\xbb\x96\xadp\xd7N>\x85\xf4,G\x98D\xccq\x7f\t\b\xdb\x10f\b\x88LV\x02\x038v\x1ec\x17\x8e\xb8Nò\xd4I\x926\xfbm\x01Q\x15i\x04X\xa2\xa401\x1a\xe9iW\xffH\x19\x7f\t\xb6\x99\xa1,\xb6X9mN\x84\x14\xb7vB^A\xbf\xb2\xa2*\b-,\x8fИ\xb3\x02\xbaLo\x12\xdfl\v4\x13F\xda\x19Sr0\xe0\x93\xd7\x12ǐI\xa1Y\x0e\xb5q\xf5\x82 \x05\xa1dC\x19\xafT\xa2\x06\x9cE\xde9K\x11\xaf\tη\xc6H\xeb|\x89\xa4H\x88\xe6&\xfa\x8a\xe3ڸT\xe9\x1eߔ\x9b\xa5`\xbe\x97U*&1-\xf0̎\x96O\xa4\xa4\xe2\xf0\xdd\xd3J\x1d\xeawOk\xac|\xf7\xb4&\xcawO뻧\x95R\xf3\xbb\xa7\xf5\xdd\xd3j\x97\xff\x13\x9e\xd6Ԉ\xdcy\xbe\x81\x8f\x93\xa3Hت\x1e\x1b\xe2\b|\x9f\\\xe1s\xc0\x9f\x95\x8b\xb9\x8a\x83\x8a$\xfe\x0f\xa4uǔVc<\xea\xe4L;k\x82̻\xe3E\x13\xae\xe43\xb2\xeeC\xa7\xe7˺_\x8dBB\xbe\x89\x8c\xc0i\xe4;Ɂ~\x13\x1a\fݿ\xbd\xec~1ҧ\n\x92'fv\x11@O;\x10\xb8\xc3.\xb6\xed\x03\x00\xe1>\x02\x7f0\xffX\xc0\"\x80\xa4\"\x82q'y\xf5m\x06m\xb9#\x9fK\x17{\x9a\xedw\x8c\xc7TҒ\tON!\xec\xa6\b\x0e\xf8\xa5sw\xbb\xcfrd\xe2wI\r\x9c\x9f\x10\x98\x12\x11\x9bH\xfe;!\xe5/1\xb7\xf8\xd9\xdb\xf3)I}sV\xcc/\x96\xc0w\xfe\xb4\xbd$\xfaL\xa7\xe8͡\u038b\xa7\xe3\xbdb\x12\xde\xeb\xa4\xde%&ܝ/s>-\x1e{R\xe6\xd8t\xe8`8in2Un2\xb40\x85\xd8l\x94&S\xe0\xe6$\xbeMr'm\x9a\xbdZj۫%\xb4\xbdn\x1aۨ\x14\x8d~\x9c\x93\xa8\x16\xbf\x96\x86L\x1a[\xfeZ\xc2v*\x19\xa4긯'\xad\xaf>\x1f\xc1\xb0\x8c\x0f\xae\xdd+\xf9\xc8E\xc5\r+9n\xa4\xeeY\x1e\r6\x98\x1d\x1c\xea\v4~\x93x\xf4\xd4\xdf\x04\xf3\xf9K-\xb5\x97G\x9e>\xd5\xe4\t8'46\xafz\x98g\xee&\xa6L.\xc1\xda#;;\xfd\xc5 \xfe\xfa\xa6\x85\x13w<]\x8bV\xad\x88\x85\x98\xa8\x18\xbeEf\xd0p\xa4蛞\a\xeb\xfcp\xfc\xed\xef\x15\xa8\x03\xc1{lj?\xa79\x04\xe6'\xa6\xb6\v\xb1\xa0*\xbc\xda\x1a\x8a\x9f\xf7\x9c\xfef*\x93w\xc2Y\xdd\xe3\xf1`\x1b\xab#\x9aE\x8dU|v\xbd\x12\xedc\xa0\xb9\x90u\xebH\xb3)\a9\xf5\xb4\xd4\xcb.q\xe6/r&\xbd\x8at\xcf\xefw:\x05u\xca駴\x04\x80\xc9\xd3N/\xb5\xe4\x99Z\xf4$\xfbyi\xa7\x99\xe6m\x16\xbe\xe0饗8\xb5\x94H\xa9\x94SJ\xf3\xe8\xf4\n\xa7\x92^\xf54\xd2k\x9dBJ>}\x94\x94Ⓖ\v\x9c\x9a\xa2r\xe2q\x9a\xe9=\xde\xf1\xd3D\t\xa7\x88\x12v\x7f\xa7\x91<\x01\xbd\x84SB\xf3N\a%\xf0,u*\xbe\xe2)\xa0W<\xfd\xf3ڧ~&$k\xe2\xf3\xbc\xd3='oYH\x95\x83\x1a\xdd\xf6I\x95\xc2Q\xf9KY\xdbt\ar\xb4\xdf\x11n\xfd\xb3\xb5:\xfe2\x9a\a\x7f\xd1(^);\xb4}i%\xad\xe5mt\xf6\xa2\x1a\xf7\xa7\xebL\xfa{f\xddv\x95\x86\x92*\xbc\xbbx}p\xe9,Q\xd3\xfc\x81f\xbb#\xe8;\xaa\xc9F\xaa\x82\x1arQo\x00^9\xe0\xf6\xef\x8bKB>\xca:'\xa2}/\x8ffE\xc9\x0fv\x85B.\xda\rN\x93\x80\xa8\xb4\x85\xden%gY\xc4w\x8b\xde\xcd\xe4*\xf7.\xcb\xc0\x1b\xa3\xb2v\xca@i+\xc6]7t\xf3\xbaW`n$\xe7\xf2i\xe6ڟ\x96\xec\xafxs\xf73\xa2C\xefnW\b#\x88\a^\x05^'g\xd5ج\xc1\x9a\xe5\x06ϡ\xb9\xbf\xdat v\xf3\x1cۗ\xe3B\xee\xeeA\x0en\x81W\x9d\x99\xb4\xda\xe5v\xe5\xc61ԋ\x95\x19*\x0eDbF\x8d\xd91\x95/K\xaa\xcc\xc1%j,:c\b\xb6t,\xba3h=\xfaw;G\xc9\x1b\xaet\xc6\x1d\xcaC\xd9\xdd\xf4=\xa6\xdd)\xe3\x18>\xbd8yn\xf1\x8c\xe3\x18vK\x96H\xa9\xc8\xcf\xd1̯\xb3Eʹ\xbf\x99\xf8\x17\xb9\x87\xf7\xd1\xe8Y\x872\xb3\x19\xfe葫mT\x80\x7f\xe6\x1c\x04\xeb\xf3p\x0e\xfc#\xe3\xa0ݰ\x12\x14\xf0m\xbfU\xad\x8f\xabb\xed|\xb8\x8d\xfdXw0`\xe3\x1cZ\x18\x8a.AY/\xca\x05\xad+\x1ddu\x18\xf1\x86#L\x18\xd8B\x7f\x158\xa2\x81ݭ\xd2h>\x83:\xc1\xb5\xccO\xb1\xf8V\a\xf9\x87\xe1\x96G\x9cl\x85\xbcb7\xee9'\xe4\xf6\xe1F\x93J\xe4\x18.~\xf8\xeb\xdd,\xa9\xdbwn\xae\x0f\xb3uJ\xa9>\xc4[\xb5\x9c㖾pޱ\xdcD\x10\x18\x82\xd3z\a\xe4\x89\x19\x7fq\xd7yoZ\x1dZ\xf2\f\xbdp\x80W\xfaO\xbfq\xe0n\xfe\xf7/\xa3\xf8\xe9X)\xbc&տ\n\x80\u05ca\x9e\xf4\xcc\xc1\xbaNت\x93\xbf\xf4;c\xa0(M\xccטV\x87\x7f\x19\x03X\xfbi\xd2Pޚ\x954T\x88y\xda\xfa \xb2\xb1\xc42\xaf\x8dF\xb896\x1fc\x04\xb8\xf1\xe7!\xceF\x80\x1a\xe0\x10\x01t\x95e\xa0\xf5\xa6\xe2\xfcP\x1f\xc7\xf8F\xa8\xf1\x912~>R8h\x83\x82`\xd1\x1b\x854\x89\xb0O\xf7\x06\x91\x87\x99\x1e\x8e*\xcd#\x85\xe7\x82φԆ\x16'=\xd8p\xd3\a\x83O\xf6\xa8\xbc\x95TI\xeb\xb1Sݰ?f\\\x1ap\xae%.\xb2,4\xc8\t\xecA\x10k\x9d\x1d\x89ÛS3\xa1\xf8\x13\xae\xce\xc2\x05{\x17B!ч\x89\x88\x8fvh|\x00\xe7\a]\xc3\xc4\\Q|ϤO\x84\xbe\xf3\xeb\xa2\x15\xd7\xd6\xfb\x87\xa5\x05q\x9a\xd7\x1a\xd5͙f]\xbb\xf0<%ws\xb7\x1a\x02w\x8a\x8a\xeb?\xf7\xf2\xcci\xdcG\xf7Y*\xad\x8f\xee,\x85\x16\x81X\xcb\xf8\xf9qǩ~ڥ\xee\xd8\xd29\x1cY8CG9\xf7\a\x1d\vКn\xc3m\xeeOv\xe9\xb1\x05\x01.<\xe76O\"@\x9bSqݻ\xccݔ\xa1\x99\xa9\xa8\xef $\xf8\xb6j\xfd\xa0\t\x971\xa8\xf8\xa0\v\v/\x85\x855\xd9LB}-\x99JY\xc3}\xa8+Zڠ'\x8c\xdci\xdev\x03ζ̮u,\xe7\xb6T\xad\xe9\x16\x96\x99\xe4\x1cP[\xf7\xc7\xf5\x92sݟ=\xfc\x02TO\xa2\xf6\xb1]\xd7\xef\x00:n\xbb\x8do\xea\xd2\xdd\xf1\xf5.\xc3\x144\x0f\xe9\xf5\x06$\xb1\xe3Y\x8e\xb2\xa3B\xf4\x95\xb9\xfeH\xdbuì\xf3j\xd9\xc7y\xfd#s\v\x1f\x17\x88\xcbcA\x7f\x93jA\n&\xec?T\xe4n\x03/4\x9e5\xfe\x9d\x94\x8fw\x11'\xb67\xf8\x1f\xeb\x8a\xcdV\a\x13n\xd8x`t-+\xbf\xfb^;\xb4\xf1m\x15\xbc\x99\xff\xcc\xcbM\x849b\x0fz\xe8\fFt\x7f\xec@\x9a4\x05\xae\xe7\x01Xw\xe1%3\xce\x0f\x8bc\xc8G\xaf&6\xb0[/\x17x7\xa0\xb9\x8f`\xa0\xa3\xb0#\x15\x05R_|\xd1V觬z=\x99\x87\x9c\xc9\x1e\x8d\x7flj\x0f\xd1\xd1\r\xb3\xe5\xee\r \xd8q\x02ϻ`\xc7g*&\x84\xff\xd6֩\xef.h-\xdcB\x96\xd8`\x94.~\xf6}I>A\x7f\xbb\xc2\x1dg\x87\x1c33pVE\xaa\xacĭ\x92[\x05\xba/tK\xf27\xca\f\x13ۏR\xdd\xf2j\xcb\xc4\xe7\xe1\xa3;c\x95o\xa92\xcc\n\xad\x1bOl\xa0LP\xce\xfe\x11\xd3O\xed\x8fӀn\x06\x17JK\x920\x8c\xa1\x0f\xef\xc1\xfa\xaa\x83\xeb\xfb\xa8*,=]O\xf1;\x02O\xa6tc\xed\x134>E\xe8\xf6\x92|\x92\xd1\t\xeeӚX\x17\xa6u\xad@\x9b%l6R\x19\xb7\xeb\xbc\\\x12\xb6\tA\x04\xab;0\xfe\xe5ޜ$,\xb6]\\'\x8c4f\b\x83\xd7\n\xad)^I_Ѓ\xdba\xa2YVYO\xe9J\x1b\xca#\x8eʳ\x148Fk\xec$\x82\xfc\xd7g\xedȭڀ\xfa\xc1C\xecǑ\x14/\xc5p\xde\x1b\xb7(\x82 O\x8a\x19c}#9\x92\x12\xe0Ie\xac\x8f\xc49і\xd4'E\x11\x89S\x87\xab\xe1Ԛ4\x94\xefk(Cj\xd6c\x8d/,\xae\x916\xc4\xfa\xaf\x98E\xe4kY6g;*\xb6\x837\r씬\xb6\xbb \xc9\x03N1\xc9+\xc0\xa0+\xaa\x14\x1d\x1e\b6\x95\x12\xad\x94\x80\x91\xe3\xdb$\b\x03\x0e\x97f\x8f\xa4*\x17\xfe\x01^\xff\xbe\xf2\x95\x7f\xcbd\xb9Q\xb2X\xfa~1&\xba\xf0;\xf2\x8aI끘]\x94\xea\xc4y\xdf\xfe\xb9\x00\x94\x84\xb2\x04A\xa8\xf6='\xdc\xf8t\xb2\xb9ц*\xf3\xacp\xc4]\a\xc2D$\x02\xbb\x8b#q\xe7\x13\x13ܕW7\xfe\xa1\xd1\x1a\xf0\x82h&\xc2\x13\xcf.\xc9\xc1\xc9Gt\xcfK\xe0\x93\x8cR\xc5\xf3\x0e\xc7C\v]\x84^7\xaa\xb0\xafm퇓\x17\x9d\x0fG0\x8e\x8e/\xe3\v\x9cu\x95\xb0P\xfc\x03\x8bE\xbe1a5\xb3\xa8\xfc\xf1w?\x96\xbcOZ\xd4\xc4)2\xb6\xc6\xc1\xe5\xcb\xf0b\xa5\xfb\xe2\xe6-\a\xebzi\x80\xee\xf2i\xd62y\x7fƸ\xd19\x83F\xe11\xf3\xf3DM\xf6g\f\x17\xbdX\xac\xe8\xbc(?Q|\n\xf9\xa4Y\xfb7\xdf6\x12,\xf2`\xcf\x1d.jE\x8b\xc2\xc0_5^\x14\xb5J\xbd\x1fQO\xe7-m\xe1{\xf2\xbf\xfco\x00\x00\x00\xff\xff\xaf\x05\xd1\xf3\xa2\x81\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\xdb8\x92\xef\xf9\x15(\xdd\xc3\xecnI\xf6\xa6\ue8ee\xfc\x96q\x92\x1d\xd5\xcc$\xde\xd8\xe3}\x86Ȗ\x841\bp\x00P\xb6\xf6\xee\xfe\xfb\x15\x1a\x00?DP\x04eٓ\xdd\r_\x12\x8b`\x03\xfd\xddh4\x80\xc5b\xf1\x86\x96\xec\x1e\x94fR\\\x11Z2x2 \xec_\xfa\xe2\xe1\xbf\xf5\x05\x93\x97\xbb\xb7o\x1e\x98ȯ\xc8u\xa5\x8d,\xbe\x80\x96\x95\xca\xe0=\xac\x99`\x86I\xf1\xa6\x00Csj\xe8\xd5\x1bB\xa8\x10\xd2P\xfb\xb3\xb6\x7f\x12\x92Ia\x94\xe4\x1c\xd4b\x03\xe2\xe2\xa1Z\xc1\xaab<\a\x85\xc0C\u05fb?_\xbc\xfd\xaf\x8b\xff|C\x88\xa0\x05\\\x91\x15\xcd\x1e\xaaR_쀃\x92\x17L\xbe\xd1%d\x16\xe4Fɪ\xbc\"\xcd\v\xf7\x89\xef\xce\r\xf5{\xfc\x1a\x7f\xe0L\x9b\x1f[?\xfeĴ\xc1\x17%\xaf\x14\xe5uO\xf8\x9bfbSq\xaa¯o\bљ,\xe1\x8a|\xb2]\x944\x83\xfc\r!~\xd4\xd8\xe5\xc2\x0fx\xf7\xd6AȶPP7\x16Bd\t\xe2\xdd\xcd\xf2\xfe\xdfo;?\x13\x92\x83\xce\x14+\r\xe2\xfe\xbf\x8b\xfaw\xe2GI\x98&\x94\xdc#\x8eDy\x92\x13\xb3\xa5\x86((\x15h\x10F\x13\xb3\x05\x92\xd1\xd2T\n\x88\\\x93\x1f\xab\x15(\x01\x06t\v^\xc6+m@\x11m\xa8\x01B\r\xa1\xa4\x94L\x18\xc2\x041\xac\x00\xf2\x87w7K\"W\xbfBf4\xa1\"'Tk\x991j ';ɫ\x02ܷ\x7f\xbc\xa8\xa1\x96J\x96\xa0\f\vDwOK\x92Z\xbf\x1e\xc3\xd5>\x96<\xee+\x92[\x91\x02\x87\x96'1䞢\x16?\xb3e\xbaA\x1f\x85\xcc\xfeL\x85\x1f\xfe\xc5\x01\xe8[P\x16\f\xd1[Y\xf1\xdcJ\xe2\x0e\x94%`&7\x82\xfd\xbd\x86\xad\x89\x91\xd8)\xa7\x06\xb4\xa5\x8c\x01%(';\xca+\x98[\xa2\x1c@.\xe8\x9e(\xb0}\x92J\xb4\xe0\xe1\a\xfap\x1c?K\x05\x84\x89\xb5\xbc\"[cJ}uy\xb9a&\xe8W&\x8b\xa2\x12\xcc\xec/QUت2R\xe9\xcb\x1cv\xc0/5\xdb,\xa8ʶ\xcc@f\xd9|IK\xb6@D\x04\xea\xd8E\x91\xff[\x10\x0f\xdd\xe9\xd6\xec\xad\xd8j\xa3\x98ش^\xa0~L`\x8fU\x1d'\x8c\x0e\x94C\xb1\xe1\x82\xfdɒ\xeeˇۻ\xb6\xa02\xed\x99Ғ\xd7!\xfeXj2\xb1\x06\xe5\xbe[+Y L\x10\xb9\x13U\x94s\xce@\x18\xa2\xabU\xc1\x8c\x15\x83\xdf*\xd0V\a\xe4!\xd8k\xb4Ad\x05\xa4*s+Ƈ\r\x96\x82\\\xd3\x02\xf85\xd5\xf0ʼ\xb2\\\xd1\v˄$n\xb5-\xebacG\xde\u058b` \aX\xeb\f\xcbm\tYG\xd1\xecWl\xcd2\xa7Nk\xa9\x1a\xbb\xe3l`\x97BqշO\xa6٭\xa0\xa5\xdeJs\xc7\n\x90\x959l1&kȼ\xdb\xe5\x01\x940B?^\xb4Y\x95\x86\xdc*\xed#e\x06\xc7|}\xbb$\xf7h\xac\xc2\xd7h\xb4*ML\xa5\x84\x95\x92H__\x80\xe6\xfb;\xf9\x8b\x06\x92W(ܙ\x02\xa4Ü\xac`m%A\x81\xfd\u07be\x02\xa5,m4\x0e@V=cc\x9f\xbb-X\xdaҊ\x1b\xaf'L\x93\xb7\x7f&\x05\x13\x95\xe9\x89\xda בR\xd4\xd0B\xee@\x9dB\xc4\xf7\xd4П\xed\xc7\a\xb4\xb3@\tB\xb5\xc4[y:\xae\xf6\xf82\xc6m\xf7,\xd7-\x88L\x93ٌHEf\xce\x03\xcf\xe6\xee\xeb\x8aq\xb3`\xa2\xdd\xc7#\xe3<\xf42\ryGC\xc7P}'?j'\xbc'\xd1b\x00V\x8b4\x8f[0[P\xa4\x94\xb5\xc7[3\x0eD﵁\xc2\x13&x\x11\x8fO\xa4'\xd4\x1d\xce=\bm\xe9\xea\x11\xe9#/*\xce\xe9\x8a\xc3\x151\xaa\x82\x01ڬ\xa4\xe4@\xc5\bq\xbe\x806,;\ai\x1c\xa4\ba\x94\x7fѡ\x00:M\xfa\x00\x84F@{\x9aY\xef\xccy\x8b\xb0]\xaaD\xc7T*Ȭվ\xf2ހ\x01G\x0f$$\xe1Rl@\xb9\xdem\xa4\x12\x04L\x81\x15\xb8\x9cXC\xab\x80[oB֕\xb5\xc1\x17\xc4j\xf7\xa0\f0\xa1\rЈp>\x83?\xf0\x94\xf1*\x87\xfc\xda\x05^\xb76~\xccC\xd4ܳ\x9a)|\xfap\x14\xa2\xf7Μe\x18\x04\xfaxo\x81qkLL\x1b'\xbd/\xc1\x85Ζ\x95~؍\xf7=j\x0f4\x18\xfb\xd1\xecO\xb39r\xb8\xdbk\xb7\x0fM\xa8\x82\x9a,\xc9v\x13\x8a\xd2\xec\xfb\xad\x99\x81\"Bţ\xf6$\x91\x9fT)\xba\x1f\xe0f\x1d\xff\x9f\x91\x9fC0\x0f8*B\xb3W\xe6\xe9a\xbf\xff\xcc\\=\x0f\x1f5\xcev)\x13\x96\x7fv\xe2\xd9a\x9fv\xf37K6!M\x04\x1e\x13\x0e\x1eN͎p\xebw\"\xd6Yd~H\xc8k\xd9\xf2\xc2\xfb\x0fI\xa9\xad\x94\x0fc\xd4\xf9\xc1\xb6i&E$ì\nY\xc1\x96\xee\x98T\x1e\xf5\xc6\xd5\xc2\x13d\x95\x89j=5$g\xeb5(\v\xa7\xdcR\r\xdaM\x93\x87\t2\x1c\xbe\x93\x96\x19\x89\xbe<\xc0\xa3a\xa4e\x13b>4t\x1bG\x1cz\xc9\xf0\u0601\xda\xf0\x1a\x9dq\xcev,\xaf(G\xbfLE\xe6\xf0\xa1\xf5\xb8bV\xe6\b\x93{c\x8eJ\xa6{\\@\x10\x90\xb2L\xea̔\xa4\x00\x1b\xf3\x16vN\xd0o:\x8c\xf9\x8a\xdaXE\x0eaO\x90Y\xaa\xe2\xa0}W9\x86\x91\x8d͘7L\xc1D\x04\xe1t\x05\x9ch\xe0\x90\x19\xa9\xe2\x14\x19\xe3\xb3{R\x8c\xe0\x00!#\x96\xaf;\xd3h\x108\x02\x92\xe0\x14n˲\xad\v\xf5\xac\x10!\x1c\x92K\xb0\x01\x9f!\xb4,y\xc4]4\xcfQ\xe6\xfbN\x8e\xe9z\xf3\x8ch\xfd!\xbc\x98\xfe7O\x82\xcdl\x9e(i\x1b\xfd\xeaR\xb6\x16\x87\xf8\x9c\xb6y\xfe9\t\x1b,\xff\tB{D\xfb\tf\x85\x92ezPn-U\x19\xe8\v\x1bNa\xa43'̄_\xc74\xa1\x13s\xf5\x92e\x1d\"|ݼ\x99.\xf4\x89\xacIщ\x17bL\xdd\xc5? _\xd0e\xdcz\x8f\x91̓\x9f\xda_\xcd\t[\xd7D\xcf\xe7d\u0378\x01u@\xfd\x93L}\xe0\xcc9\x88\x91\xe2\xf5\b\xa6\xefM\xb6\xfd\xf0dC0ݬT%\xd2\xe5\xf0c\x17Ȇh\xbf\xeb\x9eG\xe0\x12Lc3\x05\x05\xa6\xc7q\xc6\xd4\xfe\x05C\xabw\x9f\xde\xc7\xe7W\xed'A\xf2z\x88\x8c(\x9d{\xde\x1d`\xd4\x1e\x9f\x0f\xe1\xc3\x1b\x8c\x81\xea\t\x90[\n\x99\x13J\x1e`\xefB\x17*\x88\xe5\x0f\r\x8d\x13\xbaW\x80k2(g\x0f\xb0G0\xf1E\x96\xfe\x93*\r\xeey\x80}J\xb3\x03\x1a\xda11\xed\x17\x8f,\x9d\xec\x0fH\b̭\xa7\x8a\x81{\xbc*D\x964\xe2O\xa2-\tO\xa0\xfd\th&\x89J\xbb\x8f\xf6*%J\xc0w\xda\xf1\xd2j̖\x95hV1\xe3 \xd7\xc9\fu\xcf=\xe5,\xaf;r:\xb2\x14s\xf2I\x1a\xfbχ'\xa6\xfdB\xe6{\t\xfa\x934\xf8ˋP\xd4\r\xfc%\xe9\xe9z@E\x13\xce\xca[\x82\xb5\x97\xe2\x9cO\xb3\xd2VӞi\xb2\x14v\xba\xe2H\x92\xd8\x15\xae\xba\xba\xee\\GE\xa5q\x15MH\xb1pi\x9bXO\x9e\xdeRu\xc8\xfd\xecN}\x87w\xd6Y\xb87n\xed\x97\xd3\f\xf2\xb0\\\x83\x8b\x92\xd4\xc0\x86e\x89\xfd\x15\xa06@Jk\xc2\xd3$\"Ѱzl\xa6\x89O\x9a\xf7n?O\x8b\x87z\x8d\x7fa]\xce\xc2C0\xb2H\xa0\x81\xb7\xdd\xf98>\v\xab\xb3\t\xad\x82$\x8c6\x1dX\xb3\x1cn\x9aB\x94g\x90\x03\xbd8\x868\xa3ܥy\x8eu.\x94\xdfL\xf0(\x13da\xaaih\x8dݹ\xe0\x82\xe2R\xcb\xffXO\x8b\xda\xf4\x7f\xa4\xa4L\xe9\v\xf2\x0eKZ8t\xde\xf9\xa4Y\vLB\x97X\x92b\xe5gG\xb9\xf5\xfdր\v\x02\xdcE\x02r\u074b\x8b\xe6\xe4q+\xb5s\xdb\xf5\"\xce\xec\x01\xf6n\xc5p\xb4˶\x91\x99-\xc5\xcc\xc5\x10=\x83Q\a\x1cR\xf0=\x99\xe1\xbb\xd9sB\xa9DIMl\xd6\x11т\x96i\x12\x8a%E\xa9\x81\xba\x9d\xb0\x86 \xc4~X\x97\xca\xd8 \xfb\x18\xb6I\"ZJ\x1dY\xc8\x1f\x18ʈ\xf0\xdeHm\\\xbe\xac\x133G\x13j2$\xd1\b]\xbb\xfa%\xa9B\xb1\x895\xcac\xa9\xdf\xf6s\xb7\x05\r~\xbd\xc2'\xe6\x1cP;\xb3\x9b5\xfa\xed\xac\xfḓ\x97`'4È\x05\xbf-\x95\xcc@Gײ\x9b'\xc1_D\xaa2ڸ\xd79G\xeafI\xae$\xe3x\n4<\xe9!\xaf%\xc4\xc4\xf9\u0087\xa7VB\xd4\xea\xbe\xfd{LƦ\x8e\x8b`\xc9`Q\xd0\xc32\xa5\xa4!^\xbb/\x836x@n\xf2\xa16\x15Z\x82T_^\v\xe0\xd7\x10(\x14L,\xb1\x03\xf2\xf6\x05\x02\voCc\xc5&\xb1\xe7\xb4P\xf6:t\xd2p\xa7\xfe\xc1\xa9r)q\xa9@A\x87y\xfd\xac:ơB\x9aVBbB\xb8Y\xca\xfc;M\xd6Li\xd3\x1e\x82\x1e(S\x89\x82\x998\xf1\x12\x1f\x94:i\xde\xf5\xd9}\xd9Jwm\xe5c(\xcfr\x84I\xc4\x1cח\x80\xb05a\x86\x80\xc8d%0\x81c\xf5\x18\xbbp\xc4u\x16\x96\xa5*I\x9a\xf6\xdb\aDU\xa4\x11`\x81\x92\xc2\xc4\xd1LO\xbb\xf9G\xca\xf8K\xb0\xcd\fU\xb1Ş\xd3t\"\x94\xb8\xb5\v\xf2\n\xfaĊ\xaa \xb4\xb0\x1f\xc0\xb0\x8c\x0f\xa1\xdd+\xc5\xc8E\xc5\r+9.\xa4\xeeX\x1eM6\x98-\xec\xeb\x034~\x95\xb8\xf5ԟ\x04\xf3\xf9K-\xb5\x17\a\x91>\xd5\xe4\x118'4\xa6W=\xcc3w\x12S&\x17`\xfd\x91\xd5N\x7f0\x88?\xbei\xee\xc4\x1dwעW+b)&*\x86O\x91\x19t\x1c)\xf6\xa6\x17\xc1\xba8\x1c\x7f\xfb\xad\x02\xb5'x\x8eM\x1d\xe74\x9b\xc0\xbcbj;\x11\v\xa6\u009b\xad\xa1\xfcy/\xe8oT\x99\xbc\x13\xce\xeb\x1e\x8e\a\xbf\xb16\xa2\x99\xd4X\xc3g\xe7+\xd1>\x06>\x17\xb2\xfe:\xf2\xd9X\x80\x9c\xba[\xeae\xa78\xd3'9\xa3QEz\xe4\xf7;\xed\x82:e\xf7SZ\x01\xc0\xe8n\xa7\x97\x9a\xf2\x8cMz\x92㼴\xddL\xd3\x16\v_p\xf7\xd2K\xecZJ\xa4T\xca.\xa5itz\x85]I\xaf\xba\x1b\xe9\xb5v!%\xef>J*qI^\x05N-Q9q;\xcd\xf8\x1a\xef\xf1\xddD\t\xbb\x88\x12V\x7fǑ<\x01\xbd\x84]B\xd3v\a%\xf0,U\x15_q\x17\xd0+\xee\xfey\xed]?#\x925\xf2z\xda\ue793\x97,\xa4\xcaA\x1d]\xf6I\x95£\xf2\x972\xb7\xe9\x0e\xe4`\xbd#\x9c\xfag[u\xe2et\x0f\xfe\xa0Q\x97G\xb3\xa2\xe4{;C!\xb3\xf6\a\xa7I@T\xdaBo7\x92\xb3,\x12\xbbE\xcffr\x8d{\x87e\xe0\x89QY\xbbd\xa0\xb4\r\xe3\xa1\x1b\x86y\xdd#0גs\xf98q\xeeOK\xf6\x17<\xb9\xfb\x19١w7K\x84\x11\xc4\x03\x8f\x02\xaf\x8b\xb3jlV`\xddr\x83\xe7\x90\xee/\xd7\x1d\x88\xdd:\xc7\xf6Ḑ\xbbs\x90CX\xe0Mg&\xadu\xb9Y\xbaq\f\xf5be\x86\x8a=\x91XQc\xb6L勒*\xb3w\x85\x1a\xf3\xce\x18\x82/=\x96\xdd\x19\xf4\x1e\xfd\xb3\x9d\xa3\xe4\rG:\xe3\n\xe5\xbe\xec.\xfa\x1e\xd2\xee\x94q\f\xef^\x1cݷx\xc6q\f\x87%\v\xa4T\xe4\xe7h\xe5\xd7ٲfڟL\xfc\xb3\xdc\xc1\xfbh\xf6\xacC\x9eۃ\xe6\x91\xf2\xac\x00\xd1\x1d\xba;X\xa5\xba\x02<\x90\xb7\xff\xea\x19\xf5V\xa1k\x7f\xa6\xea)\x89\xb2\xdb.\x88\b~\xe1\x84\xd9\xd0Y\xcc>\xe1\x01\xf0{rs\x8fs\xb4ڴy\x15\xf5s\xb4\x90*\v\x8b\xc1\x118\xfe\x83\xef\xcf_\x9a\xa6\x8dTt\x03?Iw\xc6\xf6\x18ۻ\xad;g\xaf\xfb\xa8'ԏ\x06\xa5\x89\x1d\xc0\xebO\xfb>\x00\xd6\xd4|\xf7\x0e5\xb6\xa3\x9cxL\xb31\xfc\x14\xbe\xdf\xdd\xfd\xe4\xb02\xac\x80\x8b\xf7\x95+w\xb06Q\x83%q\xc0\xd6AZ\xd9\xffn\xe5#\x1e\xfe\x1b\xcfc\x86;\x13\x1ad\x14`\xb19\x96 NB\xa9*\xb9\xa49\xa8k)\xd6l3\x82\xdd/\x9d\xc6\an6\xc3\x1f=r\xb5\x8f\n\xf0\xcf\\\x83`c\x1e\u0381\x7fd\x1c\xb4\x1bV\x82\x01\xbe\xe9\x7fU\xdb\xe3\xaaX\xb9\x18nm_\xd6\x1d\f\xf88\x87\x16\xa6\xa2KP6\x8arI\xebJ\aY\x1dF\xbc\xe1\b\x13\x066П\x05\x1e\xb1\xc0\xeeTit\x9f\xc1\x9c\xe0\\\xe6\xc7X~\xab\x83\xfc\xfd\xf0\x97\a\x9cl\xa5\xbcb'\xee\xb9 \xe4\xe6\xfeZ\x93J\xe4\x98.\xbe\xff\xcb\xed$\xa9\xdbuN\xae\x0f\xda:fT\xef\xe3_\xb5\x82㖽pѱ\\G\x10\x18\x82Ӻ\a\xe4\x91\x19\x7fp\xd7yOZ\x1d\x9a\xf2\f\xddp\x80G\xfa\x8f\xdfq\xe0N\xfe\xf77\xa3xu\xac\x14\x1e\x93\xeao\x05\xc0cEO\xba\xe6`U\x17l\xd5\xc5_\xfa\x9d1P\x94&\x16k\x8c\x9b\xc3\xef\x8f\x01\xac\xe34i(oi%\r\rb\x91\xb6ދ\xecXa\x99\xb7FG\xb8yL\x1fc\x04\xb8\xf6\xfb!\xceF\x80\x1a\xe0\x10\x01t\x95e\xa0\xf5\xba\xe2|_o\xc7\xf8J\xa8\xf1\x912~>R8h\x83\x82`\xd1;\ni\x14a_\xee\r\"\x0f\x9a\x1e\xb6*M#\x85炯\x86Ԇ\x16']\xd8p\xdd\a\x83W\xf6\xa8\xbcUTI\xeb\xb1Sݰ?\xe6\\\x1ap\xeeK\x9cdYh\x90\x13\u0601 \xd6;;\x12\x87;\xa7&B\xf1;\\\x9d\x87\v\xfe.\xa4B\xa2\x17\x13\x11\x9f\xed\xd0x\x01\xcew\xba\x86\x89\xb5\xa2x\x9fI\x9f\b\xfd\xe0\xd7e+\xael\xf4\x0f\v\v\u2d285j\x9b3ͺ~\xe1yF\xee\xfav9\x04\xee\x14\x13\u05ff\xee\xe5\x99j\xdcG\xf7Y&\xad\x8f\xee$\x83\x16\x81X\xcb\xf8\xf9qGU?\xedPw\xfc\xd2\x05\x1cY\xd8CG9\xf7\x1b\x1d\vКn\xc2i\xee\x8fv\xea\xb1\x01\x01.=\xe7\x16O\"@\x9b]qݳ̝\xca\xd0\xccT\xd4w\x10\n|[\xad\xbeӄ\xcb\x18T\xbcЅ\x85\x9b\xc2\u009cl\"\xa1\x9eJ\xa6R\xe6p\x1fꆖ6\x18\t#w\x9a\xbb݀\xb3\r\xb3s\x1d˹\rU+\xba\x81E&9\a\xb4\xd6\xfdq\xbd\xa4\xae\xfb\xbd\x87_\x80\xeaQ\xd4>\xb6\xdb\xfa\x15@\xc7m\xb7\xf0M]\xb9;\xde\xdee\x98\x82\xe6\"\xbdހ$v<)PvT\x88\xde2\xd7\x1fi\xbbm\xd0:o\x96}\x9e\xd7_27\xf7y\x81\xb8<\x16\xf4W\xa9\xe6\xa4`\xc2\xfeCE\xee\x16\xf0\xc2Ǔƿ\x95\xf2\xe16\x12\xc4\xf6\x06\xffCݰY\xea`\xc2\r\x1b7\x8c\xaed\xe5W\xdf\xeb\x806\xbe\xac\x82'\xf3\x9fy\xba\x890\x8f\xf8\x83\x1e:\x83\x19\xdd\x1f:\x90F]\x81\xeby\x00\xd6m\xb8Ɍ\xf3\xfd\xfc\x10\xf2\xc1\xad\x89\r\xec\xd6\xcd\x05>\fh\xce#\x18\xe8(\xacHE\x81\xd4\a_\xb4\r\xfa)\xb3^O\xe6\xa1`\xb2G\xe3\x1f\x9a\xd6Ctt\xc3l\x85{\x03\bv\x82\xc0\xf3N\xd8\xf1\x9a\x8a\x11\u1ff1m\xea\xb3\vZ\x13\xb7P%6\x98\xa5\x8b\xef}_\x90O\xd0_\xaeX\x90\xbfVPEh\xb0\b\x17\xc3\xdd\x1a\xaa\xfa)_\xb7\r\x1er\xac\xe8@m\x8c4Y\x8a\x1b%7\nt_X\x17\xe4o\x94\x19&6\x1f\xa5\xba\xe1Ն\x89\xcf\xc3[~\x8e5\xbe\xa1\xca0+\xecn<\xb1\x812A9\xfb{̮\xb5_\x8e\x03\xba\x1e\x9c`-H\xc20\x86^\xbc\a\x1b\xe3\x0e\xe6\x05\xa2&\xb4\xf4t=%^\t<\x19\xb3\xa9u,\xd1\xc4\"\xa1\xdb\v\xf2IF\r\x83/\x87b]\x986$\x03m\x16\xb0^Ke\xdcj\xf5bA\xd8:$\x1f\xac\xcd\xc1\xbc\x99\xbb\xab\x92\xb0\xd82s]hҸ/Lz+\xf4\xc2x\x94}A\xf7ne\x8afYe#\xacKm(\x8f\x048\xcf2\xfc\x98\xe5\xb1\xca\a\xf9/\xcfZ\xc9[\xb6\x01\xf5\x93\x8e؏#)\x1e\xa6\xe1\xa2>nQ\x04A\x1e\x153\xc6\xc6T\xf2H)\x81'\x95\xb1\xb1\x15\xe7D[R\x9f\x94}$Ό.\x87Kr\xd2P\xbe\xab\xa1\f\x99g\x8f5\xde̸B\xda\x10\x1b\xf7b\xf5\x91oeٜm\xa9\xd8\f\x9eP\xb0U\xb2\xdal\x83$\x0f\x04\xd3$\xaf\x00\x93\xb5hRt\xb8X\xd8TJ\xb4J\t\x8el\xfb&A\x18p\xb84{ U9\xf7\x17\xf7\xfa{\x99/\xfd\x1d(\x8b\xb5\x92\xc5\xc2\xf7\x8b\xb9Թ_\xc9WL\xda\xc8\xc5l\xa3T'.j\xf7\xd7\f\xa0$\x94%\bB\xb5\xef9ᤨ\x93\xdd\xd4o\xd65\xdcH\xcd\x12\xa2\xfd(\xc7\xff\xda\x06\x10\x18^\x86\xbf\xbb\xcc\xf03\x18\xec3\x86\xc7g\xbf\x05\x1fvT\x187\x9d\xa8]\xe4\xcc9\xb1٤\x89\x8c\xb6\x8e\xedYI\x9a\xdb\x0e\x84\x91\xfc\fv\x17gѭ/\xd7p\a\x81]\xfb\xebWk\xc0s\xa2\x99\b\x17_\xbb\xd2\x0f'\xfdѕ@\x81\x17UJ\x15\xaf\xc6<\x9ep\xe9\"\xf4\xba\xb9\x96]\x1dI|8y*~\x7f\x00\xe3`S7\xdeKZ7\t\xd3\xe7?\xb0\xd8z\x00\x96\xf1f\x16\x95?\xfe\ue6f5wIS\xbd8E\x8e\xcd\xfcpR7<\x85\xeb\xdeCz\xc3\xc1j\x9b\x06\xe8N*'\xe9\xdc\xee\x8cٴs\xa6\xd2\xc2\x15\xef\xe7\xc9%\xedΘD{\xb1\f\xdayQ~\xa4xA\xf4IZ\xfb7\xffm$\x85\xe6\xc1\x9e;\x89\xd6ʡ\x85\x81\xbfj\x16-\xeas{?\xa2\x9d\xce[\xd6\xc2\xf7\xe4\x7f\xf9\xff\x00\x00\x00\xff\xff<\x82OF\xb8\x82\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccYK\x8f\x1b\xb9\x11\xbe\xebW\x14v\x0f{ٖ\xec\x04\t\x02\xdd\xc6r\x02\x18\x19\xc7\x03k2\xb9.EVK\\\xb1\xc9\x0e\x1f\x92\x95\xc7\x7f\x0f\x8a\x0f\xa9\xd5\x0fK\xe3\x04\x9b\xe5eF|\x14\xeb\xf9U\x15\xbb\xaa\xaa\x19k\xe5\vZ'\x8d^\x02k%~\xf1\xa8闛\xef\xff\xe0\xe6\xd2,\x0eog{\xa9\xc5\x12V\xc1y\xd3|Fg\x82\xe5\xf8\x1ek\xa9\xa5\x97F\xcf\x1a\xf4L0ϖ3\x00\xa6\xb5\xf1\x8c\xa6\x1d\xfd\x04\xe0F{k\x94B[mQ\xcf\xf7a\x83\x9b \x95@\x1b\x89\x97\xab\x0fo\xe6o\x7f?\xff\xdd\f@\xb3\x06\x97\xb0a|\x1fZ\xe7\x8de[T\x86'\x92\xf3\x03*\xb4f.\xcd̵\xc8醭5\xa1]\xc2e!Qȷ'\xce\xdfEb\xebD\xec1\x13\x8b\xebJ:\xff\xe7\xe9=\x8f\xd2\xf9\xb8\xafU\xc125\xc5V\xdc\xe2v\xc6\xfa\xbf\\\xae\xae`\xe3TZ\x91z\x1b\x14\xb3\x13\xc7g\x00\x8e\x9b\x16\x97\x10O\xb7\x8c\xa3\x98\x01d\xd5Dj\x150!\xa2\xb2\x99z\xb2R{\xb4+\xa3B\xa3\xcfw\tt\xdc\xca\xd6Ge&Y \v\x03E\x1ap\x9e\xf9\xe0\xc0\x05\xbe\x03\xe6\xe0\xe1\xc0\xa4b\x1b\x85\x8b\xbfjV\xfe\x8f\xf4\x00~vF?1\xbf[\xc2<\x9d\x9a\xb7;\xe6\xcaj\xb2\xd1SgƟH\x00\xe7\xad\xd4\xdb1\x96\x1e\x99\xf3/LI\x119y\x96\r\x82t\xe0w\b\x8a9\x0f\x9e&\xe8W\xd2\x10\x90\x8a\x10\x8a\x86\xe0\xc8\\\xbe\a\xe0\x90\xa8D\x1d\x8ds\xaa\x06w]\xb1M\xac\xc0K\x8fJ\xe2\x9ff2\xf7\x1d\xb2ſ\xe7\xdc♤\xf3\xaci\xaf\xe8>lq\x8aؕ*\xdec͂\xf2]Q\xc9J\xaa\xeb\x97\xd7b\xb5\xc8\xe7\"\x9d\xba\xba\xf1\xfd\xd5\\\xbauc\x8cB\x96\xa8\xa4]\x87\xb7\xc9\v\xf9\x0e\x1b\xb6̛M\x8b\xfa\xe1\xe9\xc3\xcbo\xd7W\xd30\xe6H\xbd\xa0 ñ\x8emvh\x11^b\xfc%\xbb\xb9,ڙ&\x80\xd9\xfc\x8c\xdc_\x8c\xd8ZӢ\xf5\xb2\x04K\x1a\x1d,\xea\xcc\xf6x\xfaWu\xb5\x06@b\xa4S \b\x940\xf9U\x8e\x1f\x14Yr05\xf8\x9dt`\xb1\xb5\xe8P'\x98\xa2i\xa63\x83\xf3\x1e\xe95Z\"C\xb1\x1d\x94 ,;\xa0\xf5`\x91\x9b\xad\x96\xff8\xd3v\xe0Mvf\x8f\xceC\x8cP\xcd\x149k\xc0\x1f\x81iѣܰ\x13X\xa4;!\xe8\x0e\xbdx\xc0\xf5\xf9\xf8H\xd1 um\x96\xb0\xf3\xbeu\xcb\xc5b+}Ahn\x9a&h\xe9O\x8b\b\xb6r\x13\xbc\xb1n!\xf0\x80j\xe1\xe4\xb6b\x96\xef\xa4G\xee\x83\xc5\x05ke\x15\x05\xd1\tR\x1b\xf1\xbd͘\uebae\x1d\x84t\x1a\x11R_a\x1e\x82\xd7\xe42\x89T\x12\xf1b\x05\x9a\"\xd5}\xfe\xe3\xfa\x19\n'\xc9R\xc9(\x97\xad\x03\xbd\x14\xfb\x906\xa5\xaeѦs\xb55M\xa4\x89Z\xb4Fj\x1f\x7fp%Q{pa\xd3HOn\xf0\xf7\x80Γ\xe9\xfadW1\x8b\xc1\x06!\xb4\x11$\xfa\x1b>hX\xb1\x06Պ9\xfc\x85mEVq\x15\x19\xe1.kuss\x7fsRog\xa1\xe4\xd4\tӎ\xa2\xc1\xbaE~\x15w\x02\x9d\xb4\x14\x19\x9ey\x8c\xd1\xd5SP\x86\x8a\xe9\xa4\\\xc68H\xd0`\x9c\xa3s\x1f\x8d\xc0\xfeJ\x8f\xe5\x87\xf3\xc6+\x1e[\xb4\x8dt1\xbdBml?\xf3\xb03\x92wGA\xbc\xbe\xc1\x01P\x87f\xc8H\x05\x9f\x91\x89OZ\x9d&\x96\xfef\xa5\x1f^4aH\x1a\x89\xc5\xf5I\xf3'\xb4҈\x1b¿\xebm?\xab`g\x8ePG\xff\xd7^\x9d\b\xbb\xdcI\xf3!j\x97\xf1\xf0\xf4\xa1 x\x8a\xad\x1c\x98YWsx\xc8Amjx\x03B:*$\\$:T\x96\x0e*\x16\x1aK\xf06\xbcJ|nt-\xb7C\xa1\xbb\xb5є\xc7\xdc \xdd\xd3\xdc*\xdeD\xa8E\xde\xd1Zs\x90\x02mE\xf1!k\xc93'\xc1\xa6\fRKTb\x80M\x93Q\x16E\xb1((\xa8\x99\xbaa\xc3\xd5yc\xac\xa4\x99\xd4Ƀ/\x04\"\xd6\xd8&\xa7f\xedQ\v\xecg\x9bȍ\x89\x80\xe6P\xc0Q\xfa]BJ5\x16w\xf0\xd5أ\xb1\xc7\xd3\xd8t\x8f\xf7\xe7\x1d\xd2Δx\x11\x1cr\x8b>z\x1b*r\x1fr\xa59\xc0\xc7\xe0\"\xd6\xf6q\xa2\x8cX\xf0\x95\xd3{<\r\x15\r\xb7\x8c\x9bK\xa1\t\x96c\x11\xb5\x84ﾻ-\xd2 \xbb\x95A\xa5{\x11\xd4b\x8d\x16\xf5\xa0\x9a(\xe39\xe6(r\x1a\xf20\xack\xe4^\x1eP\x9dbN\"\xf0\xfc\x116\xc1\x83\b\x18\xad\xc6\xf8\xfeȬp\xc0M\xd32/7RI\x7f\x02\xe9&\xe83\xa5\xcc\x11E\xb686\xad?\xcd\xe1\x83v\x9ei\x8e\xee\\\a\x91ƒ+0\x9dv\xe5(\x8e\x05\x1d\xb3c\x18\x98\xc87\xc6y\xe0h\xc9\x1d\xd5\t\x8e\xd6\xe8픰#\xe9\x90z@\xab\xd1c̈\xc2pGɐc\xeb\xdd\xc2\x1c\xd0\x1e$\x1e\x17Gc\xf7Ro+b\xb0\xcaೈ\x9d\xdd\xe2\xfb\xf8\xe7[\xbc\xc0\xb4\t'\xeep\xdeu\x8c\xf5\x13\x95\xb7~\x87)E\xac\x93\x0f\x1a\vT@\x90k7\xd9w\x13\xb2\x8e\x85\xddX]\xde\x1d\xc5\xe4c\xf9c\x8f\xc3\xd4\xf1\x15P\x01\xf8R]t[5\xac\xad\xd2n\xe6M#\xf9\xac/m\xf2\xfb\xaf\xe3OiV\xa4\x16\x92Sq{\x8d\x1b\xa5\x89\x13W=͈\x1a\xfa]\xce\x14Z\x8e\xab)\x89\x9bk\x85\x1b\x1c\x7f\xea\uef74\xbe\t\xbas\xfew\xe8\xa9\xeet\xa0\x91\xea\x03f\x87z\x8e\x80ɍքT\xde\x00;\xa7\x81\x1f\\?\xff\xbd\x12=7\x81\xefqD\xf1\x03Q\xdeōE\xc7\xe9\x18\xf1\x12\x1c\xc6\xc4t\x8b\r\xb8\x1d\x11\x9c\xad\xd0\xde\xc3\xcb\xea\x816\x9eK\b\x06\xab\a\xd8\x04-\x14\x16\x8e\x8e;\xd4\xd4u\xc9\xfa4~\x17\x8d\xe7\xc7u\xd1j\xac\xber\xdfTt;.C\xcaoK\u061cF\xea\xa5;\x84l-\xd6\xf2\xcb\x1dB>ōE\xe1-\xf3;\x90\xdaI\x81\xc0Fԟ\n\xd9\tAϵѧ\x8c9\xdf`\x9e\xafaCb\xe75\xf0Pt|#~\x9e\xf2\xb6\xb3\x16\xca\xef\x9cݮ\xeb\xe4\xa98\x1e\x95\xe8p~\x94\xf9S\xaa>\xf9H\x19q\xc5\xcc\xcb\xf0\xc4W\xaa\xd8\xf244\x16\xccT3\x19kѵF\v\xea9\xef\xaba/,\xff\xef*\xd9q\xb3V\xd7(\xd7[+V\xb8\xab\x8d\x8b\xcf`\xafn\xe4\xd2\xe3`\xb7M2\x1bG\r\xf6\xa5\x97\xeb\xc9\xf8\x8b\xb4p\xa3%W\xa7\xaf\x93\x8eꗠce\x1b\xab\xaa\xf9l\xe4\xc4{l-R\x06\x13K\x92\xcdƃ\xda\x1c\xe9p\x87Z*ˌN\xf9\x9ez[\xa6E~U\xa0\xa5\x11\xcaG\xa9\x14\xd5\x00\x16\x1bCʢ\xb2\xdcR5\xc7b\xadu\xf8\xcd\xfc\xcd\xff\xafeT\xccy\xea\x00Q|ƃ\x1c>\xadݧ\xee\xc7\x01\x95\x82\x0e瘡\x1f?\x95׆\x85\xcd\xdb~\x82Z*\xaa\xff:\xd0qGu0\xf20\xfcn\xfd\xf8\x83\x8b=\x10j\xef\xe0H\x16t\x91%jzL~\xe1\t\xceS\x12\xb9i\xffn\x01\xae\r(\xa3\xb7h\xcbk\x0f\x15xɛ\x8c\x05\x81\x9er\x95\xde\x02\xdf1\xbd\xa5\xc8\x18\x83\xfc\xc8p\xe6\xbe\xcb'yϤ\x83H=\xe1\x1dw\x19\xf4Y\x8e\xb54\xaf1\xe6\xf43\xfc\x99\xffl\xd9\xcbkoO\xefSP[,\xd1_,\xa9\x9c\x14]\xf9\xcb\xd3\xfce|\xfb\xfb\xc0\xf0\xdd\xff[\xd5\xf3_}\xa9\x18|\xa1\xf8U(\xa7\xa1:\xf7f\xf1\xfc1\xedJ\xef\xb5\xf9\b\xb0\x8d\t~$\xf7w\x1c~4\xa6\xe3ǘ\xd7\xf0\x18?1\xdd*OhO\xb1\b\x0f\xd6\xc67\xdd\xf2\xd6\x18\x91b,+ݏ\xc0\x0f\xbd/aݵ\xe1w\xb2;\xe4\x1a\xcd҃ɔi;v\xcdJ\xee΄\xcd\xf9\xa5~\t\xff\xfc\xf7\xec?\x01\x00\x00\xff\xff\x03f\x86Y\xc0\x1d\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVMo\x1b7\x10\xbd\xebW\f\xd0kwU\xa3hQ\xec\xadqr0\xda\x06\x82\x1d\xe4N\x91#-c.\xc9\xce\f\xe5\xba\x1f\xff\xbd \xb9+K\xab\x95\x93\\\xb27\x91Ù\xc7\xf7f\x1e\xd54\xcdJE\xfb\x11\x89m\xf0\x1d\xa8h\xf1/A\x9f\x7fq\xfb\xf8\v\xb76\xac\x0f7\xabG\xebM\a\xb7\x89%\f\xf7\xc8!\x91Ʒ\xb8\xb3ފ\r~5\xa0(\xa3Du+\x00\xe5}\x10\x95\x979\xff\x04\xd0\xc1\v\x05琚=\xfa\xf61mq\x9b\xac3H%\xf9T\xfa\xf0C{\xf3s\xfb\xd3\n\xc0\xab\x01;0\xe8Pp\xab\xf4c\x8a\x84\x7f&d\xe1\xf6\x80\x0e)\xb46\xac8\xa2\xce\xf9\xf7\x14R\xec\xe0e\xa3\x9e\x1fkW\xdcoK\xaa7%\xd5}MUv\x9de\xf9\xedZ\xc4\xefv\x8c\x8a.\x91rˀJ\x00[\xbfON\xd1b\xc8\n\x80u\x88\xd8\xc1\xfb\f+*\x8df\x050^\xbb\xc0l@\x19S\x88TnC\xd6\v\xd2mpi\x98\bl\xc0 k\xb2Q\nQ\x1fz,W\x84\xb0\x03\xe9\x11j9\x90\x00[\x1c\x11\x98r\x0e\xe0\x13\a\xbfQ\xd2w\xd0f\xbe\xda\x1a\x9a\x81\x8c\x01\x95\xea7\xf3ey\u0380Y\xc8\xfa\xfd5\b,J\x12O J]\x1b<\xd0\t\xbf\xe7\x00J|\x1b{\xc5\xe7\xd5\x1f\xcaƵ\xca5\xe6pS\x99\xd6=\x0e\xaa\x1bcCD\xff\xeb\xe6\xee\xe3\x8f\x0fg\xcbp\x8euAZ\xb0\fjB\x9a\x89\xab\xacA\xf0\b\x81`\b4\xb1\xca\xed1i\xa4\x10\x91\xc4N\xadU\xbf\x93\xe19Y\x9dA\xf8\xb79\xdb\x03Ȩ\xeb)0y\x8a\x90\v\x89cS\xa0\x19/Zɵ\f\x84\x91\x90\xd1\u05f9\xca\xcb\xcaC\xd8~B-\xed,\xf5\x03RN\x03܇\xe4L\x1e\xbe\x03\x92\x00\xa1\x0e{o\xff>\xe6\xe6|\xef\\\xd4))\x94\xe4\xb6\xf3\xca\xc1A\xb9\x84߃\xf2f\x96yP\xcf@\x98kB\xf2'\xf9\xca\x01\x9e\xe3\xf8#\x93h\xfd.tЋD\xee\xd6뽕\xc9Rt\x18\x86\xe4\xad<\xaf\x8b;\xd8m\x92@\xbc6x@\xb7f\xbbo\x14\xe9\xde\njI\x84k\x15mS.⋭\xb4\x83\xf9\x8eF\x13Ⳳ\x17\xddS\xbf\xe2\x02_!O\xf6\x84\xda#5U\xbd\xe2\x8b\ny)Sw\xff\xee\xe1\x03LH\xaaRU\x94\x97\xd0\v^&}2\x9b\xd6\xef\x90\xea\xb9\x1d\x85\xa1\xe4Dob\xb0^\xca\x0f\xed,z\x01N\xdb\xc1\nO\x1d\x9b\xa5\x9b\xa7\xbd-\xb6\x9b\x1d E\xa3\x04\xcd<\xe0\xceí\x1a\xd0\xdd*\xc6o\xacUV\x85\x9b,\xc2\x17\xa9u\xfa\x98̃+\xbd'\x1b\xd33pEڅ\xe1\x7f\x88\xa8\xb3\xb8\x99\xdf|\xda\ueb2ec\xb5\v\x04O\xbd\xd5\xfd4\xfc3\x9a\x8eFq\xce߲1\xe4\xef\xc5n\xe7;W/\x0fEdK8k\xd8\x06.\xbc\xfbu^\x8a\xa9~%3\xd5\xd1Gnt\"*\xcdw\xf4y\xb5t\xe8K\xb9@\xa2@\x17\xab3P\xefJP\xf9Ǡ\xacgP\xfey<\b\xd2+\x81'\xa4\r\x97\x95\x1ax\x8fO\v\xabw~CaO\xc8\xf3\x96ϛ\x9b\xca\x1e\xce߃WXZlʋE\xceVhNXd\t\xa4\xf6\xa7\xbcr\xda\x1e\x9d\xbe\x83\x7f\xfe[\xfd\x1f\x00\x00\xff\xff\xbeM\x1a\xea\xb1\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcWMo\xe36\x10\xbd\xfbW\f\xd0K\v\xac\xe4\x06E\x8b·\xd6\xd9C\xb0\xe96\x88\xb7\xb9S\xd4HbC\x91,9t6E\x7f|1\xa4\xe4\x0fYv\x9c\xcb\xea\xe6\xe1p\xf8\xe6\xcd\xcc#]\x14\xc5B8\xf5\x84>(kV \x9c¯\x84\x86\x7f\x85\xf2\xf9\xd7P*\xbb\xdc\xde,\x9e\x95\xa9W\xb0\x8e\x81l\xff\x88\xc1F/\xf1\x16\x1be\x14)k\x16=\x92\xa8\x05\x89\xd5\x02@\x18cI\xb09\xf0O\x00i\ry\xab5\xfa\xa2ES>\xc7\n\xab\xa8t\x8d>\x05\x1f\x8f\xde\xfeX\xde\xfcR\xfe\xbc\x000\xa2\xc7\x15\xd4\xf6\xc5h+j\x8f\xffD\f\x14\xca-j\xf4\xb6Tv\x11\x1cJ\x8e\xddz\x1b\xdd\n\xf6\vy\xefpn\xc6|;\x84y\xccaҊV\x81>ͭޫ\xc1\xc3\xe9\xe8\x85>\x05\x91\x16\x832m\xd4\u009f,/\x00\x82\xb4\x0eW\xf0\x99a8!\xb1^\x00\f)&XŐ\xdd\xf6&\x87\x92\x1d\xf6\"\xe3\x05\xb0\x0e\xcdo\x0fwO?m\x8e\xcc\x005\x06镣D\xd4\x7f\xc5\xce\x0e\xd3\x04@\x05\x100\xc0\x01\xb2;\x84 \f\bO\xaa\x11\x92\xa0\xf1\xb6\x87J\xc8\xe7\xe8\xc0V\x7f\xa3$\bd\xbdh\xf1\x03\x84(;\x10\x1c%;\x1c\x9c\xa5m\v\x8d\xd2X\xeel\xce[\x87\x9e\xd4Hy\xfe\x0e\x1a\xea\xc0z)\v\xfe8\xf1\xbc\vj\xee,\f@\x1d\x8e\xe4a=p\x05\xb6\x01\xeaT\x00\x8f\xcec@\x93{\x8d\xcd\xc2\fٔ\x93\xd0\x1b\xf4\x1c\x06Bg\xa3\xae\xb9!\xb7\xe8\t\x1aE\xaf\xcb41\xaa\x8ad}XָE\xbd\f\xaa-\x84\x97\x9d\"\x94\x14=.\x85SEJĤQ+\xfb\xfa;?\ff8:\x96^\xb9!\x03yeڃ\x854\x1d\xef(\x0f\xcfK\xee\xae\x1c*\xa7\xb8\xaf\x02\x9b\x98\xbaǏ\x9b/0\"ɕ\x1aZl\xe7z\xc2\xcbX\x1ffS\x99\x06}ޗڔc\xa2\xa9\x9dU\x86\xd2\x0f\xa9\x15\x1a\x82\x10\xab^Q\x18{\x9dK7\r\xbbNR\x04\x15Bt\xb5 \xac\xa7\x0ew\x06֢G\xbd\x16\x01\xbfq\xad\xb8*\xa1\xe0\"\\U\xadC\x81\x9d:gz\x0f\x16Fy&j^\x01\x128\xe1[\xa4\xa9u\x82\xe5Kr\xe2\xe3_:q,X\xdfcٖ\xac9a\x00\x92\xf5\xe8\x87i\xa1.a\x80\xd9F\x9fE2\xf67\xd3\xc0\xbc\xb2\xa0\xb0\xd8\x1db:=\x9a?4\xb1\x9f?\xa0\x80\xdf\x13\xe6{\xdb^\\_[C<\x17\x17\x9d\x9e\xac\x8e=n\x8cp\xa1\xb3o\xf8\xde\x11\xf6\x7f:\xf4\xf9\x1a\xbe\xe8:\xde滫\xef\x82c\xd4g\xcf}D\xbeA\xf0|\xa6\x83\xc3UQ\xae\xc04x^\x95\xe8zs\xf7\x1e\nϸ\xbf\xa3Hw\xa6\xb1o\xa4\xb8w\x9c\xf5;#\x03\xe3\x97\xde\x10o\xf74\xbfBƞ\xe6-\xf9\xeeD\xf8\x14+\xf4\x06\t\xc3^\xa9_\x14u\xb3\x11\x01^:%\xbb\xb41\r\x04_\x02!X\xa9\xe6$\xf5\n\xf8\xac#\xca\xe3\xccP\x16iXg\xcc\f\xfe\xc4|F\xfd\xce\x1dP\f\x8at\x95\x82\x92\xa0\x18ޡ\xa1\xc9\x7f\xa4ZF\xef\xd3\x15\x95\xad\xfc2\x99n\xb8VDG\xe5\xf9\xeb\xf1\xfe\r%\xbd\xdd{\xa6\x17\xb7P&\xa3q\x1e\x8b\xa0Z~A\xf1\x1akiҸS2\xf2w\xfc\xc2;&j\xb6\xa2\xf8թ<\x80o@\xfc\xb8ŝ\x8f&\xdf\xf3\xd37l\n\x88\x81\x9f[ \x85\x99\xc1X!Ԩ\x91\xb0\x86\xea5\xdf\\\xaf\x81\xb0?\xc5\xddX\xdf\vZ\x01\xdf\xff\x05\xa9\x9962QkQi\\\x01\xf9x\xae\xcbf\x13w\x9d\b3cx\x94\xf3\x03\xfb\xcc5\xc6n\x18/v\x06\x9c\xbd_\n\xf8\x8c/3\xd6\ao%\x86\x80\xa7ct6\x93\xd9!81\x06~\xa4\xd5\a,\r\x7f\x19\x06\xcb\xff\x01\x00\x00\xff\xffx\xae@\xbaJ\x0e\x00\x00"), diff --git a/pkg/apis/velero/v1/backup_types.go b/pkg/apis/velero/v1/backup_types.go index 5ba3b2175..24af8132d 100644 --- a/pkg/apis/velero/v1/backup_types.go +++ b/pkg/apis/velero/v1/backup_types.go @@ -288,7 +288,7 @@ const ( // BackupPhase is a string representation of the lifecycle phase // of a Velero backup. -// +kubebuilder:validation:Enum=New;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;Finalizing;FinalizingPartiallyFailed;Completed;PartiallyFailed;Failed;Deleting +// +kubebuilder:validation:Enum=New;Queued;ReadyToStart;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;Finalizing;FinalizingPartiallyFailed;Completed;PartiallyFailed;Failed;Deleting type BackupPhase string const ( @@ -296,6 +296,12 @@ const ( // yet processed by the BackupController. BackupPhaseNew BackupPhase = "New" + // BackupPhaseQueued means the backup has been added to the queue and is waiting for the Queue to move it out of the queue. + BackupPhaseQueued BackupPhase = "Queued" + + // BackupPhaseReadyToStart means the backup has been pulled from the queue and is ready to start. + BackupPhaseReadyToStart BackupPhase = "ReadyToStart" + // BackupPhaseFailedValidation means the backup has failed // the controller's validations and therefore will not run. BackupPhaseFailedValidation BackupPhase = "FailedValidation" @@ -371,6 +377,11 @@ type BackupStatus struct { // +optional Phase BackupPhase `json:"phase,omitempty"` + // QueuePosition is the position of the backup in the queue. + // Only relevant when Phase is "Queued" + // +optional + QueuePosition int `json:"queuePosition,omitempty"` + // ValidationErrors is a slice of all validation errors (if // applicable). // +optional diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index b0d0937dd..de4947afb 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -117,7 +117,6 @@ type kubernetesBackupper struct { podCommandExecutor podexec.PodCommandExecutor podVolumeBackupperFactory podvolume.BackupperFactory podVolumeTimeout time.Duration - podVolumeContext context.Context defaultVolumesToFsBackup bool clientPageSize int uploaderType string @@ -365,12 +364,12 @@ func (kb *kubernetesBackupper) BackupWithResolvers( } var podVolumeCancelFunc context.CancelFunc - kb.podVolumeContext, podVolumeCancelFunc = context.WithTimeout(context.Background(), podVolumeTimeout) + podVolumeContext, podVolumeCancelFunc := context.WithTimeout(context.Background(), podVolumeTimeout) defer podVolumeCancelFunc() var podVolumeBackupper podvolume.Backupper if kb.podVolumeBackupperFactory != nil { - podVolumeBackupper, err = kb.podVolumeBackupperFactory.NewBackupper(kb.podVolumeContext, log, backupRequest.Backup, kb.uploaderType) + podVolumeBackupper, err = kb.podVolumeBackupperFactory.NewBackupper(podVolumeContext, log, backupRequest.Backup, kb.uploaderType) if err != nil { log.WithError(errors.WithStack(err)).Debugf("Error from NewBackupper") return errors.WithStack(err) @@ -416,6 +415,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers( kbClient: kb.kbClient, discoveryHelper: kb.discoveryHelper, podVolumeBackupper: podVolumeBackupper, + podVolumeContext: podVolumeContext, podVolumeSnapshotTracker: podvolume.NewTracker(), volumeSnapshotterCache: NewVolumeSnapshotterCache(volumeSnapshotterGetter), itemHookHandler: &hook.DefaultItemHookHandler{ @@ -597,7 +597,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers( log.Infof("Backing Up Item Block including %s %s/%s (%v items in block)", items[i].groupResource.String(), items[i].namespace, items[i].name, len(itemBlock.Items)) wg.Add(1) - backupRequest.ItemBlockChannel <- ItemBlockInput{ + backupRequest.WorkerPool.GetInputChannel() <- ItemBlockInput{ itemBlock: itemBlock, returnChan: itemBlockReturn, } @@ -848,7 +848,7 @@ func (kb *kubernetesBackupper) handleItemBlockPostHooks(itemBlock *BackupItemBlo log := itemBlock.Log // the post hooks will not execute until all PVBs of the item block pods are processed - if err := kb.waitUntilPVBsProcessed(kb.podVolumeContext, log, itemBlock, hookPods); err != nil { + if err := kb.waitUntilPVBsProcessed(itemBlock.itemBackupper.podVolumeContext, log, itemBlock, hookPods); err != nil { log.WithError(err).Error("failed to wait PVBs processed for the ItemBlock") return } diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 6918c99f5..f9351245c 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -79,7 +79,7 @@ func TestBackedUpItemsMatchesTarballContents(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: h.itemBlockPool.GetInputChannel(), + WorkerPool: &h.itemBlockPool, } backupFile := bytes.NewBuffer([]byte{}) @@ -141,7 +141,7 @@ func TestBackupProgressIsUpdated(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: h.itemBlockPool.GetInputChannel(), + WorkerPool: &h.itemBlockPool, } backupFile := bytes.NewBuffer([]byte{}) @@ -881,7 +881,7 @@ func TestBackupOldResourceFiltering(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1062,7 +1062,7 @@ func TestCRDInclusion(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1161,7 +1161,7 @@ func TestBackupResourceCohabitation(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1190,7 +1190,7 @@ func TestBackupUsesNewCohabitatingResourcesForEachBackup(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: h.itemBlockPool.GetInputChannel(), + WorkerPool: &h.itemBlockPool, } backup1File := bytes.NewBuffer([]byte{}) @@ -1206,7 +1206,7 @@ func TestBackupUsesNewCohabitatingResourcesForEachBackup(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: h.itemBlockPool.GetInputChannel(), + WorkerPool: &h.itemBlockPool, } backup2File := bytes.NewBuffer([]byte{}) @@ -1260,7 +1260,7 @@ func TestBackupResourceOrdering(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1381,7 +1381,7 @@ func TestBackupItemActionsForSkippedPV(t *testing.T) { Backup: defaultBackup().SnapshotVolumes(false).Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, resPolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", @@ -1428,8 +1428,8 @@ func TestBackupItemActionsForSkippedPV(t *testing.T) { }, includedPVs: map[string]struct{}{}, }, - BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + BackedUpItems: NewBackedUpItemsMap(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVCs( @@ -1679,7 +1679,7 @@ func TestBackupActionsRunForCorrectItems(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1764,7 +1764,7 @@ func TestBackupWithInvalidActions(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1918,7 +1918,7 @@ func TestBackupActionModifications(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -2178,7 +2178,7 @@ func TestBackupActionAdditionalItems(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -2439,7 +2439,7 @@ func TestItemBlockActionsRunForCorrectItems(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -2524,7 +2524,7 @@ func TestBackupWithInvalidItemBlockActions(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -2780,7 +2780,7 @@ func TestItemBlockActionRelatedItems(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -2948,7 +2948,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -2984,7 +2984,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3021,7 +3021,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3058,7 +3058,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3095,7 +3095,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3130,7 +3130,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3148,7 +3148,7 @@ func TestBackupWithSnapshots(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3169,7 +3169,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3188,7 +3188,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3210,7 +3210,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3344,7 +3344,7 @@ func TestBackupWithAsyncOperations(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.Pods( @@ -3376,7 +3376,7 @@ func TestBackupWithAsyncOperations(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.Pods( @@ -3408,7 +3408,7 @@ func TestBackupWithAsyncOperations(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.Pods( @@ -3494,7 +3494,7 @@ func TestBackupWithInvalidHooks(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -3968,7 +3968,7 @@ func TestBackupWithHooks(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) podCommandExecutor = new(test.MockPodCommandExecutor) @@ -4193,7 +4193,7 @@ func TestBackupWithPodVolume(t *testing.T) { SnapshotLocations: []*velerov1.VolumeSnapshotLocation{tc.vsl}, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -5312,7 +5312,7 @@ func TestBackupNewResourceFiltering(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -5477,7 +5477,7 @@ func TestBackupNamespaces(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 3622ac7e1..7b1ea69bd 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -69,6 +69,7 @@ type itemBackupper struct { kbClient kbClient.Client discoveryHelper discovery.Helper podVolumeBackupper podvolume.Backupper + podVolumeContext context.Context podVolumeSnapshotTracker *podvolume.Tracker kubernetesBackupper *kubernetesBackupper volumeSnapshotterCache *VolumeSnapshotterCache diff --git a/pkg/backup/request.go b/pkg/backup/request.go index 4643142b1..eb9edcbe8 100644 --- a/pkg/backup/request.go +++ b/pkg/backup/request.go @@ -69,7 +69,7 @@ type Request struct { ResPolicies *resourcepolicies.Policies SkippedPVTracker *skipPVTracker VolumesInformation volume.BackupVolumesInformation - ItemBlockChannel chan ItemBlockInput + WorkerPool *ItemBlockWorkerPool } // BackupVolumesInformation contains the information needs by generating @@ -103,3 +103,7 @@ func (r *Request) FillVolumesInformation() { r.VolumesInformation.BackupOperations = *r.GetItemOperationsList() r.VolumesInformation.BackupName = r.Backup.Name } + +func (r *Request) StopWorkerPool() { + r.WorkerPool.Stop() +} diff --git a/pkg/builder/backup_builder.go b/pkg/builder/backup_builder.go index 2e5dbf24f..d5b955e43 100644 --- a/pkg/builder/backup_builder.go +++ b/pkg/builder/backup_builder.go @@ -222,6 +222,12 @@ func (b *BackupBuilder) Phase(phase velerov1api.BackupPhase) *BackupBuilder { return b } +// Phase sets the Backup's queue position. +func (b *BackupBuilder) QueuePosition(queuePos int) *BackupBuilder { + b.object.Status.QueuePosition = queuePos + return b +} + // StorageLocation sets the Backup's storage location. func (b *BackupBuilder) StorageLocation(location string) *BackupBuilder { b.object.Spec.StorageLocation = location diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 6698010bd..283bfa557 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -89,6 +89,7 @@ type Options struct { RepoMaintenanceJobConfigMap string NodeAgentConfigMap string ItemBlockWorkerCount int + ConcurrentBackups int NodeAgentDisableHostPath bool kubeletRootDir string ServerPriorityClassName string @@ -196,6 +197,12 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { o.ItemBlockWorkerCount, "Number of worker threads to process ItemBlocks. Default is one. Optional.", ) + flags.IntVar( + &o.ConcurrentBackups, + "concurrent-backups", + o.ConcurrentBackups, + "Number of backups to process concurrently. Default is one. Optional.", + ) flags.StringVar( &o.ServerPriorityClassName, "server-priority-class-name", @@ -313,6 +320,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { RepoMaintenanceJobConfigMap: o.RepoMaintenanceJobConfigMap, NodeAgentConfigMap: o.NodeAgentConfigMap, ItemBlockWorkerCount: o.ItemBlockWorkerCount, + ConcurrentBackups: o.ConcurrentBackups, KubeletRootDir: o.kubeletRootDir, NodeAgentDisableHostPath: o.NodeAgentDisableHostPath, ServerPriorityClassName: o.ServerPriorityClassName, diff --git a/pkg/cmd/server/config/config.go b/pkg/cmd/server/config/config.go index 401a89632..5adb9946b 100644 --- a/pkg/cmd/server/config/config.go +++ b/pkg/cmd/server/config/config.go @@ -47,11 +47,13 @@ const ( defaultDisableInformerCache = false DefaultItemBlockWorkerCount = 1 + DefaultConcurrentBackups = 1 ) var ( // DisableableControllers is a list of controllers that can be disabled DisableableControllers = []string{ + constant.ControllerBackupQueue, constant.ControllerBackup, constant.ControllerBackupOperations, constant.ControllerBackupDeletion, @@ -174,6 +176,7 @@ type Config struct { BackupRepoConfig string RepoMaintenanceJobConfig string ItemBlockWorkerCount int + ConcurrentBackups int } func GetDefaultConfig() *Config { @@ -206,6 +209,7 @@ func GetDefaultConfig() *Config { ScheduleSkipImmediately: false, CredentialsDirectory: credentials.DefaultStoreDirectory(), ItemBlockWorkerCount: DefaultItemBlockWorkerCount, + ConcurrentBackups: DefaultConcurrentBackups, } return config @@ -261,4 +265,10 @@ func (c *Config) BindFlags(flags *pflag.FlagSet) { c.ItemBlockWorkerCount, "Number of worker threads to process ItemBlocks. Default is one. Optional.", ) + flags.IntVar( + &c.ConcurrentBackups, + "concurrent-backups", + c.ConcurrentBackups, + "Number of backups to process concurrently. Default is one. Optional.", + ) } diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 15c4afd95..68330670d 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -581,6 +581,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string constant.ControllerSchedule: {}, constant.ControllerServerStatusRequest: {}, constant.ControllerRestoreFinalizer: {}, + constant.ControllerBackupQueue: {}, } if s.config.RestoreOnly { @@ -668,6 +669,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.config.MaxConcurrentK8SConnections, s.config.DefaultSnapshotMoveData, s.config.ItemBlockWorkerCount, + s.config.ConcurrentBackups, s.crClient, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackup) @@ -909,6 +911,18 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string } } + if _, ok := enabledRuntimeControllers[constant.ControllerBackupQueue]; ok { + if err := controller.NewBackupQueueReconciler( + s.mgr.GetClient(), + s.mgr.GetScheme(), + s.logger, + s.config.ConcurrentBackups, + backupTracker, + ).SetupWithManager(s.mgr); err != nil { + s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackupQueue) + } + } + s.logger.Info("Server starting...") if err := s.mgr.Start(s.ctx); err != nil { diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index 932b77cfb..e0637a4bd 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -75,6 +75,7 @@ func DescribeBackup( case velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed: case velerov1api.BackupPhaseInProgress: case velerov1api.BackupPhaseNew: + case velerov1api.BackupPhaseQueued, velerov1api.BackupPhaseReadyToStart: } logsNote := "" @@ -83,6 +84,9 @@ func DescribeBackup( } d.Printf("Phase:\t%s%s\n", phaseString, logsNote) + if phase == velerov1api.BackupPhaseQueued { + d.Printf("Queue position:\t%v\n", backup.Status.QueuePosition) + } if backup.Spec.ResourcePolicy != nil { d.Println() diff --git a/pkg/cmd/util/output/backup_printer.go b/pkg/cmd/util/output/backup_printer.go index bae077a29..873bc9fc3 100644 --- a/pkg/cmd/util/output/backup_printer.go +++ b/pkg/cmd/util/output/backup_printer.go @@ -20,6 +20,7 @@ import ( "fmt" "regexp" "sort" + "strconv" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,6 +41,7 @@ var ( {Name: "Created"}, {Name: "Expires"}, {Name: "Storage Location"}, + {Name: "Queue Position"}, {Name: "Selector"}, } ) @@ -108,6 +110,7 @@ func printBackup(backup *velerov1api.Backup) []metav1.TableRow { backup.Status.StartTimestamp, humanReadableTimeFromNow(expiration), backup.Spec.StorageLocation, + queuePosition(backup.Status.QueuePosition), metav1.FormatLabelSelector(backup.Spec.LabelSelector), ) @@ -127,3 +130,11 @@ func humanReadableTimeFromNow(when time.Time) string { return fmt.Sprintf("%s ago", duration.ShortHumanDuration(now.Sub(when))) } } + +func queuePosition(pos int) string { + if pos == 0 { + return "" + } else { + return strconv.Itoa(pos) + } +} diff --git a/pkg/constant/constant.go b/pkg/constant/constant.go index cfadcc5dc..8eac99573 100644 --- a/pkg/constant/constant.go +++ b/pkg/constant/constant.go @@ -1,6 +1,7 @@ package constant const ( + ControllerBackupQueue = "backup-queue" ControllerBackup = "backup" ControllerBackupOperations = "backup-operations" ControllerBackupDeletion = "backup-deletion" diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index c9728ea82..37faafd68 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -36,7 +36,11 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/resourcepolicies" @@ -105,7 +109,7 @@ type backupReconciler struct { defaultSnapshotMoveData bool globalCRClient kbclient.Client itemBlockWorkerCount int - workerPool *pkgbackup.ItemBlockWorkerPool + concurrentBackups int } func NewBackupReconciler( @@ -132,6 +136,7 @@ func NewBackupReconciler( maxConcurrentK8SConnections int, defaultSnapshotMoveData bool, itemBlockWorkerCount int, + concurrentBackups int, globalCRClient kbclient.Client, ) *backupReconciler { b := &backupReconciler{ @@ -159,8 +164,8 @@ func NewBackupReconciler( maxConcurrentK8SConnections: maxConcurrentK8SConnections, defaultSnapshotMoveData: defaultSnapshotMoveData, itemBlockWorkerCount: itemBlockWorkerCount, + concurrentBackups: max(concurrentBackups, 1), globalCRClient: globalCRClient, - workerPool: pkgbackup.StartItemBlockWorkerPool(ctx, itemBlockWorkerCount, logger), } b.updateTotalBackupMetric() return b @@ -168,7 +173,24 @@ func NewBackupReconciler( func (b *backupReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). - For(&velerov1api.Backup{}). + For(&velerov1api.Backup{}, builder.WithPredicates(predicate.Funcs{ + UpdateFunc: func(ue event.UpdateEvent) bool { + backup := ue.ObjectNew.(*velerov1api.Backup) + return backup.Status.Phase == velerov1api.BackupPhaseReadyToStart + }, + CreateFunc: func(ce event.CreateEvent) bool { + return false + }, + DeleteFunc: func(de event.DeleteEvent) bool { + return false + }, + GenericFunc: func(ge event.GenericEvent) bool { + return false + }, + })). + WithOptions(controller.Options{ + MaxConcurrentReconciles: b.concurrentBackups, + }). Named(constant.ControllerBackup). Complete(b) } @@ -254,8 +276,8 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr // InProgress, we still need this check so we can return nil to indicate we've finished processing // this key (even though it was a no-op). switch original.Status.Phase { - case "", velerov1api.BackupPhaseNew: - // only process new backups + case velerov1api.BackupPhaseReadyToStart: + // only process ReadytToStart backups default: b.logger.WithFields(logrus.Fields{ "backup": kubeutil.NamespaceAndName(original), @@ -265,7 +287,9 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } log.Debug("Preparing backup request") - request := b.prepareBackupRequest(original, log) + request := b.prepareBackupRequest(ctx, original, log) + // delete worker pool after reconcile + defer request.StopWorkerPool() if len(request.Status.ValidationErrors) > 0 { request.Status.Phase = velerov1api.BackupPhaseFailedValidation } else { @@ -299,6 +323,8 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr switch request.Status.Phase { case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed, velerov1api.BackupPhaseFailedValidation: b.backupTracker.Delete(request.Namespace, request.Name) + case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed, velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed: + b.backupTracker.AddPostProcessing(request.Namespace, request.Name) } }() @@ -347,12 +373,12 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, nil } -func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logger logrus.FieldLogger) *pkgbackup.Request { +func (b *backupReconciler) prepareBackupRequest(ctx context.Context, backup *velerov1api.Backup, logger logrus.FieldLogger) *pkgbackup.Request { request := &pkgbackup.Request{ Backup: backup.DeepCopy(), // don't modify items in the cache SkippedPVTracker: pkgbackup.NewSkipPVTracker(), BackedUpItems: pkgbackup.NewBackedUpItemsMap(), - ItemBlockChannel: b.workerPool.GetInputChannel(), + WorkerPool: pkgbackup.StartItemBlockWorkerPool(ctx, b.itemBlockWorkerCount, logger), } request.VolumesInformation.Init() diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 715f5068a..e0b554ff1 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -95,7 +95,11 @@ func (b *fakeBackupper) FinalizeBackup( } func defaultBackup() *builder.BackupBuilder { - return builder.ForBackup(velerov1api.DefaultNamespace, "backup-1") + return builder.ForBackup(velerov1api.DefaultNamespace, "backup-1").Phase(velerov1api.BackupPhaseReadyToStart) +} + +func namedBackup(name string) *builder.BackupBuilder { + return builder.ForBackup(velerov1api.DefaultNamespace, name).Phase(velerov1api.BackupPhaseReadyToStart) } func TestProcessBackupNonProcessedItems(t *testing.T) { @@ -104,6 +108,16 @@ func TestProcessBackupNonProcessedItems(t *testing.T) { key string backup *velerov1api.Backup }{ + { + name: "New backup is not processed", + key: "velero/backup-1", + backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).Result(), + }, + { + name: "Queued backup is not processed", + key: "velero/backup-1", + backup: defaultBackup().Phase(velerov1api.BackupPhaseQueued).Result(), + }, { name: "FailedValidation backup is not processed", key: "velero/backup-1", @@ -135,9 +149,7 @@ func TestProcessBackupNonProcessedItems(t *testing.T) { kbClient: velerotest.NewFakeControllerRuntimeClient(t), formatFlag: formatFlag, logger: logger, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() if test.backup != nil { require.NoError(t, c.kbClient.Create(t.Context(), test.backup)) } @@ -234,9 +246,7 @@ func TestProcessBackupValidationFailures(t *testing.T) { clock: &clock.RealClock{}, formatFlag: formatFlag, metrics: metrics.NewServerMetrics(), - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() require.NotNil(t, test.backup) require.NoError(t, c.kbClient.Create(t.Context(), test.backup)) @@ -299,11 +309,10 @@ func TestBackupLocationLabel(t *testing.T) { defaultBackupLocation: test.backupLocation.Name, clock: &clock.RealClock{}, formatFlag: formatFlag, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() - res := c.prepareBackupRequest(test.backup, logger) + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.Equal(t, test.expectedBackupLocation, res.Labels[velerov1api.StorageLocationLabel]) }) @@ -331,7 +340,7 @@ func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) { }{ { name: "BackupLocation is specified in backup CR'spec and it can be found in ApiServer", - backup: builder.ForBackup("velero", "backup-1").Result(), + backup: defaultBackup().Result(), backupLocationNameInBackup: "test-backup-location", backupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "test-backup-location").Result(), defaultBackupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "default-location").Result(), @@ -340,7 +349,7 @@ func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) { }, { name: "BackupLocation is specified in backup CR'spec and it can't be found in ApiServer", - backup: builder.ForBackup("velero", "backup-1").Result(), + backup: defaultBackup().Result(), backupLocationNameInBackup: "test-backup-location", backupLocationInAPIServer: nil, defaultBackupLocationInAPIServer: nil, @@ -349,7 +358,7 @@ func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) { }, { name: "Using default BackupLocation and it can be found in ApiServer", - backup: builder.ForBackup("velero", "backup-1").Result(), + backup: defaultBackup().Result(), backupLocationNameInBackup: "", backupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "test-backup-location").Result(), defaultBackupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "default-location").Result(), @@ -358,7 +367,7 @@ func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) { }, { name: "Using default BackupLocation and it can't be found in ApiServer", - backup: builder.ForBackup("velero", "backup-1").Result(), + backup: defaultBackup().Result(), backupLocationNameInBackup: "", backupLocationInAPIServer: nil, defaultBackupLocationInAPIServer: nil, @@ -396,14 +405,13 @@ func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) { defaultBackupTTL: defaultBackupTTL.Duration, clock: testclocks.NewFakeClock(now), formatFlag: formatFlag, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() test.backup.Spec.StorageLocation = test.backupLocationNameInBackup // Run - res := c.prepareBackupRequest(test.backup, logger) + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() // Assert if test.expectedSuccess { @@ -472,11 +480,10 @@ func TestDefaultBackupTTL(t *testing.T) { defaultBackupTTL: defaultBackupTTL.Duration, clock: testclocks.NewFakeClock(now), formatFlag: formatFlag, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() - res := c.prepareBackupRequest(test.backup, logger) + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.Equal(t, test.expectedTTL, res.Spec.TTL) assert.Equal(t, test.expectedExpiration, *res.Status.Expiration) @@ -497,7 +504,7 @@ func TestPrepareBackupRequest_SetsVGSLabelKey(t *testing.T) { }{ { name: "backup with spec label key set", - backup: builder.ForBackup("velero", "backup-1"). + backup: defaultBackup(). VolumeGroupSnapshotLabelKey("spec-key"). Result(), serverFlagKey: "server-key", @@ -505,13 +512,13 @@ func TestPrepareBackupRequest_SetsVGSLabelKey(t *testing.T) { }, { name: "backup with no spec key, uses server flag", - backup: builder.ForBackup("velero", "backup-2").Result(), + backup: namedBackup("backup-2").Result(), serverFlagKey: "server-key", expectedLabelKey: "server-key", }, { name: "backup with no spec or server flag, uses default", - backup: builder.ForBackup("velero", "backup-3").Result(), + backup: namedBackup("backup-3").Result(), serverFlagKey: velerov1api.DefaultVGSLabelKey, expectedLabelKey: velerov1api.DefaultVGSLabelKey, }, @@ -533,11 +540,10 @@ func TestPrepareBackupRequest_SetsVGSLabelKey(t *testing.T) { defaultVGSLabelKey: test.serverFlagKey, discoveryHelper: discoveryHelper, clock: testclocks.NewFakeClock(now), - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() - res := c.prepareBackupRequest(test.backup, logger) + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.Equal(t, test.expectedLabelKey, res.Spec.VolumeGroupSnapshotLabelKey) @@ -635,11 +641,10 @@ func TestDefaultVolumesToResticDeprecation(t *testing.T) { clock: &clock.RealClock{}, formatFlag: formatFlag, defaultVolumesToFsBackup: test.globalVal, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() - res := c.prepareBackupRequest(test.backup, logger) + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.NotNil(t, res.Spec.DefaultVolumesToFsBackup) if test.expectRemap { @@ -1390,7 +1395,7 @@ func TestProcessBackupCompletions(t *testing.T) { }, { name: "backup with namespace-scoped and cluster-scoped resource filters", - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + backup: defaultBackup(). ExcludedClusterScopedResources("clusterroles"). IncludedClusterScopedResources("storageclasses"). ExcludedNamespaceScopedResources("secrets"). @@ -1439,7 +1444,7 @@ func TestProcessBackupCompletions(t *testing.T) { }, { name: "backup's include filter overlap with default exclude resources", - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + backup: defaultBackup(). ExcludedClusterScopedResources("clusterroles"). IncludedClusterScopedResources("storageclasses", "volumesnapshotcontents.snapshot.storage.k8s.io"). ExcludedNamespaceScopedResources("secrets"). @@ -1555,9 +1560,7 @@ func TestProcessBackupCompletions(t *testing.T) { backupper: backupper, formatFlag: formatFlag, globalCRClient: fakeGlobalClient, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() pluginManager.On("GetBackupItemActionsV2").Return(nil, nil) pluginManager.On("GetItemBlockActions").Return(nil, nil) @@ -1763,9 +1766,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { logger: logger, defaultSnapshotLocations: test.defaultLocations, kbClient: velerotest.NewFakeControllerRuntimeClient(t), - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() // set up a Backup object to represent what we expect to be passed to backupper.Backup() backup := test.backup.DeepCopy() diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go new file mode 100644 index 000000000..c5cdc9461 --- /dev/null +++ b/pkg/controller/backup_queue_controller.go @@ -0,0 +1,363 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "slices" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/constant" + "github.com/vmware-tanzu/velero/pkg/util/collections" + "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// backupQueueReconciler reconciles a Backup object +type backupQueueReconciler struct { + client.Client + Scheme *runtime.Scheme + logger logrus.FieldLogger + concurrentBackups int + backupTracker BackupTracker + frequency time.Duration +} + +const ( + defaultQueuedBackupRecheckFrequency = time.Minute +) + +// NewBackupQueueReconciler returns a new backupQueueReconciler +func NewBackupQueueReconciler( + client client.Client, + scheme *runtime.Scheme, + logger logrus.FieldLogger, + concurrentBackups int, + backupTracker BackupTracker, +) *backupQueueReconciler { + return &backupQueueReconciler{ + Client: client, + Scheme: scheme, + logger: logger, + concurrentBackups: max(concurrentBackups, 1), + backupTracker: backupTracker, + frequency: defaultQueuedBackupRecheckFrequency, + } +} + +func queuePositionOrderFunc(objList client.ObjectList) client.ObjectList { + backupList := objList.(*velerov1api.BackupList) + slices.SortFunc(backupList.Items, func(backup1, backup2 velerov1api.Backup) int { + if backup1.Status.QueuePosition < backup2.Status.QueuePosition { + return -1 + } else if backup1.Status.QueuePosition == backup2.Status.QueuePosition { + return 0 + } else { + return 1 + } + }) + return backupList +} + +// SetupWithManager adds the reconciler to the manager +func (r *backupQueueReconciler) SetupWithManager(mgr ctrl.Manager) error { + // For periodic requeue, only consider Queued backups, order by QueuePosition + gp := kube.NewGenericEventPredicate(func(object client.Object) bool { + backup := object.(*velerov1api.Backup) + return backup.Status.Phase == velerov1api.BackupPhaseQueued + }) + + s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerBackupQueue), mgr.GetClient(), &velerov1api.BackupList{}, r.frequency, kube.PeriodicalEnqueueSourceOption{ + Predicates: []predicate.Predicate{gp}, + OrderFunc: queuePositionOrderFunc, + }) + + return ctrl.NewControllerManagedBy(mgr). + For(&velerov1api.Backup{}, builder.WithPredicates(predicate.Funcs{ + UpdateFunc: func(ue event.UpdateEvent) bool { + backup := ue.ObjectNew.(*velerov1api.Backup) + return backup.Status.Phase == "" || backup.Status.Phase == velerov1api.BackupPhaseNew + }, + CreateFunc: func(ce event.CreateEvent) bool { + backup := ce.Object.(*velerov1api.Backup) + return backup.Status.Phase == "" || backup.Status.Phase == velerov1api.BackupPhaseNew + }, + DeleteFunc: func(de event.DeleteEvent) bool { + return false + }, + GenericFunc: func(ge event.GenericEvent) bool { + return false + }, + })). + Watches( + &velerov1api.Backup{}, + handler.EnqueueRequestsFromMapFunc(r.findQueuedBackupsToRequeue), + builder.WithPredicates(predicate.Funcs{ + UpdateFunc: func(ue event.UpdateEvent) bool { + oldBackup := ue.ObjectOld.(*velerov1api.Backup) + newBackup := ue.ObjectNew.(*velerov1api.Backup) + return oldBackup.Status.Phase == velerov1api.BackupPhaseInProgress && + newBackup.Status.Phase != velerov1api.BackupPhaseInProgress || + oldBackup.Status.Phase != velerov1api.BackupPhaseQueued && + newBackup.Status.Phase == velerov1api.BackupPhaseQueued && + r.backupTracker.RunningCount() < r.concurrentBackups + }, + CreateFunc: func(event.CreateEvent) bool { + return false + }, + DeleteFunc: func(de event.DeleteEvent) bool { + return false + }, + GenericFunc: func(ge event.GenericEvent) bool { + return false + }, + })). + WatchesRawSource(s). + Named(constant.ControllerBackupQueue). + Complete(r) +} + +func (r *backupQueueReconciler) detectNamespaceConflict(ctx context.Context, backup *velerov1api.Backup, earlierBackups []velerov1api.Backup) (bool, string, []string, error) { + nsList := &corev1api.NamespaceList{} + if err := r.Client.List(ctx, nsList); err != nil { + return false, "", nil, err + } + var clusterNamespaces []string + for _, ns := range nsList.Items { + clusterNamespaces = append(clusterNamespaces, ns.Name) + } + foundConflict, conflictBackup := detectNSConflictsInternal(backup, earlierBackups, clusterNamespaces) + return foundConflict, conflictBackup, clusterNamespaces, nil +} + +func detectNSConflictsInternal(backup *velerov1api.Backup, earlierBackups []velerov1api.Backup, clusterNamespaces []string) (bool, string) { + backupNamespaces := sets.NewString(namespacesForBackup(backup, clusterNamespaces)...) + for _, earlierBackup := range earlierBackups { + // This will never be true for the primary backup, but for the secondary + // runnability check for queued backups ahead of the current backup, we + // only care about backups ahead of it. + // Backup isn't earlier than this one, skip + if earlierBackup.Status.Phase == velerov1api.BackupPhaseQueued && + earlierBackup.Status.QueuePosition >= backup.Status.QueuePosition { + continue + } + if backupNamespaces.HasAny(namespacesForBackup(&earlierBackup, clusterNamespaces)...) { + return true, earlierBackup.Name + } + } + return false, "" +} + +// Returns true if there are backups ahead of the current backup that are runnable +// This could happen if velero just reconciled the one earlier in the queue and rejected it +// due to too many running backups, but a backup completed in between that reconcile and this one +// so exit, as the recent completion has triggered another reconcile of all queued backups +func (r *backupQueueReconciler) checkForEarlierRunnableBackups(backup *velerov1api.Backup, earlierBackups []velerov1api.Backup, clusterNamespaces []string) (bool, string) { + for _, earlierBackup := range earlierBackups { + // if this backup is queued and ahead of current backup, check for conflicts + if earlierBackup.Status.Phase != velerov1api.BackupPhaseQueued || + earlierBackup.Status.QueuePosition >= backup.Status.QueuePosition { + continue + } + conflict, _ := detectNSConflictsInternal(&earlierBackup, earlierBackups, clusterNamespaces) + // !conflict means we've found an earlier backup that is currently runnable + // so current reconcile should exit to run this one + if !conflict { + return true, earlierBackup.Name + } + } + return false, "" +} + +func namespacesForBackup(backup *velerov1api.Backup, clusterNamespaces []string) []string { + // Ignore error here. If a backup has invalid namespace wildcards, the backup controller + // will validate and fail it. Consider the ns list empty for conflict detection purposes. + nsList, err := collections.NewNamespaceIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...).ActiveNamespaces(clusterNamespaces).ResolveNamespaceList() + if err != nil { + return []string{} + } + return nsList +} +func (r *backupQueueReconciler) getMaxQueuePosition(lister *queuedBackupsLister) int { + queuedBackups := lister.orderedQueued() + maxPos := 0 + if len(queuedBackups) > 0 { + maxPos = queuedBackups[len(queuedBackups)-1].Status.QueuePosition + } + return maxPos +} + +func (r *backupQueueReconciler) findQueuedBackupsToRequeue(ctx context.Context, obj client.Object) []reconcile.Request { + backup := obj.(*velerov1api.Backup) + requests := []reconcile.Request{} + allBackups := &velerov1api.BackupList{} + if err := r.Client.List(ctx, allBackups, &client.ListOptions{Namespace: backup.Namespace}); err != nil { + r.logger.WithError(err).Error("error listing backups") + return requests + } + backups := r.newQueuedBackupsLister(allBackups).orderedQueued() + for _, item := range backups { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: item.GetNamespace(), + Name: item.GetName(), + }, + }) + } + return requests +} + +// Reconcile reconciles a Backup object +func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.logger.WithField("backup", req.NamespacedName.String()) + + log.Debug("Getting backup") + backup := &velerov1api.Backup{} + if err := r.Get(ctx, req.NamespacedName, backup); err != nil { + log.WithError(err).Error("unable to get backup") + return ctrl.Result{}, client.IgnoreNotFound(err) + } + switch backup.Status.Phase { + case "", velerov1api.BackupPhaseNew: + // queue new backup + allBackups := &velerov1api.BackupList{} + if err := r.Client.List(ctx, allBackups, &client.ListOptions{Namespace: backup.Namespace}); err != nil { + r.logger.WithError(err).Error("error listing backups") + return ctrl.Result{}, nil + } + lister := r.newQueuedBackupsLister(allBackups) + maxQueuePosition := r.getMaxQueuePosition(lister) + original := backup.DeepCopy() + backup.Status.Phase = velerov1api.BackupPhaseQueued + backup.Status.QueuePosition = maxQueuePosition + 1 + log.Infof("Queueing backup %v, queue position %v", backup.Name, backup.Status.QueuePosition) + if err := kube.PatchResource(original, backup, r.Client); err != nil { + return ctrl.Result{}, errors.Wrapf(err, "error updating Backup status to %s", backup.Status.Phase) + } + case velerov1api.BackupPhaseQueued: + // handle queued backup + // Find backups ahead of this one (InProgress, ReadyToStart, or Queued with higher position) + allBackups := &velerov1api.BackupList{} + if err := r.Client.List(ctx, allBackups, &client.ListOptions{Namespace: backup.Namespace}); err != nil { + r.logger.WithError(err).Error("error listing backups") + return ctrl.Result{}, nil + } + lister := r.newQueuedBackupsLister(allBackups) + if r.backupTracker.RunningCount() >= r.concurrentBackups { + log.Debugf("%v concurrent backups are already running, leaving %v queued", r.concurrentBackups, backup.Name) + return ctrl.Result{}, nil + } + earlierBackups := lister.earlierThan(backup.Status.QueuePosition) + foundConflict, conflictBackup, clusterNamespaces, err := r.detectNamespaceConflict(ctx, backup, earlierBackups) + if err != nil { + log.WithError(err).Error("error listing namespaces") + return ctrl.Result{}, nil + } + if foundConflict { + log.Infof("Backup %v has namespace conflict with %v, leaving queued", backup.Name, conflictBackup) + return ctrl.Result{}, nil + } + foundEarlierRunnable, earlierRunnable := r.checkForEarlierRunnableBackups(backup, earlierBackups, clusterNamespaces) + if foundEarlierRunnable { + log.Infof("Earlier queued backup %v is runnable, leaving %v queued", earlierRunnable, backup.Name) + return ctrl.Result{}, nil + } + log.Infof("Dequeueing backup %v, moving to ReadyToStart", backup.Name) + original := backup.DeepCopy() + backup.Status.Phase = velerov1api.BackupPhaseReadyToStart + backup.Status.QueuePosition = 0 + if err := kube.PatchResource(original, backup, r.Client); err != nil { + return ctrl.Result{}, errors.Wrapf(err, "error updating Backup status to %s", backup.Status.Phase) + } + r.backupTracker.AddReadyToStart(backup.Namespace, backup.Name) + log.Debug("Updating queuePosition for remaining queued backups") + queuedBackups := lister.orderedQueued() + newQueuePos := 1 + for _, queuedBackup := range queuedBackups { + if queuedBackup.Name != backup.Name { + original := queuedBackup.DeepCopy() + queuedBackup.Status.QueuePosition = newQueuePos + if err := kube.PatchResource(original, &queuedBackup, r.Client); err != nil { + log.WithError(errors.Wrapf(err, "error updating Backup %s queuePosition to %v", queuedBackup.Name, newQueuePos)) + return ctrl.Result{}, nil + } + newQueuePos++ + } + } + return ctrl.Result{}, nil + default: + log.Debug("Backup is not New or Queued, skipping") + return ctrl.Result{}, nil + } + + return ctrl.Result{}, nil +} + +// queuedBackupsLister manages a list of all backups Queued, ReadyToStart, or InProgress +// with methods to return specific subsets as needed +type queuedBackupsLister struct { + backups *velerov1api.BackupList +} + +func (r *backupQueueReconciler) newQueuedBackupsLister(backupList *velerov1api.BackupList) *queuedBackupsLister { + backups := []velerov1api.Backup{} + for _, backup := range backupList.Items { + if backup.Status.Phase == velerov1api.BackupPhaseQueued || + backup.Status.Phase == velerov1api.BackupPhaseInProgress || + backup.Status.Phase == velerov1api.BackupPhaseReadyToStart { + backups = append(backups, backup) + } + } + backupList.Items = backups + return &queuedBackupsLister{backupList} +} + +func (l *queuedBackupsLister) earlierThan(queuePos int) []velerov1api.Backup { + backups := []velerov1api.Backup{} + for _, backup := range l.backups.Items { + // InProgress and ReadyToStart backups have QueuePosition==0 + if backup.Status.QueuePosition < queuePos { + backups = append(backups, backup) + } + } + return backups +} + +func (l *queuedBackupsLister) orderedQueued() []velerov1api.Backup { + var returnList []velerov1api.Backup + orderedBackupList := queuePositionOrderFunc(l.backups).(*velerov1api.BackupList) + for _, item := range orderedBackupList.Items { + if item.Status.Phase == velerov1api.BackupPhaseQueued { + returnList = append(returnList, item) + } + } + return returnList +} diff --git a/pkg/controller/backup_queue_controller_test.go b/pkg/controller/backup_queue_controller_test.go new file mode 100644 index 000000000..2b479ed7e --- /dev/null +++ b/pkg/controller/backup_queue_controller_test.go @@ -0,0 +1,241 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + //metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + + //"sigs.k8s.io/controller-runtime/pkg/client/fake" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +func TestBackupQueueReconciler(t *testing.T) { + scheme := runtime.NewScheme() + velerov1api.AddToScheme(scheme) + + tests := []struct { + name string + priorBackups []*velerov1api.Backup + namespaces []string + backup *velerov1api.Backup + concurrentBackups int + expectError bool + expectPhase velerov1api.BackupPhase + expectQueuePosition int + }{ + { + name: "New Backup gets queued", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "InProgress Backup is ignored", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), + expectPhase: velerov1api.BackupPhaseInProgress, + }, + { + name: "Second New Backup gets queued with queuePosition 2", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), + }, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 2, + }, + { + name: "Queued Backup moves to ReadyToStart if no others are running", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).Result(), + expectPhase: velerov1api.BackupPhaseReadyToStart, + }, + { + name: "Queued Backup remains queued if no spaces available", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), + builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), + }, + concurrentBackups: 2, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Queued Backup remains queued if no spaces available including ReadyToStart", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), + builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseReadyToStart).Result(), + }, + concurrentBackups: 2, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Queued Backup remains queued if earlier runnable backup is also queued", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), + builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), + }, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(2).Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 2, + }, + { + name: "Queued Backup remains queued if in conflict with running backup", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("foo").Result(), + }, + namespaces: []string{"foo"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Queued Backup remains queued if in conflict with ReadyToStart backup", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseReadyToStart).IncludedNamespaces("foo").Result(), + }, + namespaces: []string{"foo"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Queued Backup remains queued if in conflict with earlier queued backup", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), + }, + namespaces: []string{"foo", "bar"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(2).IncludedNamespaces("foo", "bar").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 2, + }, + { + name: "Queued Backup remains queued if earlier non-ns-conflict backup exists", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), + builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), + }, + namespaces: []string{"foo", "bar", "baz"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(2).IncludedNamespaces("baz").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 2, + }, + { + name: "Running all-namespace backup conflicts with queued one-namespace backup ", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("*").Result(), + }, + namespaces: []string{"foo", "bar"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Running one-namespace backup conflicts with queued all-namespace backup ", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), + }, + namespaces: []string{"foo", "bar"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("*").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), + builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), + }, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), + expectPhase: velerov1api.BackupPhaseReadyToStart, + }, + { + name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups and no ns conflict found", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseReadyToStart).IncludedNamespaces("foo").Result(), + }, + namespaces: []string{"foo", "bar"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("bar").Result(), + expectPhase: velerov1api.BackupPhaseReadyToStart, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.backup == nil { + return + } + + backupTracker := NewBackupTracker() + initObjs := []runtime.Object{} + for _, priorBackup := range test.priorBackups { + initObjs = append(initObjs, priorBackup) + if priorBackup.Status.Phase == velerov1api.BackupPhaseReadyToStart { + backupTracker.AddReadyToStart(priorBackup.Namespace, priorBackup.Name) + } else if priorBackup.Status.Phase == velerov1api.BackupPhaseInProgress { + backupTracker.Add(priorBackup.Namespace, priorBackup.Name) + } + } + for _, ns := range test.namespaces { + initObjs = append(initObjs, builder.ForNamespace(ns).Result()) + } + initObjs = append(initObjs, test.backup) + + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) + logger := logrus.New() + log := logger.WithField("controller", "backup-queue-test") + r := NewBackupQueueReconciler(fakeClient, scheme, log, test.concurrentBackups, backupTracker) + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}} + res, err := r.Reconcile(t.Context(), req) + gotErr := err != nil + require.NoError(t, err) + assert.Equal(t, ctrl.Result{}, res) + assert.Equal(t, test.expectError, gotErr) + backupAfter := velerov1api.Backup{} + err = fakeClient.Get(t.Context(), types.NamespacedName{ + Namespace: test.backup.Namespace, + Name: test.backup.Name, + }, &backupAfter) + + require.NoError(t, err) + assert.Equal(t, test.expectPhase, backupAfter.Status.Phase) + assert.Equal(t, test.expectQueuePosition, backupAfter.Status.QueuePosition) + }) + } +} diff --git a/pkg/controller/backup_tracker.go b/pkg/controller/backup_tracker.go index 262886d56..a330b17b0 100644 --- a/pkg/controller/backup_tracker.go +++ b/pkg/controller/backup_tracker.go @@ -25,45 +25,90 @@ import ( // BackupTracker keeps track of in-progress backups. type BackupTracker interface { + // Add informs the tracker that a backup is ReadyToStart. + AddReadyToStart(ns, name string) // Add informs the tracker that a backup is in progress. Add(ns, name string) - // Delete informs the tracker that a backup is no longer in progress. + // Add informs the tracker that a backup has moved beyond InProgress + AddPostProcessing(ns, name string) + // Delete informs the tracker that a backup has reached a terminal state. Delete(ns, name string) - // Contains returns true if the tracker is tracking the backup. + // Contains returns true if backup is InProgress or post-InProgress Contains(ns, name string) bool + // RunningCount returns the number of backups which are ReadyToStart or InProgress + RunningCount() int } type backupTracker struct { - lock sync.RWMutex - backups sets.Set[string] + lock sync.RWMutex + readyToStartBackups sets.Set[string] + inProgressBackups sets.Set[string] + postProgressBackups sets.Set[string] } // NewBackupTracker returns a new BackupTracker. func NewBackupTracker() BackupTracker { return &backupTracker{ - backups: sets.New[string](), + readyToStartBackups: sets.New[string](), + inProgressBackups: sets.New[string](), + postProgressBackups: sets.New[string](), } } +func (bt *backupTracker) AddReadyToStart(ns, name string) { + bt.lock.Lock() + defer bt.lock.Unlock() + + bt.readyToStartBackups.Insert(backupTrackerKey(ns, name)) +} + func (bt *backupTracker) Add(ns, name string) { bt.lock.Lock() defer bt.lock.Unlock() - bt.backups.Insert(backupTrackerKey(ns, name)) + key := backupTrackerKey(ns, name) + bt.readyToStartBackups.Delete(key) + bt.inProgressBackups.Insert(key) +} + +func (bt *backupTracker) AddPostProcessing(ns, name string) { + bt.lock.Lock() + defer bt.lock.Unlock() + + key := backupTrackerKey(ns, name) + bt.readyToStartBackups.Delete(key) + bt.inProgressBackups.Delete(key) + bt.postProgressBackups.Insert(key) } func (bt *backupTracker) Delete(ns, name string) { bt.lock.Lock() defer bt.lock.Unlock() - bt.backups.Delete(backupTrackerKey(ns, name)) + key := backupTrackerKey(ns, name) + bt.readyToStartBackups.Delete(key) + bt.inProgressBackups.Delete(key) + bt.postProgressBackups.Delete(key) } +// Contains returns true if backup is InProgress or post-InProgress +// ignores ReadyToStart, since this is used to determine whether +// a backup is in progress and thus not able to be deleted now. func (bt *backupTracker) Contains(ns, name string) bool { bt.lock.RLock() defer bt.lock.RUnlock() - return bt.backups.Has(backupTrackerKey(ns, name)) + key := backupTrackerKey(ns, name) + return bt.inProgressBackups.Has(key) || bt.postProgressBackups.Has(key) +} + +// RunningCount returns the number of backups which are ReadyToStart or InProgress +// used by queue controller to determine whether a new backup can be started. +func (bt *backupTracker) RunningCount() int { + bt.lock.RLock() + defer bt.lock.RUnlock() + + return bt.inProgressBackups.Len() + bt.readyToStartBackups.Len() } func backupTrackerKey(ns, name string) string { diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 91acce344..d1010d294 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -59,6 +59,7 @@ type podTemplateConfig struct { repoMaintenanceJobConfigMap string nodeAgentConfigMap string itemBlockWorkerCount int + concurrentBackups int forWindows bool kubeletRootDir string nodeAgentDisableHostPath bool @@ -224,6 +225,12 @@ func WithItemBlockWorkerCount(itemBlockWorkerCount int) podTemplateOption { } } +func WithConcurrentBackups(concurrentBackups int) podTemplateOption { + return func(c *podTemplateConfig) { + c.concurrentBackups = concurrentBackups + } +} + func WithPriorityClassName(priorityClassName string) podTemplateOption { return func(c *podTemplateConfig) { c.priorityClassName = priorityClassName @@ -337,6 +344,10 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme args = append(args, fmt.Sprintf("--item-block-worker-count=%d", c.itemBlockWorkerCount)) } + if c.concurrentBackups > 0 { + args = append(args, fmt.Sprintf("--concurrent-backups=%d", c.concurrentBackups)) + } + deployment := &appsv1api.Deployment{ ObjectMeta: objectMeta(namespace, "velero"), TypeMeta: metav1.TypeMeta{ diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 1d0ffa568..5c7534774 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -271,6 +271,7 @@ type VeleroOptions struct { RepoMaintenanceJobConfigMap string NodeAgentConfigMap string ItemBlockWorkerCount int + ConcurrentBackups int KubeletRootDir string NodeAgentDisableHostPath bool ServerPriorityClassName string @@ -362,6 +363,7 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { WithPodResources(o.PodResources), WithKeepLatestMaintenanceJobs(o.KeepLatestMaintenanceJobs), WithItemBlockWorkerCount(o.ItemBlockWorkerCount), + WithConcurrentBackups(o.ConcurrentBackups), } if o.ServerPriorityClassName != "" { diff --git a/site/content/docs/main/customize-installation.md b/site/content/docs/main/customize-installation.md index d62945a35..7c1274ebf 100644 --- a/site/content/docs/main/customize-installation.md +++ b/site/content/docs/main/customize-installation.md @@ -158,7 +158,7 @@ At installation, You could set resource requests and limits for the Velero pod, |Memory limit|512Mi|N/A| {{< /table >}} -For Velero pod, through testing, the Velero maintainers have found these defaults work well when backing up and restoring 1000 or less resources. +For Velero pod, through testing, the Velero maintainers have found these defaults work well when backing up and restoring 1000 or less resources. If you are enabling concurrent backups and your backups tend to be large, you may need to increase these limits. For node-agent pod, by default it doesn't have CPU/memory request/limit, so that the backups/restores won't break due to resource throttling. The Velero maintainers have also done some [Performance Tests][13] to show the relationship of CPU/memory usage and the scale of data being backed up/restored. For repository maintenance job, it's no limit on resources by default. You could configure the job resource limitation based on target data to be backed up, some further settings please refer to [repository maintenance job][14].