From 9c09d049792cf0ccaf04a15ce312342e194c1c46 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Fri, 25 Jul 2025 13:10:58 -0400 Subject: [PATCH 01/37] feat: Add Queued and ReadyToStart phases to BackupPhase Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- config/crd/v1/bases/velero.io_backups.yaml | 2 ++ config/crd/v1/crds/crds.go | 2 +- pkg/apis/velero/v1/backup_types.go | 8 +++++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/config/crd/v1/bases/velero.io_backups.yaml b/config/crd/v1/bases/velero.io_backups.yaml index 47cbc3734..73dcdfc06 100644 --- a/config/crd/v1/bases/velero.io_backups.yaml +++ b/config/crd/v1/bases/velero.io_backups.yaml @@ -594,6 +594,8 @@ spec: description: Phase is the current state of the Backup. enum: - New + - Queued + - ReadyToStart - FailedValidation - InProgress - WaitingForPluginOperations diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index 130cae45e..764c1e5a4 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -30,7 +30,7 @@ import ( var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccXK\x8f۶\x13\xbf\xfbS\f\xf6\x7f\xfd\xcbnP\xb4(|K\xdc\x06\b\x9a\x04\v{\x91;-\x8dlf)\x92%\x87N\xdd\xc7w/\x86\x94lY\xa2\xad\xf5\x1e\x8a\xf2&r\u07bfy\xd9EQ̄\x95_\xd0yi\xf4\x12\x84\x95\xf8;\xa1\xe6/?\x7f\xfe\xc9ϥY\x1c\xde̞\xa5\xae\x96\xb0\n\x9eL\xb3Fo\x82+\xf1g\xac\xa5\x96$\x8d\x9e5H\xa2\x12$\x963\x00\xa1\xb5!\xc1מ?\x01J\xa3\xc9\x19\xa5\xd0\x15;\xd4\xf3\xe7\xb0\xc5m\x90\xaaB\x17\x85w\xaa\x0f\xdf\xcd\xdf\xfc8\xffa\x06\xa0E\x83K؊\xf29X\x87\xd6xI\xc6I\xf4\xf3\x03*tf.\xcd\xcc[,Y\xfaΙ`\x97p~Hܭ\xe6d\xf5\xbb(h\xdd\t:\xc6'%=\xfd\x9a}\xfe(=E\x12\xab\x82\x13*gH|\xf6R\xef\x82\x12nD\xc0\n|i,.\xe13\xdbbE\x89\xd5\f\xa0\xf54\xdaV\x80\xa8\xaa\x18;\xa1\x1e\x9dԄneTh\xba\x98\x15\xf0\xd5\x1b\xfd(h\xbf\x84y\x17\xddy\xe90\x06\xf6I6\xe8I46\xd2v\x01{\xbb\xc3\xf6\x9b\x8e\xac\xbc\x12\x84ca\x1c\xb9\xf9\xd9֧\xa3\xc5\v)\xe7@@\xef-I\xf4\xe4\xa4\xde\xcd\xceć7)\x14\xe5\x1e\x1b\xb1li\x8dE\xfd\xf6\xf1×\xef7\x17\xd7\x00\xd6\x19\x8b\x8ed\aO:\xbd\xf4\xeb\xdd\x02T\xe8K'-\xc5\xe4\xf8\xab\xb8x\x03`\x05\x89\v*\xceC\xf4@{\xecb\x8cUk\x13\x98\x1ah/=8\xb4\x0e=ꔙ|-4\x98\xedW,i>\x10\xbdA\xc7b\xc0\xefMP\x15\xa7\xef\x01\x1d\x81\xc3\xd2\xec\xb4\xfc\xe3$\xdb\x03\x99\xa8T\tBO\x10Q\xd4B\xc1A\xa8\x80\xff\a\xa1\xab\x81\xe4F\x1c\xc1!넠{\xf2\"\x83\x1f\xda\xf1\xc98\x04\xa9k\xb3\x84=\x91\xf5\xcb\xc5b'\xa9+\xca\xd24MВ\x8e\x8bX_r\x1b\xc88\xbf\xa8\xf0\x80j\xe1\xe5\xae\x10\xae\xdcK\u0092\x82Å\xb0\xb2\x88\x8e\xe8X\x98\xf3\xa6\xfa\x9fk\xcb\xd8_\xa8\x1d\x01\x9dN\xac\xa4;\xe0\xe1\xd2\x02\xe9A\xb4\xa2\x92\x8bg\x14\xf8\x8aC\xb7\xfee\xf3\x04\x9d%\t\xa9\x04ʙt\x14\x97\x0e\x1f\x8e\xa6\xd45\xba\xc4W;\xd3D\x99\xa8+k\xa4\xa6\xf8Q*\x89\x9a\xc0\x87m#\x89\xd3ව\x9e\x18\xba\xa1\xd8Ul\\\xb0E\b\x96K\xa7\x1a\x12|а\x12\r\xaa\x95\xf0\xf8/cŨ\xf8\x82Ax\x11Z\xfdv<$N\xe1\xed=t\xad\xf4\n\xb4\xc3\xf6\xb8\xb1X2\xb2\x1c\\f\x95\xb5,SM\xd5Ɓ\x18\xd1_F*\xdf\x02\xf8\xa4&\xba!\xe3\xc4\x0e?\x9a$sH4\x95v|\xde\xe5\x04u\x16s\xdbJ=\x01\xf3\x84\x19\x81\xb4\x17\xd4k\x06$\xa4>\xf5\x94\xac\x937\x90\x89\xe8\b\xee\x14Z\xe8\x12\xdf\xc7|\xd4\xe5q\xc2\xd1O\x19\x16vio\xbe\x81\xa9\tu_hkkƓ-\x82\v\xfa.c\xcf>\xae\x8c\xae\xe5nlh\x7f\x90]\x03wB\xc9\xc0\xdb\xf5@'{\xca\xc9u\xb6\xa5\xe82\x8f\x01\xa9\xe5.\xb8k\xe0\xd5\x12U5j!\x00:(%\xb6\n\x97@.\xe0\x95\x88\x8cj\xe52\"<\x1f'\x80[_\x10\x83\xd4\x15WK;\xacXI\x97\x8c\x9c\xfe\xa8+p\x97kJ\xff\xa0\x0e\xcdX]\x01\xcf\xc6J\x91\xb9w\xe8I\x96\x99\x87\x87\x87\xfb2\x80\xc5|\xa8\xb8\x1d\xd5\x12\xddkjr=\x90ѕc\x1d\x94j\x15\x14\xa5i\xac \xb9U\xd8\xcd\f\xc6\\&\x9ec.i`T\x86\xf0\x14'\x01c\xce*\x8cVG\b\x1e+\xf8\xb6G=\x02\xc3\xc3C\xd2\xfdpWI\x1cxQ\xc3\xd3j\xf7\x9ax|\xb9\x14\xd1\xefN\xe9\":\x96ZbϿ\xae\xfd\xf8\x8cHk\xaaֲ\x96/\xd6\xcc\x1d\x8eq_\x91\x0e\ac\xbe\xc87\xe6\x01M\xae\xa5\rH\x06Q{\xd1d\"A\xc1\xdf3\x9b\"C\x17\xcd28\x17g\x7f\xba\xe5\x95\xef\xd5\xd3I\tO\xbd&\xcc\v\xf8\x04\xee\x1f\xc7\x1c\x9da,\f\x88/\x18\xda~\xf02\xb8\xfaP\x96\x88\xd5x\x1d\x01Ʒ\x11\x94\x16\xfd\x82彮\xcb\xe5\x87\x14z/vSN~JTi\xd3kY@lM\xa0+\b\xd0>\xe7\xe3mT&,\xb5{\xe1\xa7\xec|d\x9a\\^\f\x96\x81[&\\k\xbf\x9f\xf1[\xe6v\x8d\xa2\x1a\xb7\xf0\x02>\x1b\xca?\xdd\xec\xc0%\xea~2M\x0e\x9d\x01={~\x81A+r\x94\x7fc\xaf%a\x93\x1d\xe7\xd7k%\x1dn\xe7\n\tO\xbfU\xf3d\x03\xd3WC\xae\x13h\xe9\x81W\xb9X9Ws\xa9\vٔc\xe9L\x97P:\x13\x85\x94\xce\xcd\r\an\x15U&\x12\xf7\x96\xd6\xd5P$\xb8_\x16\x8eI\x0f\x1c\xfa\xa0\xe8E\x0e\xac#i\x87_b<\xa7\xdf\xcb\xec\xc9\xd7\\:\x05l\xba\xd6x\x95⽐\xea\xea\U000e4cde\x84\xa3\xfb\xf2ws\xc1r\xfa\x9dķ\xfd\xbc\xfdO\xe6獝\xb7{\x14Ή\xe3\xf4\xe8\x1e]z\xfe\xc9^\xf5\x8c\xf3i\x9d\xe8߄\xed\xe9\x1f\x89%\xfc\xf9\xf7\xec\x9f\x00\x00\x00\xff\xff\x18\xd9g\x90\x9b\x14\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec}_s\x1c)\x92\xf8\xbb?\x05\xa1\xdf\xc3\xecntK\xeb\xf8\xdd]\\\xe8\xcd+\xdb;\x1d3c+,\x8d\xf6\x99\xae\xca\xeefDA-P-\xf7\xde\xddw\xbf \x81\xfa\xd3EUQ\xad\x96ƻg^luAB\xfe!3I\x12X.\x97oh\xc9\x1e@i&\xc55\xa1%\x83\xaf\x06\x84\xfdK_>\xfe\xa7\xbed\xf2j\xff\xf6\xcd#\x13\xf95\xb9\xa9\xb4\x91\xc5\x17вR\x19\xbc\x87\r\x13\xcc0)\xde\x14`hN\r\xbd~C\b\x15B\x1aj\x7f\xd6\xf6OB2)\x8c\x92\x9c\x83ZnA\\>VkXW\x8c\xe7\xa0\x10x\xe8z\xff\xe7˷\xffq\xf9\xefo\b\x11\xb4\x80k\xb2\xa6\xd9cU\xea\xcb=pP\xf2\x92\xc97\xba\x84̂\xdc*Y\x95פ\xf9\xe0\x9a\xf8\xee\xdcP\xff\x82\xad\xf1\aδ\xf9\xa9\xf5\xe3\xcfL\x1b\xfcP\xf2JQ^\xf7\x84\xbfi&\xb6\x15\xa7*\xfc\xfa\x86\x10\x9d\xc9\x12\xae\xc9'\xdbEI3\xc8\xdf\x10\xe2G\x8d].\xfd\x80\xf7o\x1d\x84l\a\x05uc!D\x96 \xdeݮ\x1e\xfe\xff]\xe7gBrЙb\xa5A\xdc\xff{Y\xffN\xfc(\tӄ\x92\ađ(Orbv\xd4\x10\x05\xa5\x02\r\xc2hbv@2Z\x9aJ\x01\x91\x1b\xf2S\xb5\x06%\xc0\x80n\xc1\xcbx\xa5\r(\xa2\r5@\xa8!\x94\x94\x92\tC\x98 \x86\x15@\xfe\xf0\xeevE\xe4\xfa7Ȍ&T\xe4\x84j-3F\r\xe4d/yU\x80k\xfb\xc7\xcb\x1aj\xa9d\tʰ@tWZ\x92\xd4\xfau\fW[,y\\+\x92[\x91\x02\x87\x96'1䞢\x16?\xb3c\xbaA\x1f\x85\xcc\xfeL\x85\x1f\xfe\xe5\x11\xe8;P\x16\f\xd1;Y\xf1\xdcJ\xe2\x1e\x94%`&\xb7\x82\xfd\xa3\x86\xad\x89\x91\xd8)\xa7\x06\xb4\xa5\x8c\x01%('{\xca+XX\xa2\x1cA.\xe8\x81(\xb0}\x92J\xb4\xe0a\x03}<\x8e_\xa4\x02\xc2\xc4F^\x93\x9d1\xa5\xbe\xbe\xba\xda2\x13\xe6W&\x8b\xa2\x12\xcc\x1c\xaep\xaa\xb0ue\xa4\xd2W9\xec\x81_i\xb6]R\x95혁̲\xf9\x8a\x96l\x89\x88\b\x9cc\x97E\xfe\xff\x82x\xe8N\xb7\xe6`\xc5V\x1b\xc5Ķ\xf5\x01\xe7\xc7\f\xf6ة\xe3\x84сr(6\\\xb0?Y\xd2}\xf9pw\xdf\x16T\xa6=SZ\xf2:\xc4\x1fKM&6\xa0\\\xbb\x8d\x92\x05\xc2\x04\x91;QE9\xe7\f\x84!\xbaZ\x17\xccX1\xf8{\x05\xda\xce\x01y\f\xf6\x06u\x10Y\x03\xa9\xca܊\xf1q\x85\x95 7\xb4\x00~C5\xbc2\xaf,W\xf4\xd22!\x89[m\xcdz\\ّ\xb7\xf5!(\xc8\x01\xd6:\xc5rWB֙h\xb6\x15۰\xccM\xa7\x8dT\x8d\xdeq:\xb0K\xa1\xf8Է%\xd3\xecN\xd0R魯g\x05\xc8\xca\x1cט\x925d\xde\xdd\xea\bJ\x18\xa1\x1f/\xea\xacJCn'\xed\x13e\x06\xc7|s\xb7\"\x0f\xa8\xacBkTZ\x95&\xa6R\xc2JI\xa4\xaf/@\xf3ý\xfcU\x03\xc9+\x14\xeeL\x01\xd2aAְ\xb1\x92\xa0\xc0\xb6\xb7\x9f@)K\x1b\x8d\x03\x90UO\xd9\xd8r\xbf\x03K[Zq\xe3\xe7\t\xd3\xe4\xed\x9fI\xc1Dez\xa26\xc8u\xa4\x145\xb4\x90{P\xa7\x10\xf1=5\xf4\x17\xdb\xf8\x88v\x16(A\xa8\x96xkO\xc7\xf5\x01?Ƹ\xed\xcajӂ\xc84\xb9\xb8 R\x91\vg\x81/\x16\xaeuŸY2\xd1\xee\xe3\x89q\x1ez\x99\x87\xbc\xa3\xa1c\xa8\xbe\x97\x1f\xb5\x13ޓh1\x00\xabE\x9a\xa7\x1d\x98\x1d(R\xca\xda\xe2m\x18\a\xa2\x0f\xda@\xe1\t\x13\xac\x88\xc7'\xd2\x13\xce\x1d\xce=\bm\xe9\xea\x11\xe9#/*\xce\xe9\x9a\xc351\xaa\x82\x01ڬ\xa5\xe4@\xc5\x04q\xbe\x806,;\ai\x1c\xa4\ba\x94\xffС\x00\x1aM\xfa\b\x84F@{\x9aY\xeb\xccy\x8b\xb0]\xaaD\xc7T*Ȭ־\xf6ր\x01G\v$$\xe1RlA\xb9ޭ\xa7\x12\x04L\x81\x15\xb8\x9cXE\xab\x80[kB6\x95\xd5\xc1\x97\xc4\xce\xeeA\x19`B\x1b\xa0\x11\xe1|\x06\x7f\xe0kƫ\x1c\xf2\x1b\xe7x\xddY\xff1\x0f^sOk\xa6\xf0\xe9\xc3(Do\x9d9\xcb\xd0\t\xf4\xfe\xde\x12\xfd֘\x986F\xfaP\x82s\x9d-+\xfd\xb0\x1b\xeb;\xaa\x0f4\x18\xdb\xe8\xe2O\x17\v\xe4p\xb7\xd7n\x1f\x9aP\x055Y\x92\xf5&\x14\xa59\xf4k3\x03E\x84\x8a\xa3\xfa$\x91\x9fT)z\x18\xe0f\xed\xff\x9f\x91\x9fC0\x8f8*B\xb5W\xe6\xe9q\xbf\xff\xca\\=\x0f\x1f5\xaev)\x13\x96\x7fv\xe1\xd9a\x9fv\xeb7K6!M\x04\x1e\x13\x0e\x1e.\xcdF\xb8\xf5;\x11\xeb,2?$\xe4\xb5ly\xe1\xfd\xa7\xa4\xd4N\xca\xc7)\xea\xfch\xeb4\x8b\"\x92aT\x85\xacaG\xf7L*\x8fzcj\xe1+d\x95\x89\xcezjH\xce6\x1bP\x16N\xb9\xa3\x1a\xb4[&\x0f\x13d\xd8}'-5\x12\xfdx\x84G\xc3H\xcb&\xc4|h\xe8֏8\xb6\x92\xa1\u0601Z\xf7\x1a\x8dq\xce\xf6,\xaf(G\xbbLE\xe6\xf0\xa1\xf5\xb8bZf\x84ɽ1G%\xd3\x15\xe7\x10\x04\xa4,\x93:+%)\xc0\xfa\xbc\x85]\x13\xf4\xab\x0ec\xbe\xa6\xd6W\x91C\xd8\x13d\x96\xaa8h\xdfU\x8end\xa33\x16\rS0\x10A8]\x03'\x1a8dF\xaa8E\xa6\xf8\xecJ\x8a\x12\x1c dD\xf3uW\x1a\r\x02# \t.\xe1v,\xdb9W\xcf\n\x11\xc2!\xb9\x04\xeb\xf0\x19B˒G\xccESF\x99\xef;\x19\x9b\xebM\x99\x98\xf5\xc7\xf0b\xf3\xbf)\t:\xb3)Q\xd26\xf3\xabK\xd9Z\x1c\xe2kڦ\xfck\x126h\xfe\x13\x84vd\xf6\x13\x8c\n%\xcb\xf4\xa0\xdcZ\xaa2З֝BOgA\x98\t\xbfN̈́\x8e\xcf\xd5\v\x96u\x88\xf0m\xf3f\xbe\xd0'\xb2&eN\xbc\x10c\xea.\xfe\t\xf9\x82&\xe3\xce[\x8cd\x9e\xfc\xdcn\xb5 lS\x13=_\x90\r\xe3\x06\xd4\x11\xf5OR\xf5\x813\xe7 F\x8a\xd5#\x18\xbe7\xd9\xee\xc3W\xeb\x82\xe9f\xa7*\x91.Ǎ\x9d#\x1b\xbc\xfd\xaey\x9e\x80K0\x8c\xcd\x14\x14\x18\x1e\xc7\x15S\xfb\x17t\xad\xde}z\x1f__\xb5K\x82\xe4\xf5\x10\x99\x98t\xae\xbc;¨=>\xef\u0087/\xe8\x03\xd5\v \xb7\x15\xb2 \x94<\xc2\xc1\xb9.T\x10\xcb\x1f\x1a*'t\xaf\x00\xf7dP\xce\x1e\xe1\x80`\xe2\x9b,\xfd\x92*\r\xae<\xc2!\xa5\xda\x11\r혘\xf6\x9bG\x96N\xf6\a$\x04\xc6\xd6S\xc5\xc0\x15?\x15\"[\x1a\xf1\x92\xa8KB\t\xb4?\x01\xcd$Qi\xf7\xd1ޥD\t\xf8A;^\xda\x19\xb3c%\xaaU\x8c8\xc8M2C]y\xa0\x9c\xe5uGn\x8e\xacĂ|\x92\xc6\xfe\xf3\xe1+\xd3~#\xf3\xbd\x04\xfdI\x1a\xfc\xe5E(\xea\x06\xfe\x92\xf4t=\xe0D\x13N\xcb[\x82\xb5\xb7\xe2\x9cM\xb3\xd2VӞi\xb2\x12v\xb9\xe2H\x92\xd8\x15\ueeba\xee\\GE\xa5q\x17MH\xb1ta\x9bXO\x9e\xdeRu\xc8\xfd\xecN}\x87\xf7\xd6X\xb8/n\xef\x97\xd3\f\xf2\xb0]\x83\x9b\x92\xd4\xc0\x96e\x89\xfd\x15\xa0\xb6@J\xab\xc2\xd3$\"Q\xb1zl\xe6\x89O\x9a\xf5n\x97\xaf\xcb\xc7z\x8f\x7fiM\xce\xd2C0\xb2H\xa0\x81\xd7\xdd\xf94>K;g\x13j\x05I\x98\xac:\xb0g9\\5\x85(\xcf \aZqtq&\xb9K\xf3\x1c\xf3\\(\xbf\x9daQf\xc8\xc2\\\xd5\xd0\x1a\xbb3\xc1\x05ŭ\x96\xff\xb2\x96\x16g\xd3\xff\x90\x922\xa5/\xc9;Li\xe1\xd0\xf9\xe6\x83f-0\t]bJ\x8a\x95\x9f=\xe5\xd6\xf6[\x05.\bp\xe7\t\xc8M\xcf/Z\x90\xa7\x9d\xd4\xcelכ8\x17\x8fpp;\x86\x93]\xb6\x95\xcc\xc5J\\8\x1f\xa2\xa70j\x87C\n~ \x17\xf8\xed\xe29\xaeT\xa2\xa4&V\xeb\x88hA\xcb4\tŔ\xa2TG\xdd.X\x83\x13b\x1b֩2\xd6\xc9\x1e\xc36IDK\xa9#\x1b\xf9\x03C\x99\x10\xde[\xa9\x8d\x8b\x97u|\xe6h@M\x86 \x1a\xa1\x1b\x97\xbf$UH6\xb1Jy*\xf4\xdb.\xf7;\xd0\xe0\xf7+|`\xce\x01\xb5+\xbb\x8bf~;m\x7f\xe1\xf6K\xb0\x13\x9a\xa1ǂmK%3\xd0ѽ\xec\xa6$؋HVF\x1b\xf7:\xe6H\xdd*ɥd\x8c\x87@CIwy-!f\xae\x17>|m\x05D\xedܷ\x7fO\xc9\xd8\xdcq\x11L\x19,\nz\x9c\xa6\x944\xc4\x1b\xd72\xcc\x06\x0f\xc8->ԶBM\x90j\xcbk\x01\xfc\x16\x1c\x85\x82\x89\x15v@\u07be\x80c\xe1uh,\xd9$VNseoB'\rw\xea\x1f\xdcT.%n\x15(\xe80\xaf\x1fUG?TH\xd3\nH\xccp7K\x99\xff\xa0Ɇ)m\xdaC\xd0\x03i*Q03\x17^\xe2\x83R'\xad\xbb>\xbb\x96\xadp\xd7N>\x85\xf4,G\x98D\xccq\x7f\t\b\xdb\x10f\b\x88LV\x02\x038v\x1ec\x17\x8e\xb8Nò\xd4I\x926\xfbm\x01Q\x15i\x04X\xa2\xa401\x1a\xe9iW\xffH\x19\x7f\t\xb6\x99\xa1,\xb6X9mN\x84\x14\xb7vB^A\xbf\xb2\xa2*\b-,\x8fИ\xb3\x02\xbaLo\x12\xdfl\v4\x13F\xda\x19Sr0\xe0\x93\xd7\x12ǐI\xa1Y\x0e\xb5q\xf5\x82 \x05\xa1dC\x19\xafT\xa2\x06\x9cE\xde9K\x11\xaf\tη\xc6H\xeb|\x89\xa4H\x88\xe6&\xfa\x8a\xe3ڸT\xe9\x1eߔ\x9b\xa5`\xbe\x97U*&1-\xf0̎\x96O\xa4\xa4\xe2\xf0\xdd\xd3J\x1d\xeawOk\xac|\xf7\xb4&\xcawO뻧\x95R\xf3\xbb\xa7\xf5\xdd\xd3j\x97\xff\x13\x9e\xd6Ԉ\xdcy\xbe\x81\x8f\x93\xa3Hت\x1e\x1b\xe2\b|\x9f\\\xe1s\xc0\x9f\x95\x8b\xb9\x8a\x83\x8a$\xfe\x0f\xa4uǔVc<\xea\xe4L;k\x82̻\xe3E\x13\xae\xe43\xb2\xeeC\xa7\xe7˺_\x8dBB\xbe\x89\x8c\xc0i\xe4;Ɂ~\x13\x1a\fݿ\xbd\xec~1ҧ\n\x92'fv\x11@O;\x10\xb8\xc3.\xb6\xed\x03\x00\xe1>\x02\x7f0\xffX\xc0\"\x80\xa4\"\x82q'y\xf5m\x06m\xb9#\x9fK\x17{\x9a\xedw\x8c\xc7TҒ\tON!\xec\xa6\b\x0e\xf8\xa5sw\xbb\xcfrd\xe2wI\r\x9c\x9f\x10\x98\x12\x11\x9bH\xfe;!\xe5/1\xb7\xf8\xd9\xdb\xf3)I}sV\xcc/\x96\xc0w\xfe\xb4\xbd$\xfaL\xa7\xe8͡\u038b\xa7\xe3\xbdb\x12\xde\xeb\xa4\xde%&ܝ/s>-\x1e{R\xe6\xd8t\xe8`8in2Un2\xb40\x85\xd8l\x94&S\xe0\xe6$\xbeMr'm\x9a\xbdZj۫%\xb4\xbdn\x1aۨ\x14\x8d~\x9c\x93\xa8\x16\xbf\x96\x86L\x1a[\xfeZ\xc2v*\x19\xa4긯'\xad\xaf>\x1f\xc1\xb0\x8c\x0f\xae\xdd+\xf9\xc8E\xc5\r+9n\xa4\xeeY\x1e\r6\x98\x1d\x1c\xea\v4~\x93x\xf4\xd4\xdf\x04\xf3\xf9K-\xb5\x97G\x9e>\xd5\xe4\t8'46\xafz\x98g\xee&\xa6L.\xc1\xda#;;\xfd\xc5 \xfe\xfa\xa6\x85\x13w<]\x8bV\xad\x88\x85\x98\xa8\x18\xbeEf\xd0p\xa4蛞\a\xeb\xfcp\xfc\xed\xef\x15\xa8\x03\xc1{lj?\xa79\x04\xe6'\xa6\xb6\v\xb1\xa0*\xbc\xda\x1a\x8a\x9f\xf7\x9c\xfef*\x93w\xc2Y\xdd\xe3\xf1`\x1b\xab#\x9aE\x8dU|v\xbd\x12\xedc\xa0\xb9\x90u\xebH\xb3)\a9\xf5\xb4\xd4\xcb.q\xe6/r&\xbd\x8at\xcf\xefw:\x05u\xca駴\x04\x80\xc9\xd3N/\xb5\xe4\x99Z\xf4$\xfbyi\xa7\x99\xe6m\x16\xbe\xe0饗8\xb5\x94H\xa9\x94SJ\xf3\xe8\xf4\n\xa7\x92^\xf54\xd2k\x9dBJ>}\x94\x94Ⓖ\v\x9c\x9a\xa2r\xe2q\x9a\xe9=\xde\xf1\xd3D\t\xa7\x88\x12v\x7f\xa7\x91<\x01\xbd\x84SB\xf3N\a%\xf0,u*\xbe\xe2)\xa0W<\xfd\xf3ڧ~&$k\xe2\xf3\xbc\xd3='oYH\x95\x83\x1a\xdd\xf6I\x95\xc2Q\xf9KY\xdbt\ar\xb4\xdf\x11n\xfd\xb3\xb5:\xfe2\x9a\a\x7f\xd1(^);\xb4}i%\xad\xe5mt\xf6\xa2\x1a\xf7\xa7\xebL\xfa{f\xddv\x95\x86\x92*\xbc\xbbx}p\xe9,Q\xd3\xfc\x81f\xbb#\xe8;\xaa\xc9F\xaa\x82\x1arQo\x00^9\xe0\xf6\xef\x8bKB>\xca:'\xa2}/\x8ffE\xc9\x0fv\x85B.\xda\rN\x93\x80\xa8\xb4\x85\xden%gY\xc4w\x8b\xde\xcd\xe4*\xf7.\xcb\xc0\x1b\xa3\xb2v\xca@i+\xc6]7t\xf3\xbaW`n$\xe7\xf2i\xe6ڟ\x96\xec\xafxs\xf73\xa2C\xefnW\b#\x88\a^\x05^'g\xd5ج\xc1\x9a\xe5\x06ϡ\xb9\xbf\xdat v\xf3\x1cۗ\xe3B\xee\xeeA\x0en\x81W\x9d\x99\xb4\xda\xe5v\xe5\xc61ԋ\x95\x19*\x0eDbF\x8d\xd91\x95/K\xaa\xcc\xc1%j,:c\b\xb6t,\xba3h=\xfaw;G\xc9\x1b\xaet\xc6\x1d\xcaC\xd9\xdd\xf4=\xa6\xdd)\xe3\x18>\xbd8yn\xf1\x8c\xe3\x18vK\x96H\xa9\xc8\xcf\xd1̯\xb3Eʹ\xbf\x99\xf8\x17\xb9\x87\xf7\xd1\xe8Y\x872\xb3\x19\xfe葫mT\x80\x7f\xe6\x1c\x04\xeb\xf3p\x0e\xfc#\xe3\xa0ݰ\x12\x14\xf0m\xbfU\xad\x8f\xabb\xed|\xb8\x8d\xfdXw0`\xe3\x1cZ\x18\x8a.AY/\xca\x05\xad+\x1ddu\x18\xf1\x86#L\x18\xd8B\x7f\x158\xa2\x81ݭ\xd2h>\x83:\xc1\xb5\xccO\xb1\xf8V\a\xf9\x87\xe1\x96G\x9cl\x85\xbcb7\xee9'\xe4\xf6\xe1F\x93J\xe4\x18.~\xf8\xeb\xdd,\xa9\xdbwn\xae\x0f\xb3uJ\xa9>\xc4[\xb5\x9c㖾pޱ\xdcD\x10\x18\x82\xd3z\a\xe4\x89\x19\x7fq\xd7yoZ\x1dZ\xf2\f\xbdp\x80W\xfaO\xbfq\xe0n\xfe\xf7/\xa3\xf8\xe9X)\xbc&տ\n\x80\u05ca\x9e\xf4\xcc\xc1\xbaNت\x93\xbf\xf4;c\xa0(M\xccטV\x87\x7f\x19\x03X\xfbi\xd2Pޚ\x954T\x88y\xda\xfa \xb2\xb1\xc42\xaf\x8dF\xb896\x1fc\x04\xb8\xf1\xe7!\xceF\x80\x1a\xe0\x10\x01t\x95e\xa0\xf5\xa6\xe2\xfcP\x1f\xc7\xf8F\xa8\xf1\x912~>R8h\x83\x82`\xd1\x1b\x854\x89\xb0O\xf7\x06\x91\x87\x99\x1e\x8e*\xcd#\x85\xe7\x82φԆ\x16'=\xd8p\xd3\a\x83O\xf6\xa8\xbc\x95TI\xeb\xb1Sݰ?f\\\x1ap\xae%.\xb2,4\xc8\t\xecA\x10k\x9d\x1d\x89ÛS3\xa1\xf8\x13\xae\xce\xc2\x05{\x17B!ч\x89\x88\x8fvh|\x00\xe7\a]\xc3\xc4\\Q|ϤO\x84\xbe\xf3\xeb\xa2\x15\xd7\xd6\xfb\x87\xa5\x05q\x9a\xd7\x1a\xd5͙f]\xbb\xf0<%ws\xb7\x1a\x02w\x8a\x8a\xeb?\xf7\xf2\xcci\xdcG\xf7Y*\xad\x8f\xee,\x85\x16\x81X\xcb\xf8\xf9qǩ~ڥ\xee\xd8\xd29\x1cY8CG9\xf7\a\x1d\vКn\xc3m\xeeOv\xe9\xb1\x05\x01.<\xe76O\"@\x9bSqݻ\xccݔ\xa1\x99\xa9\xa8\xef $\xf8\xb6j\xfd\xa0\t\x971\xa8\xf8\xa0\v\v/\x85\x855\xd9LB}-\x99JY\xc3}\xa8+Zڠ'\x8c\xdci\xdev\x03ζ̮u,\xe7\xb6T\xad\xe9\x16\x96\x99\xe4\x1cP[\xf7\xc7\xf5\x92sݟ=\xfc\x02TO\xa2\xf6\xb1]\xd7\xef\x00:n\xbb\x8do\xea\xd2\xdd\xf1\xf5.\xc3\x144\x0f\xe9\xf5\x06$\xb1\xe3Y\x8e\xb2\xa3B\xf4\x95\xb9\xfeH\xdbuì\xf3j\xd9\xc7y\xfd#s\v\x1f\x17\x88\xcbcA\x7f\x93jA\n&\xec?T\xe4n\x03/4\x9e5\xfe\x9d\x94\x8fw\x11'\xb67\xf8\x1f\xeb\x8a\xcdV\a\x13n\xd8x`t-+\xbf\xfb^;\xb4\xf1m\x15\xbc\x99\xff\xcc\xcbM\x849b\x0fz\xe8\fFt\x7f\xec@\x9a4\x05\xae\xe7\x01Xw\xe1%3\xce\x0f\x8bc\xc8G\xaf&6\xb0[/\x17x7\xa0\xb9\x8f`\xa0\xa3\xb0#\x15\x05R_|\xd1V觬z=\x99\x87\x9c\xc9\x1e\x8d\x7flj\x0f\xd1\xd1\r\xb3\xe5\xee\r \xd8q\x02ϻ`\xc7g*&\x84\xff\xd6֩\xef.h-\xdcB\x96\xd8`\x94.~\xf6}I>A\x7f\xbb\xc2\x1dg\x87\x1c33pVE\xaa\xacĭ\x92[\x05\xba/tK\xf27\xca\f\x13ۏR\xdd\xf2j\xcb\xc4\xe7\xe1\xa3;c\x95o\xa92\xcc\n\xad\x1bOl\xa0LP\xce\xfe\x11\xd3O\xed\x8fӀn\x06\x17JK\x920\x8c\xa1\x0f\xef\xc1\xfa\xaa\x83\xeb\xfb\xa8*,=]O\xf1;\x02O\xa6tc\xed\x134>E\xe8\xf6\x92|\x92\xd1\t\xeeӚX\x17\xa6u\xad@\x9b%l6R\x19\xb7\xeb\xbc\\\x12\xb6\tA\x04\xab;0\xfe\xe5ޜ$,\xb6]\\'\x8c4f\b\x83\xd7\n\xad)^I_Ѓ\xdba\xa2YVYO\xe9J\x1b\xca#\x8eʳ\x148Fk\xec$\x82\xfc\xd7g\xedȭڀ\xfa\xc1C\xecǑ\x14/\xc5p\xde\x1b\xb7(\x82 O\x8a\x19c}#9\x92\x12\xe0Ie\xac\x8f\xc49і\xd4'E\x11\x89S\x87\xab\xe1Ԛ4\x94\xefk(Cj\xd6c\x8d/,\xae\x916\xc4\xfa\xaf\x98E\xe4kY6g;*\xb6\x837\r씬\xb6\xbb \xc9\x03N1\xc9+\xc0\xa0+\xaa\x14\x1d\x1e\b6\x95\x12\xad\x94\x80\x91\xe3\xdb$\b\x03\x0e\x97f\x8f\xa4*\x17\xfe\x01^\xff\xbe\xf2\x95\x7f\xcbd\xb9Q\xb2X\xfa~1&\xba\xf0;\xf2\x8aI끘]\x94\xea\xc4y\xdf\xfe\xb9\x00\x94\x84\xb2\x04A\xa8\xf6='\xdc\xf8t\xb2\xb9ц*\xf3\xacp\xc4]\a\xc2D$\x02\xbb\x8b#q\xe7\x13\x13ܕW7\xfe\xa1\xd1\x1a\xf0\x82h&\xc2\x13\xcf.\xc9\xc1\xc9Gt\xcfK\xe0\x93\x8cR\xc5\xf3\x0e\xc7C\v]\x84^7\xaa\xb0\xafm퇓\x17\x9d\x0fG0\x8e\x8e/\xe3\v\x9cu\x95\xb0P\xfc\x03\x8bE\xbe1a5\xb3\xa8\xfc\xf1w?\x96\xbcOZ\xd4\xc4)2\xb6\xc6\xc1\xe5\xcb\xf0b\xa5\xfb\xe2\xe6-\a\xebzi\x80\xee\xf2i\xd62y\x7fƸ\xd19\x83F\xe11\xf3\xf3DM\xf6g\f\x17\xbdX\xac\xe8\xbc(?Q|\n\xf9\xa4Y\xfb7\xdf6\x12,\xf2`\xcf\x1d.jE\x8b\xc2\xc0_5^\x14\xb5J\xbd\x1fQO\xe7-m\xe1{\xf2\xbf\xfco\x00\x00\x00\xff\xff\xaf\x05\xd1\xf3\xa2\x81\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c)\x92\xef\xfe\x15\x84\xeeav7\xba\xa5u\xdcG\\\xe8\xcd+\xdb;\x1d;c\xeb,\x8d\xf6\x99\xae\xca\xeefDA-P-\xf7\xde\xdd\x7f\xbf \x81\xfa袪\xa8VK\xe3\xdd3/\xb6\xba !?\xc8L\x92\x04\x96\xcb\xe5\x1bZ\xb2\aP\x9aIqMh\xc9\xe0\xab\x01a\xffҗ\x8f\xff\xa9/\x99\xbcڿ}\xf3\xc8D~Mn*md\xf1\x05\xb4\xacT\x06\xefa\xc3\x043L\x8a7\x05\x18\x9aSC\xaf\xdf\x10B\x85\x90\x86ڟ\xb5\xfd\x93\x90L\n\xa3$砖[\x10\x97\x8f\xd5\x1a\xd6\x15\xe39(\x04\x1e\xba\xde\xff\xf1\xf2\xed\x7f\\\xfe\xfb\x1bB\x04-\xe0\x9a\xaci\xf6X\x95\xfar\x0f\x1c\x94\xbcd\xf2\x8d.!\xb3 \xb7JV\xe55i>\xb8&\xbe;7\xd4?ak\xfc\x813m\xfe\xd2\xfa\xf1'\xa6\r~(y\xa5(\xaf{\xc2\xdf4\x13ۊS\x15~}C\x88\xced\t\xd7\xe4\x93\xed\xa2\xa4\x19\xe4o\b\xf1\xa3\xc6.\x97~\xc0\xfb\xb7\x0eB\xb6\x83\x82\xba\xb1\x10\"K\x10\xefnW\x0f\xffz\xd7\xf9\x99\x90\x1ct\xa6Xi\x10\xf7\xffYֿ\x13?J\xc24\xa1\xe4\x01q$ʓ\x9c\x98\x1d5DA\xa9@\x830\x9a\x98\x1d\x90\x8c\x96\xa6R@\xe4\x86\xfc\xa5Z\x83\x12`@\xb7\xe0e\xbc\xd2\x06\x14ц\x1a \xd4\x10JJɄ!L\x10\xc3\n \xbf{w\xbb\"r\xfd+dF\x13*rB\xb5\x96\x19\xa3\x06r\xb2\x97\xbc*\xc0\xb5\xfd\xfde\r\xb5T\xb2\x04eX \xba+-Ij\xfd:\x86\xab-\x96<\xae\x15ɭH\x81C˓\x18rOQ\x8b\x9f\xd91ݠ\x8fBf\x7f\xa6\xc2\x0f\xff\xf2\b\xf4\x1d(\v\x86蝬xn%q\x0f\xca\x120\x93[\xc1\xfe^\xc3\xd6\xc4H\xec\x94S\x03\xdaRƀ\x12\x94\x93=\xe5\x15,,Q\x8e \x17\xf4@\x14\xd8>I%Z\xf0\xb0\x81>\x1e\xc7\xcfR\x01ab#\xaf\xc9ΘR__]m\x99\t\xf3+\x93EQ\tf\x0eW8Uغ2R\xe9\xab\x1c\xf6\xc0\xaf4\xdb.\xa9\xcav\xcc@f\xd9|EK\xb6DD\x04α\xcb\"\xff\x97 \x1e\xbaӭ9X\xb1\xd5F1\xb1m}\xc0\xf91\x83=v\xea8at\xa0\x1c\x8a\r\x17\xecO\x96t_>\xdcݷ\x05\x95iϔ\x96\xbc\x0e\xf1\xc7R\x93\x89\r(\xd7n\xa3d\x810A\xe4NTQ\xce9\x03a\x88\xae\xd6\x053V\f\xfeV\x81\xb6s@\x1e\x83\xbdA\x1dD\xd6@\xaa2\xb7b|\\a%\xc8\r-\x80\xdfP\r\xaf\xcc+\xcb\x15\xbd\xb4LH\xe2V[\xb3\x1eWv\xe4m}\b\nr\x80\xb5N\xb1ܕ\x90u&\x9am\xc56,s\xd3i#U\xa3w\x9c\x0e\xecR(>\xf5m\xc94\xbb\x13\xb4\xd4;i\xeeY\x01\xb22\xc75\xa6d\r\x99w\xb7:\x82\x12F\xe8Nj:\xabҐ\xdbI\xfbD\x99\xc11\xdfܭ\xc8\x03*\xab\xd0\x1a\x95V\xa5\x89\xa9\x94\xb0R\x12\xe9\xeb\v\xd0\xfcp/\x7f\xd1@\xf2\n\x85;S\x80tX\x905l\xac$(\xb0\xed\xed'P\xca\xd2F\xe3\x00d\xd5S6\xb6\xdc\xef\xc0ҖV\xdc\xf8y\xc24y\xfbGR0Q\x99\x9e\xa8\rr\x1d)E\r-\xe4\x1e\xd4)D|O\r\xfd\xd96>\xa2\x9d\x05J\x10\xaa%\xde\xda\xd3q}\xc0\x8f1n\xbb\xb2ڴ 2M..\x88T\xe4\xc2Y\xe0\x8b\x85k]1n\x96L\xb4\xfbxb\x9c\x87^\xe6!\xefh\xe8\x18\xaa\xef\xe5G\xed\x84\xf7$Z\f\xc0j\x91\xe6i\af\a\x8a\x94\xb2\xb6x\x1bƁ\xe8\x836Px\xc2\x04+\xe2\xf1\x89\xf4\x84s\x87s\x0fB[\xbazD\xfaȋ\x8as\xba\xe6pM\x8c\xaa`\x806k)9P1A\x9c/\xa0\r\xcb\xceA\x1a\a)B\x18\xe5?t(\x80F\x93>\x02\xa1\x11Оf\xd6:s\xde\"l\x97*\xd11\x95\n2\xab\xb5\xaf\xbd5`\xc0\xd1\x02\tI\xb8\x14[P\xaew\xeb\xa9\x04\x01S`\x05.'V\xd1*\xe0֚\x90Meu\xf0%\xb1\xb3{P\x06\x98\xd0\x06hD8\x9f\xc1\x1f\xf8\x9a\xf1*\x87\xfc\xc69^w\xd6\x7f̃\xd7\xdcӚ)|\xfa0\n\xd1[g\xce2t\x02\xbd\xbf\xb7D\xbf5&\xa6\x8d\x91>\x94\xe0\\g\xcbJ?\xec\xc6\xfa\x8e\xea\x03\r\xc66\xba\xf8\xc3\xc5\x029\xdc\xed\xb5ۇ&TAM\x96d\xbd\tEi\x0e\xfd\xda\xcc@\x11\xa1\xe2\xa8>I\xe4'U\x8a\x1e\x06\xb8Y\xfb\xffg\xe4\xe7\x10\xcc#\x8e\x8aP\xed\x95yz\xdc\xef?3W\xcf\xc3G\x8d\xab]ʄ\xe5\x9f]xvا\xdd\xfa͒MH\x13\x81DŽ\x83\x87K\xb3\x11n\xfdF\xc4:\x8b\xcc\x0f\ty-[^x\xff!)\xb5\x93\xf2q\x8a:?\xda:͢\x88d\x18U!k\xd8\xd1=\x93ʣޘZ\xf8\nYe\xa2\xb3\x9e\x1a\x92\xb3\xcd\x06\x94\x85S\xee\xa8\x06\xed\x96\xc9\xc3\x04\x19v\xdfIK\x8dD?\x1e\xe1\xd10Ҳ\t1\x1f\x1a\xba\xf5#\x8e\xadd(v\xa0ֽFc\x9c\xb3=\xcb+\xca\xd1.S\x919|h=\xae\x98\x96\x19aro\xccQ\xc9t\xc59\x04\x01)ˤ\xceJI\n\xb0>oa\xd7\x04\xfd\xaaØ\xaf\xa9\xf5U\xe4\x10\xf6\x04\x99\xa5*\x0e\xdaw\x95\xa3\x1b\xd9\xe8\x8cE\xc3\x14\fD\x10N\xd7\xc0\x89\x06\x0e\x99\x91*N\x91)>\xbb\x92\xa2\x04\a\b\x19\xd1|ݕF\x83\xc0\bH\x82K\xb8\x1d\xcbv\xceճB\x84pH.\xc1:|\x86в\xe4\x11sєQ\xe6\xfbN\xc6\xe6zS&f\xfd1\xbc\xd8\xfcoJ\x82\xcelJ\x94\xb4\xcd\xfc\xeaR\xb6\x16\x87\xf8\x9a\xb6)\xff\x9c\x84\r\x9a\xff\x04\xa1\x1d\x99\xfd\x04\xa3B\xc92=(\xb7\x96\xaa\f\xf4\xa5u\xa7\xd0\xd3Y\x10f¯S3\xa1\xe3s\xf5\x82e\x1d\"|ۼ\x99/\xf4\x89\xacI\x99\x13/Ę\xba\x8b\x7f@\xbe\xa0ɸ\xf3\x16#\x99'?\xb5[-\b\xdb\xd4D\xcf\x17dø\x01uD\xfd\x93T}\xe0\xcc9\x88\x91b\xf5\b\x86\xefM\xb6\xfb\xf0պ`\xba٩J\xa4\xcbqc\xe7\xc8\x06o\xbfk\x9e'\xe0\x12\fc3\x05\x05\x86\xc7q\xc5\xd4\xfe\x05]\xabw\x9f\xde\xc7\xd7W\xed\x92 y=D&&\x9d+\xef\x8e0j\x8fϻ\xf0\xe1\v\xfa@\xf5\x02\xc8m\x85,\b%\x8fpp\xae\v\x15\xc4\xf2\x87\x86\xca\t\xdd+\xc0=\x19\x94\xb3G8 \x98\xf8&K\xbf\xa4J\x83+\x8fpH\xa9vDC;&\xa6\xfd摥\x93\xfd\x01\t\x81\xb1\xf5T1p\xc5O\x85ȖF\xbc$\xea\x92P\x02\xedO@3IT\xda}\xb4w)Q\x02~Ў\x97v\xc6\xecX\x89j\x15#\x0er\x93\xccPW\x1e(gyݑ\x9b#+\xb1 \x9f\xa4\xb1\xff|\xf8ʴ\xdf\xc8|/A\x7f\x92\x06\x7fy\x11\x8a\xba\x81\xbf$=]\x0f8ф\xd3\xf2\x96`\xed\xad8gӬ\xb4մg\x9a\xac\x84]\xae8\x92$v\x85\xbb\xae\xae;\xd7QQi\xdcE\x13R,]\xd8&֓\xa7\xb7T\x1dr?\xbbS\xdf\xe1\xbd5\x16\xee\x8b\xdb\xfb\xe54\x83\x8d\xcf\xd2\xceلZA\x12&\xab\x0e\xecY\x0eWM!\xca3ȁV\x1c]\x9cI\xee\xd2<\xc7<\x17\xcaogX\x94\x19\xb20W5\xb4\xc6\xeeLpAq\xab忭\xa5\xc5\xd9\xf4\xbf\xa4\xa4L\xe9K\xf2\x0eSZ8t\xbe\xf9\xa0Y\vLB\x97\x98\x92b\xe5gO\xb9\xb5\xfdV\x81\v\x02\xdcy\x02r\xd3\xf3\x8b\x16\xe4i'\xb53\xdb\xf5&\xce\xc5#\x1c\u070e\xe1d\x97m%s\xb1\x12\x17·\xe8)\x8c\xdaᐂ\x1f\xc8\x05~\xbbx\x8e+\x95(\xa9\x89\xd5:\"Z\xd02MB1\xa5(\xd5Q\xb7\v\xd6\xe0\x84؆u\xaa\x8cu\xb2ǰM\x12\xd1R\xea\xc8F\xfe\xc0P&\x84\xf7Vj\xe3\xe2e\x1d\x9f9\x1aP\x93!\x88F\xe8\xc6\xe5/I\x15\x92M\xacR\x9e\n\xfd\xb6\xcb\xfd\x0e4\xf8\xfd\n\x1f\x98s@\xed\xca\ue899\xdfN\xdb_\xb8\xfd\x12\xec\x84f\xe8\xb1`\xdbR\xc9\ftt/\xbb)\t\xf6\"\x92\x95\xd1ƽ\x8e9R\xb7Jr)\x19\xe3!\xd0P\xd2]^K\x88\x99\xeb\x85\x0f_[\x01Q;\xf7\xed\xdfS26w\\\x04S\x06\x8b\x82\x1e\xa7)%\r\xf1Ƶ\f\xb3\xc1\x03r\x8b\x0f\xb5\xadP\x13\xa4\xda\xf2Z\x00\xbf\x05G\xa1`b\x85\x1d\x90\xb7/\xe0Xx\x1d\x1aK6\x89\x95\xd3\\ٛ\xd0IÝ\xfa\a7\x95K\x89[\x05\n:\xcc\xebG\xd5\xd1\x0f\x15Ҵ\x02\x123\xdc\xcdR\xe6?h\xb2aJ\x9b\xf6\x10\xf4@\x9aJ\x14\xcc̅\x97\xf8\xa0\xd4I\xeb\xaeϮe+ܵ\x93O!=\xcb\x11&\x11s\xdc_\x02\xc26\x84\x19\x02\"\x93\x95\xc0\x00\x8e\x9d\xc7\u0605#\xaeӰ,u\x92\xa4\xcd~[@TE\x1a\x01\x96()L\x8cFz\xda\xd5?R\xc6_\x82mf(\x8b-VN\x9b\x13!ŭ\x9d\x90WЯ\xac\xa8\nB\v\xcb#4欀.ӛ\xc47\xdb\x02̈́\x91vƔ\x1c\f\xf8\xe4\xb5\xc41dRh\x96Cm\\\xbd HA(\xd9P\xc6+\x95\xa8\x01g\x91w\xceR\xc4k\x82\xf3\xad1\xd2:_\")\x12\xa2\xb9\x89\xbe\xe2\xb86.U\xba\xc77\xe5f)\x98\xefe\x95\x8aIL\v<\xb3\xa3\xe5\x13)\xa98|\xf7\xb4R\x87\xfa\xdd\xd3\x1a+\xdf=\xad\x89\xf2\xdd\xd3\xfa\xeei\xa5\xd4\xfc\xeei}\xf7\xb4\xda\xe5\xff\x85\xa755\"w\x9eo\xe0\xe3\xe4(\x12\xb6\xaadž8\x02\xdf'W\xf8\x1c\xf0g\xe5b\xae\xe2\xa0\"\x89\xff\x03i\xdd1\xa5\xd5\x18\x8f:9\xd3Κ \xf3\xeexф+\xf9\x8c\xac\xfb\xd0\xe9\xf9\xb2\xeeW\xa3\x10ϔu\xef\x87=\xedc\x9f\x94s\x1f\x882/;{\xe1\x135\n\xa0!\xac\xee\xb6\xe1cx\rI\xc8D\xff\xaf\x9c\x98\xdb\xcb\x1a;\xa3|\xbcx\x16\x7f\xb2\x8cDYz\xf1\x87\x8bo\x8f\xfc\xe7!\xf8 \x89\xfb\xb4\xf3\xe7\x9b#P\xed\n\xb4\x9d\x16\xd6\xcd\xc2\xfb6\xc5\xf8,r\x9b\x9a\x89_\x131\x02\xab+\x92GT\xfcVu\x81\x81\xe2s\xe9-\xd23N\xaa\xae\"p\x92ΪR}\x10\xd9NI!+\xed\xa3\x12\x16ֻ\xcc\x1dh\x0f c\xc2\x1a\x9d\xe1\xffFv\xb2\x8ad\x82\x8f\x90o\"#p\x1a\xf9Nr\xa0߄\x06C\xf7o/\xbb_\x8c\xf4\xa9\x82䉙]\x04\xd0\xd3\x0e\x04\uec0bm\xfb\x00@\xb8\x8f\xc0\x1f\xcc?\x16\xb0\b \xa9\x88`\xdcI^}\x9bA[\xee\xc8\xe7\xd2Şf\xfb\x1d\xe31\x95\xb4d\u0093S\b\xbb)\x82\x03~\xe9\xdc\xdd\xee\xb3\x1c\x99\xf8MR\x03\xe7'\x04\xa6D\xc4&\x92\xffNH\xf9K\xcc-~\xf6\xf6|JRߜ\x15\xf3\x8b%\xf0\x9d?m/\x89>\xd3)zs\xa8\xf3\xe2\xe9x\xaf\x98\x84\xf7:\xa9w\x89\tw\xe7˜O\x8bǞ\x9496\x1d:\x18N\x9a\x9bL\x95\x9b\f-L!6\x1b\xa5\xc9\x14\xb89\x89o\x93\xdcI\x9bf\xaf\x96\xda\xf6j\tm\xaf\x9b\xc66*E\xa3\x1f\xe7$\xaaů\xa5!\x93Ɩ\xbf\x96\xb0\x9dJ\x06\xa9:\xee\xebI\xeb\xab\xcfG0,\xe3\x83k\xf7J>rQq\xc3J\x8e\x1b\xa9{\x96G\x83\rf\a\x87\xfa\x02\x8d_%\x1e=\xf57\xc1|\xfeRK\xed呧O5y\x02\xce\t\x8dͫ\x1e晻\x89)\x93K\xb0\xf6\xc8\xceN\x7f1\x88\xbf\xbei\xe1\xc4\x1dOעU+b!&*\x86o\x91\x194\x1c)\xfa\xa6\xe7\xc1:?\x1c\x7f\xfb[\x05\xea@\xf0\x1e\x9b\xda\xcfi\x0e\x81\xf9\x89\xa9\xedB,\xa8\n\xaf\xb6\x86\xe2\xe7=\xa7\xbf\x99\xca\xe4\x9dpV\xf7x<\xd8\xc6\xea\x88fQc\x15\x9f]\xafD\xfb\x18h.d\xdd:\xd2l\xcaAN=-\xf5\xb2K\x9c\xf9\x8b\x9cI\xaf\"\xdd\xf3\xfb\x8dNA\x9dr\xfa)-\x01`\xf2\xb4\xd3K-y\xa6\x16=\xc9~^\xdai\xa6y\x9b\x85/xz\xe9%N-%R*\xe5\x94\xd2<:\xbd©\xa4W=\x8d\xf4Z\xa7\x90\x92O\x1f%\xa5\xb8$\xef\x02\xa7\xa6\xa8\x9cx\x9cfz\x8fw\xfc4Q\xc2)\xa2\x84\xdd\xdfi$O@/\xe1\x94м\xd3A\t\x80\xf3\x83\xaeab\xae(\xbeg\xd2'B\xdf\xf9uъk\xeb\xfd\xc3҂8\xcdk\x8d\xea\xe6L\xb3\xae]x\x9e\x92\xbb\xb9[\r\x81;E\xc5\xf5\x9f{y\xe64\xee\xa3\xfb,\x95\xd6Gw\x96B\x8b@\xace\xfc\xfc\xb8\xe3T?\xedRwl\xe9\x1c\x8e,\x9c\xa1\xa3\x9c\xfb\x83\x8e\x05hM\xb7\xe16\xf7'\xbb\xf4\u0602\x00\x17\x9es\x9b'\x11\xa0ͩ\xb8\xee]\xe6n\xca\xd0\xccT\xd4w\x10\x12|[\xb5~Є\xcb\x18T|Ѕ\x85\x97\xc2\u009al&\xa1\xbe\x96L\xa5\xac\xe1>\xd4\x15-m\xd0\x13F\xee4o\xbb\x01g[f\xd7:\x96s[\xaa\xd6t\v\xcbLr\x0e\xa8\xad\xfb\xe3zɹ\xee\xcf\x1e~\x01\xaa'Q\xfbخ\xebw\x00\x1d\xb7\xdd\xc67u\xe9\xee\xf8z\x97a\n\x9a\x87\xf4z\x03\x92\xd8\xf1,G\xd9Q!\xfa\xca\\\x7f\xa4\xed\xbaa\xd6y\xb5\xec\xe3\xbc\xfe\x91\xb9\x85\x8f\v\xc4屠\xbfJ\xb5 \x05\x13\xf6\x1f*r\xb7\x81\x17\x1a\xcf\x1a\xffN\xcaǻ\x88\x13\xdb\x1b\xfc\x8fu\xc5f\xab\x83\t7l<0\xba\x96\x95\xdf}\xaf\x1d\xda\xf8\xb6\n\xde\xcc\x7f\xe6\xe5&\xc2\x1c\xb1\a=t\x06#\xba?v M\x9a\x02\xd7\xf3\x00\xac\xbb\xf0\x92\x19\xe7\x87\xc51\xe4\xa3W\x13\x1bح\x97\v\xbc\x1b\xd0\xdcG0\xd0Qؑ\x8a\x02\xa9/\xbeh+\xf4SV\xbd\x9e\xccC\xced\x8f\xc6?6\xb5\x87\xe8\xe8\x86\xd9r\xf7\x06\x10\xec8\x81\xe7]\xb0\xe33\x15\x13\xc2\x7fk\xeb\xd4w\x17\xb4\x16n!Kl0J\x17?\xfb\xbe$\x9f\xa0\xbf]\xb1$\xffUA\x15\xa1\xc12<\fwg\xa8\xea\x87|\xdd1x\xc81\xa3\x03gc\xa4\xcaJ\xdc*\xb9U\xa0\xfbº$\x7f\xa5\xcc0\xb1\xfd(\xd5-\xaf\xb6L|\x1e>\xf23V\xf9\x96*ì\xb0\xbb\xf1\xc4\x06\xca\x04\xe5\xec\xef1\xbd\xd6\xfe8\r\xe8fp\x81\xb5$\t\xc3\x18\xfa\xf0\x1e\xac\x8f;\x18\x17\x88\xaa\xd0\xd2\xd3\xf5\x14\x7f%\xf0dJ\xa7־D㋄n/\xc9'\x19U\f>\x1d\x8auaZ\x97\f\xb4Y\xc2f#\x95q\xbb\xd5\xcb%a\x9b\x10|\xb0:\a\xe3f\xee\xadJ\xc2b\xdb\xccu\xa2Ic\xbe0\xe8\xad\xd0\n\xe3U\xf6\x05=\xb8\x9d)\x9ae\x95\xf5\xb0\xae\xb4\xa1<\xe2\xe0\xc8\x7fy\xd6Nު\r\xa8\x1ft\xc4~\x1cI\xf12\r\xe7\xf5q\x8b\"\b\xf2\xa4\x981֧\x92#\xa9\x04\x9eT\xc6\xfaV\x9c\x13mI}R\xf4\x9185\xba\x1aN\xc9IC\xf9\xbe\x862\xa4\x9e=\xd6\xf82\xe3\x1aiC\xacߋ\xd9G\xbe\x96es\xb6\xa3b;xC\xc1N\xc9j\xbb\v\x92<\xe0L\x93\xbc\x02\f֢J\xd1\xe1aaS)\xd1J%\x189\xf6M\x820\xe0pi\xf6H\xaar\xe1\x1f\xee\xf5\xef2_\xf97P\x96\x1b%\x8b\xa5\xef\x17c\xa9\v\xbf\x93\xaf\x98\xb4\x9e\x8b\xd9E\xa9N\x9c\xd7\xee\x9f\x19@I(K\x10\x84j\xdfs\xc2MQ'\x9b)mU\xff\xb3\xc2\x18w\x1d\b\x13\x11\f\xec.\x8eĝOhpWe\xdd\xf8\aJk\xc0\v\xa2\x99\bOC\xbb\xe4\b'\x1fѽ2\x81O9J\x15\xcfW\x1c\x0fIt\x11z\xddhľ\xb6\xb5\x1fN^\xac>\x1c\xc18:\xf6\x8c/w\xd6U\xc2\x02\xf3w,\x161\xc7D\xd7̢\xf2\xfb\xdf\xfc8\xf3>i1\x14\xa7\xc8\xd8\xda\b\x97=Ë\x9c\xeeK\x9d\xb7\x1c\xac˦\x01\xbaˮY\xcb\xeb\xfd\x19\xe3M\xe7\f6\x85G\xd0\xcf\x13mٟ1\xcc\xf4b1\xa6\xf3\xa2\xfcD\xf1\t\xe5\x93f\xed_}\xdbH\x90Ƀ=w\x98\xa9\x15e\n\x03\x7f\xd58S\xd4*\xf5~D=\x9d\xb7\xb4\x85\xef\xc9\xff\xf2\x7f\x01\x00\x00\xff\xffC\xf4\a\x86ځ\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccYK\x8f\x1b\xb9\x11\xbe\xebW\x14v\x0f{ٖ\xec\x04\t\x02\xdd\xc6r\x02\x18\x19\xc7\x03k2\xb9.EVK\\\xb1\xc9\x0e\x1f\x92\x95\xc7\x7f\x0f\x8a\x0f\xa9\xd5\x0fK\xe3\x04\x9b\xe5eF|\x14\xeb\xf9U\x15\xbb\xaa\xaa\x19k\xe5\vZ'\x8d^\x02k%~\xf1\xa8闛\xef\xff\xe0\xe6\xd2,\x0eog{\xa9\xc5\x12V\xc1y\xd3|Fg\x82\xe5\xf8\x1ek\xa9\xa5\x97F\xcf\x1a\xf4L0ϖ3\x00\xa6\xb5\xf1\x8c\xa6\x1d\xfd\x04\xe0F{k\x94B[mQ\xcf\xf7a\x83\x9b \x95@\x1b\x89\x97\xab\x0fo\xe6o\x7f?\xff\xdd\f@\xb3\x06\x97\xb0a|\x1fZ\xe7\x8de[T\x86'\x92\xf3\x03*\xb4f.\xcd̵\xc8醭5\xa1]\xc2e!Qȷ'\xce\xdfEb\xebD\xec1\x13\x8b\xebJ:\xff\xe7\xe9=\x8f\xd2\xf9\xb8\xafU\xc125\xc5V\xdc\xe2v\xc6\xfa\xbf\\\xae\xae`\xe3TZ\x91z\x1b\x14\xb3\x13\xc7g\x00\x8e\x9b\x16\x97\x10O\xb7\x8c\xa3\x98\x01d\xd5Dj\x150!\xa2\xb2\x99z\xb2R{\xb4+\xa3B\xa3\xcfw\tt\xdc\xca\xd6Ge&Y \v\x03E\x1ap\x9e\xf9\xe0\xc0\x05\xbe\x03\xe6\xe0\xe1\xc0\xa4b\x1b\x85\x8b\xbfjV\xfe\x8f\xf4\x00~vF?1\xbf[\xc2<\x9d\x9a\xb7;\xe6\xcaj\xb2\xd1SgƟH\x00\xe7\xad\xd4\xdb1\x96\x1e\x99\xf3/LI\x119y\x96\r\x82t\xe0w\b\x8a9\x0f\x9e&\xe8W\xd2\x10\x90\x8a\x10\x8a\x86\xe0\xc8\\\xbe\a\xe0\x90\xa8D\x1d\x8ds\xaa\x06w]\xb1M\xac\xc0K\x8fJ\xe2\x9ff2\xf7\x1d\xb2ſ\xe7\xdc♤\xf3\xaci\xaf\xe8>lq\x8aؕ*\xdec͂\xf2]Q\xc9J\xaa\xeb\x97\xd7b\xb5\xc8\xe7\"\x9d\xba\xba\xf1\xfd\xd5\\\xbauc\x8cB\x96\xa8\xa4]\x87\xb7\xc9\v\xf9\x0e\x1b\xb6̛M\x8b\xfa\xe1\xe9\xc3\xcbo\xd7W\xd30\xe6H\xbd\xa0 ñ\x8emvh\x11^b\xfc%\xbb\xb9,ڙ&\x80\xd9\xfc\x8c\xdc_\x8c\xd8ZӢ\xf5\xb2\x04K\x1a\x1d,\xea\xcc\xf6x\xfaWu\xb5\x06@b\xa4S \b\x940\xf9U\x8e\x1f\x14Yr05\xf8\x9dt`\xb1\xb5\xe8P'\x98\xa2i\xa63\x83\xf3\x1e\xe95Z\"C\xb1\x1d\x94 ,;\xa0\xf5`\x91\x9b\xad\x96\xff8\xd3v\xe0Mvf\x8f\xceC\x8cP\xcd\x149k\xc0\x1f\x81iѣܰ\x13X\xa4;!\xe8\x0e\xbdx\xc0\xf5\xf9\xf8H\xd1 um\x96\xb0\xf3\xbeu\xcb\xc5b+}Ahn\x9a&h\xe9O\x8b\b\xb6r\x13\xbc\xb1n!\xf0\x80j\xe1\xe4\xb6b\x96\xef\xa4G\xee\x83\xc5\x05ke\x15\x05\xd1\tR\x1b\xf1\xbd͘\uebae\x1d\x84t\x1a\x11R_a\x1e\x82\xd7\xe42\x89T\x12\xf1b\x05\x9a\"\xd5}\xfe\xe3\xfa\x19\n'\xc9R\xc9(\x97\xad\x03\xbd\x14\xfb\x906\xa5\xaeѦs\xb55M\xa4\x89Z\xb4Fj\x1f\x7fp%Q{pa\xd3HOn\xf0\xf7\x80Γ\xe9\xfadW1\x8b\xc1\x06!\xb4\x11$\xfa\x1b>hX\xb1\x06Պ9\xfc\x85mEVq\x15\x19\xe1.kuss\x7fsRog\xa1\xe4\xd4\tӎ\xa2\xc1\xbaE~\x15w\x02\x9d\xb4\x14\x19\x9ey\x8c\xd1\xd5SP\x86\x8a\xe9\xa4\\\xc68H\xd0`\x9c\xa3s\x1f\x8d\xc0\xfeJ\x8f\xe5\x87\xf3\xc6+\x1e[\xb4\x8dt1\xbdBml?\xf3\xb03\x92wGA\xbc\xbe\xc1\x01P\x87f\xc8H\x05\x9f\x91\x89OZ\x9d&\x96\xfef\xa5\x1f^4aH\x1a\x89\xc5\xf5I\xf3'\xb4҈\x1b¿\xebm?\xab`g\x8ePG\xff\xd7^\x9d\b\xbb\xdcI\xf3!j\x97\xf1\xf0\xf4\xa1 x\x8a\xad\x1c\x98YWsx\xc8Amjx\x03B:*$\\$:T\x96\x0e*\x16\x1aK\xf06\xbcJ|nt-\xb7C\xa1\xbb\xb5є\xc7\xdc \xdd\xd3\xdc*\xdeD\xa8E\xde\xd1Zs\x90\x02mE\xf1!k\xc93'\xc1\xa6\fRKTb\x80M\x93Q\x16E\xb1((\xa8\x99\xbaa\xc3\xd5yc\xac\xa4\x99\xd4Ƀ/\x04\"\xd6\xd8&\xa7f\xedQ\v\xecg\x9bȍ\x89\x80\xe6P\xc0Q\xfa]BJ5\x16w\xf0\xd5أ\xb1\xc7\xd3\xd8t\x8f\xf7\xe7\x1d\xd2Δx\x11\x1cr\x8b>z\x1b*r\x1fr\xa59\xc0\xc7\xe0\"\xd6\xf6q\xa2\x8cX\xf0\x95\xd3{<\r\x15\r\xb7\x8c\x9bK\xa1\t\x96c\x11\xb5\x84ﾻ-\xd2 \xbb\x95A\xa5{\x11\xd4b\x8d\x16\xf5\xa0\x9a(\xe39\xe6(r\x1a\xf20\xack\xe4^\x1eP\x9dbN\"\xf0\xfc\x116\xc1\x83\b\x18\xad\xc6\xf8\xfeȬp\xc0M\xd32/7RI\x7f\x02\xe9&\xe83\xa5\xcc\x11E\xb686\xad?\xcd\xe1\x83v\x9ei\x8e\xee\\\a\x91ƒ+0\x9dv\xe5(\x8e\x05\x1d\xb3c\x18\x98\xc87\xc6y\xe0h\xc9\x1d\xd5\t\x8e\xd6\xe8픰#\xe9\x90z@\xab\xd1c̈\xc2pGɐc\xeb\xdd\xc2\x1c\xd0\x1e$\x1e\x17Gc\xf7Ro+b\xb0\xcaೈ\x9d\xdd\xe2\xfb\xf8\xe7[\xbc\xc0\xb4\t'\xeep\xdeu\x8c\xf5\x13\x95\xb7~\x87)E\xac\x93\x0f\x1a\vT@\x90k7\xd9w\x13\xb2\x8e\x85\xddX]\xde\x1d\xc5\xe4c\xf9c\x8f\xc3\xd4\xf1\x15P\x01\xf8R]t[5\xac\xad\xd2n\xe6M#\xf9\xac/m\xf2\xfb\xaf\xe3OiV\xa4\x16\x92Sq{\x8d\x1b\xa5\x89\x13W=͈\x1a\xfa]\xce\x14Z\x8e\xab)\x89\x9bk\x85\x1b\x1c\x7f\xea\uef74\xbe\t\xbas\xfew\xe8\xa9\xeet\xa0\x91\xea\x03f\x87z\x8e\x80ɍքT\xde\x00;\xa7\x81\x1f\\?\xff\xbd\x12=7\x81\xefqD\xf1\x03Q\xdeōE\xc7\xe9\x18\xf1\x12\x1c\xc6\xc4t\x8b\r\xb8\x1d\x11\x9c\xad\xd0\xde\xc3\xcb\xea\x816\x9eK\b\x06\xab\a\xd8\x04-\x14\x16\x8e\x8e;\xd4\xd4u\xc9\xfa4~\x17\x8d\xe7\xc7u\xd1j\xac\xber\xdfTt;.C\xcaoK\u061cF\xea\xa5;\x84l-\xd6\xf2\xcb\x1dB>ōE\xe1-\xf3;\x90\xdaI\x81\xc0Fԟ\n\xd9\tAϵѧ\x8c9\xdf`\x9e\xafaCb\xe75\xf0Pt|#~\x9e\xf2\xb6\xb3\x16\xca\xef\x9cݮ\xeb\xe4\xa98\x1e\x95\xe8p~\x94\xf9S\xaa>\xf9H\x19q\xc5\xcc\xcb\xf0\xc4W\xaa\xd8\xf244\x16\xccT3\x19kѵF\v\xea9\xef\xaba/,\xff\xef*\xd9q\xb3V\xd7(\xd7[+V\xb8\xab\x8d\x8b\xcf`\xafn\xe4\xd2\xe3`\xb7M2\x1bG\r\xf6\xa5\x97\xeb\xc9\xf8\x8b\xb4p\xa3%W\xa7\xaf\x93\x8eꗠce\x1b\xab\xaa\xf9l\xe4\xc4{l-R\x06\x13K\x92\xcdƃ\xda\x1c\xe9p\x87Z*ˌN\xf9\x9ez[\xa6E~U\xa0\xa5\x11\xcaG\xa9\x14\xd5\x00\x16\x1bCʢ\xb2\xdcR5\xc7b\xadu\xf8\xcd\xfc\xcd\xff\xafeT\xccy\xea\x00Q|ƃ\x1c>\xadݧ\xee\xc7\x01\x95\x82\x0e瘡\x1f?\x95׆\x85\xcd\xdb~\x82Z*\xaa\xff:\xd0qGu0\xf20\xfcn\xfd\xf8\x83\x8b=\x10j\xef\xe0H\x16t\x91%jzL~\xe1\t\xceS\x12\xb9i\xffn\x01\xae\r(\xa3\xb7h\xcbk\x0f\x15xɛ\x8c\x05\x81\x9er\x95\xde\x02\xdf1\xbd\xa5\xc8\x18\x83\xfc\xc8p\xe6\xbe\xcb'yϤ\x83H=\xe1\x1dw\x19\xf4Y\x8e\xb54\xaf1\xe6\xf43\xfc\x99\xffl\xd9\xcbkoO\xefSP[,\xd1_,\xa9\x9c\x14]\xf9\xcb\xd3\xfce|\xfb\xfb\xc0\xf0\xdd\xff[\xd5\xf3_}\xa9\x18|\xa1\xf8U(\xa7\xa1:\xf7f\xf1\xfc1\xedJ\xef\xb5\xf9\b\xb0\x8d\t~$\xf7w\x1c~4\xa6\xe3ǘ\xd7\xf0\x18?1\xdd*OhO\xb1\b\x0f\xd6\xc67\xdd\xf2\xd6\x18\x91b,+ݏ\xc0\x0f\xbd/aݵ\xe1w\xb2;\xe4\x1a\xcd҃ɔi;v\xcdJ\xee΄\xcd\xf9\xa5~\t\xff\xfc\xf7\xec?\x01\x00\x00\xff\xff\x03f\x86Y\xc0\x1d\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVMo\x1b7\x10\xbd\xebW\f\xd0kwU\xa3hQ\xec\xadqr0\xda\x06\x82\x1d\xe4N\x91#-c.\xc9\xce\f\xe5\xba\x1f\xff\xbd \xb9+K\xab\x95\x93\\\xb27\x91Ù\xc7\xf7f\x1e\xd54\xcdJE\xfb\x11\x89m\xf0\x1d\xa8h\xf1/A\x9f\x7fq\xfb\xf8\v\xb76\xac\x0f7\xabG\xebM\a\xb7\x89%\f\xf7\xc8!\x91Ʒ\xb8\xb3ފ\r~5\xa0(\xa3Du+\x00\xe5}\x10\x95\x979\xff\x04\xd0\xc1\v\x05琚=\xfa\xf61mq\x9b\xac3H%\xf9T\xfa\xf0C{\xf3s\xfb\xd3\n\xc0\xab\x01;0\xe8Pp\xab\xf4c\x8a\x84\x7f&d\xe1\xf6\x80\x0e)\xb46\xac8\xa2\xce\xf9\xf7\x14R\xec\xe0e\xa3\x9e\x1fkW\xdcoK\xaa7%\xd5}MUv\x9de\xf9\xedZ\xc4\xefv\x8c\x8a.\x91rˀJ\x00[\xbfON\xd1b\xc8\n\x80u\x88\xd8\xc1\xfb\f+*\x8df\x050^\xbb\xc0l@\x19S\x88TnC\xd6\v\xd2mpi\x98\bl\xc0 k\xb2Q\nQ\x1fz,W\x84\xb0\x03\xe9\x11j9\x90\x00[\x1c\x11\x98r\x0e\xe0\x13\a\xbfQ\xd2w\xd0f\xbe\xda\x1a\x9a\x81\x8c\x01\x95\xea7\xf3ey\u0380Y\xc8\xfa\xfd5\b,J\x12O J]\x1b<\xd0\t\xbf\xe7\x00J|\x1b{\xc5\xe7\xd5\x1f\xcaƵ\xca5\xe6pS\x99\xd6=\x0e\xaa\x1bcCD\xff\xeb\xe6\xee\xe3\x8f\x0fg\xcbp\x8euAZ\xb0\fjB\x9a\x89\xab\xacA\xf0\b\x81`\b4\xb1\xca\xed1i\xa4\x10\x91\xc4N\xadU\xbf\x93\xe19Y\x9dA\xf8\xb79\xdb\x03Ȩ\xeb)0y\x8a\x90\v\x89cS\xa0\x19/Zɵ\f\x84\x91\x90\xd1\u05f9\xca\xcb\xcaC\xd8~B-\xed,\xf5\x03RN\x03܇\xe4L\x1e\xbe\x03\x92\x00\xa1\x0e{o\xff>\xe6\xe6|\xef\\\xd4))\x94\xe4\xb6\xf3\xca\xc1A\xb9\x84߃\xf2f\x96yP\xcf@\x98kB\xf2'\xf9\xca\x01\x9e\xe3\xf8#\x93h\xfd.tЋD\xee\xd6뽕\xc9Rt\x18\x86\xe4\xad<\xaf\x8b;\xd8m\x92@\xbc6x@\xb7f\xbbo\x14\xe9\xde\njI\x84k\x15mS.⋭\xb4\x83\xf9\x8eF\x13Ⳳ\x17\xddS\xbf\xe2\x02_!O\xf6\x84\xda#5U\xbd\xe2\x8b\ny)Sw\xff\xee\xe1\x03LH\xaaRU\x94\x97\xd0\v^&}2\x9b\xd6\xef\x90\xea\xb9\x1d\x85\xa1\xe4Dob\xb0^\xca\x0f\xed,z\x01N\xdb\xc1\nO\x1d\x9b\xa5\x9b\xa7\xbd-\xb6\x9b\x1d E\xa3\x04\xcd<\xe0\xceí\x1a\xd0\xdd*\xc6o\xacUV\x85\x9b,\xc2\x17\xa9u\xfa\x98̃+\xbd'\x1b\xd33pEڅ\xe1\x7f\x88\xa8\xb3\xb8\x99\xdf|\xda\ueb2ec\xb5\v\x04O\xbd\xd5\xfd4\xfc3\x9a\x8eFq\xce߲1\xe4\xef\xc5n\xe7;W/\x0fEdK8k\xd8\x06.\xbc\xfbu^\x8a\xa9~%3\xd5\xd1Gnt\"*\xcdw\xf4y\xb5t\xe8K\xb9@\xa2@\x17\xab3P\xefJP\xf9Ǡ\xacgP\xfey<\b\xd2+\x81'\xa4\r\x97\x95\x1ax\x8fO\v\xabw~CaO\xc8\xf3\x96ϛ\x9b\xca\x1e\xce߃WXZlʋE\xceVhNXd\t\xa4\xf6\xa7\xbcr\xda\x1e\x9d\xbe\x83\x7f\xfe[\xfd\x1f\x00\x00\xff\xff\xbeM\x1a\xea\xb1\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcWMo\xe36\x10\xbd\xfbW\f\xd0K\v\xac\xe4\x06E\x8b·\xd6\xd9C\xb0\xe96\x88\xb7\xb9S\xd4HbC\x91,9t6E\x7f|1\xa4\xe4\x0fYv\x9c\xcb\xea\xe6\xe1p\xf8\xe6\xcd\xcc#]\x14\xc5B8\xf5\x84>(kV \x9c¯\x84\x86\x7f\x85\xf2\xf9\xd7P*\xbb\xdc\xde,\x9e\x95\xa9W\xb0\x8e\x81l\xff\x88\xc1F/\xf1\x16\x1be\x14)k\x16=\x92\xa8\x05\x89\xd5\x02@\x18cI\xb09\xf0O\x00i\ry\xab5\xfa\xa2ES>\xc7\n\xab\xa8t\x8d>\x05\x1f\x8f\xde\xfeX\xde\xfcR\xfe\xbc\x000\xa2\xc7\x15\xd4\xf6\xc5h+j\x8f\xffD\f\x14\xca-j\xf4\xb6Tv\x11\x1cJ\x8e\xddz\x1b\xdd\n\xf6\vy\xefpn\xc6|;\x84y\xccaҊV\x81>ͭޫ\xc1\xc3\xe9\xe8\x85>\x05\x91\x16\x832m\xd4\u009f,/\x00\x82\xb4\x0eW\xf0\x99a8!\xb1^\x00\f)&XŐ\xdd\xf6&\x87\x92\x1d\xf6\"\xe3\x05\xb0\x0e\xcdo\x0fwO?m\x8e\xcc\x005\x06镣D\xd4\x7f\xc5\xce\x0e\xd3\x04@\x05\x100\xc0\x01\xb2;\x84 \f\bO\xaa\x11\x92\xa0\xf1\xb6\x87J\xc8\xe7\xe8\xc0V\x7f\xa3$\bd\xbdh\xf1\x03\x84(;\x10\x1c%;\x1c\x9c\xa5m\v\x8d\xd2X\xeel\xce[\x87\x9e\xd4Hy\xfe\x0e\x1a\xea\xc0z)\v\xfe8\xf1\xbc\vj\xee,\f@\x1d\x8e\xe4a=p\x05\xb6\x01\xeaT\x00\x8f\xcec@\x93{\x8d\xcd\xc2\fٔ\x93\xd0\x1b\xf4\x1c\x06Bg\xa3\xae\xb9!\xb7\xe8\t\x1aE\xaf\xcb41\xaa\x8ad}XָE\xbd\f\xaa-\x84\x97\x9d\"\x94\x14=.\x85SEJĤQ+\xfb\xfa;?\ff8:\x96^\xb9!\x03yeڃ\x854\x1d\xef(\x0f\xcfK\xee\xae\x1c*\xa7\xb8\xaf\x02\x9b\x98\xbaǏ\x9b/0\"ɕ\x1aZl\xe7z\xc2\xcbX\x1ffS\x99\x06}ޗڔc\xa2\xa9\x9dU\x86\xd2\x0f\xa9\x15\x1a\x82\x10\xab^Q\x18{\x9dK7\r\xbbNR\x04\x15Bt\xb5 \xac\xa7\x0ew\x06֢G\xbd\x16\x01\xbfq\xad\xb8*\xa1\xe0\"\\U\xadC\x81\x9d:gz\x0f\x16Fy&j^\x01\x128\xe1[\xa4\xa9u\x82\xe5Kr\xe2\xe3_:q,X\xdfcٖ\xac9a\x00\x92\xf5\xe8\x87i\xa1.a\x80\xd9F\x9fE2\xf67\xd3\xc0\xbc\xb2\xa0\xb0\xd8\x1db:=\x9a?4\xb1\x9f?\xa0\x80\xdf\x13\xe6{\xdb^\\_[C<\x17\x17\x9d\x9e\xac\x8e=n\x8cp\xa1\xb3o\xf8\xde\x11\xf6\x7f:\xf4\xf9\x1a\xbe\xe8:\xde滫\xef\x82c\xd4g\xcf}D\xbeA\xf0|\xa6\x83\xc3UQ\xae\xc04x^\x95\xe8zs\xf7\x1e\nϸ\xbf\xa3Hw\xa6\xb1o\xa4\xb8w\x9c\xf5;#\x03\xe3\x97\xde\x10o\xf74\xbfBƞ\xe6-\xf9\xeeD\xf8\x14+\xf4\x06\t\xc3^\xa9_\x14u\xb3\x11\x01^:%\xbb\xb41\r\x04_\x02!X\xa9\xe6$\xf5\n\xf8\xac#\xca\xe3\xccP\x16iXg\xcc\f\xfe\xc4|F\xfd\xce\x1dP\f\x8at\x95\x82\x92\xa0\x18ޡ\xa1\xc9\x7f\xa4ZF\xef\xd3\x15\x95\xad\xfc2\x99n\xb8VDG\xe5\xf9\xeb\xf1\xfe\r%\xbd\xdd{\xa6\x17\xb7P&\xa3q\x1e\x8b\xa0Z~A\xf1\x1akiҸS2\xf2w\xfc\xc2;&j\xb6\xa2\xf8թ<\x80o@\xfc\xb8ŝ\x8f&\xdf\xf3\xd37l\n\x88\x81\x9f[ \x85\x99\xc1X!Ԩ\x91\xb0\x86\xea5\xdf\\\xaf\x81\xb0?\xc5\xddX\xdf\vZ\x01\xdf\xff\x05\xa9\x9962QkQi\\\x01\xf9x\xae\xcbf\x13w\x9d\b3cx\x94\xf3\x03\xfb\xcc5\xc6n\x18/v\x06\x9c\xbd_\n\xf8\x8c/3\xd6\ao%\x86\x80\xa7ct6\x93\xd9!81\x06~\xa4\xd5\a,\r\x7f\x19\x06\xcb\xff\x01\x00\x00\xff\xffx\xae@\xbaJ\x0e\x00\x00"), diff --git a/pkg/apis/velero/v1/backup_types.go b/pkg/apis/velero/v1/backup_types.go index 5ba3b2175..2f4707d3c 100644 --- a/pkg/apis/velero/v1/backup_types.go +++ b/pkg/apis/velero/v1/backup_types.go @@ -288,7 +288,7 @@ const ( // BackupPhase is a string representation of the lifecycle phase // of a Velero backup. -// +kubebuilder:validation:Enum=New;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;Finalizing;FinalizingPartiallyFailed;Completed;PartiallyFailed;Failed;Deleting +// +kubebuilder:validation:Enum=New;Queued;ReadyToStart;FailedValidation;InProgress;WaitingForPluginOperations;WaitingForPluginOperationsPartiallyFailed;Finalizing;FinalizingPartiallyFailed;Completed;PartiallyFailed;Failed;Deleting type BackupPhase string const ( @@ -296,6 +296,12 @@ const ( // yet processed by the BackupController. BackupPhaseNew BackupPhase = "New" + // BackupPhaseQueued means the backup has been added to the queue and is waiting for the Queue to move it out of the queue. + BackupPhaseQueued BackupPhase = "Queued" + + // BackupPhaseReadyToStart means the backup has been pulled from the queue and is ready to start. + BackupPhaseReadyToStart BackupPhase = "ReadyToStart" + // BackupPhaseFailedValidation means the backup has failed // the controller's validations and therefore will not run. BackupPhaseFailedValidation BackupPhase = "FailedValidation" From cfc12dc6bf277861a859ff249dbc91aa145adc7e Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Thu, 7 Aug 2025 16:26:33 -0400 Subject: [PATCH 02/37] feat: Add install arg and config for concurrent backups Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/cmd/cli/install/install.go | 8 ++++++++ pkg/cmd/server/config/config.go | 9 +++++++++ pkg/install/deployment.go | 11 +++++++++++ pkg/install/resources.go | 2 ++ 4 files changed, 30 insertions(+) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 6698010bd..283bfa557 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -89,6 +89,7 @@ type Options struct { RepoMaintenanceJobConfigMap string NodeAgentConfigMap string ItemBlockWorkerCount int + ConcurrentBackups int NodeAgentDisableHostPath bool kubeletRootDir string ServerPriorityClassName string @@ -196,6 +197,12 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { o.ItemBlockWorkerCount, "Number of worker threads to process ItemBlocks. Default is one. Optional.", ) + flags.IntVar( + &o.ConcurrentBackups, + "concurrent-backups", + o.ConcurrentBackups, + "Number of backups to process concurrently. Default is one. Optional.", + ) flags.StringVar( &o.ServerPriorityClassName, "server-priority-class-name", @@ -313,6 +320,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { RepoMaintenanceJobConfigMap: o.RepoMaintenanceJobConfigMap, NodeAgentConfigMap: o.NodeAgentConfigMap, ItemBlockWorkerCount: o.ItemBlockWorkerCount, + ConcurrentBackups: o.ConcurrentBackups, KubeletRootDir: o.kubeletRootDir, NodeAgentDisableHostPath: o.NodeAgentDisableHostPath, ServerPriorityClassName: o.ServerPriorityClassName, diff --git a/pkg/cmd/server/config/config.go b/pkg/cmd/server/config/config.go index 401a89632..ba1b9f3bf 100644 --- a/pkg/cmd/server/config/config.go +++ b/pkg/cmd/server/config/config.go @@ -47,6 +47,7 @@ const ( defaultDisableInformerCache = false DefaultItemBlockWorkerCount = 1 + DefaultConcurrentBackups = 1 ) var ( @@ -174,6 +175,7 @@ type Config struct { BackupRepoConfig string RepoMaintenanceJobConfig string ItemBlockWorkerCount int + ConcurrentBackups int } func GetDefaultConfig() *Config { @@ -206,6 +208,7 @@ func GetDefaultConfig() *Config { ScheduleSkipImmediately: false, CredentialsDirectory: credentials.DefaultStoreDirectory(), ItemBlockWorkerCount: DefaultItemBlockWorkerCount, + ConcurrentBackups: DefaultConcurrentBackups, } return config @@ -261,4 +264,10 @@ func (c *Config) BindFlags(flags *pflag.FlagSet) { c.ItemBlockWorkerCount, "Number of worker threads to process ItemBlocks. Default is one. Optional.", ) + flags.IntVar( + &c.ConcurrentBackups, + "concurrent-backups", + c.ConcurrentBackups, + "Number of backups to process concurrently. Default is one. Optional.", + ) } diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 91acce344..d1010d294 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -59,6 +59,7 @@ type podTemplateConfig struct { repoMaintenanceJobConfigMap string nodeAgentConfigMap string itemBlockWorkerCount int + concurrentBackups int forWindows bool kubeletRootDir string nodeAgentDisableHostPath bool @@ -224,6 +225,12 @@ func WithItemBlockWorkerCount(itemBlockWorkerCount int) podTemplateOption { } } +func WithConcurrentBackups(concurrentBackups int) podTemplateOption { + return func(c *podTemplateConfig) { + c.concurrentBackups = concurrentBackups + } +} + func WithPriorityClassName(priorityClassName string) podTemplateOption { return func(c *podTemplateConfig) { c.priorityClassName = priorityClassName @@ -337,6 +344,10 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme args = append(args, fmt.Sprintf("--item-block-worker-count=%d", c.itemBlockWorkerCount)) } + if c.concurrentBackups > 0 { + args = append(args, fmt.Sprintf("--concurrent-backups=%d", c.concurrentBackups)) + } + deployment := &appsv1api.Deployment{ ObjectMeta: objectMeta(namespace, "velero"), TypeMeta: metav1.TypeMeta{ diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 1d0ffa568..5c7534774 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -271,6 +271,7 @@ type VeleroOptions struct { RepoMaintenanceJobConfigMap string NodeAgentConfigMap string ItemBlockWorkerCount int + ConcurrentBackups int KubeletRootDir string NodeAgentDisableHostPath bool ServerPriorityClassName string @@ -362,6 +363,7 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { WithPodResources(o.PodResources), WithKeepLatestMaintenanceJobs(o.KeepLatestMaintenanceJobs), WithItemBlockWorkerCount(o.ItemBlockWorkerCount), + WithConcurrentBackups(o.ConcurrentBackups), } if o.ServerPriorityClassName != "" { From dc91d6ee675503d13abdc2a4c7373b18ee8132ca Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Mon, 11 Aug 2025 16:56:12 -0400 Subject: [PATCH 03/37] feat: Add `pvc-for-tmp` install arg to use PVC for server /tmp dir Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/cmd/cli/install/install.go | 8 ++++++++ pkg/install/deployment.go | 29 +++++++++++++++++++++++++++++ pkg/install/resources.go | 5 +++++ 3 files changed, 42 insertions(+) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 283bfa557..748b6953e 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -90,6 +90,7 @@ type Options struct { NodeAgentConfigMap string ItemBlockWorkerCount int ConcurrentBackups int + PVCForTmp string NodeAgentDisableHostPath bool kubeletRootDir string ServerPriorityClassName string @@ -191,6 +192,12 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { o.NodeAgentConfigMap, "The name of ConfigMap containing node-agent configurations.", ) + flags.StringVar( + &o.PVCForTmp, + "pvc-for-tmp", + o.PVCForTmp, + "The name of PVC to be used as /tmp dir of Velero server pod. Optional.", + ) flags.IntVar( &o.ItemBlockWorkerCount, "item-block-worker-count", @@ -321,6 +328,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { NodeAgentConfigMap: o.NodeAgentConfigMap, ItemBlockWorkerCount: o.ItemBlockWorkerCount, ConcurrentBackups: o.ConcurrentBackups, + PVCForTmp: o.PVCForTmp, KubeletRootDir: o.kubeletRootDir, NodeAgentDisableHostPath: o.NodeAgentDisableHostPath, ServerPriorityClassName: o.ServerPriorityClassName, diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index d1010d294..84f94d01d 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -60,6 +60,7 @@ type podTemplateConfig struct { nodeAgentConfigMap string itemBlockWorkerCount int concurrentBackups int + pvcForTmp string forWindows bool kubeletRootDir string nodeAgentDisableHostPath bool @@ -231,6 +232,12 @@ func WithConcurrentBackups(concurrentBackups int) podTemplateOption { } } +func WithPVCForTmp(pvcForTmp string) podTemplateOption { + return func(c *podTemplateConfig) { + c.pvcForTmp = pvcForTmp + } +} + func WithPriorityClassName(priorityClassName string) podTemplateOption { return func(c *podTemplateConfig) { c.priorityClassName = priorityClassName @@ -431,6 +438,28 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme }, } + if c.pvcForTmp != "" { + deployment.Spec.Template.Spec.Volumes = append( + deployment.Spec.Template.Spec.Volumes, + corev1api.Volume{ + Name: "tmp", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: c.pvcForTmp, + }, + }, + }, + ) + + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append( + deployment.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1api.VolumeMount{ + Name: "tmp", + MountPath: "/tmp", + }, + ) + } + if c.withSecret { deployment.Spec.Template.Spec.Volumes = append( deployment.Spec.Template.Spec.Volumes, diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 5c7534774..1d550ca86 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -272,6 +272,7 @@ type VeleroOptions struct { NodeAgentConfigMap string ItemBlockWorkerCount int ConcurrentBackups int + PVCForTmp string KubeletRootDir string NodeAgentDisableHostPath bool ServerPriorityClassName string @@ -366,6 +367,10 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { WithConcurrentBackups(o.ConcurrentBackups), } + if o.PVCForTmp != "" { + deployOpts = append(deployOpts, WithPVCForTmp(o.PVCForTmp)) + } + if o.ServerPriorityClassName != "" { deployOpts = append(deployOpts, WithPriorityClassName(o.ServerPriorityClassName)) } From cd103add11c3a7182109311e31e23098d54dc123 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Tue, 12 Aug 2025 10:42:45 -0400 Subject: [PATCH 04/37] feat: Add QueuePosition field to BackupStatus Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/apis/velero/v1/backup_types.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/apis/velero/v1/backup_types.go b/pkg/apis/velero/v1/backup_types.go index 2f4707d3c..01a7bf104 100644 --- a/pkg/apis/velero/v1/backup_types.go +++ b/pkg/apis/velero/v1/backup_types.go @@ -377,6 +377,10 @@ type BackupStatus struct { // +optional Phase BackupPhase `json:"phase,omitempty"` + // QueuePosition is the position of the backup in the queue. + // +optional + QueuePosition int `json:"queuePosition,omitempty"` + // ValidationErrors is a slice of all validation errors (if // applicable). // +optional From 2944c0dad4d04c5ba36577e6a16b45018494d196 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Tue, 12 Aug 2025 10:45:40 -0400 Subject: [PATCH 05/37] update CRDs Signed-off-by: Scott Seago --- config/crd/v1/bases/velero.io_backups.yaml | 5 +++++ config/crd/v1/crds/crds.go | 2 +- pkg/apis/velero/v1/backup_types.go | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config/crd/v1/bases/velero.io_backups.yaml b/config/crd/v1/bases/velero.io_backups.yaml index 73dcdfc06..794c342c8 100644 --- a/config/crd/v1/bases/velero.io_backups.yaml +++ b/config/crd/v1/bases/velero.io_backups.yaml @@ -627,6 +627,11 @@ spec: filters that happen as items are processed. type: integer type: object + queuePosition: + description: |- + QueuePosition is the position of the backup in the queue. + Only relevant when Phase is "Queued" + type: integer startTimestamp: description: |- StartTimestamp records the time a backup was started. diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index 764c1e5a4..045f529da 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -30,7 +30,7 @@ import ( var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccXK\x8f۶\x13\xbf\xfbS\f\xf6\x7f\xfd\xcbnP\xb4(|K\xdc\x06\b\x9a\x04\v{\x91;-\x8dlf)\x92%\x87N\xdd\xc7w/\x86\x94lY\xa2\xad\xf5\x1e\x8a\xf2&r\u07bfy\xd9EQ̄\x95_\xd0yi\xf4\x12\x84\x95\xf8;\xa1\xe6/?\x7f\xfe\xc9ϥY\x1c\xde̞\xa5\xae\x96\xb0\n\x9eL\xb3Fo\x82+\xf1g\xac\xa5\x96$\x8d\x9e5H\xa2\x12$\x963\x00\xa1\xb5!\xc1מ?\x01J\xa3\xc9\x19\xa5\xd0\x15;\xd4\xf3\xe7\xb0\xc5m\x90\xaaB\x17\x85w\xaa\x0f\xdf\xcd\xdf\xfc8\xffa\x06\xa0E\x83K؊\xf29X\x87\xd6xI\xc6I\xf4\xf3\x03*tf.\xcd\xcc[,Y\xfaΙ`\x97p~Hܭ\xe6d\xf5\xbb(h\xdd\t:\xc6'%=\xfd\x9a}\xfe(=E\x12\xab\x82\x13*gH|\xf6R\xef\x82\x12nD\xc0\n|i,.\xe13\xdbbE\x89\xd5\f\xa0\xf54\xdaV\x80\xa8\xaa\x18;\xa1\x1e\x9dԄneTh\xba\x98\x15\xf0\xd5\x1b\xfd(h\xbf\x84y\x17\xddy\xe90\x06\xf6I6\xe8I46\xd2v\x01{\xbb\xc3\xf6\x9b\x8e\xac\xbc\x12\x84ca\x1c\xb9\xf9\xd9֧\xa3\xc5\v)\xe7@@\xef-I\xf4\xe4\xa4\xde\xcd\xceć7)\x14\xe5\x1e\x1b\xb1li\x8dE\xfd\xf6\xf1×\xef7\x17\xd7\x00\xd6\x19\x8b\x8ed\aO:\xbd\xf4\xeb\xdd\x02T\xe8K'-\xc5\xe4\xf8\xab\xb8x\x03`\x05\x89\v*\xceC\xf4@{\xecb\x8cUk\x13\x98\x1ah/=8\xb4\x0e=ꔙ|-4\x98\xedW,i>\x10\xbdA\xc7b\xc0\xefMP\x15\xa7\xef\x01\x1d\x81\xc3\xd2\xec\xb4\xfc\xe3$\xdb\x03\x99\xa8T\tBO\x10Q\xd4B\xc1A\xa8\x80\xff\a\xa1\xab\x81\xe4F\x1c\xc1!넠{\xf2\"\x83\x1f\xda\xf1\xc98\x04\xa9k\xb3\x84=\x91\xf5\xcb\xc5b'\xa9+\xca\xd24MВ\x8e\x8bX_r\x1b\xc88\xbf\xa8\xf0\x80j\xe1\xe5\xae\x10\xae\xdcK\u0092\x82Å\xb0\xb2\x88\x8e\xe8X\x98\xf3\xa6\xfa\x9fk\xcb\xd8_\xa8\x1d\x01\x9dN\xac\xa4;\xe0\xe1\xd2\x02\xe9A\xb4\xa2\x92\x8bg\x14\xf8\x8aC\xb7\xfee\xf3\x04\x9d%\t\xa9\x04ʙt\x14\x97\x0e\x1f\x8e\xa6\xd45\xba\xc4W;\xd3D\x99\xa8+k\xa4\xa6\xf8Q*\x89\x9a\xc0\x87m#\x89\xd3ව\x9e\x18\xba\xa1\xd8Ul\\\xb0E\b\x96K\xa7\x1a\x12|а\x12\r\xaa\x95\xf0\xf8/cŨ\xf8\x82Ax\x11Z\xfdv<$N\xe1\xed=t\xad\xf4\n\xb4\xc3\xf6\xb8\xb1X2\xb2\x1c\\f\x95\xb5,SM\xd5Ɓ\x18\xd1_F*\xdf\x02\xf8\xa4&\xba!\xe3\xc4\x0e?\x9a$sH4\x95v|\xde\xe5\x04u\x16s\xdbJ=\x01\xf3\x84\x19\x81\xb4\x17\xd4k\x06$\xa4>\xf5\x94\xac\x937\x90\x89\xe8\b\xee\x14Z\xe8\x12\xdf\xc7|\xd4\xe5q\xc2\xd1O\x19\x16vio\xbe\x81\xa9\tu_hkkƓ-\x82\v\xfa.c\xcf>\xae\x8c\xae\xe5nlh\x7f\x90]\x03wB\xc9\xc0\xdb\xf5@'{\xca\xc9u\xb6\xa5\xe82\x8f\x01\xa9\xe5.\xb8k\xe0\xd5\x12U5j!\x00:(%\xb6\n\x97@.\xe0\x95\x88\x8cj\xe52\"<\x1f'\x80[_\x10\x83\xd4\x15WK;\xacXI\x97\x8c\x9c\xfe\xa8+p\x97kJ\xff\xa0\x0e\xcdX]\x01\xcf\xc6J\x91\xb9w\xe8I\x96\x99\x87\x87\x87\xfb2\x80\xc5|\xa8\xb8\x1d\xd5\x12\xddkjr=\x90ѕc\x1d\x94j\x15\x14\xa5i\xac \xb9U\xd8\xcd\f\xc6\\&\x9ec.i`T\x86\xf0\x14'\x01c\xce*\x8cVG\b\x1e+\xf8\xb6G=\x02\xc3\xc3C\xd2\xfdpWI\x1cxQ\xc3\xd3j\xf7\x9ax|\xb9\x14\xd1\xefN\xe9\":\x96ZbϿ\xae\xfd\xf8\x8cHk\xaaֲ\x96/\xd6\xcc\x1d\x8eq_\x91\x0e\ac\xbe\xc87\xe6\x01M\xae\xa5\rH\x06Q{\xd1d\"A\xc1\xdf3\x9b\"C\x17\xcd28\x17g\x7f\xba\xe5\x95\xef\xd5\xd3I\tO\xbd&\xcc\v\xf8\x04\xee\x1f\xc7\x1c\x9da,\f\x88/\x18\xda~\xf02\xb8\xfaP\x96\x88\xd5x\x1d\x01Ʒ\x11\x94\x16\xfd\x82彮\xcb\xe5\x87\x14z/vSN~JTi\xd3kY@lM\xa0+\b\xd0>\xe7\xe3mT&,\xb5{\xe1\xa7\xec|d\x9a\\^\f\x96\x81[&\\k\xbf\x9f\xf1[\xe6v\x8d\xa2\x1a\xb7\xf0\x02>\x1b\xca?\xdd\xec\xc0%\xea~2M\x0e\x9d\x01={~\x81A+r\x94\x7fc\xaf%a\x93\x1d\xe7\xd7k%\x1dn\xe7\n\tO\xbfU\xf3d\x03\xd3WC\xae\x13h\xe9\x81W\xb9X9Ws\xa9\vٔc\xe9L\x97P:\x13\x85\x94\xce\xcd\r\an\x15U&\x12\xf7\x96\xd6\xd5P$\xb8_\x16\x8eI\x0f\x1c\xfa\xa0\xe8E\x0e\xac#i\x87_b<\xa7\xdf\xcb\xec\xc9\xd7\\:\x05l\xba\xd6x\x95⽐\xea\xea\U000e4cde\x84\xa3\xfb\xf2ws\xc1r\xfa\x9dķ\xfd\xbc\xfdO\xe6獝\xb7{\x14Ή\xe3\xf4\xe8\x1e]z\xfe\xc9^\xf5\x8c\xf3i\x9d\xe8߄\xed\xe9\x1f\x89%\xfc\xf9\xf7\xec\x9f\x00\x00\x00\xff\xff\x18\xd9g\x90\x9b\x14\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c)\x92\xef\xfe\x15\x84\xeeav7\xba\xa5u\xdcG\\\xe8\xcd+\xdb;\x1d;c\xeb,\x8d\xf6\x99\xae\xca\xeefDA-P-\xf7\xde\xdd\x7f\xbf \x81\xfa袪\xa8VK\xe3\xdd3/\xb6\xba !?\xc8L\x92\x04\x96\xcb\xe5\x1bZ\xb2\aP\x9aIqMh\xc9\xe0\xab\x01a\xffҗ\x8f\xff\xa9/\x99\xbcڿ}\xf3\xc8D~Mn*md\xf1\x05\xb4\xacT\x06\xefa\xc3\x043L\x8a7\x05\x18\x9aSC\xaf\xdf\x10B\x85\x90\x86ڟ\xb5\xfd\x93\x90L\n\xa3$砖[\x10\x97\x8f\xd5\x1a\xd6\x15\xe39(\x04\x1e\xba\xde\xff\xf1\xf2\xed\x7f\\\xfe\xfb\x1bB\x04-\xe0\x9a\xaci\xf6X\x95\xfar\x0f\x1c\x94\xbcd\xf2\x8d.!\xb3 \xb7JV\xe55i>\xb8&\xbe;7\xd4?ak\xfc\x813m\xfe\xd2\xfa\xf1'\xa6\r~(y\xa5(\xaf{\xc2\xdf4\x13ۊS\x15~}C\x88\xced\t\xd7\xe4\x93\xed\xa2\xa4\x19\xe4o\b\xf1\xa3\xc6.\x97~\xc0\xfb\xb7\x0eB\xb6\x83\x82\xba\xb1\x10\"K\x10\xefnW\x0f\xffz\xd7\xf9\x99\x90\x1ct\xa6Xi\x10\xf7\xffYֿ\x13?J\xc24\xa1\xe4\x01q$ʓ\x9c\x98\x1d5DA\xa9@\x830\x9a\x98\x1d\x90\x8c\x96\xa6R@\xe4\x86\xfc\xa5Z\x83\x12`@\xb7\xe0e\xbc\xd2\x06\x14ц\x1a \xd4\x10JJɄ!L\x10\xc3\n \xbf{w\xbb\"r\xfd+dF\x13*rB\xb5\x96\x19\xa3\x06r\xb2\x97\xbc*\xc0\xb5\xfd\xfde\r\xb5T\xb2\x04eX \xba+-Ij\xfd:\x86\xab-\x96<\xae\x15ɭH\x81C˓\x18rOQ\x8b\x9f\xd91ݠ\x8fBf\x7f\xa6\xc2\x0f\xff\xf2\b\xf4\x1d(\v\x86蝬xn%q\x0f\xca\x120\x93[\xc1\xfe^\xc3\xd6\xc4H\xec\x94S\x03\xdaRƀ\x12\x94\x93=\xe5\x15,,Q\x8e \x17\xf4@\x14\xd8>I%Z\xf0\xb0\x81>\x1e\xc7\xcfR\x01ab#\xaf\xc9ΘR__]m\x99\t\xf3+\x93EQ\tf\x0eW8Uغ2R\xe9\xab\x1c\xf6\xc0\xaf4\xdb.\xa9\xcav\xcc@f\xd9|EK\xb6DD\x04α\xcb\"\xff\x97 \x1e\xbaӭ9X\xb1\xd5F1\xb1m}\xc0\xf91\x83=v\xea8at\xa0\x1c\x8a\r\x17\xecO\x96t_>\xdcݷ\x05\x95iϔ\x96\xbc\x0e\xf1\xc7R\x93\x89\r(\xd7n\xa3d\x810A\xe4NTQ\xce9\x03a\x88\xae\xd6\x053V\f\xfeV\x81\xb6s@\x1e\x83\xbdA\x1dD\xd6@\xaa2\xb7b|\\a%\xc8\r-\x80\xdfP\r\xaf\xcc+\xcb\x15\xbd\xb4LH\xe2V[\xb3\x1eWv\xe4m}\b\nr\x80\xb5N\xb1ܕ\x90u&\x9am\xc56,s\xd3i#U\xa3w\x9c\x0e\xecR(>\xf5m\xc94\xbb\x13\xb4\xd4;i\xeeY\x01\xb22\xc75\xa6d\r\x99w\xb7:\x82\x12F\xe8Nj:\xabҐ\xdbI\xfbD\x99\xc11\xdfܭ\xc8\x03*\xab\xd0\x1a\x95V\xa5\x89\xa9\x94\xb0R\x12\xe9\xeb\v\xd0\xfcp/\x7f\xd1@\xf2\n\x85;S\x80tX\x905l\xac$(\xb0\xed\xed'P\xca\xd2F\xe3\x00d\xd5S6\xb6\xdc\xef\xc0ҖV\xdc\xf8y\xc24y\xfbGR0Q\x99\x9e\xa8\rr\x1d)E\r-\xe4\x1e\xd4)D|O\r\xfd\xd96>\xa2\x9d\x05J\x10\xaa%\xde\xda\xd3q}\xc0\x8f1n\xbb\xb2ڴ 2M..\x88T\xe4\xc2Y\xe0\x8b\x85k]1n\x96L\xb4\xfbxb\x9c\x87^\xe6!\xefh\xe8\x18\xaa\xef\xe5G\xed\x84\xf7$Z\f\xc0j\x91\xe6i\af\a\x8a\x94\xb2\xb6x\x1bƁ\xe8\x836Px\xc2\x04+\xe2\xf1\x89\xf4\x84s\x87s\x0fB[\xbazD\xfaȋ\x8as\xba\xe6pM\x8c\xaa`\x806k)9P1A\x9c/\xa0\r\xcb\xceA\x1a\a)B\x18\xe5?t(\x80F\x93>\x02\xa1\x11Оf\xd6:s\xde\"l\x97*\xd11\x95\n2\xab\xb5\xaf\xbd5`\xc0\xd1\x02\tI\xb8\x14[P\xaew\xeb\xa9\x04\x01S`\x05.'V\xd1*\xe0֚\x90Meu\xf0%\xb1\xb3{P\x06\x98\xd0\x06hD8\x9f\xc1\x1f\xf8\x9a\xf1*\x87\xfc\xc69^w\xd6\x7f̃\xd7\xdcӚ)|\xfa0\n\xd1[g\xce2t\x02\xbd\xbf\xb7D\xbf5&\xa6\x8d\x91>\x94\xe0\\g\xcbJ?\xec\xc6\xfa\x8e\xea\x03\r\xc66\xba\xf8\xc3\xc5\x029\xdc\xed\xb5ۇ&TAM\x96d\xbd\tEi\x0e\xfd\xda\xcc@\x11\xa1\xe2\xa8>I\xe4'U\x8a\x1e\x06\xb8Y\xfb\xffg\xe4\xe7\x10\xcc#\x8e\x8aP\xed\x95yz\xdc\xef?3W\xcf\xc3G\x8d\xab]ʄ\xe5\x9f]xvا\xdd\xfa͒MH\x13\x81DŽ\x83\x87K\xb3\x11n\xfdF\xc4:\x8b\xcc\x0f\ty-[^x\xff!)\xb5\x93\xf2q\x8a:?\xda:͢\x88d\x18U!k\xd8\xd1=\x93ʣޘZ\xf8\nYe\xa2\xb3\x9e\x1a\x92\xb3\xcd\x06\x94\x85S\xee\xa8\x06\xed\x96\xc9\xc3\x04\x19v\xdfIK\x8dD?\x1e\xe1\xd10Ҳ\t1\x1f\x1a\xba\xf5#\x8e\xadd(v\xa0ֽFc\x9c\xb3=\xcb+\xca\xd1.S\x919|h=\xae\x98\x96\x19aro\xccQ\xc9t\xc59\x04\x01)ˤ\xceJI\n\xb0>oa\xd7\x04\xfd\xaaØ\xaf\xa9\xf5U\xe4\x10\xf6\x04\x99\xa5*\x0e\xdaw\x95\xa3\x1b\xd9\xe8\x8cE\xc3\x14\fD\x10N\xd7\xc0\x89\x06\x0e\x99\x91*N\x91)>\xbb\x92\xa2\x04\a\b\x19\xd1|ݕF\x83\xc0\bH\x82K\xb8\x1d\xcbv\xceճB\x84pH.\xc1:|\x86в\xe4\x11sєQ\xe6\xfbN\xc6\xe6zS&f\xfd1\xbc\xd8\xfcoJ\x82\xcelJ\x94\xb4\xcd\xfc\xeaR\xb6\x16\x87\xf8\x9a\xb6)\xff\x9c\x84\r\x9a\xff\x04\xa1\x1d\x99\xfd\x04\xa3B\xc92=(\xb7\x96\xaa\f\xf4\xa5u\xa7\xd0\xd3Y\x10f¯S3\xa1\xe3s\xf5\x82e\x1d\"|ۼ\x99/\xf4\x89\xacI\x99\x13/Ę\xba\x8b\x7f@\xbe\xa0ɸ\xf3\x16#\x99'?\xb5[-\b\xdb\xd4D\xcf\x17dø\x01uD\xfd\x93T}\xe0\xcc9\x88\x91b\xf5\b\x86\xefM\xb6\xfb\xf0պ`\xba٩J\xa4\xcbqc\xe7\xc8\x06o\xbfk\x9e'\xe0\x12\fc3\x05\x05\x86\xc7q\xc5\xd4\xfe\x05]\xabw\x9f\xde\xc7\xd7W\xed\x92 y=D&&\x9d+\xef\x8e0j\x8fϻ\xf0\xe1\v\xfa@\xf5\x02\xc8m\x85,\b%\x8fpp\xae\v\x15\xc4\xf2\x87\x86\xca\t\xdd+\xc0=\x19\x94\xb3G8 \x98\xf8&K\xbf\xa4J\x83+\x8fpH\xa9vDC;&\xa6\xfd摥\x93\xfd\x01\t\x81\xb1\xf5T1p\xc5O\x85ȖF\xbc$\xea\x92P\x02\xedO@3IT\xda}\xb4w)Q\x02~Ў\x97v\xc6\xecX\x89j\x15#\x0er\x93\xccPW\x1e(gyݑ\x9b#+\xb1 \x9f\xa4\xb1\xff|\xf8ʴ\xdf\xc8|/A\x7f\x92\x06\x7fy\x11\x8a\xba\x81\xbf$=]\x0f8ф\xd3\xf2\x96`\xed\xad8gӬ\xb4մg\x9a\xac\x84]\xae8\x92$v\x85\xbb\xae\xae;\xd7QQi\xdcE\x13R,]\xd8&֓\xa7\xb7T\x1dr?\xbbS\xdf\xe1\xbd5\x16\xee\x8b\xdb\xfb\xe54\x83\x8d\xcf\xd2\xceلZA\x12&\xab\x0e\xecY\x0eWM!\xca3ȁV\x1c]\x9cI\xee\xd2<\xc7<\x17\xcaogX\x94\x19\xb20W5\xb4\xc6\xeeLpAq\xab忭\xa5\xc5\xd9\xf4\xbf\xa4\xa4L\xe9K\xf2\x0eSZ8t\xbe\xf9\xa0Y\vLB\x97\x98\x92b\xe5gO\xb9\xb5\xfdV\x81\v\x02\xdcy\x02r\xd3\xf3\x8b\x16\xe4i'\xb53\xdb\xf5&\xce\xc5#\x1c\u070e\xe1d\x97m%s\xb1\x12\x17·\xe8)\x8c\xdaᐂ\x1f\xc8\x05~\xbbx\x8e+\x95(\xa9\x89\xd5:\"Z\xd02MB1\xa5(\xd5Q\xb7\v\xd6\xe0\x84؆u\xaa\x8cu\xb2ǰM\x12\xd1R\xea\xc8F\xfe\xc0P&\x84\xf7Vj\xe3\xe2e\x1d\x9f9\x1aP\x93!\x88F\xe8\xc6\xe5/I\x15\x92M\xacR\x9e\n\xfd\xb6\xcb\xfd\x0e4\xf8\xfd\n\x1f\x98s@\xed\xca\ue899\xdfN\xdb_\xb8\xfd\x12\xec\x84f\xe8\xb1`\xdbR\xc9\ftt/\xbb)\t\xf6\"\x92\x95\xd1ƽ\x8e9R\xb7Jr)\x19\xe3!\xd0P\xd2]^K\x88\x99\xeb\x85\x0f_[\x01Q;\xf7\xed\xdfS26w\\\x04S\x06\x8b\x82\x1e\xa7)%\r\xf1Ƶ\f\xb3\xc1\x03r\x8b\x0f\xb5\xadP\x13\xa4\xda\xf2Z\x00\xbf\x05G\xa1`b\x85\x1d\x90\xb7/\xe0Xx\x1d\x1aK6\x89\x95\xd3\\ٛ\xd0IÝ\xfa\a7\x95K\x89[\x05\n:\xcc\xebG\xd5\xd1\x0f\x15Ҵ\x02\x123\xdc\xcdR\xe6?h\xb2aJ\x9b\xf6\x10\xf4@\x9aJ\x14\xcc̅\x97\xf8\xa0\xd4I\xeb\xaeϮe+ܵ\x93O!=\xcb\x11&\x11s\xdc_\x02\xc26\x84\x19\x02\"\x93\x95\xc0\x00\x8e\x9d\xc7\u0605#\xaeӰ,u\x92\xa4\xcd~[@TE\x1a\x01\x96()L\x8cFz\xda\xd5?R\xc6_\x82mf(\x8b-VN\x9b\x13!ŭ\x9d\x90WЯ\xac\xa8\nB\v\xcb#4欀.ӛ\xc47\xdb\x02̈́\x91vƔ\x1c\f\xf8\xe4\xb5\xc41dRh\x96Cm\\\xbd HA(\xd9P\xc6+\x95\xa8\x01g\x91w\xceR\xc4k\x82\xf3\xad1\xd2:_\")\x12\xa2\xb9\x89\xbe\xe2\xb86.U\xba\xc77\xe5f)\x98\xefe\x95\x8aIL\v<\xb3\xa3\xe5\x13)\xa98|\xf7\xb4R\x87\xfa\xdd\xd3\x1a+\xdf=\xad\x89\xf2\xdd\xd3\xfa\xeei\xa5\xd4\xfc\xeei}\xf7\xb4\xda\xe5\xff\x85\xa755\"w\x9eo\xe0\xe3\xe4(\x12\xb6\xaadž8\x02\xdf'W\xf8\x1c\xf0g\xe5b\xae\xe2\xa0\"\x89\xff\x03i\xdd1\xa5\xd5\x18\x8f:9\xd3Κ \xf3\xeexф+\xf9\x8c\xac\xfb\xd0\xe9\xf9\xb2\xeeW\xa3\x10ϔu\xef\x87=\xedc\x9f\x94s\x1f\x882/;{\xe1\x135\n\xa0!\xac\xee\xb6\xe1cx\rI\xc8D\xff\xaf\x9c\x98\xdb\xcb\x1a;\xa3|\xbcx\x16\x7f\xb2\x8cDYz\xf1\x87\x8bo\x8f\xfc\xe7!\xf8 \x89\xfb\xb4\xf3\xe7\x9b#P\xed\n\xb4\x9d\x16\xd6\xcd\xc2\xfb6\xc5\xf8,r\x9b\x9a\x89_\x131\x02\xab+\x92GT\xfcVu\x81\x81\xe2s\xe9-\xd23N\xaa\xae\"p\x92ΪR}\x10\xd9NI!+\xed\xa3\x12\x16ֻ\xcc\x1dh\x0f c\xc2\x1a\x9d\xe1\xffFv\xb2\x8ad\x82\x8f\x90o\"#p\x1a\xf9Nr\xa0߄\x06C\xf7o/\xbb_\x8c\xf4\xa9\x82䉙]\x04\xd0\xd3\x0e\x04\uec0bm\xfb\x00@\xb8\x8f\xc0\x1f\xcc?\x16\xb0\b \xa9\x88`\xdcI^}\x9bA[\xee\xc8\xe7\xd2Şf\xfb\x1d\xe31\x95\xb4d\u0093S\b\xbb)\x82\x03~\xe9\xdc\xdd\xee\xb3\x1c\x99\xf8MR\x03\xe7'\x04\xa6D\xc4&\x92\xffNH\xf9K\xcc-~\xf6\xf6|JRߜ\x15\xf3\x8b%\xf0\x9d?m/\x89>\xd3)zs\xa8\xf3\xe2\xe9x\xaf\x98\x84\xf7:\xa9w\x89\tw\xe7˜O\x8bǞ\x9496\x1d:\x18N\x9a\x9bL\x95\x9b\f-L!6\x1b\xa5\xc9\x14\xb89\x89o\x93\xdcI\x9bf\xaf\x96\xda\xf6j\tm\xaf\x9b\xc66*E\xa3\x1f\xe7$\xaaů\xa5!\x93Ɩ\xbf\x96\xb0\x9dJ\x06\xa9:\xee\xebI\xeb\xab\xcfG0,\xe3\x83k\xf7J>rQq\xc3J\x8e\x1b\xa9{\x96G\x83\rf\a\x87\xfa\x02\x8d_%\x1e=\xf57\xc1|\xfeRK\xed呧O5y\x02\xce\t\x8dͫ\x1e晻\x89)\x93K\xb0\xf6\xc8\xceN\x7f1\x88\xbf\xbei\xe1\xc4\x1dOעU+b!&*\x86o\x91\x194\x1c)\xfa\xa6\xe7\xc1:?\x1c\x7f\xfb[\x05\xea@\xf0\x1e\x9b\xda\xcfi\x0e\x81\xf9\x89\xa9\xedB,\xa8\n\xaf\xb6\x86\xe2\xe7=\xa7\xbf\x99\xca\xe4\x9dpV\xf7x<\xd8\xc6\xea\x88fQc\x15\x9f]\xafD\xfb\x18h.d\xdd:\xd2l\xcaAN=-\xf5\xb2K\x9c\xf9\x8b\x9cI\xaf\"\xdd\xf3\xfb\x8dNA\x9dr\xfa)-\x01`\xf2\xb4\xd3K-y\xa6\x16=\xc9~^\xdai\xa6y\x9b\x85/xz\xe9%N-%R*\xe5\x94\xd2<:\xbd©\xa4W=\x8d\xf4Z\xa7\x90\x92O\x1f%\xa5\xb8$\xef\x02\xa7\xa6\xa8\x9cx\x9cfz\x8fw\xfc4Q\xc2)\xa2\x84\xdd\xdfi$O@/\xe1\x94м\xd3A\t\x80\xf3\x83\xaeab\xae(\xbeg\xd2'B\xdf\xf9uъk\xeb\xfd\xc3҂8\xcdk\x8d\xea\xe6L\xb3\xae]x\x9e\x92\xbb\xb9[\r\x81;E\xc5\xf5\x9f{y\xe64\xee\xa3\xfb,\x95\xd6Gw\x96B\x8b@\xace\xfc\xfc\xb8\xe3T?\xedRwl\xe9\x1c\x8e,\x9c\xa1\xa3\x9c\xfb\x83\x8e\x05hM\xb7\xe16\xf7'\xbb\xf4\u0602\x00\x17\x9es\x9b'\x11\xa0ͩ\xb8\xee]\xe6n\xca\xd0\xccT\xd4w\x10\x12|[\xb5~Є\xcb\x18T|Ѕ\x85\x97\xc2\u009al&\xa1\xbe\x96L\xa5\xac\xe1>\xd4\x15-m\xd0\x13F\xee4o\xbb\x01g[f\xd7:\x96s[\xaa\xd6t\v\xcbLr\x0e\xa8\xad\xfb\xe3zɹ\xee\xcf\x1e~\x01\xaa'Q\xfbخ\xebw\x00\x1d\xb7\xdd\xc67u\xe9\xee\xf8z\x97a\n\x9a\x87\xf4z\x03\x92\xd8\xf1,G\xd9Q!\xfa\xca\\\x7f\xa4\xed\xbaa\xd6y\xb5\xec\xe3\xbc\xfe\x91\xb9\x85\x8f\v\xc4屠\xbfJ\xb5 \x05\x13\xf6\x1f*r\xb7\x81\x17\x1a\xcf\x1a\xffN\xcaǻ\x88\x13\xdb\x1b\xfc\x8fu\xc5f\xab\x83\t7l<0\xba\x96\x95\xdf}\xaf\x1d\xda\xf8\xb6\n\xde\xcc\x7f\xe6\xe5&\xc2\x1c\xb1\a=t\x06#\xba?v M\x9a\x02\xd7\xf3\x00\xac\xbb\xf0\x92\x19\xe7\x87\xc51\xe4\xa3W\x13\x1bح\x97\v\xbc\x1b\xd0\xdcG0\xd0Qؑ\x8a\x02\xa9/\xbeh+\xf4SV\xbd\x9e\xccC\xced\x8f\xc6?6\xb5\x87\xe8\xe8\x86\xd9r\xf7\x06\x10\xec8\x81\xe7]\xb0\xe33\x15\x13\xc2\x7fk\xeb\xd4w\x17\xb4\x16n!Kl0J\x17?\xfb\xbe$\x9f\xa0\xbf]\xb1$\xffUA\x15\xa1\xc12<\fwg\xa8\xea\x87|\xdd1x\xc81\xa3\x03gc\xa4\xcaJ\xdc*\xb9U\xa0\xfbº$\x7f\xa5\xcc0\xb1\xfd(\xd5-\xaf\xb6L|\x1e>\xf23V\xf9\x96*ì\xb0\xbb\xf1\xc4\x06\xca\x04\xe5\xec\xef1\xbd\xd6\xfe8\r\xe8fp\x81\xb5$\t\xc3\x18\xfa\xf0\x1e\xac\x8f;\x18\x17\x88\xaa\xd0\xd2\xd3\xf5\x14\x7f%\xf0dJ\xa7־D㋄n/\xc9'\x19U\f>\x1d\x8auaZ\x97\f\xb4Y\xc2f#\x95q\xbb\xd5\xcb%a\x9b\x10|\xb0:\a\xe3f\xee\xadJ\xc2b\xdb\xccu\xa2Ic\xbe0\xe8\xad\xd0\n\xe3U\xf6\x05=\xb8\x9d)\x9ae\x95\xf5\xb0\xae\xb4\xa1<\xe2\xe0\xc8\x7fy\xd6Nު\r\xa8\x1ft\xc4~\x1cI\xf12\r\xe7\xf5q\x8b\"\b\xf2\xa4\x981֧\x92#\xa9\x04\x9eT\xc6\xfaV\x9c\x13mI}R\xf4\x9185\xba\x1aN\xc9IC\xf9\xbe\x862\xa4\x9e=\xd6\xf82\xe3\x1aiC\xacߋ\xd9G\xbe\x96es\xb6\xa3b;xC\xc1N\xc9j\xbb\v\x92<\xe0L\x93\xbc\x02\f֢J\xd1\xe1aaS)\xd1J%\x189\xf6M\x820\xe0pi\xf6H\xaar\xe1\x1f\xee\xf5\xef2_\xf97P\x96\x1b%\x8b\xa5\xef\x17c\xa9\v\xbf\x93\xaf\x98\xb4\x9e\x8b\xd9E\xa9N\x9c\xd7\xee\x9f\x19@I(K\x10\x84j\xdfs\xc2MQ'\x9b)mU\xff\xb3\xc2\x18w\x1d\b\x13\x11\f\xec.\x8eĝOhpWe\xdd\xf8\aJk\xc0\v\xa2\x99\bOC\xbb\xe4\b'\x1fѽ2\x81O9J\x15\xcfW\x1c\x0fIt\x11z\xddhľ\xb6\xb5\x1fN^\xac>\x1c\xc18:\xf6\x8c/w\xd6U\xc2\x02\xf3w,\x161\xc7D\xd7̢\xf2\xfb\xdf\xfc8\xf3>i1\x14\xa7\xc8\xd8\xda\b\x97=Ë\x9c\xeeK\x9d\xb7\x1c\xac˦\x01\xbaˮY\xcb\xeb\xfd\x19\xe3M\xe7\f6\x85G\xd0\xcf\x13mٟ1\xcc\xf4b1\xa6\xf3\xa2\xfcD\xf1\t\xe5\x93f\xed_}\xdbH\x90Ƀ=w\x98\xa9\x15e\n\x03\x7f\xd58S\xd4*\xf5~D=\x9d\xb7\xb4\x85\xef\xc9\xff\xf2\x7f\x01\x00\x00\xff\xffC\xf4\a\x86ځ\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\xdb8\x92\xef\xf9\x15(\xdd\xc3\xecnI\xf6\xa6\ue8ee\xfc\x96q\x92\x1d\xd5\xcc$\xde\xd8\xe3}\x86Ȗ\x841\bp\x00P\xb6\xf6\xee\xfe\xfb\x15\x1a\x00?DP\x04eٓ\xdd\r_\x12\x8b`\x03\xfd\xddh4\x80\xc5b\xf1\x86\x96\xec\x1e\x94fR\\\x11Z2x2 \xec_\xfa\xe2\xe1\xbf\xf5\x05\x93\x97\xbb\xb7o\x1e\x98ȯ\xc8u\xa5\x8d,\xbe\x80\x96\x95\xca\xe0=\xac\x99`\x86I\xf1\xa6\x00Csj\xe8\xd5\x1bB\xa8\x10\xd2P\xfb\xb3\xb6\x7f\x12\x92Ia\x94\xe4\x1c\xd4b\x03\xe2\xe2\xa1Z\xc1\xaab<\a\x85\xc0C\u05fb?_\xbc\xfd\xaf\x8b\xff|C\x88\xa0\x05\\\x91\x15\xcd\x1e\xaaR_쀃\x92\x17L\xbe\xd1%d\x16\xe4Fɪ\xbc\"\xcd\v\xf7\x89\xef\xce\r\xf5{\xfc\x1a\x7f\xe0L\x9b\x1f[?\xfeĴ\xc1\x17%\xaf\x14\xe5uO\xf8\x9bfbSq\xaa¯o\bљ,\xe1\x8a|\xb2]\x944\x83\xfc\r!~\xd4\xd8\xe5\xc2\x0fx\xf7\xd6AȶPP7\x16Bd\t\xe2\xdd\xcd\xf2\xfe\xdfo;?\x13\x92\x83\xce\x14+\r\xe2\xfe\xbf\x8b\xfaw\xe2GI\x98&\x94\xdc#\x8eDy\x92\x13\xb3\xa5\x86((\x15h\x10F\x13\xb3\x05\x92\xd1\xd2T\n\x88\\\x93\x1f\xab\x15(\x01\x06t\v^\xc6+m@\x11m\xa8\x01B\r\xa1\xa4\x94L\x18\xc2\x041\xac\x00\xf2\x87w7K\"W\xbfBf4\xa1\"'Tk\x991j ';ɫ\x02ܷ\x7f\xbc\xa8\xa1\x96J\x96\xa0\f\vDwOK\x92Z\xbf\x1e\xc3\xd5>\x96<\xee+\x92[\x91\x02\x87\x96'1䞢\x16?\xb3e\xbaA\x1f\x85\xcc\xfeL\x85\x1f\xfe\xc5\x01\xe8[P\x16\f\xd1[Y\xf1\xdcJ\xe2\x0e\x94%`&7\x82\xfd\xbd\x86\xad\x89\x91\xd8)\xa7\x06\xb4\xa5\x8c\x01%(';\xca+\x98[\xa2\x1c@.\xe8\x9e(\xb0}\x92J\xb4\xe0\xe1\a\xfap\x1c?K\x05\x84\x89\xb5\xbc\"[cJ}uy\xb9a&\xe8W&\x8b\xa2\x12\xcc\xec/QUت2R\xe9\xcb\x1cv\xc0/5\xdb,\xa8ʶ\xcc@f\xd9|IK\xb6@D\x04\xea\xd8E\x91\xff[\x10\x0f\xdd\xe9\xd6\xec\xad\xd8j\xa3\x98ش^\xa0~L`\x8fU\x1d'\x8c\x0e\x94C\xb1\xe1\x82\xfdɒ\xeeˇۻ\xb6\xa02\xed\x99Ғ\xd7!\xfeXj2\xb1\x06\xe5\xbe[+Y L\x10\xb9\x13U\x94s\xce@\x18\xa2\xabU\xc1\x8c\x15\x83\xdf*\xd0V\a\xe4!\xd8k\xb4Ad\x05\xa4*s+Ƈ\r\x96\x82\\\xd3\x02\xf85\xd5\xf0ʼ\xb2\\\xd1\v˄$n\xb5-\xebacG\xde\u058b` \aX\xeb\f\xcbm\tYG\xd1\xecWl\xcd2\xa7Nk\xa9\x1a\xbb\xe3l`\x97BqշO\xa6٭\xa0\xa5\xdeJs\xc7\n\x90\x959l1&kȼ\xdb\xe5\x01\x940B?^\xb4Y\x95\x86\xdc*\xed#e\x06\xc7|}\xbb$\xf7h\xac\xc2\xd7h\xb4*ML\xa5\x84\x95\x92H__\x80\xe6\xfb;\xf9\x8b\x06\x92W(ܙ\x02\xa4Ü\xac`m%A\x81\xfd\u07be\x02\xa5,m4\x0e@V=cc\x9f\xbb-X\xdaҊ\x1b\xaf'L\x93\xb7\x7f&\x05\x13\x95\xe9\x89\xda בR\xd4\xd0B\xee@\x9dB\xc4\xf7\xd4П\xed\xc7\a\xb4\xb3@\tB\xb5\xc4[y:\xae\xf6\xf82\xc6m\xf7,\xd7-\x88L\x93ٌHEf\xce\x03\xcf\xe6\xee\xeb\x8aq\xb3`\xa2\xdd\xc7#\xe3<\xf42\ryGC\xc7P}'?j'\xbc'\xd1b\x00V\x8b4\x8f[0[P\xa4\x94\xb5\xc7[3\x0eD﵁\xc2\x13&x\x11\x8fO\xa4'\xd4\x1d\xce=\bm\xe9\xea\x11\xe9#/*\xce\xe9\x8a\xc3\x151\xaa\x82\x01ڬ\xa4\xe4@\xc5\bq\xbe\x806,;\ai\x1c\xa4\ba\x94\x7fѡ\x00:M\xfa\x00\x84F@{\x9aY\xef\xccy\x8b\xb0]\xaaD\xc7T*Ȭվ\xf2ހ\x01G\x0f$$\xe1Rl@\xb9\xdem\xa4\x12\x04L\x81\x15\xb8\x9cXC\xab\x80[oB֕\xb5\xc1\x17\xc4j\xf7\xa0\f0\xa1\rЈp>\x83?\xf0\x94\xf1*\x87\xfc\xda\x05^\xb76~\xccC\xd4ܳ\x9a)|\xfap\x14\xa2\xf7Μe\x18\x04\xfaxo\x81qkLL\x1b'\xbd/\xc1\x85Ζ\x95~؍\xf7=j\x0f4\x18\xfb\xd1\xecO\xb39r\xb8\xdbk\xb7\x0fM\xa8\x82\x9a,\xc9v\x13\x8a\xd2\xec\xfb\xad\x99\x81\"Bţ\xf6$\x91\x9fT)\xba\x1f\xe0f\x1d\xff\x9f\x91\x9fC0\x0f8*B\xb3W\xe6\xe9a\xbf\xff\xcc\\=\x0f\x1f5\xcev)\x13\x96\x7fv\xe2\xd9a\x9fv\xf37K6!M\x04\x1e\x13\x0e\x1eN͎p\xebw\"\xd6Yd~H\xc8k\xd9\xf2\xc2\xfb\x0fI\xa9\xad\x94\x0fc\xd4\xf9\xc1\xb6i&E$ì\nY\xc1\x96\xee\x98T\x1e\xf5\xc6\xd5\xc2\x13d\x95\x89j=5$g\xeb5(\v\xa7\xdcR\r\xdaM\x93\x87\t2\x1c\xbe\x93\x96\x19\x89\xbe<\xc0\xa3a\xa4e\x13b>4t\x1bG\x1cz\xc9\xf0\u0601\xda\xf0\x1a\x9dq\xcev,\xaf(G\xbfLE\xe6\xf0\xa1\xf5\xb8bV\xe6\b\x93{c\x8eJ\xa6{\\@\x10\x90\xb2L\xea̔\xa4\x00\x1b\xf3\x16vN\xd0o:\x8c\xf9\x8a\xdaXE\x0eaO\x90Y\xaa\xe2\xa0}W9\x86\x91\x8d͘7L\xc1D\x04\xe1t\x05\x9ch\xe0\x90\x19\xa9\xe2\x14\x19\xe3\xb3{R\x8c\xe0\x00!#\x96\xaf;\xd3h\x108\x02\x92\xe0\x14n˲\xad\v\xf5\xac\x10!\x1c\x92K\xb0\x01\x9f!\xb4,y\xc4]4\xcfQ\xe6\xfbN\x8e\xe9z\xf3\x8ch\xfd!\xbc\x98\xfe7O\x82\xcdl\x9e(i\x1b\xfd\xeaR\xb6\x16\x87\xf8\x9c\xb6y\xfe9\t\x1b,\xff\tB{D\xfb\tf\x85\x92ezPn-U\x19\xe8\v\x1bNa\xa43'̄_\xc74\xa1\x13s\xf5\x92e\x1d\"|ݼ\x99.\xf4\x89\xacIщ\x17bL\xdd\xc5? _\xd0e\xdcz\x8f\x91̓\x9f\xda_\xcd\t[\xd7D\xcf\xe7d\u0378\x01u@\xfd\x93L}\xe0\xcc9\x88\x91\xe2\xf5\b\xa6\xefM\xb6\xfd\xf0dC0ݬT%\xd2\xe5\xf0c\x17Ȇh\xbf\xeb\x9eG\xe0\x12Lc3\x05\x05\xa6\xc7q\xc6\xd4\xfe\x05C\xabw\x9f\xde\xc7\xe7W\xed'A\xf2z\x88\x8c(\x9d{\xde\x1d`\xd4\x1e\x9f\x0f\xe1\xc3\x1b\x8c\x81\xea\t\x90[\n\x99\x13J\x1e`\xefB\x17*\x88\xe5\x0f\r\x8d\x13\xbaW\x80k2(g\x0f\xb0G0\xf1E\x96\xfe\x93*\r\xeey\x80}J\xb3\x03\x1a\xda11\xed\x17\x8f,\x9d\xec\x0fH\b̭\xa7\x8a\x81{\xbc*D\x964\xe2O\xa2-\tO\xa0\xfd\th&\x89J\xbb\x8f\xf6*%J\xc0w\xda\xf1\xd2j̖\x95hV1\xe3 \xd7\xc9\fu\xcf=\xe5,\xaf;r:\xb2\x14s\xf2I\x1a\xfbχ'\xa6\xfdB\xe6{\t\xfa\x934\xf8ˋP\xd4\r\xfc%\xe9\xe9z@E\x13\xce\xca[\x82\xb5\x97\xe2\x9cO\xb3\xd2VӞi\xb2\x14v\xba\xe2H\x92\xd8\x15\xae\xba\xba\xee\\GE\xa5q\x15MH\xb1pi\x9bXO\x9e\xdeRu\xc8\xfd\xecN}\x87w\xd6Y\xb87n\xed\x97\xd3\f\xf2\xb0\\\x83\x8b\x92\xd4\xc0\x86e\x89\xfd\x15\xa06@Jk\xc2\xd3$\"Ѱzl\xa6\x89O\x9a\xf7n?O\x8b\x87z\x8d\x7fa]\xce\xc2C0\xb2H\xa0\x81\xb7\xdd\xf98>\v\xab\xb3\t\xad\x82$\x8c6\x1dX\xb3\x1cn\x9aB\x94g\x90\x03\xbd8\x868\xa3ܥy\x8eu.\x94\xdfL\xf0(\x13da\xaaih\x8dݹ\xe0\x82\xe2R\xcb\xffXO\x8b\xda\xf4\x7f\xa4\xa4L\xe9\v\xf2\x0eKZ8t\xde\xf9\xa4Y\vLB\x97X\x92b\xe5gG\xb9\xf5\xfdր\v\x02\xdcE\x02r\u074b\x8b\xe6\xe4q+\xb5s\xdb\xf5\"\xce\xec\x01\xf6n\xc5p\xb4˶\x91\x99-\xc5\xcc\xc5\x10=\x83Q\a\x1cR\xf0=\x99\xe1\xbb\xd9sB\xa9DIMl\xd6\x11т\x96i\x12\x8a%E\xa9\x81\xba\x9d\xb0\x86 \xc4~X\x97\xca\xd8 \xfb\x18\xb6I\"ZJ\x1dY\xc8\x1f\x18ʈ\xf0\xdeHm\\\xbe\xac\x133G\x13j2$\xd1\b]\xbb\xfa%\xa9B\xb1\x895\xcac\xa9\xdf\xf6s\xb7\x05\r~\xbd\xc2'\xe6\x1cP;\xb3\x9b5\xfa\xed\xac\xfḓ\x97`'4È\x05\xbf-\x95\xcc@Gײ\x9b'\xc1_D\xaa2ڸ\xd79G\xeafI\xae$\xe3x\n4<\xe9!\xaf%\xc4\xc4\xf9\u0087\xa7VB\xd4\xea\xbe\xfd{LƦ\x8e\x8b`\xc9`Q\xd0\xc32\xa5\xa4!^\xbb/\x836x@n\xf2\xa16\x15Z\x82T_^\v\xe0\xd7\x10(\x14L,\xb1\x03\xf2\xf6\x05\x02\voCc\xc5&\xb1\xe7\xb4P\xf6:t\xd2p\xa7\xfe\xc1\xa9r)q\xa9@A\x87y\xfd\xac:ơB\x9aVBbB\xb8Y\xca\xfc;M\xd6Li\xd3\x1e\x82\x1e(S\x89\x82\x998\xf1\x12\x1f\x94:i\xde\xf5\xd9}\xd9Jwm\xe5c(\xcfr\x84I\xc4\x1cח\x80\xb05a\x86\x80\xc8d%0\x81c\xf5\x18\xbbp\xc4u\x16\x96\xa5*I\x9a\xf6\xdb\aDU\xa4\x11`\x81\x92\xc2\xc4\xd1LO\xbb\xf9G\xca\xf8K\xb0\xcd\fU\xb1Ş\xd3t\"\x94\xb8\xb5\v\xf2\n\xfaĊ\xaa \xb4\xb0\x1f\xc0\xb0\x8c\x0f\xa1\xdd+\xc5\xc8E\xc5\r+9.\xa4\xeeX\x1eM6\x98-\xec\xeb\x034~\x95\xb8\xf5ԟ\x04\xf3\xf9K-\xb5\x17\a\x91>\xd5\xe4\x118'4\xa6W=\xcc3w\x12S&\x17`\xfd\x91\xd5N\x7f0\x88?\xbei\xee\xc4\x1dwעW+b)&*\x86O\x91\x19t\x1c)\xf6\xa6\x17\xc1\xba8\x1c\x7f\xfb\xad\x02\xb5'x\x8eM\x1d\xe74\x9b\xc0\xbcbj;\x11\v\xa6\u009b\xad\xa1\xfcy/\xe8oT\x99\xbc\x13\xce\xeb\x1e\x8e\a\xbf\xb16\xa2\x99\xd4X\xc3g\xe7+\xd1>\x06>\x17\xb2\xfe:\xf2\xd9X\x80\x9c\xba[\xeae\xa78\xd3'9\xa3QEz\xe4\xf7;\xed\x82:e\xf7SZ\x01\xc0\xe8n\xa7\x97\x9a\xf2\x8cMz\x92㼴\xddL\xd3\x16\v_p\xf7\xd2K\xecZJ\xa4T\xca.\xa5itz\x85]I\xaf\xba\x1b\xe9\xb5v!%\xef>J*qI^\x05N-Q9q;\xcd\xf8\x1a\xef\xf1\xddD\t\xbb\x88\x12V\x7fǑ<\x01\xbd\x84]B\xd3v\a%\xf0,U\x15_q\x17\xd0+\xee\xfey\xed]?#\x925\xf2z\xda\ue793\x97,\xa4\xcaA\x1d]\xf6I\x95£\xf2\x972\xb7\xe9\x0e\xe4`\xbd#\x9c\xfag[u\xe2et\x0f\xfe\xa0Q\x97G\xb3\xa2\xe4{;C!\xb3\xf6\a\xa7I@T\xdaBo7\x92\xb3,\x12\xbbE\xcffr\x8d{\x87e\xe0\x89QY\xbbd\xa0\xb4\r\xe3\xa1\x1b\x86y\xdd#0גs\xf98q\xeeOK\xf6\x17<\xb9\xfb\x19١w7K\x84\x11\xc4\x03\x8f\x02\xaf\x8b\xb3jlV`\xddr\x83\xe7\x90\xee/\xd7\x1d\x88\xdd:\xc7\xf6Ḑ\xbbs\x90CX\xe0Mg&\xadu\xb9Y\xbaq\f\xf5be\x86\x8a=\x91XQc\xb6L勒*\xb3w\x85\x1a\xf3\xce\x18\x82/=\x96\xdd\x19\xf4\x1e\xfd\xb3\x9d\xa3\xe4\rG:\xe3\n\xe5\xbe\xec.\xfa\x1e\xd2\xee\x94q\f\xef^\x1cݷx\xc6q\f\x87%\v\xa4T\xe4\xe7h\xe5\xd7ٲfڟL\xfc\xb3\xdc\xc1\xfbh\xf6\xacC\x9eۃ\xe6\x91\xf2\xac\x00\xd1\x1d\xba;X\xa5\xba\x02<\x90\xb7\xff\xea\x19\xf5V\xa1k\x7f\xa6\xea)\x89\xb2\xdb.\x88\b~\xe1\x84\xd9\xd0Y\xcc>\xe1\x01\xf0{rs\x8fs\xb4ڴy\x15\xf5s\xb4\x90*\v\x8b\xc1\x118\xfe\x83\xef\xcf_\x9a\xa6\x8dTt\x03?Iw\xc6\xf6\x18ۻ\xad;g\xaf\xfb\xa8'ԏ\x06\xa5\x89\x1d\xc0\xebO\xfb>\x00\xd6\xd4|\xf7\x0e5\xb6\xa3\x9cxL\xb31\xfc\x14\xbe\xdf\xdd\xfd\xe4\xb02\xac\x80\x8b\xf7\x95+w\xb06Q\x83%q\xc0\xd6AZ\xd9\xffn\xe5#\x1e\xfe\x1b\xcfc\x86;\x13\x1ad\x14`\xb19\x96 NB\xa9*\xb9\xa49\xa8k)\xd6l3\x82\xdd/\x9d\xc6\an6\xc3\x1f=r\xb5\x8f\n\xf0\xcf\\\x83`c\x1e\u0381\x7fd\x1c\xb4\x1bV\x82\x01\xbe\xe9\x7fU\xdb\xe3\xaaX\xb9\x18nm_\xd6\x1d\f\xf88\x87\x16\xa6\xa2KP6\x8arI\xebJ\aY\x1dF\xbc\xe1\b\x13\x066П\x05\x1e\xb1\xc0\xeeTit\x9f\xc1\x9c\xe0\\\xe6\xc7X~\xab\x83\xfc\xfd\xf0\x97\a\x9cl\xa5\xbcb'\xee\xb9 \xe4\xe6\xfeZ\x93J\xe4\x98.\xbe\xff\xcb\xed$\xa9\xdbuN\xae\x0f\xda:fT\xef\xe3_\xb5\x82㖽pѱ\\G\x10\x18\x82Ӻ\a\xe4\x91\x19\x7fp\xd7yOZ\x1d\x9a\xf2\f\xddp\x80G\xfa\x8f\xdfq\xe0N\xfe\xf77\xa3xu\xac\x14\x1e\x93\xeao\x05\xc0cEO\xba\xe6`U\x17l\xd5\xc5_\xfa\x9d1P\x94&\x16k\x8c\x9b\xc3\xef\x8f\x01\xac\xe34i(oi%\r\rb\x91\xb6ދ\xecXa\x99\xb7FG\xb8yL\x1fc\x04\xb8\xf6\xfb!\xceF\x80\x1a\xe0\x10\x01t\x95e\xa0\xf5\xba\xe2|_o\xc7\xf8J\xa8\xf1\x912~>R8h\x83\x82`\xd1;\ni\x14a_\xee\r\"\x0f\x9a\x1e\xb6*M#\x85炯\x86Ԇ\x16']\xd8p\xdd\a\x83W\xf6\xa8\xbcUTI\xeb\xb1Sݰ?\xe6\\\x1ap\xeeK\x9cdYh\x90\x13\u0601 \xd6;;\x12\x87;\xa7&B\xf1;\\\x9d\x87\v\xfe.\xa4B\xa2\x17\x13\x11\x9f\xed\xd0x\x01\xcew\xba\x86\x89\xb5\xa2x\x9fI\x9f\b\xfd\xe0\xd7e+\xael\xf4\x0f\v\v\u2d285j\x9b3ͺ~\xe1yF\xee\xfav9\x04\xee\x14\x13\u05ff\xee\xe5\x99j\xdcG\xf7Y&\xad\x8f\xee$\x83\x16\x81X\xcb\xf8\xf9qGU?\xedPw\xfc\xd2\x05\x1cY\xd8CG9\xf7\x1b\x1d\vКn\xc2i\xee\x8fv\xea\xb1\x01\x01.=\xe7\x16O\"@\x9b]qݳ̝\xca\xd0\xccT\xd4w\x10\n|[\xad\xbeӄ\xcb\x18T\xbcЅ\x85\x9b\xc2\u009cl\"\xa1\x9eJ\xa6R\xe6p\x1fꆖ6\x18\t#w\x9a\xbb݀\xb3\r\xb3s\x1d˹\rU+\xba\x81E&9\a\xb4\xd6\xfdq\xbd\xa4\xae\xfb\xbd\x87_\x80\xeaQ\xd4>\xb6\xdb\xfa\x15@\xc7m\xb7\xf0M]\xb9;\xde\xdee\x98\x82\xe6\"\xbdހ$v<)PvT\x88\xde2\xd7\x1fi\xbbm\xd0:o\x96}\x9e\xd7_27\xf7y\x81\xb8<\x16\xf4W\xa9\xe6\xa4`\xc2\xfeCE\xee\x16\xf0\xc2Ǔƿ\x95\xf2\xe16\x12\xc4\xf6\x06\xffCݰY\xea`\xc2\r\x1b7\x8c\xaed\xe5W\xdf\xeb\x806\xbe\xac\x82'\xf3\x9fy\xba\x890\x8f\xf8\x83\x1e:\x83\x19\xdd\x1f:\x90F]\x81\xeby\x00\xd6m\xb8Ɍ\xf3\xfd\xfc\x10\xf2\xc1\xad\x89\r\xec\xd6\xcd\x05>\fh\xce#\x18\xe8(\xacHE\x81\xd4\a_\xb4\r\xfa)\xb3^O\xe6\xa1`\xb2G\xe3\x1f\x9a\xd6Ctt\xc3l\x85{\x03\bv\x82\xc0\xf3N\xd8\xf1\x9a\x8a\x11\u1ff1m\xea\xb3\vZ\x13\xb7P%6\x98\xa5\x8b\xef}_\x90O\xd0_\xaeX\x90\xbfVPEh\xb0\b\x17\xc3\xdd\x1a\xaa\xfa)_\xb7\r\x1er\xac\xe8@m\x8c4Y\x8a\x1b%7\nt_X\x17\xe4o\x94\x19&6\x1f\xa5\xba\xe1Ն\x89\xcf\xc3[~\x8e5\xbe\xa1\xca0+\xecn<\xb1\x812A9\xfb{̮\xb5_\x8e\x03\xba\x1e\x9c`-H\xc20\x86^\xbc\a\x1b\xe3\x0e\xe6\x05\xa2&\xb4\xf4t=%^\t<\x19\xb3\xa9u,\xd1\xc4\"\xa1\xdb\v\xf2IF\r\x83/\x87b]\x986$\x03m\x16\xb0^Ke\xdcj\xf5bA\xd8:$\x1f\xac\xcd\xc1\xbc\x99\xbb\xab\x92\xb0\xd82s]hҸ/Lz+\xf4\xc2x\x94}A\xf7ne\x8afYe#\xacKm(\x8f\x048\xcf2\xfc\x98\xe5\xb1\xca\a\xf9/\xcfZ\xc9[\xb6\x01\xf5\x93\x8e؏#)\x1e\xa6\xe1\xa2>nQ\x04A\x1e\x153\xc6\xc6T\xf2H)\x81'\x95\xb1\xb1\x15\xe7D[R\x9f\x94}$Ό.\x87Kr\xd2P\xbe\xab\xa1\f\x99g\x8f5\xde̸B\xda\x10\x1b\xf7b\xf5\x91oeٜm\xa9\xd8\f\x9eP\xb0U\xb2\xdal\x83$\x0f\x04\xd3$\xaf\x00\x93\xb5hRt\xb8X\xd8TJ\xb4J\t\x8el\xfb&A\x18p\xb84{ U9\xf7\x17\xf7\xfa{\x99/\xfd\x1d(\x8b\xb5\x92\xc5\xc2\xf7\x8b\xb9Թ_\xc9WL\xda\xc8\xc5l\xa3T'.j\xf7\xd7\f\xa0$\x94%\bB\xb5\xef9ᤨ\x93\xdd\xd4o\xd65\xdcH\xcd\x12\xa2\xfd(\xc7\xff\xda\x06\x10\x18^\x86\xbf\xbb\xcc\xf03\x18\xec3\x86\xc7g\xbf\x05\x1fvT\x187\x9d\xa8]\xe4\xcc9\xb1٤\x89\x8c\xb6\x8e\xedYI\x9a\xdb\x0e\x84\x91\xfc\fv\x17gѭ/\xd7p\a\x81]\xfb\xebWk\xc0s\xa2\x99\b\x17_\xbb\xd2\x0f'\xfdѕ@\x81\x17UJ\x15\xaf\xc6<\x9ep\xe9\"\xf4\xba\xb9\x96]\x1dI|8y*~\x7f\x00\xe3`S7\xdeKZ7\t\xd3\xe7?\xb0\xd8z\x00\x96\xf1f\x16\x95?\xfe\ue6f5wIS\xbd8E\x8e\xcd\xfcpR7<\x85\xeb\xdeCz\xc3\xc1j\x9b\x06\xe8N*'\xe9\xdc\xee\x8cٴs\xa6\xd2\xc2\x15\xef\xe7\xc9%\xedΘD{\xb1\f\xdayQ~\xa4xA\xf4IZ\xfb7\xffm$\x85\xe6\xc1\x9e;\x89\xd6ʡ\x85\x81\xbfj\x16-\xeas{?\xa2\x9d\xce[\xd6\xc2\xf7\xe4\x7f\xf9\xff\x00\x00\x00\xff\xff<\x82OF\xb8\x82\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xccYK\x8f\x1b\xb9\x11\xbe\xebW\x14v\x0f{ٖ\xec\x04\t\x02\xdd\xc6r\x02\x18\x19\xc7\x03k2\xb9.EVK\\\xb1\xc9\x0e\x1f\x92\x95\xc7\x7f\x0f\x8a\x0f\xa9\xd5\x0fK\xe3\x04\x9b\xe5eF|\x14\xeb\xf9U\x15\xbb\xaa\xaa\x19k\xe5\vZ'\x8d^\x02k%~\xf1\xa8闛\xef\xff\xe0\xe6\xd2,\x0eog{\xa9\xc5\x12V\xc1y\xd3|Fg\x82\xe5\xf8\x1ek\xa9\xa5\x97F\xcf\x1a\xf4L0ϖ3\x00\xa6\xb5\xf1\x8c\xa6\x1d\xfd\x04\xe0F{k\x94B[mQ\xcf\xf7a\x83\x9b \x95@\x1b\x89\x97\xab\x0fo\xe6o\x7f?\xff\xdd\f@\xb3\x06\x97\xb0a|\x1fZ\xe7\x8de[T\x86'\x92\xf3\x03*\xb4f.\xcd̵\xc8醭5\xa1]\xc2e!Qȷ'\xce\xdfEb\xebD\xec1\x13\x8b\xebJ:\xff\xe7\xe9=\x8f\xd2\xf9\xb8\xafU\xc125\xc5V\xdc\xe2v\xc6\xfa\xbf\\\xae\xae`\xe3TZ\x91z\x1b\x14\xb3\x13\xc7g\x00\x8e\x9b\x16\x97\x10O\xb7\x8c\xa3\x98\x01d\xd5Dj\x150!\xa2\xb2\x99z\xb2R{\xb4+\xa3B\xa3\xcfw\tt\xdc\xca\xd6Ge&Y \v\x03E\x1ap\x9e\xf9\xe0\xc0\x05\xbe\x03\xe6\xe0\xe1\xc0\xa4b\x1b\x85\x8b\xbfjV\xfe\x8f\xf4\x00~vF?1\xbf[\xc2<\x9d\x9a\xb7;\xe6\xcaj\xb2\xd1SgƟH\x00\xe7\xad\xd4\xdb1\x96\x1e\x99\xf3/LI\x119y\x96\r\x82t\xe0w\b\x8a9\x0f\x9e&\xe8W\xd2\x10\x90\x8a\x10\x8a\x86\xe0\xc8\\\xbe\a\xe0\x90\xa8D\x1d\x8ds\xaa\x06w]\xb1M\xac\xc0K\x8fJ\xe2\x9ff2\xf7\x1d\xb2ſ\xe7\xdc♤\xf3\xaci\xaf\xe8>lq\x8aؕ*\xdec͂\xf2]Q\xc9J\xaa\xeb\x97\xd7b\xb5\xc8\xe7\"\x9d\xba\xba\xf1\xfd\xd5\\\xbauc\x8cB\x96\xa8\xa4]\x87\xb7\xc9\v\xf9\x0e\x1b\xb6̛M\x8b\xfa\xe1\xe9\xc3\xcbo\xd7W\xd30\xe6H\xbd\xa0 ñ\x8emvh\x11^b\xfc%\xbb\xb9,ڙ&\x80\xd9\xfc\x8c\xdc_\x8c\xd8ZӢ\xf5\xb2\x04K\x1a\x1d,\xea\xcc\xf6x\xfaWu\xb5\x06@b\xa4S \b\x940\xf9U\x8e\x1f\x14Yr05\xf8\x9dt`\xb1\xb5\xe8P'\x98\xa2i\xa63\x83\xf3\x1e\xe95Z\"C\xb1\x1d\x94 ,;\xa0\xf5`\x91\x9b\xad\x96\xff8\xd3v\xe0Mvf\x8f\xceC\x8cP\xcd\x149k\xc0\x1f\x81iѣܰ\x13X\xa4;!\xe8\x0e\xbdx\xc0\xf5\xf9\xf8H\xd1 um\x96\xb0\xf3\xbeu\xcb\xc5b+}Ahn\x9a&h\xe9O\x8b\b\xb6r\x13\xbc\xb1n!\xf0\x80j\xe1\xe4\xb6b\x96\xef\xa4G\xee\x83\xc5\x05ke\x15\x05\xd1\tR\x1b\xf1\xbd͘\uebae\x1d\x84t\x1a\x11R_a\x1e\x82\xd7\xe42\x89T\x12\xf1b\x05\x9a\"\xd5}\xfe\xe3\xfa\x19\n'\xc9R\xc9(\x97\xad\x03\xbd\x14\xfb\x906\xa5\xaeѦs\xb55M\xa4\x89Z\xb4Fj\x1f\x7fp%Q{pa\xd3HOn\xf0\xf7\x80Γ\xe9\xfadW1\x8b\xc1\x06!\xb4\x11$\xfa\x1b>hX\xb1\x06Պ9\xfc\x85mEVq\x15\x19\xe1.kuss\x7fsRog\xa1\xe4\xd4\tӎ\xa2\xc1\xbaE~\x15w\x02\x9d\xb4\x14\x19\x9ey\x8c\xd1\xd5SP\x86\x8a\xe9\xa4\\\xc68H\xd0`\x9c\xa3s\x1f\x8d\xc0\xfeJ\x8f\xe5\x87\xf3\xc6+\x1e[\xb4\x8dt1\xbdBml?\xf3\xb03\x92wGA\xbc\xbe\xc1\x01P\x87f\xc8H\x05\x9f\x91\x89OZ\x9d&\x96\xfef\xa5\x1f^4aH\x1a\x89\xc5\xf5I\xf3'\xb4҈\x1b¿\xebm?\xab`g\x8ePG\xff\xd7^\x9d\b\xbb\xdcI\xf3!j\x97\xf1\xf0\xf4\xa1 x\x8a\xad\x1c\x98YWsx\xc8Amjx\x03B:*$\\$:T\x96\x0e*\x16\x1aK\xf06\xbcJ|nt-\xb7C\xa1\xbb\xb5є\xc7\xdc \xdd\xd3\xdc*\xdeD\xa8E\xde\xd1Zs\x90\x02mE\xf1!k\xc93'\xc1\xa6\fRKTb\x80M\x93Q\x16E\xb1((\xa8\x99\xbaa\xc3\xd5yc\xac\xa4\x99\xd4Ƀ/\x04\"\xd6\xd8&\xa7f\xedQ\v\xecg\x9bȍ\x89\x80\xe6P\xc0Q\xfa]BJ5\x16w\xf0\xd5أ\xb1\xc7\xd3\xd8t\x8f\xf7\xe7\x1d\xd2Δx\x11\x1cr\x8b>z\x1b*r\x1fr\xa59\xc0\xc7\xe0\"\xd6\xf6q\xa2\x8cX\xf0\x95\xd3{<\r\x15\r\xb7\x8c\x9bK\xa1\t\x96c\x11\xb5\x84ﾻ-\xd2 \xbb\x95A\xa5{\x11\xd4b\x8d\x16\xf5\xa0\x9a(\xe39\xe6(r\x1a\xf20\xack\xe4^\x1eP\x9dbN\"\xf0\xfc\x116\xc1\x83\b\x18\xad\xc6\xf8\xfeȬp\xc0M\xd32/7RI\x7f\x02\xe9&\xe83\xa5\xcc\x11E\xb686\xad?\xcd\xe1\x83v\x9ei\x8e\xee\\\a\x91ƒ+0\x9dv\xe5(\x8e\x05\x1d\xb3c\x18\x98\xc87\xc6y\xe0h\xc9\x1d\xd5\t\x8e\xd6\xe8픰#\xe9\x90z@\xab\xd1c̈\xc2pGɐc\xeb\xdd\xc2\x1c\xd0\x1e$\x1e\x17Gc\xf7Ro+b\xb0\xcaೈ\x9d\xdd\xe2\xfb\xf8\xe7[\xbc\xc0\xb4\t'\xeep\xdeu\x8c\xf5\x13\x95\xb7~\x87)E\xac\x93\x0f\x1a\vT@\x90k7\xd9w\x13\xb2\x8e\x85\xddX]\xde\x1d\xc5\xe4c\xf9c\x8f\xc3\xd4\xf1\x15P\x01\xf8R]t[5\xac\xad\xd2n\xe6M#\xf9\xac/m\xf2\xfb\xaf\xe3OiV\xa4\x16\x92Sq{\x8d\x1b\xa5\x89\x13W=͈\x1a\xfa]\xce\x14Z\x8e\xab)\x89\x9bk\x85\x1b\x1c\x7f\xea\uef74\xbe\t\xbas\xfew\xe8\xa9\xeet\xa0\x91\xea\x03f\x87z\x8e\x80ɍքT\xde\x00;\xa7\x81\x1f\\?\xff\xbd\x12=7\x81\xefqD\xf1\x03Q\xdeōE\xc7\xe9\x18\xf1\x12\x1c\xc6\xc4t\x8b\r\xb8\x1d\x11\x9c\xad\xd0\xde\xc3\xcb\xea\x816\x9eK\b\x06\xab\a\xd8\x04-\x14\x16\x8e\x8e;\xd4\xd4u\xc9\xfa4~\x17\x8d\xe7\xc7u\xd1j\xac\xber\xdfTt;.C\xcaoK\u061cF\xea\xa5;\x84l-\xd6\xf2\xcb\x1dB>ōE\xe1-\xf3;\x90\xdaI\x81\xc0Fԟ\n\xd9\tAϵѧ\x8c9\xdf`\x9e\xafaCb\xe75\xf0Pt|#~\x9e\xf2\xb6\xb3\x16\xca\xef\x9cݮ\xeb\xe4\xa98\x1e\x95\xe8p~\x94\xf9S\xaa>\xf9H\x19q\xc5\xcc\xcb\xf0\xc4W\xaa\xd8\xf244\x16\xccT3\x19kѵF\v\xea9\xef\xaba/,\xff\xef*\xd9q\xb3V\xd7(\xd7[+V\xb8\xab\x8d\x8b\xcf`\xafn\xe4\xd2\xe3`\xb7M2\x1bG\r\xf6\xa5\x97\xeb\xc9\xf8\x8b\xb4p\xa3%W\xa7\xaf\x93\x8eꗠce\x1b\xab\xaa\xf9l\xe4\xc4{l-R\x06\x13K\x92\xcdƃ\xda\x1c\xe9p\x87Z*ˌN\xf9\x9ez[\xa6E~U\xa0\xa5\x11\xcaG\xa9\x14\xd5\x00\x16\x1bCʢ\xb2\xdcR5\xc7b\xadu\xf8\xcd\xfc\xcd\xff\xafeT\xccy\xea\x00Q|ƃ\x1c>\xadݧ\xee\xc7\x01\x95\x82\x0e瘡\x1f?\x95׆\x85\xcd\xdb~\x82Z*\xaa\xff:\xd0qGu0\xf20\xfcn\xfd\xf8\x83\x8b=\x10j\xef\xe0H\x16t\x91%jzL~\xe1\t\xceS\x12\xb9i\xffn\x01\xae\r(\xa3\xb7h\xcbk\x0f\x15xɛ\x8c\x05\x81\x9er\x95\xde\x02\xdf1\xbd\xa5\xc8\x18\x83\xfc\xc8p\xe6\xbe\xcb'yϤ\x83H=\xe1\x1dw\x19\xf4Y\x8e\xb54\xaf1\xe6\xf43\xfc\x99\xffl\xd9\xcbkoO\xefSP[,\xd1_,\xa9\x9c\x14]\xf9\xcb\xd3\xfce|\xfb\xfb\xc0\xf0\xdd\xff[\xd5\xf3_}\xa9\x18|\xa1\xf8U(\xa7\xa1:\xf7f\xf1\xfc1\xedJ\xef\xb5\xf9\b\xb0\x8d\t~$\xf7w\x1c~4\xa6\xe3ǘ\xd7\xf0\x18?1\xdd*OhO\xb1\b\x0f\xd6\xc67\xdd\xf2\xd6\x18\x91b,+ݏ\xc0\x0f\xbd/aݵ\xe1w\xb2;\xe4\x1a\xcd҃ɔi;v\xcdJ\xee΄\xcd\xf9\xa5~\t\xff\xfc\xf7\xec?\x01\x00\x00\xff\xff\x03f\x86Y\xc0\x1d\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVMo\x1b7\x10\xbd\xebW\f\xd0kwU\xa3hQ\xec\xadqr0\xda\x06\x82\x1d\xe4N\x91#-c.\xc9\xce\f\xe5\xba\x1f\xff\xbd \xb9+K\xab\x95\x93\\\xb27\x91Ù\xc7\xf7f\x1e\xd54\xcdJE\xfb\x11\x89m\xf0\x1d\xa8h\xf1/A\x9f\x7fq\xfb\xf8\v\xb76\xac\x0f7\xabG\xebM\a\xb7\x89%\f\xf7\xc8!\x91Ʒ\xb8\xb3ފ\r~5\xa0(\xa3Du+\x00\xe5}\x10\x95\x979\xff\x04\xd0\xc1\v\x05琚=\xfa\xf61mq\x9b\xac3H%\xf9T\xfa\xf0C{\xf3s\xfb\xd3\n\xc0\xab\x01;0\xe8Pp\xab\xf4c\x8a\x84\x7f&d\xe1\xf6\x80\x0e)\xb46\xac8\xa2\xce\xf9\xf7\x14R\xec\xe0e\xa3\x9e\x1fkW\xdcoK\xaa7%\xd5}MUv\x9de\xf9\xedZ\xc4\xefv\x8c\x8a.\x91rˀJ\x00[\xbfON\xd1b\xc8\n\x80u\x88\xd8\xc1\xfb\f+*\x8df\x050^\xbb\xc0l@\x19S\x88TnC\xd6\v\xd2mpi\x98\bl\xc0 k\xb2Q\nQ\x1fz,W\x84\xb0\x03\xe9\x11j9\x90\x00[\x1c\x11\x98r\x0e\xe0\x13\a\xbfQ\xd2w\xd0f\xbe\xda\x1a\x9a\x81\x8c\x01\x95\xea7\xf3ey\u0380Y\xc8\xfa\xfd5\b,J\x12O J]\x1b<\xd0\t\xbf\xe7\x00J|\x1b{\xc5\xe7\xd5\x1f\xcaƵ\xca5\xe6pS\x99\xd6=\x0e\xaa\x1bcCD\xff\xeb\xe6\xee\xe3\x8f\x0fg\xcbp\x8euAZ\xb0\fjB\x9a\x89\xab\xacA\xf0\b\x81`\b4\xb1\xca\xed1i\xa4\x10\x91\xc4N\xadU\xbf\x93\xe19Y\x9dA\xf8\xb79\xdb\x03Ȩ\xeb)0y\x8a\x90\v\x89cS\xa0\x19/Zɵ\f\x84\x91\x90\xd1\u05f9\xca\xcb\xcaC\xd8~B-\xed,\xf5\x03RN\x03܇\xe4L\x1e\xbe\x03\x92\x00\xa1\x0e{o\xff>\xe6\xe6|\xef\\\xd4))\x94\xe4\xb6\xf3\xca\xc1A\xb9\x84߃\xf2f\x96yP\xcf@\x98kB\xf2'\xf9\xca\x01\x9e\xe3\xf8#\x93h\xfd.tЋD\xee\xd6뽕\xc9Rt\x18\x86\xe4\xad<\xaf\x8b;\xd8m\x92@\xbc6x@\xb7f\xbbo\x14\xe9\xde\njI\x84k\x15mS.⋭\xb4\x83\xf9\x8eF\x13Ⳳ\x17\xddS\xbf\xe2\x02_!O\xf6\x84\xda#5U\xbd\xe2\x8b\ny)Sw\xff\xee\xe1\x03LH\xaaRU\x94\x97\xd0\v^&}2\x9b\xd6\xef\x90\xea\xb9\x1d\x85\xa1\xe4Dob\xb0^\xca\x0f\xed,z\x01N\xdb\xc1\nO\x1d\x9b\xa5\x9b\xa7\xbd-\xb6\x9b\x1d E\xa3\x04\xcd<\xe0\xceí\x1a\xd0\xdd*\xc6o\xacUV\x85\x9b,\xc2\x17\xa9u\xfa\x98̃+\xbd'\x1b\xd33pEڅ\xe1\x7f\x88\xa8\xb3\xb8\x99\xdf|\xda\ueb2ec\xb5\v\x04O\xbd\xd5\xfd4\xfc3\x9a\x8eFq\xce߲1\xe4\xef\xc5n\xe7;W/\x0fEdK8k\xd8\x06.\xbc\xfbu^\x8a\xa9~%3\xd5\xd1Gnt\"*\xcdw\xf4y\xb5t\xe8K\xb9@\xa2@\x17\xab3P\xefJP\xf9Ǡ\xacgP\xfey<\b\xd2+\x81'\xa4\r\x97\x95\x1ax\x8fO\v\xabw~CaO\xc8\xf3\x96ϛ\x9b\xca\x1e\xce߃WXZlʋE\xceVhNXd\t\xa4\xf6\xa7\xbcr\xda\x1e\x9d\xbe\x83\x7f\xfe[\xfd\x1f\x00\x00\xff\xff\xbeM\x1a\xea\xb1\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcWMo\xe36\x10\xbd\xfbW\f\xd0K\v\xac\xe4\x06E\x8b·\xd6\xd9C\xb0\xe96\x88\xb7\xb9S\xd4HbC\x91,9t6E\x7f|1\xa4\xe4\x0fYv\x9c\xcb\xea\xe6\xe1p\xf8\xe6\xcd\xcc#]\x14\xc5B8\xf5\x84>(kV \x9c¯\x84\x86\x7f\x85\xf2\xf9\xd7P*\xbb\xdc\xde,\x9e\x95\xa9W\xb0\x8e\x81l\xff\x88\xc1F/\xf1\x16\x1be\x14)k\x16=\x92\xa8\x05\x89\xd5\x02@\x18cI\xb09\xf0O\x00i\ry\xab5\xfa\xa2ES>\xc7\n\xab\xa8t\x8d>\x05\x1f\x8f\xde\xfeX\xde\xfcR\xfe\xbc\x000\xa2\xc7\x15\xd4\xf6\xc5h+j\x8f\xffD\f\x14\xca-j\xf4\xb6Tv\x11\x1cJ\x8e\xddz\x1b\xdd\n\xf6\vy\xefpn\xc6|;\x84y\xccaҊV\x81>ͭޫ\xc1\xc3\xe9\xe8\x85>\x05\x91\x16\x832m\xd4\u009f,/\x00\x82\xb4\x0eW\xf0\x99a8!\xb1^\x00\f)&XŐ\xdd\xf6&\x87\x92\x1d\xf6\"\xe3\x05\xb0\x0e\xcdo\x0fwO?m\x8e\xcc\x005\x06镣D\xd4\x7f\xc5\xce\x0e\xd3\x04@\x05\x100\xc0\x01\xb2;\x84 \f\bO\xaa\x11\x92\xa0\xf1\xb6\x87J\xc8\xe7\xe8\xc0V\x7f\xa3$\bd\xbdh\xf1\x03\x84(;\x10\x1c%;\x1c\x9c\xa5m\v\x8d\xd2X\xeel\xce[\x87\x9e\xd4Hy\xfe\x0e\x1a\xea\xc0z)\v\xfe8\xf1\xbc\vj\xee,\f@\x1d\x8e\xe4a=p\x05\xb6\x01\xeaT\x00\x8f\xcec@\x93{\x8d\xcd\xc2\fٔ\x93\xd0\x1b\xf4\x1c\x06Bg\xa3\xae\xb9!\xb7\xe8\t\x1aE\xaf\xcb41\xaa\x8ad}XָE\xbd\f\xaa-\x84\x97\x9d\"\x94\x14=.\x85SEJĤQ+\xfb\xfa;?\ff8:\x96^\xb9!\x03yeڃ\x854\x1d\xef(\x0f\xcfK\xee\xae\x1c*\xa7\xb8\xaf\x02\x9b\x98\xbaǏ\x9b/0\"ɕ\x1aZl\xe7z\xc2\xcbX\x1ffS\x99\x06}ޗڔc\xa2\xa9\x9dU\x86\xd2\x0f\xa9\x15\x1a\x82\x10\xab^Q\x18{\x9dK7\r\xbbNR\x04\x15Bt\xb5 \xac\xa7\x0ew\x06֢G\xbd\x16\x01\xbfq\xad\xb8*\xa1\xe0\"\\U\xadC\x81\x9d:gz\x0f\x16Fy&j^\x01\x128\xe1[\xa4\xa9u\x82\xe5Kr\xe2\xe3_:q,X\xdfcٖ\xac9a\x00\x92\xf5\xe8\x87i\xa1.a\x80\xd9F\x9fE2\xf67\xd3\xc0\xbc\xb2\xa0\xb0\xd8\x1db:=\x9a?4\xb1\x9f?\xa0\x80\xdf\x13\xe6{\xdb^\\_[C<\x17\x17\x9d\x9e\xac\x8e=n\x8cp\xa1\xb3o\xf8\xde\x11\xf6\x7f:\xf4\xf9\x1a\xbe\xe8:\xde滫\xef\x82c\xd4g\xcf}D\xbeA\xf0|\xa6\x83\xc3UQ\xae\xc04x^\x95\xe8zs\xf7\x1e\nϸ\xbf\xa3Hw\xa6\xb1o\xa4\xb8w\x9c\xf5;#\x03\xe3\x97\xde\x10o\xf74\xbfBƞ\xe6-\xf9\xeeD\xf8\x14+\xf4\x06\t\xc3^\xa9_\x14u\xb3\x11\x01^:%\xbb\xb41\r\x04_\x02!X\xa9\xe6$\xf5\n\xf8\xac#\xca\xe3\xccP\x16iXg\xcc\f\xfe\xc4|F\xfd\xce\x1dP\f\x8at\x95\x82\x92\xa0\x18ޡ\xa1\xc9\x7f\xa4ZF\xef\xd3\x15\x95\xad\xfc2\x99n\xb8VDG\xe5\xf9\xeb\xf1\xfe\r%\xbd\xdd{\xa6\x17\xb7P&\xa3q\x1e\x8b\xa0Z~A\xf1\x1akiҸS2\xf2w\xfc\xc2;&j\xb6\xa2\xf8թ<\x80o@\xfc\xb8ŝ\x8f&\xdf\xf3\xd37l\n\x88\x81\x9f[ \x85\x99\xc1X!Ԩ\x91\xb0\x86\xea5\xdf\\\xaf\x81\xb0?\xc5\xddX\xdf\vZ\x01\xdf\xff\x05\xa9\x9962QkQi\\\x01\xf9x\xae\xcbf\x13w\x9d\b3cx\x94\xf3\x03\xfb\xcc5\xc6n\x18/v\x06\x9c\xbd_\n\xf8\x8c/3\xd6\ao%\x86\x80\xa7ct6\x93\xd9!81\x06~\xa4\xd5\a,\r\x7f\x19\x06\xcb\xff\x01\x00\x00\xff\xffx\xae@\xbaJ\x0e\x00\x00"), diff --git a/pkg/apis/velero/v1/backup_types.go b/pkg/apis/velero/v1/backup_types.go index 01a7bf104..24af8132d 100644 --- a/pkg/apis/velero/v1/backup_types.go +++ b/pkg/apis/velero/v1/backup_types.go @@ -378,6 +378,7 @@ type BackupStatus struct { Phase BackupPhase `json:"phase,omitempty"` // QueuePosition is the position of the backup in the queue. + // Only relevant when Phase is "Queued" // +optional QueuePosition int `json:"queuePosition,omitempty"` From 5d02af3ce31dc6062c2532406f21a09a98224477 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Tue, 12 Aug 2025 15:41:52 -0400 Subject: [PATCH 06/37] feat: Create backup queue controller and add to disableable list Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/cmd/server/config/config.go | 1 + pkg/controller/backup_queue_controller.go | 69 +++++++++++++++++++ .../backup_queue_controller_test.go | 61 ++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 pkg/controller/backup_queue_controller.go create mode 100644 pkg/controller/backup_queue_controller_test.go diff --git a/pkg/cmd/server/config/config.go b/pkg/cmd/server/config/config.go index ba1b9f3bf..dda3498cc 100644 --- a/pkg/cmd/server/config/config.go +++ b/pkg/cmd/server/config/config.go @@ -53,6 +53,7 @@ const ( var ( // DisableableControllers is a list of controllers that can be disabled DisableableControllers = []string{ + "backup-queue", constant.ControllerBackup, constant.ControllerBackupOperations, constant.ControllerBackupDeletion, diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go new file mode 100644 index 000000000..8ff5571f1 --- /dev/null +++ b/pkg/controller/backup_queue_controller.go @@ -0,0 +1,69 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" +) + +// backupQueueReconciler reconciles a Backup object +type backupQueueReconciler struct { + client.Client + Scheme *runtime.Scheme + logger logrus.FieldLogger +} + +// NewBackupQueueReconciler returns a new backupQueueReconciler +func NewBackupQueueReconciler( + client client.Client, + scheme *runtime.Scheme, + logger logrus.FieldLogger, +) *backupQueueReconciler { + return &backupQueueReconciler{ + Client: client, + Scheme: scheme, + logger: logger, + } +} + +// SetupWithManager adds the reconciler to the manager +func (r *backupQueueReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&velerov1api.Backup{}). + Complete(r) +} + +// Reconcile reconciles a Backup object +func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.logger.WithField("backup", req.NamespacedName.String()) + + log.Debug("Getting backup") + backup := &velerov1api.Backup{} + if err := r.Get(ctx, req.NamespacedName, backup); err != nil { + log.WithError(err).Error("unable to get backup") + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + return ctrl.Result{}, nil +} diff --git a/pkg/controller/backup_queue_controller_test.go b/pkg/controller/backup_queue_controller_test.go new file mode 100644 index 000000000..159163300 --- /dev/null +++ b/pkg/controller/backup_queue_controller_test.go @@ -0,0 +1,61 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" +) + +func TestBackupQueueReconciler(t *testing.T) { + scheme := runtime.NewScheme() + velerov1api.AddToScheme(scheme) + + backup := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "test-backup", + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(backup).Build() + logger := logrus.New() + log := logger.WithField("controller", "backup-queue-test") + + r := NewBackupQueueReconciler(fakeClient, scheme, log) + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "velero", + Name: "test-backup", + }, + } + + res, err := r.Reconcile(context.Background(), req) + require.NoError(t, err) + assert.Equal(t, ctrl.Result{}, res) +} From 4cac891fb9211a8cc2d5526740ba3c879e52b220 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Tue, 12 Aug 2025 15:48:02 -0400 Subject: [PATCH 07/37] refactor: Extract backup-queue controller name to constant Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/cmd/server/config/config.go | 2 +- pkg/constant/constant.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/cmd/server/config/config.go b/pkg/cmd/server/config/config.go index dda3498cc..5adb9946b 100644 --- a/pkg/cmd/server/config/config.go +++ b/pkg/cmd/server/config/config.go @@ -53,7 +53,7 @@ const ( var ( // DisableableControllers is a list of controllers that can be disabled DisableableControllers = []string{ - "backup-queue", + constant.ControllerBackupQueue, constant.ControllerBackup, constant.ControllerBackupOperations, constant.ControllerBackupDeletion, diff --git a/pkg/constant/constant.go b/pkg/constant/constant.go index cfadcc5dc..8eac99573 100644 --- a/pkg/constant/constant.go +++ b/pkg/constant/constant.go @@ -1,6 +1,7 @@ package constant const ( + ControllerBackupQueue = "backup-queue" ControllerBackup = "backup" ControllerBackupOperations = "backup-operations" ControllerBackupDeletion = "backup-deletion" From 9dfa10857923aebe487baa0e13094c05c0063161 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Tue, 12 Aug 2025 15:53:52 -0400 Subject: [PATCH 08/37] feat: initialize backup queue controller Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/cmd/server/server.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 15c4afd95..bbfa70150 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -581,6 +581,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string constant.ControllerSchedule: {}, constant.ControllerServerStatusRequest: {}, constant.ControllerRestoreFinalizer: {}, + constant.ControllerBackupQueue: {}, } if s.config.RestoreOnly { @@ -909,6 +910,16 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string } } + if _, ok := enabledRuntimeControllers[constant.ControllerBackupQueue]; ok { + if err := controller.NewBackupQueueReconciler( + s.mgr.GetClient(), + s.mgr.GetScheme(), + s.logger, + ).SetupWithManager(s.mgr); err != nil { + s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackupQueue) + } + } + s.logger.Info("Server starting...") if err := s.mgr.Start(s.ctx); err != nil { From d91d50f696d2310a0c7dd7903e5a063269036179 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Wed, 13 Aug 2025 14:23:45 -0400 Subject: [PATCH 09/37] feat: Add concurrentBackups to backupQueueReconciler Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/cmd/server/server.go | 1 + pkg/controller/backup_queue_controller.go | 13 ++++++++----- pkg/controller/backup_queue_controller_test.go | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index bbfa70150..83203ca06 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -915,6 +915,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.mgr.GetClient(), s.mgr.GetScheme(), s.logger, + s.config.ConcurrentBackups, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackupQueue) } diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 8ff5571f1..467609f5d 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -30,8 +30,9 @@ import ( // backupQueueReconciler reconciles a Backup object type backupQueueReconciler struct { client.Client - Scheme *runtime.Scheme - logger logrus.FieldLogger + Scheme *runtime.Scheme + logger logrus.FieldLogger + concurrentBackups int } // NewBackupQueueReconciler returns a new backupQueueReconciler @@ -39,11 +40,13 @@ func NewBackupQueueReconciler( client client.Client, scheme *runtime.Scheme, logger logrus.FieldLogger, + concurrentBackups int, ) *backupQueueReconciler { return &backupQueueReconciler{ - Client: client, - Scheme: scheme, - logger: logger, + Client: client, + Scheme: scheme, + logger: logger, + concurrentBackups: concurrentBackups, } } diff --git a/pkg/controller/backup_queue_controller_test.go b/pkg/controller/backup_queue_controller_test.go index 159163300..70e315123 100644 --- a/pkg/controller/backup_queue_controller_test.go +++ b/pkg/controller/backup_queue_controller_test.go @@ -47,7 +47,7 @@ func TestBackupQueueReconciler(t *testing.T) { logger := logrus.New() log := logger.WithField("controller", "backup-queue-test") - r := NewBackupQueueReconciler(fakeClient, scheme, log) + r := NewBackupQueueReconciler(fakeClient, scheme, log, 1) req := ctrl.Request{ NamespacedName: types.NamespacedName{ Namespace: "velero", From fe799d7546247e1a47822827abac90ae42cb98da Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Wed, 13 Aug 2025 14:30:44 -0400 Subject: [PATCH 10/37] feat: Add concurrent backups configuration to backup reconciler Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/cmd/server/server.go | 1 + pkg/controller/backup_controller.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 83203ca06..fbff1086d 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -669,6 +669,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.config.MaxConcurrentK8SConnections, s.config.DefaultSnapshotMoveData, s.config.ItemBlockWorkerCount, + s.config.ConcurrentBackups, s.crClient, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackup) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index c9728ea82..22762bff7 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -105,6 +105,7 @@ type backupReconciler struct { defaultSnapshotMoveData bool globalCRClient kbclient.Client itemBlockWorkerCount int + concurrentBackups int workerPool *pkgbackup.ItemBlockWorkerPool } @@ -132,6 +133,7 @@ func NewBackupReconciler( maxConcurrentK8SConnections int, defaultSnapshotMoveData bool, itemBlockWorkerCount int, + concurrentBackups int, globalCRClient kbclient.Client, ) *backupReconciler { b := &backupReconciler{ @@ -159,6 +161,7 @@ func NewBackupReconciler( maxConcurrentK8SConnections: maxConcurrentK8SConnections, defaultSnapshotMoveData: defaultSnapshotMoveData, itemBlockWorkerCount: itemBlockWorkerCount, + concurrentBackups: concurrentBackups, globalCRClient: globalCRClient, workerPool: pkgbackup.StartItemBlockWorkerPool(ctx, itemBlockWorkerCount, logger), } From 4ffb29d7502319dca7544a7063711d701aa5b16a Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Wed, 20 Aug 2025 16:14:43 -0400 Subject: [PATCH 11/37] feat: Remove pvc-for-tmp install arg Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/cmd/cli/install/install.go | 8 -------- pkg/install/deployment.go | 29 ----------------------------- pkg/install/resources.go | 5 ----- 3 files changed, 42 deletions(-) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 748b6953e..283bfa557 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -90,7 +90,6 @@ type Options struct { NodeAgentConfigMap string ItemBlockWorkerCount int ConcurrentBackups int - PVCForTmp string NodeAgentDisableHostPath bool kubeletRootDir string ServerPriorityClassName string @@ -192,12 +191,6 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { o.NodeAgentConfigMap, "The name of ConfigMap containing node-agent configurations.", ) - flags.StringVar( - &o.PVCForTmp, - "pvc-for-tmp", - o.PVCForTmp, - "The name of PVC to be used as /tmp dir of Velero server pod. Optional.", - ) flags.IntVar( &o.ItemBlockWorkerCount, "item-block-worker-count", @@ -328,7 +321,6 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { NodeAgentConfigMap: o.NodeAgentConfigMap, ItemBlockWorkerCount: o.ItemBlockWorkerCount, ConcurrentBackups: o.ConcurrentBackups, - PVCForTmp: o.PVCForTmp, KubeletRootDir: o.kubeletRootDir, NodeAgentDisableHostPath: o.NodeAgentDisableHostPath, ServerPriorityClassName: o.ServerPriorityClassName, diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 84f94d01d..d1010d294 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -60,7 +60,6 @@ type podTemplateConfig struct { nodeAgentConfigMap string itemBlockWorkerCount int concurrentBackups int - pvcForTmp string forWindows bool kubeletRootDir string nodeAgentDisableHostPath bool @@ -232,12 +231,6 @@ func WithConcurrentBackups(concurrentBackups int) podTemplateOption { } } -func WithPVCForTmp(pvcForTmp string) podTemplateOption { - return func(c *podTemplateConfig) { - c.pvcForTmp = pvcForTmp - } -} - func WithPriorityClassName(priorityClassName string) podTemplateOption { return func(c *podTemplateConfig) { c.priorityClassName = priorityClassName @@ -438,28 +431,6 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme }, } - if c.pvcForTmp != "" { - deployment.Spec.Template.Spec.Volumes = append( - deployment.Spec.Template.Spec.Volumes, - corev1api.Volume{ - Name: "tmp", - VolumeSource: corev1api.VolumeSource{ - PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ - ClaimName: c.pvcForTmp, - }, - }, - }, - ) - - deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append( - deployment.Spec.Template.Spec.Containers[0].VolumeMounts, - corev1api.VolumeMount{ - Name: "tmp", - MountPath: "/tmp", - }, - ) - } - if c.withSecret { deployment.Spec.Template.Spec.Volumes = append( deployment.Spec.Template.Spec.Volumes, diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 1d550ca86..5c7534774 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -272,7 +272,6 @@ type VeleroOptions struct { NodeAgentConfigMap string ItemBlockWorkerCount int ConcurrentBackups int - PVCForTmp string KubeletRootDir string NodeAgentDisableHostPath bool ServerPriorityClassName string @@ -367,10 +366,6 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { WithConcurrentBackups(o.ConcurrentBackups), } - if o.PVCForTmp != "" { - deployOpts = append(deployOpts, WithPVCForTmp(o.PVCForTmp)) - } - if o.ServerPriorityClassName != "" { deployOpts = append(deployOpts, WithPriorityClassName(o.ServerPriorityClassName)) } From 13041b40c2c50c836cff218321ed021870973adf Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Wed, 17 Sep 2025 12:58:37 -0400 Subject: [PATCH 12/37] Backup queue controller implementation Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 298 +++++++++++++++++++++- 1 file changed, 297 insertions(+), 1 deletion(-) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 467609f5d..8fa679542 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -18,13 +18,26 @@ package controller import ( "context" + "slices" + "time" + "github.com/pkg/errors" "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/constant" + "github.com/vmware-tanzu/velero/pkg/util/collections" + "github.com/vmware-tanzu/velero/pkg/util/kube" ) // backupQueueReconciler reconciles a Backup object @@ -33,8 +46,13 @@ type backupQueueReconciler struct { Scheme *runtime.Scheme logger logrus.FieldLogger concurrentBackups int + frequency time.Duration } +const ( + defaultQueuedBackupRecheckFrequency = time.Minute +) + // NewBackupQueueReconciler returns a new backupQueueReconciler func NewBackupQueueReconciler( client client.Client, @@ -47,16 +65,223 @@ func NewBackupQueueReconciler( Scheme: scheme, logger: logger, concurrentBackups: concurrentBackups, + frequency: defaultQueuedBackupRecheckFrequency, } } +func queuePositionOrderFunc(objList client.ObjectList) client.ObjectList { + backupList := objList.(*velerov1api.BackupList) + slices.SortFunc(backupList.Items, func(backup1, backup2 velerov1api.Backup) int { + if backup1.Status.QueuePosition < backup2.Status.QueuePosition { + return -1 + } else if backup1.Status.QueuePosition == backup2.Status.QueuePosition { + return 0 + } else { + return 1 + } + }) + return backupList +} + // SetupWithManager adds the reconciler to the manager func (r *backupQueueReconciler) SetupWithManager(mgr ctrl.Manager) error { + // For periodic requeue, only consider Queued backups, order by QueuePosition + gp := kube.NewGenericEventPredicate(func(object client.Object) bool { + backup := object.(*velerov1api.Backup) + return backup.Status.Phase == velerov1api.BackupPhaseQueued + }) + + s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerBackupQueue), mgr.GetClient(), &velerov1api.BackupList{}, r.frequency, kube.PeriodicalEnqueueSourceOption{ + Predicates: []predicate.Predicate{gp}, + OrderFunc: queuePositionOrderFunc, + }) + return ctrl.NewControllerManagedBy(mgr). - For(&velerov1api.Backup{}). + For(&velerov1api.Backup{}, builder.WithPredicates(predicate.Funcs{ + UpdateFunc: func(ue event.UpdateEvent) bool { + backup := ue.ObjectNew.(*velerov1api.Backup) + return backup.Status.Phase == "" || backup.Status.Phase == velerov1api.BackupPhaseNew + }, + CreateFunc: func(ce event.CreateEvent) bool { + backup := ce.Object.(*velerov1api.Backup) + return backup.Status.Phase == "" || backup.Status.Phase == velerov1api.BackupPhaseNew + }, + DeleteFunc: func(de event.DeleteEvent) bool { + return false + }, + GenericFunc: func(ge event.GenericEvent) bool { + return false + }, + })). + Watches( + &velerov1api.Backup{}, + handler.EnqueueRequestsFromMapFunc(r.findQueuedBackupsToRequeue), + builder.WithPredicates(predicate.Funcs{ + UpdateFunc: func(ue event.UpdateEvent) bool { + oldBackup := ue.ObjectOld.(*velerov1api.Backup) + newBackup := ue.ObjectNew.(*velerov1api.Backup) + return oldBackup.Status.Phase == velerov1api.BackupPhaseInProgress && + newBackup.Status.Phase != velerov1api.BackupPhaseInProgress || + oldBackup.Status.Phase != velerov1api.BackupPhaseQueued && + newBackup.Status.Phase == velerov1api.BackupPhaseQueued + }, + CreateFunc: func(event.CreateEvent) bool { + return false + }, + DeleteFunc: func(de event.DeleteEvent) bool { + return false + }, + GenericFunc: func(ge event.GenericEvent) bool { + return false + }, + })). + WatchesRawSource(s). + Named(constant.ControllerBackupQueue). Complete(r) } +func (r *backupQueueReconciler) listQueuedBackups(ctx context.Context, ns string) ([]velerov1api.Backup, error) { + backupList := &velerov1api.BackupList{} + backups := []velerov1api.Backup{} + if err := r.Client.List(ctx, backupList, &client.ListOptions{Namespace: ns}); err != nil { + r.logger.WithError(err).Error("error listing queued backups") + return nil, err + } + for _, backup := range backupList.Items { + if backup.Status.Phase == velerov1api.BackupPhaseQueued { + backups = append(backups, backup) + } + + } + return backups, nil +} + +func (r *backupQueueReconciler) listEarlierBackups(ctx context.Context, ns string, queuePos int) ([]velerov1api.Backup, int, error) { + backupList := &velerov1api.BackupList{} + backups := []velerov1api.Backup{} + runningCount := 0 + if err := r.Client.List(ctx, backupList, &client.ListOptions{Namespace: ns}); err != nil { + return nil, 0, err + } + for _, backup := range backupList.Items { + if backup.Status.Phase == velerov1api.BackupPhaseInProgress || + backup.Status.Phase == velerov1api.BackupPhaseReadyToStart || + (backup.Status.Phase == velerov1api.BackupPhaseQueued && + backup.Status.QueuePosition < queuePos) { + backups = append(backups, backup) + } + // InProgress and ReadyToStart backups count towards the concurrentBackups limit + if backup.Status.Phase == velerov1api.BackupPhaseInProgress || + backup.Status.Phase == velerov1api.BackupPhaseReadyToStart { + runningCount += 1 + } + } + return backups, runningCount, nil +} + +func (r *backupQueueReconciler) detectNamespaceConflict(ctx context.Context, backup *velerov1api.Backup, earlierBackups []velerov1api.Backup) (bool, string, []string, error) { + nsList := &corev1api.NamespaceList{} + if err := r.Client.List(ctx, nsList); err != nil { + return false, "", nil, err + } + var clusterNamespaces []string + for _, ns := range nsList.Items { + clusterNamespaces = append(clusterNamespaces, ns.Name) + } + foundConflict, conflictBackup := detectNSConflictsInternal(ctx, backup, earlierBackups, clusterNamespaces) + return foundConflict, conflictBackup, clusterNamespaces, nil +} + +func detectNSConflictsInternal(ctx context.Context, backup *velerov1api.Backup, earlierBackups []velerov1api.Backup, clusterNamespaces []string) (bool, string) { + backupNamespaces := namespacesForBackup(backup, clusterNamespaces) + backupNSMap := make(map[string]struct{}) + for _, ns := range backupNamespaces { + backupNSMap[ns] = struct{}{} + } + for _, earlierBackup := range earlierBackups { + // This will never be true for the primary backup, but for the secondary + // runnability check for queued backups ahead of the current backup, we + // only care about backups ahead of it. + // Backup isn't earlier than this one, skip + if earlierBackup.Status.Phase == velerov1api.BackupPhaseQueued && + earlierBackup.Status.QueuePosition >= backup.Status.QueuePosition { + continue + } + runningNSList := namespacesForBackup(&earlierBackup, clusterNamespaces) + for _, runningNS := range runningNSList { + if _, found := backupNSMap[runningNS]; found { + return true, earlierBackup.Name + } + } + } + return false, "" +} +// Returns true if there are backups ahead of the current backup that are runnable +// This could happen if velero just reconciled the one earlier in the queue and rejected it +// due to too many running backups, but a backup completed in between that reconcile and this one +// so exit, as the recent completion has triggered another reconcile of all queued backups +func (r *backupQueueReconciler) checkForEarlierRunnableBackups(ctx context.Context, backup *velerov1api.Backup, earlierBackups []velerov1api.Backup, clusterNamespaces []string) (bool, string) { + for _, earlierBackup := range earlierBackups { + // if this backup is queued and ahead of current backup, check for conflicts + if earlierBackup.Status.Phase != velerov1api.BackupPhaseQueued || + earlierBackup.Status.QueuePosition >= backup.Status.QueuePosition { + continue + } + conflict, _ := detectNSConflictsInternal(ctx, &earlierBackup, earlierBackups, clusterNamespaces) + // !conflict means we've found an earlier backup that is currently runnable + // so current reconcile should exit to run this one + if !conflict { + return true, earlierBackup.Name + } + } + return false, "" +} + +func namespacesForBackup(backup *velerov1api.Backup, clusterNamespaces []string) []string { + return collections.NewNamespaceIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...).ActiveNamespaces(clusterNamespaces).ResolveNamespaceList() +} +func (r *backupQueueReconciler) getMaxQueuePosition(ctx context.Context, ns string) (int, error) { + queuedBackups, err := r.listQueuedBackups(ctx, ns) + if err != nil { + return 0, err + } + maxPos := 0 + for _, backup := range queuedBackups { + maxPos = max(maxPos, backup.Status.QueuePosition) + } + return maxPos, nil +} + +func (r *backupQueueReconciler) orderedQueuedBackups(ctx context.Context, backup *velerov1api.Backup) ([]velerov1api.Backup, error) { + backupList := &velerov1api.BackupList{} + var returnList []velerov1api.Backup + if err := r.Client.List(ctx, backupList, &client.ListOptions{Namespace: backup.Namespace}); err != nil { + r.logger.WithError(err).Error("error listing backups") + return nil, err + } + orderedBackupList := queuePositionOrderFunc(backupList).(*velerov1api.BackupList) + for _, item := range orderedBackupList.Items { + if item.Status.Phase == velerov1api.BackupPhaseQueued { + returnList = append(returnList, item) + } + } + return returnList, nil +} +func (r *backupQueueReconciler) findQueuedBackupsToRequeue(ctx context.Context, obj client.Object) []reconcile.Request { + backup := obj.(*velerov1api.Backup) + requests := []reconcile.Request{} + backups, _ := r.orderedQueuedBackups(ctx, backup) + for _, item := range backups { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: item.GetNamespace(), + Name: item.GetName(), + }, + }) + } + return requests +} + // Reconcile reconciles a Backup object func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.logger.WithField("backup", req.NamespacedName.String()) @@ -67,6 +292,77 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.WithError(err).Error("unable to get backup") return ctrl.Result{}, client.IgnoreNotFound(err) } + switch backup.Status.Phase { + case "", velerov1api.BackupPhaseNew: + // queue new backup + maxQueuePosition, err := r.getMaxQueuePosition(ctx, backup.Namespace) + if err != nil { + // error is logged in getMaxQueuePosition func + return ctrl.Result{}, nil + } + log.Debug("Queueing backup") + original := backup.DeepCopy() + backup.Status.Phase = velerov1api.BackupPhaseQueued + backup.Status.QueuePosition = maxQueuePosition + 1 + if err := kube.PatchResource(original, backup, r.Client); err != nil { + return ctrl.Result{}, errors.Wrapf(err, "error updating Backup status to %s", backup.Status.Phase) + } + case velerov1api.BackupPhaseQueued: + // handle queued backup + // Find backups ahead of this one (InProgress, ReadyToStart, or Queued with higher position) + earlierBackups, runningCount, err := r.listEarlierBackups(ctx, backup.Namespace, backup.Status.QueuePosition) + if err != nil { + log.WithError(err).Error("error listing earlier backups") + return ctrl.Result{}, nil + } + if runningCount >= r.concurrentBackups { + log.Debugf("%v concurrent backups are already running, leaving %v queued", r.concurrentBackups, backup.Name) + return ctrl.Result{}, nil + } + foundConflict, conflictBackup, clusterNamespaces, err := r.detectNamespaceConflict(ctx, backup, earlierBackups) + if err != nil { + log.WithError(err).Error("error listing namespaces") + return ctrl.Result{}, nil + } + if foundConflict { + log.Debugf("Backup has namespace conflict with %v, leaving %v queued", conflictBackup, backup.Name) + return ctrl.Result{}, nil + } + foundEarlierRunnable, earlierRunnable := r.checkForEarlierRunnableBackups(ctx, backup, earlierBackups, clusterNamespaces) + if foundEarlierRunnable { + log.Debugf("Earlier queued backup %v is runnable, leaving %v queued", earlierRunnable, backup.Name) + return ctrl.Result{}, nil + } + log.Debug("Moving backup to ReadyToStart") + original := backup.DeepCopy() + backup.Status.Phase = velerov1api.BackupPhaseReadyToStart + backup.Status.QueuePosition = 0 + if err := kube.PatchResource(original, backup, r.Client); err != nil { + return ctrl.Result{}, errors.Wrapf(err, "error updating Backup status to %s", backup.Status.Phase) + } + log.Debug("Updating queuePosition for remaining queued backups") + queuedBackups, err := r.orderedQueuedBackups(ctx, backup) + if err != nil { + log.WithError(err).Error("error listing queued backups") + return ctrl.Result{}, nil + } + newQueuePos := 1 + for _, queuedBackup := range queuedBackups { + if queuedBackup.Name != backup.Name { + original := queuedBackup.DeepCopy() + queuedBackup.Status.QueuePosition = newQueuePos + if err := kube.PatchResource(original, &queuedBackup, r.Client); err != nil { + log.WithError(errors.Wrapf(err, "error updating Backup %s queuePosition to %v", queuedBackup.Name, newQueuePos)) + return ctrl.Result{}, nil + } + newQueuePos += 1 + } + } + return ctrl.Result{}, nil + default: + log.Debug("Backup is not New or Queued, skipping") + return ctrl.Result{}, nil + } return ctrl.Result{}, nil } From 300bc70c686f8114dc9cf1a25c4e78a6913ea9f2 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Wed, 17 Sep 2025 17:05:50 -0400 Subject: [PATCH 13/37] Add queue position to backup list/describe Signed-off-by: Scott Seago --- pkg/cmd/util/output/backup_describer.go | 4 ++++ pkg/cmd/util/output/backup_printer.go | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index 932b77cfb..e0637a4bd 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -75,6 +75,7 @@ func DescribeBackup( case velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed: case velerov1api.BackupPhaseInProgress: case velerov1api.BackupPhaseNew: + case velerov1api.BackupPhaseQueued, velerov1api.BackupPhaseReadyToStart: } logsNote := "" @@ -83,6 +84,9 @@ func DescribeBackup( } d.Printf("Phase:\t%s%s\n", phaseString, logsNote) + if phase == velerov1api.BackupPhaseQueued { + d.Printf("Queue position:\t%v\n", backup.Status.QueuePosition) + } if backup.Spec.ResourcePolicy != nil { d.Println() diff --git a/pkg/cmd/util/output/backup_printer.go b/pkg/cmd/util/output/backup_printer.go index bae077a29..873bc9fc3 100644 --- a/pkg/cmd/util/output/backup_printer.go +++ b/pkg/cmd/util/output/backup_printer.go @@ -20,6 +20,7 @@ import ( "fmt" "regexp" "sort" + "strconv" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -40,6 +41,7 @@ var ( {Name: "Created"}, {Name: "Expires"}, {Name: "Storage Location"}, + {Name: "Queue Position"}, {Name: "Selector"}, } ) @@ -108,6 +110,7 @@ func printBackup(backup *velerov1api.Backup) []metav1.TableRow { backup.Status.StartTimestamp, humanReadableTimeFromNow(expiration), backup.Spec.StorageLocation, + queuePosition(backup.Status.QueuePosition), metav1.FormatLabelSelector(backup.Spec.LabelSelector), ) @@ -127,3 +130,11 @@ func humanReadableTimeFromNow(when time.Time) string { return fmt.Sprintf("%s ago", duration.ShortHumanDuration(now.Sub(when))) } } + +func queuePosition(pos int) string { + if pos == 0 { + return "" + } else { + return strconv.Itoa(pos) + } +} From d39ad6f208ddd6352909c89f0f45e287a457dcfa Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Wed, 17 Sep 2025 18:00:09 -0400 Subject: [PATCH 14/37] run multiple backup reconcilers, only reconcile ReadyToStart backups Signed-off-by: Scott Seago --- pkg/controller/backup_controller.go | 29 ++++++++++++++++--- pkg/controller/backup_controller_test.go | 34 ++++++++++++++++------- pkg/controller/backup_queue_controller.go | 2 +- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 22762bff7..90085fa1f 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -36,7 +36,11 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/vmware-tanzu/velero/internal/credentials" "github.com/vmware-tanzu/velero/internal/resourcepolicies" @@ -161,7 +165,7 @@ func NewBackupReconciler( maxConcurrentK8SConnections: maxConcurrentK8SConnections, defaultSnapshotMoveData: defaultSnapshotMoveData, itemBlockWorkerCount: itemBlockWorkerCount, - concurrentBackups: concurrentBackups, + concurrentBackups: max(concurrentBackups, 1), globalCRClient: globalCRClient, workerPool: pkgbackup.StartItemBlockWorkerPool(ctx, itemBlockWorkerCount, logger), } @@ -171,7 +175,24 @@ func NewBackupReconciler( func (b *backupReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). - For(&velerov1api.Backup{}). + For(&velerov1api.Backup{}, builder.WithPredicates(predicate.Funcs{ + UpdateFunc: func(ue event.UpdateEvent) bool { + backup := ue.ObjectNew.(*velerov1api.Backup) + return backup.Status.Phase == velerov1api.BackupPhaseReadyToStart + }, + CreateFunc: func(ce event.CreateEvent) bool { + return false + }, + DeleteFunc: func(de event.DeleteEvent) bool { + return false + }, + GenericFunc: func(ge event.GenericEvent) bool { + return false + }, + })). + WithOptions(controller.Options{ + MaxConcurrentReconciles: b.concurrentBackups, + }). Named(constant.ControllerBackup). Complete(b) } @@ -257,8 +278,8 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr // InProgress, we still need this check so we can return nil to indicate we've finished processing // this key (even though it was a no-op). switch original.Status.Phase { - case "", velerov1api.BackupPhaseNew: - // only process new backups + case velerov1api.BackupPhaseReadyToStart: + // only process ReadytToStart backups default: b.logger.WithFields(logrus.Fields{ "backup": kubeutil.NamespaceAndName(original), diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 715f5068a..524db4b24 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -95,7 +95,11 @@ func (b *fakeBackupper) FinalizeBackup( } func defaultBackup() *builder.BackupBuilder { - return builder.ForBackup(velerov1api.DefaultNamespace, "backup-1") + return builder.ForBackup(velerov1api.DefaultNamespace, "backup-1").Phase(velerov1api.BackupPhaseReadyToStart) +} + +func namedBackup(name string) *builder.BackupBuilder { + return builder.ForBackup(velerov1api.DefaultNamespace, name).Phase(velerov1api.BackupPhaseReadyToStart) } func TestProcessBackupNonProcessedItems(t *testing.T) { @@ -104,6 +108,16 @@ func TestProcessBackupNonProcessedItems(t *testing.T) { key string backup *velerov1api.Backup }{ + { + name: "New backup is not processed", + key: "velero/backup-1", + backup: defaultBackup().Phase(velerov1api.BackupPhaseNew).Result(), + }, + { + name: "Queued backup is not processed", + key: "velero/backup-1", + backup: defaultBackup().Phase(velerov1api.BackupPhaseQueued).Result(), + }, { name: "FailedValidation backup is not processed", key: "velero/backup-1", @@ -331,7 +345,7 @@ func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) { }{ { name: "BackupLocation is specified in backup CR'spec and it can be found in ApiServer", - backup: builder.ForBackup("velero", "backup-1").Result(), + backup: defaultBackup().Result(), backupLocationNameInBackup: "test-backup-location", backupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "test-backup-location").Result(), defaultBackupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "default-location").Result(), @@ -340,7 +354,7 @@ func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) { }, { name: "BackupLocation is specified in backup CR'spec and it can't be found in ApiServer", - backup: builder.ForBackup("velero", "backup-1").Result(), + backup: defaultBackup().Result(), backupLocationNameInBackup: "test-backup-location", backupLocationInAPIServer: nil, defaultBackupLocationInAPIServer: nil, @@ -349,7 +363,7 @@ func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) { }, { name: "Using default BackupLocation and it can be found in ApiServer", - backup: builder.ForBackup("velero", "backup-1").Result(), + backup: defaultBackup().Result(), backupLocationNameInBackup: "", backupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "test-backup-location").Result(), defaultBackupLocationInAPIServer: builder.ForBackupStorageLocation("velero", "default-location").Result(), @@ -358,7 +372,7 @@ func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) { }, { name: "Using default BackupLocation and it can't be found in ApiServer", - backup: builder.ForBackup("velero", "backup-1").Result(), + backup: defaultBackup().Result(), backupLocationNameInBackup: "", backupLocationInAPIServer: nil, defaultBackupLocationInAPIServer: nil, @@ -497,7 +511,7 @@ func TestPrepareBackupRequest_SetsVGSLabelKey(t *testing.T) { }{ { name: "backup with spec label key set", - backup: builder.ForBackup("velero", "backup-1"). + backup: defaultBackup(). VolumeGroupSnapshotLabelKey("spec-key"). Result(), serverFlagKey: "server-key", @@ -505,13 +519,13 @@ func TestPrepareBackupRequest_SetsVGSLabelKey(t *testing.T) { }, { name: "backup with no spec key, uses server flag", - backup: builder.ForBackup("velero", "backup-2").Result(), + backup: namedBackup("backup-2").Result(), serverFlagKey: "server-key", expectedLabelKey: "server-key", }, { name: "backup with no spec or server flag, uses default", - backup: builder.ForBackup("velero", "backup-3").Result(), + backup: namedBackup("backup-3").Result(), serverFlagKey: velerov1api.DefaultVGSLabelKey, expectedLabelKey: velerov1api.DefaultVGSLabelKey, }, @@ -1390,7 +1404,7 @@ func TestProcessBackupCompletions(t *testing.T) { }, { name: "backup with namespace-scoped and cluster-scoped resource filters", - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + backup: defaultBackup(). ExcludedClusterScopedResources("clusterroles"). IncludedClusterScopedResources("storageclasses"). ExcludedNamespaceScopedResources("secrets"). @@ -1439,7 +1453,7 @@ func TestProcessBackupCompletions(t *testing.T) { }, { name: "backup's include filter overlap with default exclude resources", - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + backup: defaultBackup(). ExcludedClusterScopedResources("clusterroles"). IncludedClusterScopedResources("storageclasses", "volumesnapshotcontents.snapshot.storage.k8s.io"). ExcludedNamespaceScopedResources("secrets"). diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 8fa679542..7fe295da3 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -64,7 +64,7 @@ func NewBackupQueueReconciler( Client: client, Scheme: scheme, logger: logger, - concurrentBackups: concurrentBackups, + concurrentBackups: max(concurrentBackups, 1), frequency: defaultQueuedBackupRecheckFrequency, } } From a56ab10f23c21b6ffe9f4e359dd8da4fadcde787 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Thu, 18 Sep 2025 14:36:11 -0400 Subject: [PATCH 15/37] Move debug logs to info Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 7fe295da3..93ef08713 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -300,10 +300,10 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) // error is logged in getMaxQueuePosition func return ctrl.Result{}, nil } - log.Debug("Queueing backup") original := backup.DeepCopy() backup.Status.Phase = velerov1api.BackupPhaseQueued backup.Status.QueuePosition = maxQueuePosition + 1 + log.Infof("Queueing backup %v, queue position %v", backup.Name, backup.Status.QueuePosition) if err := kube.PatchResource(original, backup, r.Client); err != nil { return ctrl.Result{}, errors.Wrapf(err, "error updating Backup status to %s", backup.Status.Phase) } @@ -325,15 +325,15 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } if foundConflict { - log.Debugf("Backup has namespace conflict with %v, leaving %v queued", conflictBackup, backup.Name) + log.Infof("Backup %v has namespace conflict with %v, leaving queued", backup.Name, conflictBackup) return ctrl.Result{}, nil } foundEarlierRunnable, earlierRunnable := r.checkForEarlierRunnableBackups(ctx, backup, earlierBackups, clusterNamespaces) if foundEarlierRunnable { - log.Debugf("Earlier queued backup %v is runnable, leaving %v queued", earlierRunnable, backup.Name) + log.Infof("Earlier queued backup %v is runnable, leaving %v queued", earlierRunnable, backup.Name) return ctrl.Result{}, nil } - log.Debug("Moving backup to ReadyToStart") + log.Infof("Dequeueing backup %v, moving to ReadyToStart", backup.Name) original := backup.DeepCopy() backup.Status.Phase = velerov1api.BackupPhaseReadyToStart backup.Status.QueuePosition = 0 From e0c08f03cf7f65872fc96ec0e493d8c46c7df5b8 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Thu, 18 Sep 2025 14:36:26 -0400 Subject: [PATCH 16/37] add various scenarios to queue controller unit tests Signed-off-by: Scott Seago --- pkg/builder/backup_builder.go | 6 + .../backup_queue_controller_test.go | 211 ++++++++++++++++-- 2 files changed, 199 insertions(+), 18 deletions(-) diff --git a/pkg/builder/backup_builder.go b/pkg/builder/backup_builder.go index 2e5dbf24f..d5b955e43 100644 --- a/pkg/builder/backup_builder.go +++ b/pkg/builder/backup_builder.go @@ -222,6 +222,12 @@ func (b *BackupBuilder) Phase(phase velerov1api.BackupPhase) *BackupBuilder { return b } +// Phase sets the Backup's queue position. +func (b *BackupBuilder) QueuePosition(queuePos int) *BackupBuilder { + b.object.Status.QueuePosition = queuePos + return b +} + // StorageLocation sets the Backup's storage location. func (b *BackupBuilder) StorageLocation(location string) *BackupBuilder { b.object.Spec.StorageLocation = location diff --git a/pkg/controller/backup_queue_controller_test.go b/pkg/controller/backup_queue_controller_test.go index 70e315123..1a5eb582f 100644 --- a/pkg/controller/backup_queue_controller_test.go +++ b/pkg/controller/backup_queue_controller_test.go @@ -23,39 +23,214 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + //metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client/fake" + //"sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestBackupQueueReconciler(t *testing.T) { scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) - backup := &velerov1api.Backup{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "velero", - Name: "test-backup", + tests := []struct { + name string + priorBackups []*velerov1api.Backup + namespaces []string + backup *velerov1api.Backup + concurrentBackups int + expectError bool + expectPhase velerov1api.BackupPhase + expectQueuePosition int + }{ + { + name: "New Backup gets queued", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "InProgress Backup is ignored", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), + expectPhase: velerov1api.BackupPhaseInProgress, + }, + { + name: "Second New Backup gets queued with queuePosition 2", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), + }, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 2, + }, + { + name: "Queued Backup moves to ReadyToStart if no others are running", + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).Result(), + expectPhase: velerov1api.BackupPhaseReadyToStart, + }, + { + name: "Queued Backup remains queued if no spaces available", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), + builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), + }, + concurrentBackups: 2, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Queued Backup remains queued if no spaces available including ReadyToStart", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), + builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseReadyToStart).Result(), + }, + concurrentBackups: 2, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Queued Backup remains queued if earlier runnable backup is also queued", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), + builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), + }, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(2).Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 2, + }, + { + name: "Queued Backup remains queued if in conflict with running backup", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("foo").Result(), + }, + namespaces: []string{"foo"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Queued Backup remains queued if in conflict with ReadyToStart backup", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseReadyToStart).IncludedNamespaces("foo").Result(), + }, + namespaces: []string{"foo"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Queued Backup remains queued if in conflict with earlier queued backup", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), + }, + namespaces: []string{"foo", "bar"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(2).IncludedNamespaces("foo", "bar").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 2, + }, + { + name: "Queued Backup remains queued if earlier non-ns-conflict backup exists", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), + builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), + }, + namespaces: []string{"foo", "bar", "baz"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(2).IncludedNamespaces("baz").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 2, + }, + { + name: "Running all-namespace backup conflicts with queued one-namespace backup ", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("*").Result(), + }, + namespaces: []string{"foo", "bar"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Running one-namespace backup conflicts with queued all-namespace backup ", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), + }, + namespaces: []string{"foo", "bar"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("*").Result(), + expectPhase: velerov1api.BackupPhaseQueued, + expectQueuePosition: 1, + }, + { + name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), + builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), + }, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), + expectPhase: velerov1api.BackupPhaseReadyToStart, + }, + { + name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups and no ns conflict found", + priorBackups: []*velerov1api.Backup{ + builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseReadyToStart).IncludedNamespaces("foo").Result(), + }, + namespaces: []string{"foo", "bar"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("bar").Result(), + expectPhase: velerov1api.BackupPhaseReadyToStart, }, } - fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(backup).Build() - logger := logrus.New() - log := logger.WithField("controller", "backup-queue-test") + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.backup == nil { + return + } - r := NewBackupQueueReconciler(fakeClient, scheme, log, 1) - req := ctrl.Request{ - NamespacedName: types.NamespacedName{ - Namespace: "velero", - Name: "test-backup", - }, + initObjs := []runtime.Object{} + for _, priorBackup := range test.priorBackups { + initObjs = append(initObjs, priorBackup) + } + for _, ns := range test.namespaces { + initObjs = append(initObjs, builder.ForNamespace(ns).Result()) + } + initObjs = append(initObjs, test.backup) + + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) + logger := logrus.New() + log := logger.WithField("controller", "backup-queue-test") + r := NewBackupQueueReconciler(fakeClient, scheme, log, test.concurrentBackups) + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace,Name: test.backup.Name}} + res, err := r.Reconcile(context.Background(), req) + gotErr := err != nil + require.NoError(t, err) + assert.Equal(t, ctrl.Result{}, res) + assert.Equal(t, test.expectError, gotErr) + backupAfter := velerov1api.Backup{} + err = fakeClient.Get(t.Context(), types.NamespacedName{ + Namespace: test.backup.Namespace, + Name: test.backup.Name, + }, &backupAfter) + + require.NoError(t, err) + assert.Equal(t, test.expectPhase, backupAfter.Status.Phase) + assert.Equal(t, test.expectQueuePosition, backupAfter.Status.QueuePosition) + }) } - res, err := r.Reconcile(context.Background(), req) - require.NoError(t, err) - assert.Equal(t, ctrl.Result{}, res) + } From 91357b28c4d3cdcb6af0c6ac841b5a867b705c92 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Mon, 29 Sep 2025 16:20:43 -0400 Subject: [PATCH 17/37] Move worker pool creation to backup reconcile. ItemBlockWorkerPool is now created for each backup. Signed-off-by: Scott Seago --- pkg/backup/backup.go | 2 +- pkg/backup/backup_test.go | 70 ++++++++++++------------ pkg/backup/request.go | 2 +- pkg/controller/backup_controller.go | 10 ++-- pkg/controller/backup_controller_test.go | 33 ++++------- 5 files changed, 52 insertions(+), 65 deletions(-) diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index b0d0937dd..8d009de90 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -597,7 +597,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers( log.Infof("Backing Up Item Block including %s %s/%s (%v items in block)", items[i].groupResource.String(), items[i].namespace, items[i].name, len(itemBlock.Items)) wg.Add(1) - backupRequest.ItemBlockChannel <- ItemBlockInput{ + backupRequest.WorkerPool.GetInputChannel() <- ItemBlockInput{ itemBlock: itemBlock, returnChan: itemBlockReturn, } diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 6918c99f5..1db3cff6b 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -79,7 +79,7 @@ func TestBackedUpItemsMatchesTarballContents(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: h.itemBlockPool.GetInputChannel(), + WorkerPool: &h.itemBlockPool, } backupFile := bytes.NewBuffer([]byte{}) @@ -141,7 +141,7 @@ func TestBackupProgressIsUpdated(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: h.itemBlockPool.GetInputChannel(), + WorkerPool: &h.itemBlockPool, } backupFile := bytes.NewBuffer([]byte{}) @@ -881,7 +881,7 @@ func TestBackupOldResourceFiltering(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1062,7 +1062,7 @@ func TestCRDInclusion(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1161,7 +1161,7 @@ func TestBackupResourceCohabitation(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1190,7 +1190,7 @@ func TestBackupUsesNewCohabitatingResourcesForEachBackup(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: h.itemBlockPool.GetInputChannel(), + WorkerPool: &h.itemBlockPool, } backup1File := bytes.NewBuffer([]byte{}) @@ -1206,7 +1206,7 @@ func TestBackupUsesNewCohabitatingResourcesForEachBackup(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: h.itemBlockPool.GetInputChannel(), + WorkerPool: &h.itemBlockPool, } backup2File := bytes.NewBuffer([]byte{}) @@ -1260,7 +1260,7 @@ func TestBackupResourceOrdering(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1381,7 +1381,7 @@ func TestBackupItemActionsForSkippedPV(t *testing.T) { Backup: defaultBackup().SnapshotVolumes(false).Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, resPolicies: &resourcepolicies.ResourcePolicies{ Version: "v1", @@ -1429,7 +1429,7 @@ func TestBackupItemActionsForSkippedPV(t *testing.T) { includedPVs: map[string]struct{}{}, }, BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVCs( @@ -1679,7 +1679,7 @@ func TestBackupActionsRunForCorrectItems(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1764,7 +1764,7 @@ func TestBackupWithInvalidActions(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -1918,7 +1918,7 @@ func TestBackupActionModifications(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -2178,7 +2178,7 @@ func TestBackupActionAdditionalItems(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -2439,7 +2439,7 @@ func TestItemBlockActionsRunForCorrectItems(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -2524,7 +2524,7 @@ func TestBackupWithInvalidItemBlockActions(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -2780,7 +2780,7 @@ func TestItemBlockActionRelatedItems(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -2948,7 +2948,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -2984,7 +2984,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3021,7 +3021,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3058,7 +3058,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3095,7 +3095,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3130,7 +3130,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3148,7 +3148,7 @@ func TestBackupWithSnapshots(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3169,7 +3169,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3188,7 +3188,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3210,7 +3210,7 @@ func TestBackupWithSnapshots(t *testing.T) { }, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVs( @@ -3344,7 +3344,7 @@ func TestBackupWithAsyncOperations(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.Pods( @@ -3376,7 +3376,7 @@ func TestBackupWithAsyncOperations(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.Pods( @@ -3408,7 +3408,7 @@ func TestBackupWithAsyncOperations(t *testing.T) { Backup: defaultBackup().Result(), SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.Pods( @@ -3494,7 +3494,7 @@ func TestBackupWithInvalidHooks(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -3968,7 +3968,7 @@ func TestBackupWithHooks(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) podCommandExecutor = new(test.MockPodCommandExecutor) @@ -4193,7 +4193,7 @@ func TestBackupWithPodVolume(t *testing.T) { SnapshotLocations: []*velerov1.VolumeSnapshotLocation{tc.vsl}, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -5312,7 +5312,7 @@ func TestBackupNewResourceFiltering(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) @@ -5477,7 +5477,7 @@ func TestBackupNamespaces(t *testing.T) { Backup: tc.backup, SkippedPVTracker: NewSkipPVTracker(), BackedUpItems: NewBackedUpItemsMap(), - ItemBlockChannel: itemBlockPool.GetInputChannel(), + WorkerPool: itemBlockPool, } backupFile = bytes.NewBuffer([]byte{}) ) diff --git a/pkg/backup/request.go b/pkg/backup/request.go index 4643142b1..b012992c0 100644 --- a/pkg/backup/request.go +++ b/pkg/backup/request.go @@ -69,7 +69,7 @@ type Request struct { ResPolicies *resourcepolicies.Policies SkippedPVTracker *skipPVTracker VolumesInformation volume.BackupVolumesInformation - ItemBlockChannel chan ItemBlockInput + WorkerPool *ItemBlockWorkerPool } // BackupVolumesInformation contains the information needs by generating diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 90085fa1f..1d3b9b9eb 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -110,7 +110,6 @@ type backupReconciler struct { globalCRClient kbclient.Client itemBlockWorkerCount int concurrentBackups int - workerPool *pkgbackup.ItemBlockWorkerPool } func NewBackupReconciler( @@ -167,7 +166,6 @@ func NewBackupReconciler( itemBlockWorkerCount: itemBlockWorkerCount, concurrentBackups: max(concurrentBackups, 1), globalCRClient: globalCRClient, - workerPool: pkgbackup.StartItemBlockWorkerPool(ctx, itemBlockWorkerCount, logger), } b.updateTotalBackupMetric() return b @@ -289,7 +287,9 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } log.Debug("Preparing backup request") - request := b.prepareBackupRequest(original, log) + request := b.prepareBackupRequest(ctx, original, log) + // delete worker pool after reconcile + defer request.WorkerPool.Stop() if len(request.Status.ValidationErrors) > 0 { request.Status.Phase = velerov1api.BackupPhaseFailedValidation } else { @@ -371,12 +371,12 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, nil } -func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logger logrus.FieldLogger) *pkgbackup.Request { +func (b *backupReconciler) prepareBackupRequest(ctx context.Context, backup *velerov1api.Backup, logger logrus.FieldLogger) *pkgbackup.Request { request := &pkgbackup.Request{ Backup: backup.DeepCopy(), // don't modify items in the cache SkippedPVTracker: pkgbackup.NewSkipPVTracker(), BackedUpItems: pkgbackup.NewBackedUpItemsMap(), - ItemBlockChannel: b.workerPool.GetInputChannel(), + WorkerPool: pkgbackup.StartItemBlockWorkerPool(ctx, b.itemBlockWorkerCount, logger), } request.VolumesInformation.Init() diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 524db4b24..e0b554ff1 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -149,9 +149,7 @@ func TestProcessBackupNonProcessedItems(t *testing.T) { kbClient: velerotest.NewFakeControllerRuntimeClient(t), formatFlag: formatFlag, logger: logger, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() if test.backup != nil { require.NoError(t, c.kbClient.Create(t.Context(), test.backup)) } @@ -248,9 +246,7 @@ func TestProcessBackupValidationFailures(t *testing.T) { clock: &clock.RealClock{}, formatFlag: formatFlag, metrics: metrics.NewServerMetrics(), - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() require.NotNil(t, test.backup) require.NoError(t, c.kbClient.Create(t.Context(), test.backup)) @@ -313,11 +309,10 @@ func TestBackupLocationLabel(t *testing.T) { defaultBackupLocation: test.backupLocation.Name, clock: &clock.RealClock{}, formatFlag: formatFlag, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() - res := c.prepareBackupRequest(test.backup, logger) + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.Equal(t, test.expectedBackupLocation, res.Labels[velerov1api.StorageLocationLabel]) }) @@ -410,14 +405,13 @@ func Test_prepareBackupRequest_BackupStorageLocation(t *testing.T) { defaultBackupTTL: defaultBackupTTL.Duration, clock: testclocks.NewFakeClock(now), formatFlag: formatFlag, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() test.backup.Spec.StorageLocation = test.backupLocationNameInBackup // Run - res := c.prepareBackupRequest(test.backup, logger) + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() // Assert if test.expectedSuccess { @@ -486,11 +480,10 @@ func TestDefaultBackupTTL(t *testing.T) { defaultBackupTTL: defaultBackupTTL.Duration, clock: testclocks.NewFakeClock(now), formatFlag: formatFlag, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() - res := c.prepareBackupRequest(test.backup, logger) + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.Equal(t, test.expectedTTL, res.Spec.TTL) assert.Equal(t, test.expectedExpiration, *res.Status.Expiration) @@ -547,11 +540,10 @@ func TestPrepareBackupRequest_SetsVGSLabelKey(t *testing.T) { defaultVGSLabelKey: test.serverFlagKey, discoveryHelper: discoveryHelper, clock: testclocks.NewFakeClock(now), - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() - res := c.prepareBackupRequest(test.backup, logger) + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.Equal(t, test.expectedLabelKey, res.Spec.VolumeGroupSnapshotLabelKey) @@ -649,11 +641,10 @@ func TestDefaultVolumesToResticDeprecation(t *testing.T) { clock: &clock.RealClock{}, formatFlag: formatFlag, defaultVolumesToFsBackup: test.globalVal, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() - res := c.prepareBackupRequest(test.backup, logger) + res := c.prepareBackupRequest(ctx, test.backup, logger) + defer res.WorkerPool.Stop() assert.NotNil(t, res) assert.NotNil(t, res.Spec.DefaultVolumesToFsBackup) if test.expectRemap { @@ -1569,9 +1560,7 @@ func TestProcessBackupCompletions(t *testing.T) { backupper: backupper, formatFlag: formatFlag, globalCRClient: fakeGlobalClient, - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() pluginManager.On("GetBackupItemActionsV2").Return(nil, nil) pluginManager.On("GetItemBlockActions").Return(nil, nil) @@ -1777,9 +1766,7 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { logger: logger, defaultSnapshotLocations: test.defaultLocations, kbClient: velerotest.NewFakeControllerRuntimeClient(t), - workerPool: pkgbackup.StartItemBlockWorkerPool(t.Context(), 1, logger), } - defer c.workerPool.Stop() // set up a Backup object to represent what we expect to be passed to backupper.Backup() backup := test.backup.DeepCopy() From dbaa25405ddcb6fb179dcfc128e757c8978bd91f Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Fri, 3 Oct 2025 12:33:37 -0400 Subject: [PATCH 18/37] move podVolumeContext into backupRequest Signed-off-by: Scott Seago --- pkg/backup/backup.go | 8 ++++---- pkg/backup/item_backupper.go | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 8d009de90..de4947afb 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -117,7 +117,6 @@ type kubernetesBackupper struct { podCommandExecutor podexec.PodCommandExecutor podVolumeBackupperFactory podvolume.BackupperFactory podVolumeTimeout time.Duration - podVolumeContext context.Context defaultVolumesToFsBackup bool clientPageSize int uploaderType string @@ -365,12 +364,12 @@ func (kb *kubernetesBackupper) BackupWithResolvers( } var podVolumeCancelFunc context.CancelFunc - kb.podVolumeContext, podVolumeCancelFunc = context.WithTimeout(context.Background(), podVolumeTimeout) + podVolumeContext, podVolumeCancelFunc := context.WithTimeout(context.Background(), podVolumeTimeout) defer podVolumeCancelFunc() var podVolumeBackupper podvolume.Backupper if kb.podVolumeBackupperFactory != nil { - podVolumeBackupper, err = kb.podVolumeBackupperFactory.NewBackupper(kb.podVolumeContext, log, backupRequest.Backup, kb.uploaderType) + podVolumeBackupper, err = kb.podVolumeBackupperFactory.NewBackupper(podVolumeContext, log, backupRequest.Backup, kb.uploaderType) if err != nil { log.WithError(errors.WithStack(err)).Debugf("Error from NewBackupper") return errors.WithStack(err) @@ -416,6 +415,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers( kbClient: kb.kbClient, discoveryHelper: kb.discoveryHelper, podVolumeBackupper: podVolumeBackupper, + podVolumeContext: podVolumeContext, podVolumeSnapshotTracker: podvolume.NewTracker(), volumeSnapshotterCache: NewVolumeSnapshotterCache(volumeSnapshotterGetter), itemHookHandler: &hook.DefaultItemHookHandler{ @@ -848,7 +848,7 @@ func (kb *kubernetesBackupper) handleItemBlockPostHooks(itemBlock *BackupItemBlo log := itemBlock.Log // the post hooks will not execute until all PVBs of the item block pods are processed - if err := kb.waitUntilPVBsProcessed(kb.podVolumeContext, log, itemBlock, hookPods); err != nil { + if err := kb.waitUntilPVBsProcessed(itemBlock.itemBackupper.podVolumeContext, log, itemBlock, hookPods); err != nil { log.WithError(err).Error("failed to wait PVBs processed for the ItemBlock") return } diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 3622ac7e1..7b1ea69bd 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -69,6 +69,7 @@ type itemBackupper struct { kbClient kbClient.Client discoveryHelper discovery.Helper podVolumeBackupper podvolume.Backupper + podVolumeContext context.Context podVolumeSnapshotTracker *podvolume.Tracker kubernetesBackupper *kubernetesBackupper volumeSnapshotterCache *VolumeSnapshotterCache From aedc0fe5e25fd8446af13c4e17d6b51043dbb60f Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Fri, 3 Oct 2025 14:04:38 -0400 Subject: [PATCH 19/37] make update, changelog Signed-off-by: Scott Seago --- changelogs/unreleased/9307-sseago | 1 + pkg/backup/backup_test.go | 4 +- pkg/controller/backup_queue_controller.go | 41 +++++++++--------- .../backup_queue_controller_test.go | 43 ++++++++++--------- 4 files changed, 46 insertions(+), 43 deletions(-) create mode 100644 changelogs/unreleased/9307-sseago diff --git a/changelogs/unreleased/9307-sseago b/changelogs/unreleased/9307-sseago new file mode 100644 index 000000000..c69891510 --- /dev/null +++ b/changelogs/unreleased/9307-sseago @@ -0,0 +1 @@ +Concurrent backup processing diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 1db3cff6b..f9351245c 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -1428,8 +1428,8 @@ func TestBackupItemActionsForSkippedPV(t *testing.T) { }, includedPVs: map[string]struct{}{}, }, - BackedUpItems: NewBackedUpItemsMap(), - WorkerPool: itemBlockPool, + BackedUpItems: NewBackedUpItemsMap(), + WorkerPool: itemBlockPool, }, apiResources: []*test.APIResource{ test.PVCs( diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 93ef08713..2aff4e105 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -85,7 +85,7 @@ func queuePositionOrderFunc(objList client.ObjectList) client.ObjectList { // SetupWithManager adds the reconciler to the manager func (r *backupQueueReconciler) SetupWithManager(mgr ctrl.Manager) error { - // For periodic requeue, only consider Queued backups, order by QueuePosition + // For periodic requeue, only consider Queued backups, order by QueuePosition gp := kube.NewGenericEventPredicate(func(object client.Object) bool { backup := object.(*velerov1api.Backup) return backup.Status.Phase == velerov1api.BackupPhaseQueued @@ -93,7 +93,7 @@ func (r *backupQueueReconciler) SetupWithManager(mgr ctrl.Manager) error { s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerBackupQueue), mgr.GetClient(), &velerov1api.BackupList{}, r.frequency, kube.PeriodicalEnqueueSourceOption{ Predicates: []predicate.Predicate{gp}, - OrderFunc: queuePositionOrderFunc, + OrderFunc: queuePositionOrderFunc, }) return ctrl.NewControllerManagedBy(mgr). @@ -123,7 +123,7 @@ func (r *backupQueueReconciler) SetupWithManager(mgr ctrl.Manager) error { return oldBackup.Status.Phase == velerov1api.BackupPhaseInProgress && newBackup.Status.Phase != velerov1api.BackupPhaseInProgress || oldBackup.Status.Phase != velerov1api.BackupPhaseQueued && - newBackup.Status.Phase == velerov1api.BackupPhaseQueued + newBackup.Status.Phase == velerov1api.BackupPhaseQueued }, CreateFunc: func(event.CreateEvent) bool { return false @@ -148,9 +148,9 @@ func (r *backupQueueReconciler) listQueuedBackups(ctx context.Context, ns string return nil, err } for _, backup := range backupList.Items { - if backup.Status.Phase == velerov1api.BackupPhaseQueued { + if backup.Status.Phase == velerov1api.BackupPhaseQueued { backups = append(backups, backup) - } + } } return backups, nil @@ -164,17 +164,17 @@ func (r *backupQueueReconciler) listEarlierBackups(ctx context.Context, ns strin return nil, 0, err } for _, backup := range backupList.Items { - if backup.Status.Phase == velerov1api.BackupPhaseInProgress || + if backup.Status.Phase == velerov1api.BackupPhaseInProgress || backup.Status.Phase == velerov1api.BackupPhaseReadyToStart || (backup.Status.Phase == velerov1api.BackupPhaseQueued && - backup.Status.QueuePosition < queuePos) { + backup.Status.QueuePosition < queuePos) { backups = append(backups, backup) - } + } // InProgress and ReadyToStart backups count towards the concurrentBackups limit - if backup.Status.Phase == velerov1api.BackupPhaseInProgress || + if backup.Status.Phase == velerov1api.BackupPhaseInProgress || backup.Status.Phase == velerov1api.BackupPhaseReadyToStart { runningCount += 1 - } + } } return backups, runningCount, nil } @@ -216,6 +216,7 @@ func detectNSConflictsInternal(ctx context.Context, backup *velerov1api.Backup, } return false, "" } + // Returns true if there are backups ahead of the current backup that are runnable // This could happen if velero just reconciled the one earlier in the queue and rejected it // due to too many running backups, but a backup completed in between that reconcile and this one @@ -261,9 +262,9 @@ func (r *backupQueueReconciler) orderedQueuedBackups(ctx context.Context, backup } orderedBackupList := queuePositionOrderFunc(backupList).(*velerov1api.BackupList) for _, item := range orderedBackupList.Items { - if item.Status.Phase == velerov1api.BackupPhaseQueued { - returnList = append(returnList, item) - } + if item.Status.Phase == velerov1api.BackupPhaseQueued { + returnList = append(returnList, item) + } } return returnList, nil } @@ -272,13 +273,13 @@ func (r *backupQueueReconciler) findQueuedBackupsToRequeue(ctx context.Context, requests := []reconcile.Request{} backups, _ := r.orderedQueuedBackups(ctx, backup) for _, item := range backups { - requests = append(requests, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: item.GetNamespace(), - Name: item.GetName(), - }, - }) - } + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: item.GetNamespace(), + Name: item.GetName(), + }, + }) + } return requests } diff --git a/pkg/controller/backup_queue_controller_test.go b/pkg/controller/backup_queue_controller_test.go index 1a5eb582f..c559dc8e0 100644 --- a/pkg/controller/backup_queue_controller_test.go +++ b/pkg/controller/backup_queue_controller_test.go @@ -23,10 +23,12 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + //metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + //"sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -60,7 +62,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectPhase: velerov1api.BackupPhaseInProgress, }, { - name: "Second New Backup gets queued with queuePosition 2", + name: "Second New Backup gets queued with queuePosition 2", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), }, @@ -74,7 +76,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectPhase: velerov1api.BackupPhaseReadyToStart, }, { - name: "Queued Backup remains queued if no spaces available", + name: "Queued Backup remains queued if no spaces available", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), @@ -85,7 +87,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if no spaces available including ReadyToStart", + name: "Queued Backup remains queued if no spaces available including ReadyToStart", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseReadyToStart).Result(), @@ -96,7 +98,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if earlier runnable backup is also queued", + name: "Queued Backup remains queued if earlier runnable backup is also queued", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), @@ -107,7 +109,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 2, }, { - name: "Queued Backup remains queued if in conflict with running backup", + name: "Queued Backup remains queued if in conflict with running backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("foo").Result(), }, @@ -118,7 +120,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if in conflict with ReadyToStart backup", + name: "Queued Backup remains queued if in conflict with ReadyToStart backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseReadyToStart).IncludedNamespaces("foo").Result(), }, @@ -129,7 +131,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if in conflict with earlier queued backup", + name: "Queued Backup remains queued if in conflict with earlier queued backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), }, @@ -140,7 +142,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 2, }, { - name: "Queued Backup remains queued if earlier non-ns-conflict backup exists", + name: "Queued Backup remains queued if earlier non-ns-conflict backup exists", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), @@ -152,7 +154,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 2, }, { - name: "Running all-namespace backup conflicts with queued one-namespace backup ", + name: "Running all-namespace backup conflicts with queued one-namespace backup ", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("*").Result(), }, @@ -163,7 +165,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Running one-namespace backup conflicts with queued all-namespace backup ", + name: "Running one-namespace backup conflicts with queued all-namespace backup ", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), }, @@ -174,24 +176,24 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups", + name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), }, - concurrentBackups: 3, - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), - expectPhase: velerov1api.BackupPhaseReadyToStart, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), + expectPhase: velerov1api.BackupPhaseReadyToStart, }, { - name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups and no ns conflict found", + name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups and no ns conflict found", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseReadyToStart).IncludedNamespaces("foo").Result(), }, - namespaces: []string{"foo", "bar"}, - concurrentBackups: 3, - backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("bar").Result(), - expectPhase: velerov1api.BackupPhaseReadyToStart, + namespaces: []string{"foo", "bar"}, + concurrentBackups: 3, + backup: builder.ForBackup(velerov1api.DefaultNamespace, "backup-20").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("bar").Result(), + expectPhase: velerov1api.BackupPhaseReadyToStart, }, } @@ -214,7 +216,7 @@ func TestBackupQueueReconciler(t *testing.T) { logger := logrus.New() log := logger.WithField("controller", "backup-queue-test") r := NewBackupQueueReconciler(fakeClient, scheme, log, test.concurrentBackups) - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace,Name: test.backup.Name}} + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}} res, err := r.Reconcile(context.Background(), req) gotErr := err != nil require.NoError(t, err) @@ -232,5 +234,4 @@ func TestBackupQueueReconciler(t *testing.T) { }) } - } From e7b5d20f4cf592a236bcb0de441e26f1470a7738 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Fri, 3 Oct 2025 14:31:04 -0400 Subject: [PATCH 20/37] Fix linters Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 17 ++++++++--------- pkg/controller/backup_queue_controller_test.go | 4 +--- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 2aff4e105..7817685c2 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -151,7 +151,6 @@ func (r *backupQueueReconciler) listQueuedBackups(ctx context.Context, ns string if backup.Status.Phase == velerov1api.BackupPhaseQueued { backups = append(backups, backup) } - } return backups, nil } @@ -173,7 +172,7 @@ func (r *backupQueueReconciler) listEarlierBackups(ctx context.Context, ns strin // InProgress and ReadyToStart backups count towards the concurrentBackups limit if backup.Status.Phase == velerov1api.BackupPhaseInProgress || backup.Status.Phase == velerov1api.BackupPhaseReadyToStart { - runningCount += 1 + runningCount++ } } return backups, runningCount, nil @@ -188,11 +187,11 @@ func (r *backupQueueReconciler) detectNamespaceConflict(ctx context.Context, bac for _, ns := range nsList.Items { clusterNamespaces = append(clusterNamespaces, ns.Name) } - foundConflict, conflictBackup := detectNSConflictsInternal(ctx, backup, earlierBackups, clusterNamespaces) + foundConflict, conflictBackup := detectNSConflictsInternal(backup, earlierBackups, clusterNamespaces) return foundConflict, conflictBackup, clusterNamespaces, nil } -func detectNSConflictsInternal(ctx context.Context, backup *velerov1api.Backup, earlierBackups []velerov1api.Backup, clusterNamespaces []string) (bool, string) { +func detectNSConflictsInternal(backup *velerov1api.Backup, earlierBackups []velerov1api.Backup, clusterNamespaces []string) (bool, string) { backupNamespaces := namespacesForBackup(backup, clusterNamespaces) backupNSMap := make(map[string]struct{}) for _, ns := range backupNamespaces { @@ -221,14 +220,14 @@ func detectNSConflictsInternal(ctx context.Context, backup *velerov1api.Backup, // This could happen if velero just reconciled the one earlier in the queue and rejected it // due to too many running backups, but a backup completed in between that reconcile and this one // so exit, as the recent completion has triggered another reconcile of all queued backups -func (r *backupQueueReconciler) checkForEarlierRunnableBackups(ctx context.Context, backup *velerov1api.Backup, earlierBackups []velerov1api.Backup, clusterNamespaces []string) (bool, string) { +func (r *backupQueueReconciler) checkForEarlierRunnableBackups(backup *velerov1api.Backup, earlierBackups []velerov1api.Backup, clusterNamespaces []string) (bool, string) { for _, earlierBackup := range earlierBackups { // if this backup is queued and ahead of current backup, check for conflicts if earlierBackup.Status.Phase != velerov1api.BackupPhaseQueued || earlierBackup.Status.QueuePosition >= backup.Status.QueuePosition { continue } - conflict, _ := detectNSConflictsInternal(ctx, &earlierBackup, earlierBackups, clusterNamespaces) + conflict, _ := detectNSConflictsInternal(&earlierBackup, earlierBackups, clusterNamespaces) // !conflict means we've found an earlier backup that is currently runnable // so current reconcile should exit to run this one if !conflict { @@ -299,7 +298,7 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) maxQueuePosition, err := r.getMaxQueuePosition(ctx, backup.Namespace) if err != nil { // error is logged in getMaxQueuePosition func - return ctrl.Result{}, nil + return ctrl.Result{}, nil //nolint:nilerr // We want to return nil to avoid requeue } original := backup.DeepCopy() backup.Status.Phase = velerov1api.BackupPhaseQueued @@ -329,7 +328,7 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.Infof("Backup %v has namespace conflict with %v, leaving queued", backup.Name, conflictBackup) return ctrl.Result{}, nil } - foundEarlierRunnable, earlierRunnable := r.checkForEarlierRunnableBackups(ctx, backup, earlierBackups, clusterNamespaces) + foundEarlierRunnable, earlierRunnable := r.checkForEarlierRunnableBackups(backup, earlierBackups, clusterNamespaces) if foundEarlierRunnable { log.Infof("Earlier queued backup %v is runnable, leaving %v queued", earlierRunnable, backup.Name) return ctrl.Result{}, nil @@ -356,7 +355,7 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.WithError(errors.Wrapf(err, "error updating Backup %s queuePosition to %v", queuedBackup.Name, newQueuePos)) return ctrl.Result{}, nil } - newQueuePos += 1 + newQueuePos++ } } return ctrl.Result{}, nil diff --git a/pkg/controller/backup_queue_controller_test.go b/pkg/controller/backup_queue_controller_test.go index c559dc8e0..8022d7140 100644 --- a/pkg/controller/backup_queue_controller_test.go +++ b/pkg/controller/backup_queue_controller_test.go @@ -17,7 +17,6 @@ limitations under the License. package controller import ( - "context" "testing" "github.com/sirupsen/logrus" @@ -217,7 +216,7 @@ func TestBackupQueueReconciler(t *testing.T) { log := logger.WithField("controller", "backup-queue-test") r := NewBackupQueueReconciler(fakeClient, scheme, log, test.concurrentBackups) req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}} - res, err := r.Reconcile(context.Background(), req) + res, err := r.Reconcile(t.Context(), req) gotErr := err != nil require.NoError(t, err) assert.Equal(t, ctrl.Result{}, res) @@ -233,5 +232,4 @@ func TestBackupQueueReconciler(t *testing.T) { assert.Equal(t, test.expectQueuePosition, backupAfter.Status.QueuePosition) }) } - } From 7bc57b5a5f396099a7fc02b1bd1f48871c7a109f Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Thu, 13 Nov 2025 14:04:15 -0500 Subject: [PATCH 21/37] Refactor queue controller to reduce apiserver list calls Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 145 ++++++++++++---------- 1 file changed, 79 insertions(+), 66 deletions(-) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 7817685c2..1926f60cb 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -140,44 +140,6 @@ func (r *backupQueueReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func (r *backupQueueReconciler) listQueuedBackups(ctx context.Context, ns string) ([]velerov1api.Backup, error) { - backupList := &velerov1api.BackupList{} - backups := []velerov1api.Backup{} - if err := r.Client.List(ctx, backupList, &client.ListOptions{Namespace: ns}); err != nil { - r.logger.WithError(err).Error("error listing queued backups") - return nil, err - } - for _, backup := range backupList.Items { - if backup.Status.Phase == velerov1api.BackupPhaseQueued { - backups = append(backups, backup) - } - } - return backups, nil -} - -func (r *backupQueueReconciler) listEarlierBackups(ctx context.Context, ns string, queuePos int) ([]velerov1api.Backup, int, error) { - backupList := &velerov1api.BackupList{} - backups := []velerov1api.Backup{} - runningCount := 0 - if err := r.Client.List(ctx, backupList, &client.ListOptions{Namespace: ns}); err != nil { - return nil, 0, err - } - for _, backup := range backupList.Items { - if backup.Status.Phase == velerov1api.BackupPhaseInProgress || - backup.Status.Phase == velerov1api.BackupPhaseReadyToStart || - (backup.Status.Phase == velerov1api.BackupPhaseQueued && - backup.Status.QueuePosition < queuePos) { - backups = append(backups, backup) - } - // InProgress and ReadyToStart backups count towards the concurrentBackups limit - if backup.Status.Phase == velerov1api.BackupPhaseInProgress || - backup.Status.Phase == velerov1api.BackupPhaseReadyToStart { - runningCount++ - } - } - return backups, runningCount, nil -} - func (r *backupQueueReconciler) detectNamespaceConflict(ctx context.Context, backup *velerov1api.Backup, earlierBackups []velerov1api.Backup) (bool, string, []string, error) { nsList := &corev1api.NamespaceList{} if err := r.Client.List(ctx, nsList); err != nil { @@ -240,37 +202,28 @@ func (r *backupQueueReconciler) checkForEarlierRunnableBackups(backup *velerov1a func namespacesForBackup(backup *velerov1api.Backup, clusterNamespaces []string) []string { return collections.NewNamespaceIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...).ActiveNamespaces(clusterNamespaces).ResolveNamespaceList() } -func (r *backupQueueReconciler) getMaxQueuePosition(ctx context.Context, ns string) (int, error) { - queuedBackups, err := r.listQueuedBackups(ctx, ns) - if err != nil { - return 0, err - } +func (r *backupQueueReconciler) getMaxQueuePosition(lister *queuedBackupsLister) int { + queuedBackups := lister.queued() maxPos := 0 for _, backup := range queuedBackups { maxPos = max(maxPos, backup.Status.QueuePosition) } - return maxPos, nil + return maxPos } -func (r *backupQueueReconciler) orderedQueuedBackups(ctx context.Context, backup *velerov1api.Backup) ([]velerov1api.Backup, error) { - backupList := &velerov1api.BackupList{} - var returnList []velerov1api.Backup - if err := r.Client.List(ctx, backupList, &client.ListOptions{Namespace: backup.Namespace}); err != nil { +func (r *backupQueueReconciler) orderedQueuedBackupsForEnqueue(ctx context.Context, backup *velerov1api.Backup) ([]velerov1api.Backup, error) { + lister, err := r.newQueuedBackupsLister(ctx, backup.Namespace) + if err != nil { r.logger.WithError(err).Error("error listing backups") return nil, err } - orderedBackupList := queuePositionOrderFunc(backupList).(*velerov1api.BackupList) - for _, item := range orderedBackupList.Items { - if item.Status.Phase == velerov1api.BackupPhaseQueued { - returnList = append(returnList, item) - } - } - return returnList, nil + return lister.orderedQueued(), nil } + func (r *backupQueueReconciler) findQueuedBackupsToRequeue(ctx context.Context, obj client.Object) []reconcile.Request { backup := obj.(*velerov1api.Backup) requests := []reconcile.Request{} - backups, _ := r.orderedQueuedBackups(ctx, backup) + backups, _ := r.orderedQueuedBackupsForEnqueue(ctx, backup) for _, item := range backups { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ @@ -295,11 +248,12 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) switch backup.Status.Phase { case "", velerov1api.BackupPhaseNew: // queue new backup - maxQueuePosition, err := r.getMaxQueuePosition(ctx, backup.Namespace) + lister, err := r.newQueuedBackupsLister(ctx, backup.Namespace) if err != nil { - // error is logged in getMaxQueuePosition func + // error is logged in newQueuedBackupsLister func return ctrl.Result{}, nil //nolint:nilerr // We want to return nil to avoid requeue } + maxQueuePosition := r.getMaxQueuePosition(lister) original := backup.DeepCopy() backup.Status.Phase = velerov1api.BackupPhaseQueued backup.Status.QueuePosition = maxQueuePosition + 1 @@ -310,11 +264,12 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) case velerov1api.BackupPhaseQueued: // handle queued backup // Find backups ahead of this one (InProgress, ReadyToStart, or Queued with higher position) - earlierBackups, runningCount, err := r.listEarlierBackups(ctx, backup.Namespace, backup.Status.QueuePosition) + lister, err := r.newQueuedBackupsLister(ctx, backup.Namespace) if err != nil { - log.WithError(err).Error("error listing earlier backups") - return ctrl.Result{}, nil + // error is logged in newQueuedBackupsLister func + return ctrl.Result{}, nil //nolint:nilerr // We want to return nil to avoid requeue } + earlierBackups, runningCount := lister.earlierThan(backup.Status.QueuePosition) if runningCount >= r.concurrentBackups { log.Debugf("%v concurrent backups are already running, leaving %v queued", r.concurrentBackups, backup.Name) return ctrl.Result{}, nil @@ -341,11 +296,7 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, errors.Wrapf(err, "error updating Backup status to %s", backup.Status.Phase) } log.Debug("Updating queuePosition for remaining queued backups") - queuedBackups, err := r.orderedQueuedBackups(ctx, backup) - if err != nil { - log.WithError(err).Error("error listing queued backups") - return ctrl.Result{}, nil - } + queuedBackups := lister.orderedQueued() newQueuePos := 1 for _, queuedBackup := range queuedBackups { if queuedBackup.Name != backup.Name { @@ -366,3 +317,65 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + +// queuedBackupsLister manages a list of all backups Queued, ReadyToStart, or InProgress +// with methods to return specific subsets as needed +type queuedBackupsLister struct { + backups *velerov1api.BackupList +} + +func (r *backupQueueReconciler) newQueuedBackupsLister(ctx context.Context, ns string) (*queuedBackupsLister, error) { + backupList := &velerov1api.BackupList{} + backups := []velerov1api.Backup{} + if err := r.Client.List(ctx, backupList, &client.ListOptions{Namespace: ns}); err != nil { + r.logger.WithError(err).Error("error listing backups") + return nil, err + } + for _, backup := range backupList.Items { + if backup.Status.Phase == velerov1api.BackupPhaseQueued || + backup.Status.Phase == velerov1api.BackupPhaseInProgress || + backup.Status.Phase == velerov1api.BackupPhaseReadyToStart { + backups = append(backups, backup) + } + } + backupList.Items = backups + return &queuedBackupsLister{backupList}, nil +} + +func (l *queuedBackupsLister) earlierThan(queuePos int) ([]velerov1api.Backup, int){ + backups := []velerov1api.Backup{} + runningCount := 0 + for _, backup := range l.backups.Items { + // InProgress and ReadyToStart backups have QueuePosition==0 + if backup.Status.QueuePosition < queuePos { + backups = append(backups, backup) + } + // InProgress and ReadyToStart backups count towards the concurrentBackups limit + if backup.Status.Phase == velerov1api.BackupPhaseInProgress || + backup.Status.Phase == velerov1api.BackupPhaseReadyToStart { + runningCount++ + } + } + return backups, runningCount +} + +func (l *queuedBackupsLister) queued() []velerov1api.Backup { + backups := []velerov1api.Backup{} + for _, backup := range l.backups.Items { + if backup.Status.Phase == velerov1api.BackupPhaseQueued { + backups = append(backups, backup) + } + } + return backups +} + +func (l *queuedBackupsLister) orderedQueued() []velerov1api.Backup { + var returnList []velerov1api.Backup + orderedBackupList := queuePositionOrderFunc(l.backups).(*velerov1api.BackupList) + for _, item := range orderedBackupList.Items { + if item.Status.Phase == velerov1api.BackupPhaseQueued { + returnList = append(returnList, item) + } + } + return returnList +} From 9c1c7d20ff65dc7c8b6d02f2b5eefb1b60f0717e Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Thu, 13 Nov 2025 14:35:21 -0500 Subject: [PATCH 22/37] Minor refactoring Signed-off-by: Scott Seago --- pkg/backup/request.go | 4 ++++ pkg/controller/backup_controller.go | 2 +- pkg/controller/backup_queue_controller.go | 14 ++++---------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pkg/backup/request.go b/pkg/backup/request.go index b012992c0..eb9edcbe8 100644 --- a/pkg/backup/request.go +++ b/pkg/backup/request.go @@ -103,3 +103,7 @@ func (r *Request) FillVolumesInformation() { r.VolumesInformation.BackupOperations = *r.GetItemOperationsList() r.VolumesInformation.BackupName = r.Backup.Name } + +func (r *Request) StopWorkerPool() { + r.WorkerPool.Stop() +} diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 1d3b9b9eb..f269353cb 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -289,7 +289,7 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr log.Debug("Preparing backup request") request := b.prepareBackupRequest(ctx, original, log) // delete worker pool after reconcile - defer request.WorkerPool.Stop() + defer request.StopWorkerPool() if len(request.Status.ValidationErrors) > 0 { request.Status.Phase = velerov1api.BackupPhaseFailedValidation } else { diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 1926f60cb..bd57f3c95 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -25,6 +25,7 @@ import ( "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -154,11 +155,7 @@ func (r *backupQueueReconciler) detectNamespaceConflict(ctx context.Context, bac } func detectNSConflictsInternal(backup *velerov1api.Backup, earlierBackups []velerov1api.Backup, clusterNamespaces []string) (bool, string) { - backupNamespaces := namespacesForBackup(backup, clusterNamespaces) - backupNSMap := make(map[string]struct{}) - for _, ns := range backupNamespaces { - backupNSMap[ns] = struct{}{} - } + backupNamespaces := sets.NewString(namespacesForBackup(backup, clusterNamespaces)...) for _, earlierBackup := range earlierBackups { // This will never be true for the primary backup, but for the secondary // runnability check for queued backups ahead of the current backup, we @@ -168,11 +165,8 @@ func detectNSConflictsInternal(backup *velerov1api.Backup, earlierBackups []vele earlierBackup.Status.QueuePosition >= backup.Status.QueuePosition { continue } - runningNSList := namespacesForBackup(&earlierBackup, clusterNamespaces) - for _, runningNS := range runningNSList { - if _, found := backupNSMap[runningNS]; found { - return true, earlierBackup.Name - } + if backupNamespaces.HasAny(namespacesForBackup(&earlierBackup, clusterNamespaces)...) { + return true, earlierBackup.Name } } return false, "" From 84571bc54de22e4d67c1004d8a2dd75df283c573 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Thu, 13 Nov 2025 14:51:38 -0500 Subject: [PATCH 23/37] Added doc note around parallel backups and resource limits Signed-off-by: Scott Seago --- site/content/docs/main/customize-installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/docs/main/customize-installation.md b/site/content/docs/main/customize-installation.md index d62945a35..7c1274ebf 100644 --- a/site/content/docs/main/customize-installation.md +++ b/site/content/docs/main/customize-installation.md @@ -158,7 +158,7 @@ At installation, You could set resource requests and limits for the Velero pod, |Memory limit|512Mi|N/A| {{< /table >}} -For Velero pod, through testing, the Velero maintainers have found these defaults work well when backing up and restoring 1000 or less resources. +For Velero pod, through testing, the Velero maintainers have found these defaults work well when backing up and restoring 1000 or less resources. If you are enabling concurrent backups and your backups tend to be large, you may need to increase these limits. For node-agent pod, by default it doesn't have CPU/memory request/limit, so that the backups/restores won't break due to resource throttling. The Velero maintainers have also done some [Performance Tests][13] to show the relationship of CPU/memory usage and the scale of data being backed up/restored. For repository maintenance job, it's no limit on resources by default. You could configure the job resource limitation based on target data to be backed up, some further settings please refer to [repository maintenance job][14]. From 29b01c31702003c5e2f8c88a98a2185097cdfd42 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Thu, 13 Nov 2025 14:57:26 -0500 Subject: [PATCH 24/37] make update Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index bd57f3c95..1929a2414 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -25,8 +25,8 @@ import ( "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -336,7 +336,7 @@ func (r *backupQueueReconciler) newQueuedBackupsLister(ctx context.Context, ns s return &queuedBackupsLister{backupList}, nil } -func (l *queuedBackupsLister) earlierThan(queuePos int) ([]velerov1api.Backup, int){ +func (l *queuedBackupsLister) earlierThan(queuePos int) ([]velerov1api.Backup, int) { backups := []velerov1api.Backup{} runningCount := 0 for _, backup := range l.backups.Items { From 34dc381182d65cb12056cbaf6596b00dc1f81ff4 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Mon, 17 Nov 2025 15:51:15 -0500 Subject: [PATCH 25/37] Refactor after review Signed-off-by: Scott Seago Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 55 ++++++++--------------- 1 file changed, 19 insertions(+), 36 deletions(-) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 1929a2414..3a292ae33 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -197,27 +197,23 @@ func namespacesForBackup(backup *velerov1api.Backup, clusterNamespaces []string) return collections.NewNamespaceIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...).ActiveNamespaces(clusterNamespaces).ResolveNamespaceList() } func (r *backupQueueReconciler) getMaxQueuePosition(lister *queuedBackupsLister) int { - queuedBackups := lister.queued() + queuedBackups := lister.orderedQueued() maxPos := 0 - for _, backup := range queuedBackups { - maxPos = max(maxPos, backup.Status.QueuePosition) + if len(queuedBackups) > 0 { + maxPos = queuedBackups[len(queuedBackups)-1].Status.QueuePosition } return maxPos } -func (r *backupQueueReconciler) orderedQueuedBackupsForEnqueue(ctx context.Context, backup *velerov1api.Backup) ([]velerov1api.Backup, error) { - lister, err := r.newQueuedBackupsLister(ctx, backup.Namespace) - if err != nil { - r.logger.WithError(err).Error("error listing backups") - return nil, err - } - return lister.orderedQueued(), nil -} - func (r *backupQueueReconciler) findQueuedBackupsToRequeue(ctx context.Context, obj client.Object) []reconcile.Request { backup := obj.(*velerov1api.Backup) requests := []reconcile.Request{} - backups, _ := r.orderedQueuedBackupsForEnqueue(ctx, backup) + allBackups := &velerov1api.BackupList{} + if err := r.Client.List(ctx, allBackups, &client.ListOptions{Namespace: backup.Namespace}); err != nil { + r.logger.WithError(err).Error("error listing backups") + return requests + } + backups := r.newQueuedBackupsLister(allBackups).orderedQueued() for _, item := range backups { requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ @@ -242,11 +238,12 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) switch backup.Status.Phase { case "", velerov1api.BackupPhaseNew: // queue new backup - lister, err := r.newQueuedBackupsLister(ctx, backup.Namespace) - if err != nil { - // error is logged in newQueuedBackupsLister func + allBackups := &velerov1api.BackupList{} + if err := r.Client.List(ctx, allBackups, &client.ListOptions{Namespace: backup.Namespace}); err != nil { + r.logger.WithError(err).Error("error listing backups") return ctrl.Result{}, nil //nolint:nilerr // We want to return nil to avoid requeue } + lister := r.newQueuedBackupsLister(allBackups) maxQueuePosition := r.getMaxQueuePosition(lister) original := backup.DeepCopy() backup.Status.Phase = velerov1api.BackupPhaseQueued @@ -258,11 +255,12 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) case velerov1api.BackupPhaseQueued: // handle queued backup // Find backups ahead of this one (InProgress, ReadyToStart, or Queued with higher position) - lister, err := r.newQueuedBackupsLister(ctx, backup.Namespace) - if err != nil { - // error is logged in newQueuedBackupsLister func + allBackups := &velerov1api.BackupList{} + if err := r.Client.List(ctx, allBackups, &client.ListOptions{Namespace: backup.Namespace}); err != nil { + r.logger.WithError(err).Error("error listing backups") return ctrl.Result{}, nil //nolint:nilerr // We want to return nil to avoid requeue } + lister := r.newQueuedBackupsLister(allBackups) earlierBackups, runningCount := lister.earlierThan(backup.Status.QueuePosition) if runningCount >= r.concurrentBackups { log.Debugf("%v concurrent backups are already running, leaving %v queued", r.concurrentBackups, backup.Name) @@ -318,13 +316,8 @@ type queuedBackupsLister struct { backups *velerov1api.BackupList } -func (r *backupQueueReconciler) newQueuedBackupsLister(ctx context.Context, ns string) (*queuedBackupsLister, error) { - backupList := &velerov1api.BackupList{} +func (r *backupQueueReconciler) newQueuedBackupsLister(backupList *velerov1api.BackupList) *queuedBackupsLister { backups := []velerov1api.Backup{} - if err := r.Client.List(ctx, backupList, &client.ListOptions{Namespace: ns}); err != nil { - r.logger.WithError(err).Error("error listing backups") - return nil, err - } for _, backup := range backupList.Items { if backup.Status.Phase == velerov1api.BackupPhaseQueued || backup.Status.Phase == velerov1api.BackupPhaseInProgress || @@ -333,7 +326,7 @@ func (r *backupQueueReconciler) newQueuedBackupsLister(ctx context.Context, ns s } } backupList.Items = backups - return &queuedBackupsLister{backupList}, nil + return &queuedBackupsLister{backupList} } func (l *queuedBackupsLister) earlierThan(queuePos int) ([]velerov1api.Backup, int) { @@ -353,16 +346,6 @@ func (l *queuedBackupsLister) earlierThan(queuePos int) ([]velerov1api.Backup, i return backups, runningCount } -func (l *queuedBackupsLister) queued() []velerov1api.Backup { - backups := []velerov1api.Backup{} - for _, backup := range l.backups.Items { - if backup.Status.Phase == velerov1api.BackupPhaseQueued { - backups = append(backups, backup) - } - } - return backups -} - func (l *queuedBackupsLister) orderedQueued() []velerov1api.Backup { var returnList []velerov1api.Backup orderedBackupList := queuePositionOrderFunc(l.backups).(*velerov1api.BackupList) From 6a3f821606e9c6d174352dbb0e5699539dc9aeb3 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Mon, 17 Nov 2025 18:30:33 -0500 Subject: [PATCH 26/37] fix lint Signed-off-by: Scott Seago Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 3a292ae33..5a99c6328 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -241,7 +241,7 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) allBackups := &velerov1api.BackupList{} if err := r.Client.List(ctx, allBackups, &client.ListOptions{Namespace: backup.Namespace}); err != nil { r.logger.WithError(err).Error("error listing backups") - return ctrl.Result{}, nil //nolint:nilerr // We want to return nil to avoid requeue + return ctrl.Result{}, nil } lister := r.newQueuedBackupsLister(allBackups) maxQueuePosition := r.getMaxQueuePosition(lister) @@ -258,7 +258,7 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) allBackups := &velerov1api.BackupList{} if err := r.Client.List(ctx, allBackups, &client.ListOptions{Namespace: backup.Namespace}); err != nil { r.logger.WithError(err).Error("error listing backups") - return ctrl.Result{}, nil //nolint:nilerr // We want to return nil to avoid requeue + return ctrl.Result{}, nil } lister := r.newQueuedBackupsLister(allBackups) earlierBackups, runningCount := lister.earlierThan(backup.Status.QueuePosition) From c50ab4a6eab2112bee02e07dd09efc95f567f757 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Mon, 11 Aug 2025 16:56:12 -0400 Subject: [PATCH 27/37] feat: Add `pvc-for-tmp` install arg to use PVC for server /tmp dir Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/cmd/cli/install/install.go | 8 ++++++++ pkg/install/deployment.go | 29 +++++++++++++++++++++++++++++ pkg/install/resources.go | 5 +++++ 3 files changed, 42 insertions(+) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 283bfa557..748b6953e 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -90,6 +90,7 @@ type Options struct { NodeAgentConfigMap string ItemBlockWorkerCount int ConcurrentBackups int + PVCForTmp string NodeAgentDisableHostPath bool kubeletRootDir string ServerPriorityClassName string @@ -191,6 +192,12 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { o.NodeAgentConfigMap, "The name of ConfigMap containing node-agent configurations.", ) + flags.StringVar( + &o.PVCForTmp, + "pvc-for-tmp", + o.PVCForTmp, + "The name of PVC to be used as /tmp dir of Velero server pod. Optional.", + ) flags.IntVar( &o.ItemBlockWorkerCount, "item-block-worker-count", @@ -321,6 +328,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { NodeAgentConfigMap: o.NodeAgentConfigMap, ItemBlockWorkerCount: o.ItemBlockWorkerCount, ConcurrentBackups: o.ConcurrentBackups, + PVCForTmp: o.PVCForTmp, KubeletRootDir: o.kubeletRootDir, NodeAgentDisableHostPath: o.NodeAgentDisableHostPath, ServerPriorityClassName: o.ServerPriorityClassName, diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index d1010d294..84f94d01d 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -60,6 +60,7 @@ type podTemplateConfig struct { nodeAgentConfigMap string itemBlockWorkerCount int concurrentBackups int + pvcForTmp string forWindows bool kubeletRootDir string nodeAgentDisableHostPath bool @@ -231,6 +232,12 @@ func WithConcurrentBackups(concurrentBackups int) podTemplateOption { } } +func WithPVCForTmp(pvcForTmp string) podTemplateOption { + return func(c *podTemplateConfig) { + c.pvcForTmp = pvcForTmp + } +} + func WithPriorityClassName(priorityClassName string) podTemplateOption { return func(c *podTemplateConfig) { c.priorityClassName = priorityClassName @@ -431,6 +438,28 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme }, } + if c.pvcForTmp != "" { + deployment.Spec.Template.Spec.Volumes = append( + deployment.Spec.Template.Spec.Volumes, + corev1api.Volume{ + Name: "tmp", + VolumeSource: corev1api.VolumeSource{ + PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ + ClaimName: c.pvcForTmp, + }, + }, + }, + ) + + deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append( + deployment.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1api.VolumeMount{ + Name: "tmp", + MountPath: "/tmp", + }, + ) + } + if c.withSecret { deployment.Spec.Template.Spec.Volumes = append( deployment.Spec.Template.Spec.Volumes, diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 5c7534774..1d550ca86 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -272,6 +272,7 @@ type VeleroOptions struct { NodeAgentConfigMap string ItemBlockWorkerCount int ConcurrentBackups int + PVCForTmp string KubeletRootDir string NodeAgentDisableHostPath bool ServerPriorityClassName string @@ -366,6 +367,10 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { WithConcurrentBackups(o.ConcurrentBackups), } + if o.PVCForTmp != "" { + deployOpts = append(deployOpts, WithPVCForTmp(o.PVCForTmp)) + } + if o.ServerPriorityClassName != "" { deployOpts = append(deployOpts, WithPriorityClassName(o.ServerPriorityClassName)) } From 845eee4e6053fefbaeb1d50f1c2d70e265096e77 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Tue, 12 Aug 2025 15:41:52 -0400 Subject: [PATCH 28/37] feat: Create backup queue controller and add to disableable list Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/controller/backup_queue_controller_test.go b/pkg/controller/backup_queue_controller_test.go index 8022d7140..f1ffb8185 100644 --- a/pkg/controller/backup_queue_controller_test.go +++ b/pkg/controller/backup_queue_controller_test.go @@ -17,22 +17,19 @@ limitations under the License. package controller import ( + "context" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - //metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" - - //"sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/builder" - velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestBackupQueueReconciler(t *testing.T) { From 7111f3cea2f79071bbb76dcd31057e69aab4c2bc Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Wed, 20 Aug 2025 16:14:43 -0400 Subject: [PATCH 29/37] feat: Remove pvc-for-tmp install arg Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- pkg/cmd/cli/install/install.go | 8 -------- pkg/install/deployment.go | 29 ----------------------------- pkg/install/resources.go | 5 ----- 3 files changed, 42 deletions(-) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 748b6953e..283bfa557 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -90,7 +90,6 @@ type Options struct { NodeAgentConfigMap string ItemBlockWorkerCount int ConcurrentBackups int - PVCForTmp string NodeAgentDisableHostPath bool kubeletRootDir string ServerPriorityClassName string @@ -192,12 +191,6 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { o.NodeAgentConfigMap, "The name of ConfigMap containing node-agent configurations.", ) - flags.StringVar( - &o.PVCForTmp, - "pvc-for-tmp", - o.PVCForTmp, - "The name of PVC to be used as /tmp dir of Velero server pod. Optional.", - ) flags.IntVar( &o.ItemBlockWorkerCount, "item-block-worker-count", @@ -328,7 +321,6 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { NodeAgentConfigMap: o.NodeAgentConfigMap, ItemBlockWorkerCount: o.ItemBlockWorkerCount, ConcurrentBackups: o.ConcurrentBackups, - PVCForTmp: o.PVCForTmp, KubeletRootDir: o.kubeletRootDir, NodeAgentDisableHostPath: o.NodeAgentDisableHostPath, ServerPriorityClassName: o.ServerPriorityClassName, diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 84f94d01d..d1010d294 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -60,7 +60,6 @@ type podTemplateConfig struct { nodeAgentConfigMap string itemBlockWorkerCount int concurrentBackups int - pvcForTmp string forWindows bool kubeletRootDir string nodeAgentDisableHostPath bool @@ -232,12 +231,6 @@ func WithConcurrentBackups(concurrentBackups int) podTemplateOption { } } -func WithPVCForTmp(pvcForTmp string) podTemplateOption { - return func(c *podTemplateConfig) { - c.pvcForTmp = pvcForTmp - } -} - func WithPriorityClassName(priorityClassName string) podTemplateOption { return func(c *podTemplateConfig) { c.priorityClassName = priorityClassName @@ -438,28 +431,6 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme }, } - if c.pvcForTmp != "" { - deployment.Spec.Template.Spec.Volumes = append( - deployment.Spec.Template.Spec.Volumes, - corev1api.Volume{ - Name: "tmp", - VolumeSource: corev1api.VolumeSource{ - PersistentVolumeClaim: &corev1api.PersistentVolumeClaimVolumeSource{ - ClaimName: c.pvcForTmp, - }, - }, - }, - ) - - deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append( - deployment.Spec.Template.Spec.Containers[0].VolumeMounts, - corev1api.VolumeMount{ - Name: "tmp", - MountPath: "/tmp", - }, - ) - } - if c.withSecret { deployment.Spec.Template.Spec.Volumes = append( deployment.Spec.Template.Spec.Volumes, diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 1d550ca86..5c7534774 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -272,7 +272,6 @@ type VeleroOptions struct { NodeAgentConfigMap string ItemBlockWorkerCount int ConcurrentBackups int - PVCForTmp string KubeletRootDir string NodeAgentDisableHostPath bool ServerPriorityClassName string @@ -367,10 +366,6 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { WithConcurrentBackups(o.ConcurrentBackups), } - if o.PVCForTmp != "" { - deployOpts = append(deployOpts, WithPVCForTmp(o.PVCForTmp)) - } - if o.ServerPriorityClassName != "" { deployOpts = append(deployOpts, WithPriorityClassName(o.ServerPriorityClassName)) } From f73443659a455505c18d5bfd15480dc4c5536b56 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Wed, 17 Sep 2025 12:58:37 -0400 Subject: [PATCH 30/37] Backup queue controller implementation Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 5a99c6328..723e324ee 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -124,7 +124,7 @@ func (r *backupQueueReconciler) SetupWithManager(mgr ctrl.Manager) error { return oldBackup.Status.Phase == velerov1api.BackupPhaseInProgress && newBackup.Status.Phase != velerov1api.BackupPhaseInProgress || oldBackup.Status.Phase != velerov1api.BackupPhaseQueued && - newBackup.Status.Phase == velerov1api.BackupPhaseQueued + newBackup.Status.Phase == velerov1api.BackupPhaseQueued }, CreateFunc: func(event.CreateEvent) bool { return false From 4d1802233a0f302098d5941f98ba191c18d9e023 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Thu, 18 Sep 2025 14:36:26 -0400 Subject: [PATCH 31/37] add various scenarios to queue controller unit tests Signed-off-by: Scott Seago --- .../backup_queue_controller_test.go | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/controller/backup_queue_controller_test.go b/pkg/controller/backup_queue_controller_test.go index f1ffb8185..73aecf8f2 100644 --- a/pkg/controller/backup_queue_controller_test.go +++ b/pkg/controller/backup_queue_controller_test.go @@ -23,13 +23,15 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + //metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client/fake" + //"sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestBackupQueueReconciler(t *testing.T) { @@ -58,7 +60,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectPhase: velerov1api.BackupPhaseInProgress, }, { - name: "Second New Backup gets queued with queuePosition 2", + name: "Second New Backup gets queued with queuePosition 2", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), }, @@ -72,7 +74,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectPhase: velerov1api.BackupPhaseReadyToStart, }, { - name: "Queued Backup remains queued if no spaces available", + name: "Queued Backup remains queued if no spaces available", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), @@ -83,7 +85,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if no spaces available including ReadyToStart", + name: "Queued Backup remains queued if no spaces available including ReadyToStart", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseReadyToStart).Result(), @@ -94,7 +96,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if earlier runnable backup is also queued", + name: "Queued Backup remains queued if earlier runnable backup is also queued", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), @@ -105,7 +107,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 2, }, { - name: "Queued Backup remains queued if in conflict with running backup", + name: "Queued Backup remains queued if in conflict with running backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("foo").Result(), }, @@ -116,7 +118,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if in conflict with ReadyToStart backup", + name: "Queued Backup remains queued if in conflict with ReadyToStart backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseReadyToStart).IncludedNamespaces("foo").Result(), }, @@ -127,7 +129,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if in conflict with earlier queued backup", + name: "Queued Backup remains queued if in conflict with earlier queued backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), }, @@ -138,7 +140,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 2, }, { - name: "Queued Backup remains queued if earlier non-ns-conflict backup exists", + name: "Queued Backup remains queued if earlier non-ns-conflict backup exists", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), @@ -150,7 +152,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 2, }, { - name: "Running all-namespace backup conflicts with queued one-namespace backup ", + name: "Running all-namespace backup conflicts with queued one-namespace backup ", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("*").Result(), }, @@ -161,7 +163,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Running one-namespace backup conflicts with queued all-namespace backup ", + name: "Running one-namespace backup conflicts with queued all-namespace backup ", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), }, @@ -172,7 +174,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups", + name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), From 8a95d512b331508a7ec68a7163564143a509b30c Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Fri, 3 Oct 2025 14:04:38 -0400 Subject: [PATCH 32/37] make update, changelog Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 2 +- .../backup_queue_controller_test.go | 24 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 723e324ee..5a99c6328 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -124,7 +124,7 @@ func (r *backupQueueReconciler) SetupWithManager(mgr ctrl.Manager) error { return oldBackup.Status.Phase == velerov1api.BackupPhaseInProgress && newBackup.Status.Phase != velerov1api.BackupPhaseInProgress || oldBackup.Status.Phase != velerov1api.BackupPhaseQueued && - newBackup.Status.Phase == velerov1api.BackupPhaseQueued + newBackup.Status.Phase == velerov1api.BackupPhaseQueued }, CreateFunc: func(event.CreateEvent) bool { return false diff --git a/pkg/controller/backup_queue_controller_test.go b/pkg/controller/backup_queue_controller_test.go index 73aecf8f2..ed85282d6 100644 --- a/pkg/controller/backup_queue_controller_test.go +++ b/pkg/controller/backup_queue_controller_test.go @@ -23,10 +23,12 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + //metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + //"sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -60,7 +62,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectPhase: velerov1api.BackupPhaseInProgress, }, { - name: "Second New Backup gets queued with queuePosition 2", + name: "Second New Backup gets queued with queuePosition 2", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), }, @@ -74,7 +76,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectPhase: velerov1api.BackupPhaseReadyToStart, }, { - name: "Queued Backup remains queued if no spaces available", + name: "Queued Backup remains queued if no spaces available", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), @@ -85,7 +87,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if no spaces available including ReadyToStart", + name: "Queued Backup remains queued if no spaces available including ReadyToStart", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseReadyToStart).Result(), @@ -96,7 +98,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if earlier runnable backup is also queued", + name: "Queued Backup remains queued if earlier runnable backup is also queued", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).Result(), @@ -107,7 +109,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 2, }, { - name: "Queued Backup remains queued if in conflict with running backup", + name: "Queued Backup remains queued if in conflict with running backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("foo").Result(), }, @@ -118,7 +120,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if in conflict with ReadyToStart backup", + name: "Queued Backup remains queued if in conflict with ReadyToStart backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseReadyToStart).IncludedNamespaces("foo").Result(), }, @@ -129,7 +131,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup remains queued if in conflict with earlier queued backup", + name: "Queued Backup remains queued if in conflict with earlier queued backup", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), }, @@ -140,7 +142,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 2, }, { - name: "Queued Backup remains queued if earlier non-ns-conflict backup exists", + name: "Queued Backup remains queued if earlier non-ns-conflict backup exists", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseQueued).QueuePosition(1).IncludedNamespaces("foo").Result(), @@ -152,7 +154,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 2, }, { - name: "Running all-namespace backup conflicts with queued one-namespace backup ", + name: "Running all-namespace backup conflicts with queued one-namespace backup ", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("*").Result(), }, @@ -163,7 +165,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Running one-namespace backup conflicts with queued all-namespace backup ", + name: "Running one-namespace backup conflicts with queued all-namespace backup ", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).IncludedNamespaces("bar").Result(), }, @@ -174,7 +176,7 @@ func TestBackupQueueReconciler(t *testing.T) { expectQueuePosition: 1, }, { - name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups", + name: "Queued Backup moves to ReadyToStart if running count < concurrentBackups", priorBackups: []*velerov1api.Backup{ builder.ForBackup(velerov1api.DefaultNamespace, "backup-11").Phase(velerov1api.BackupPhaseInProgress).Result(), builder.ForBackup(velerov1api.DefaultNamespace, "backup-12").Phase(velerov1api.BackupPhaseInProgress).Result(), From 73864e31ff9d8c9f0d6069d0bd2bdc1fe5615da0 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Fri, 3 Oct 2025 14:31:04 -0400 Subject: [PATCH 33/37] Fix linters Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/controller/backup_queue_controller_test.go b/pkg/controller/backup_queue_controller_test.go index ed85282d6..8022d7140 100644 --- a/pkg/controller/backup_queue_controller_test.go +++ b/pkg/controller/backup_queue_controller_test.go @@ -17,7 +17,6 @@ limitations under the License. package controller import ( - "context" "testing" "github.com/sirupsen/logrus" From 0b2e7d12386ba9b2ecdbd410e9216ade9b0d1169 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Thu, 13 Nov 2025 14:35:21 -0500 Subject: [PATCH 34/37] Minor refactoring Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 5a99c6328..87da3cb12 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -25,6 +25,7 @@ import ( "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" From f238a7e47b46b5948671f293c9c07059d5507b27 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Thu, 13 Nov 2025 14:57:26 -0500 Subject: [PATCH 35/37] make update Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 87da3cb12..5a99c6328 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -25,7 +25,6 @@ import ( "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" From 7e4797f5883dbfc6c54c6896557bfbed627df7d1 Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Mon, 1 Dec 2025 17:13:28 -0500 Subject: [PATCH 36/37] Track running backup count via BackupTracker This avoids an unnecessary apiserver List call when the backup reconciler is already at capacity. Signed-off-by: Scott Seago --- pkg/cmd/server/server.go | 1 + pkg/controller/backup_controller.go | 2 + pkg/controller/backup_queue_controller.go | 18 +++--- .../backup_queue_controller_test.go | 8 ++- pkg/controller/backup_tracker.go | 61 ++++++++++++++++--- 5 files changed, 71 insertions(+), 19 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index fbff1086d..68330670d 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -917,6 +917,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.mgr.GetScheme(), s.logger, s.config.ConcurrentBackups, + backupTracker, ).SetupWithManager(s.mgr); err != nil { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerBackupQueue) } diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index f269353cb..37faafd68 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -323,6 +323,8 @@ func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr switch request.Status.Phase { case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed, velerov1api.BackupPhaseFailedValidation: b.backupTracker.Delete(request.Namespace, request.Name) + case velerov1api.BackupPhaseWaitingForPluginOperations, velerov1api.BackupPhaseWaitingForPluginOperationsPartiallyFailed, velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed: + b.backupTracker.AddPostProcessing(request.Namespace, request.Name) } }() diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 5a99c6328..556250b3e 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -47,6 +47,7 @@ type backupQueueReconciler struct { Scheme *runtime.Scheme logger logrus.FieldLogger concurrentBackups int + backupTracker BackupTracker frequency time.Duration } @@ -60,12 +61,14 @@ func NewBackupQueueReconciler( scheme *runtime.Scheme, logger logrus.FieldLogger, concurrentBackups int, + backupTracker BackupTracker, ) *backupQueueReconciler { return &backupQueueReconciler{ Client: client, Scheme: scheme, logger: logger, concurrentBackups: max(concurrentBackups, 1), + backupTracker: backupTracker, frequency: defaultQueuedBackupRecheckFrequency, } } @@ -261,11 +264,11 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } lister := r.newQueuedBackupsLister(allBackups) - earlierBackups, runningCount := lister.earlierThan(backup.Status.QueuePosition) - if runningCount >= r.concurrentBackups { + if r.backupTracker.RunningCount() >= r.concurrentBackups { log.Debugf("%v concurrent backups are already running, leaving %v queued", r.concurrentBackups, backup.Name) return ctrl.Result{}, nil } + earlierBackups := lister.earlierThan(backup.Status.QueuePosition) foundConflict, conflictBackup, clusterNamespaces, err := r.detectNamespaceConflict(ctx, backup, earlierBackups) if err != nil { log.WithError(err).Error("error listing namespaces") @@ -287,6 +290,7 @@ func (r *backupQueueReconciler) Reconcile(ctx context.Context, req ctrl.Request) if err := kube.PatchResource(original, backup, r.Client); err != nil { return ctrl.Result{}, errors.Wrapf(err, "error updating Backup status to %s", backup.Status.Phase) } + r.backupTracker.AddReadyToStart(backup.Namespace, backup.Name) log.Debug("Updating queuePosition for remaining queued backups") queuedBackups := lister.orderedQueued() newQueuePos := 1 @@ -329,21 +333,15 @@ func (r *backupQueueReconciler) newQueuedBackupsLister(backupList *velerov1api.B return &queuedBackupsLister{backupList} } -func (l *queuedBackupsLister) earlierThan(queuePos int) ([]velerov1api.Backup, int) { +func (l *queuedBackupsLister) earlierThan(queuePos int) []velerov1api.Backup { backups := []velerov1api.Backup{} - runningCount := 0 for _, backup := range l.backups.Items { // InProgress and ReadyToStart backups have QueuePosition==0 if backup.Status.QueuePosition < queuePos { backups = append(backups, backup) } - // InProgress and ReadyToStart backups count towards the concurrentBackups limit - if backup.Status.Phase == velerov1api.BackupPhaseInProgress || - backup.Status.Phase == velerov1api.BackupPhaseReadyToStart { - runningCount++ - } } - return backups, runningCount + return backups } func (l *queuedBackupsLister) orderedQueued() []velerov1api.Backup { diff --git a/pkg/controller/backup_queue_controller_test.go b/pkg/controller/backup_queue_controller_test.go index 8022d7140..2b479ed7e 100644 --- a/pkg/controller/backup_queue_controller_test.go +++ b/pkg/controller/backup_queue_controller_test.go @@ -202,9 +202,15 @@ func TestBackupQueueReconciler(t *testing.T) { return } + backupTracker := NewBackupTracker() initObjs := []runtime.Object{} for _, priorBackup := range test.priorBackups { initObjs = append(initObjs, priorBackup) + if priorBackup.Status.Phase == velerov1api.BackupPhaseReadyToStart { + backupTracker.AddReadyToStart(priorBackup.Namespace, priorBackup.Name) + } else if priorBackup.Status.Phase == velerov1api.BackupPhaseInProgress { + backupTracker.Add(priorBackup.Namespace, priorBackup.Name) + } } for _, ns := range test.namespaces { initObjs = append(initObjs, builder.ForNamespace(ns).Result()) @@ -214,7 +220,7 @@ func TestBackupQueueReconciler(t *testing.T) { fakeClient := velerotest.NewFakeControllerRuntimeClient(t, initObjs...) logger := logrus.New() log := logger.WithField("controller", "backup-queue-test") - r := NewBackupQueueReconciler(fakeClient, scheme, log, test.concurrentBackups) + r := NewBackupQueueReconciler(fakeClient, scheme, log, test.concurrentBackups, backupTracker) req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}} res, err := r.Reconcile(t.Context(), req) gotErr := err != nil diff --git a/pkg/controller/backup_tracker.go b/pkg/controller/backup_tracker.go index 262886d56..a330b17b0 100644 --- a/pkg/controller/backup_tracker.go +++ b/pkg/controller/backup_tracker.go @@ -25,45 +25,90 @@ import ( // BackupTracker keeps track of in-progress backups. type BackupTracker interface { + // Add informs the tracker that a backup is ReadyToStart. + AddReadyToStart(ns, name string) // Add informs the tracker that a backup is in progress. Add(ns, name string) - // Delete informs the tracker that a backup is no longer in progress. + // Add informs the tracker that a backup has moved beyond InProgress + AddPostProcessing(ns, name string) + // Delete informs the tracker that a backup has reached a terminal state. Delete(ns, name string) - // Contains returns true if the tracker is tracking the backup. + // Contains returns true if backup is InProgress or post-InProgress Contains(ns, name string) bool + // RunningCount returns the number of backups which are ReadyToStart or InProgress + RunningCount() int } type backupTracker struct { - lock sync.RWMutex - backups sets.Set[string] + lock sync.RWMutex + readyToStartBackups sets.Set[string] + inProgressBackups sets.Set[string] + postProgressBackups sets.Set[string] } // NewBackupTracker returns a new BackupTracker. func NewBackupTracker() BackupTracker { return &backupTracker{ - backups: sets.New[string](), + readyToStartBackups: sets.New[string](), + inProgressBackups: sets.New[string](), + postProgressBackups: sets.New[string](), } } +func (bt *backupTracker) AddReadyToStart(ns, name string) { + bt.lock.Lock() + defer bt.lock.Unlock() + + bt.readyToStartBackups.Insert(backupTrackerKey(ns, name)) +} + func (bt *backupTracker) Add(ns, name string) { bt.lock.Lock() defer bt.lock.Unlock() - bt.backups.Insert(backupTrackerKey(ns, name)) + key := backupTrackerKey(ns, name) + bt.readyToStartBackups.Delete(key) + bt.inProgressBackups.Insert(key) +} + +func (bt *backupTracker) AddPostProcessing(ns, name string) { + bt.lock.Lock() + defer bt.lock.Unlock() + + key := backupTrackerKey(ns, name) + bt.readyToStartBackups.Delete(key) + bt.inProgressBackups.Delete(key) + bt.postProgressBackups.Insert(key) } func (bt *backupTracker) Delete(ns, name string) { bt.lock.Lock() defer bt.lock.Unlock() - bt.backups.Delete(backupTrackerKey(ns, name)) + key := backupTrackerKey(ns, name) + bt.readyToStartBackups.Delete(key) + bt.inProgressBackups.Delete(key) + bt.postProgressBackups.Delete(key) } +// Contains returns true if backup is InProgress or post-InProgress +// ignores ReadyToStart, since this is used to determine whether +// a backup is in progress and thus not able to be deleted now. func (bt *backupTracker) Contains(ns, name string) bool { bt.lock.RLock() defer bt.lock.RUnlock() - return bt.backups.Has(backupTrackerKey(ns, name)) + key := backupTrackerKey(ns, name) + return bt.inProgressBackups.Has(key) || bt.postProgressBackups.Has(key) +} + +// RunningCount returns the number of backups which are ReadyToStart or InProgress +// used by queue controller to determine whether a new backup can be started. +func (bt *backupTracker) RunningCount() int { + bt.lock.RLock() + defer bt.lock.RUnlock() + + return bt.inProgressBackups.Len() + bt.readyToStartBackups.Len() } func backupTrackerKey(ns, name string) string { From 7286d24c352ec4598269918c61270763c9c5665f Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Wed, 3 Dec 2025 16:55:59 -0500 Subject: [PATCH 37/37] Updates for merge conflict and to refine reconciler queueing logic Signed-off-by: Scott Seago --- pkg/controller/backup_queue_controller.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/controller/backup_queue_controller.go b/pkg/controller/backup_queue_controller.go index 556250b3e..c5cdc9461 100644 --- a/pkg/controller/backup_queue_controller.go +++ b/pkg/controller/backup_queue_controller.go @@ -127,7 +127,8 @@ func (r *backupQueueReconciler) SetupWithManager(mgr ctrl.Manager) error { return oldBackup.Status.Phase == velerov1api.BackupPhaseInProgress && newBackup.Status.Phase != velerov1api.BackupPhaseInProgress || oldBackup.Status.Phase != velerov1api.BackupPhaseQueued && - newBackup.Status.Phase == velerov1api.BackupPhaseQueued + newBackup.Status.Phase == velerov1api.BackupPhaseQueued && + r.backupTracker.RunningCount() < r.concurrentBackups }, CreateFunc: func(event.CreateEvent) bool { return false @@ -197,7 +198,13 @@ func (r *backupQueueReconciler) checkForEarlierRunnableBackups(backup *velerov1a } func namespacesForBackup(backup *velerov1api.Backup, clusterNamespaces []string) []string { - return collections.NewNamespaceIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...).ActiveNamespaces(clusterNamespaces).ResolveNamespaceList() + // Ignore error here. If a backup has invalid namespace wildcards, the backup controller + // will validate and fail it. Consider the ns list empty for conflict detection purposes. + nsList, err := collections.NewNamespaceIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...).ActiveNamespaces(clusterNamespaces).ResolveNamespaceList() + if err != nil { + return []string{} + } + return nsList } func (r *backupQueueReconciler) getMaxQueuePosition(lister *queuedBackupsLister) int { queuedBackups := lister.orderedQueued()