From 0416b93b07ba3d902eb12f95d6ba21ae86e23cc3 Mon Sep 17 00:00:00 2001 From: Zhiqiang Zhang Date: Sat, 1 Jul 2023 21:50:20 +0800 Subject: [PATCH 1/8] fix doc typo Signed-off-by: zhangzhiqiang02 --- site/content/docs/main/file-system-backup.md | 2 +- site/content/docs/v1.10/file-system-backup.md | 2 +- site/content/docs/v1.11/file-system-backup.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/site/content/docs/main/file-system-backup.md b/site/content/docs/main/file-system-backup.md index 89c43ce7b..e6a1d0e8f 100644 --- a/site/content/docs/main/file-system-backup.md +++ b/site/content/docs/main/file-system-backup.md @@ -74,7 +74,7 @@ Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. **RancherOS** -Update the host path for volumes in the nonde-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to +Update the host path for volumes in the node-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to `/opt/rke/var/lib/kubelet/pods`. ```yaml diff --git a/site/content/docs/v1.10/file-system-backup.md b/site/content/docs/v1.10/file-system-backup.md index 6fcd1a33b..f8badb109 100644 --- a/site/content/docs/v1.10/file-system-backup.md +++ b/site/content/docs/v1.10/file-system-backup.md @@ -74,7 +74,7 @@ Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. **RancherOS** -Update the host path for volumes in the nonde-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to +Update the host path for volumes in the node-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to `/opt/rke/var/lib/kubelet/pods`. ```yaml diff --git a/site/content/docs/v1.11/file-system-backup.md b/site/content/docs/v1.11/file-system-backup.md index 881747895..dc51cab84 100644 --- a/site/content/docs/v1.11/file-system-backup.md +++ b/site/content/docs/v1.11/file-system-backup.md @@ -74,7 +74,7 @@ Integrated Edition (formerly VMware Enterprise PKS), or Microsoft Azure. **RancherOS** -Update the host path for volumes in the nonde-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to +Update the host path for volumes in the node-agent DaemonSet in the Velero namespace from `/var/lib/kubelet/pods` to `/opt/rke/var/lib/kubelet/pods`. ```yaml From d7f1ea4fbd5ccdbf4327f855e461bca9caa89507 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Fri, 30 Jun 2023 17:53:27 +0800 Subject: [PATCH 2/8] Add exit code log and possible memory shortage warning log for Restic command failure. Signed-off-by: Xun Jiang --- changelogs/unreleased/6459-blackpiglet | 1 + pkg/repository/restic/repository.go | 2 +- pkg/restic/exec_commands.go | 4 +++- pkg/util/exec/exec.go | 23 +++++++++++++++++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 changelogs/unreleased/6459-blackpiglet diff --git a/changelogs/unreleased/6459-blackpiglet b/changelogs/unreleased/6459-blackpiglet new file mode 100644 index 000000000..26e0e3856 --- /dev/null +++ b/changelogs/unreleased/6459-blackpiglet @@ -0,0 +1 @@ +Add exit code log and possible memory shortage warning log for Restic command failure. \ No newline at end of file diff --git a/pkg/repository/restic/repository.go b/pkg/repository/restic/repository.go index 392caf284..3c15e0b37 100644 --- a/pkg/repository/restic/repository.go +++ b/pkg/repository/restic/repository.go @@ -112,7 +112,7 @@ func (r *RepositoryService) exec(cmd *restic.Command, bsl *velerov1api.BackupSto cmd.ExtraFlags = append(cmd.ExtraFlags, skipTLSRet) } - stdout, stderr, err := veleroexec.RunCommand(cmd.Cmd()) + stdout, stderr, err := veleroexec.RunCommandWithLog(cmd.Cmd(), r.log) r.log.WithFields(logrus.Fields{ "repository": cmd.RepoName(), "command": cmd.String(), diff --git a/pkg/restic/exec_commands.go b/pkg/restic/exec_commands.go index 0cbc42802..94c17c04a 100644 --- a/pkg/restic/exec_commands.go +++ b/pkg/restic/exec_commands.go @@ -86,6 +86,7 @@ func RunBackup(backupCmd *Command, log logrus.FieldLogger, updater uploader.Prog err := cmd.Start() if err != nil { + exec.LogErrorAsExitCode(err, log) return stdoutBuf.String(), stderrBuf.String(), err } @@ -119,6 +120,7 @@ func RunBackup(backupCmd *Command, log logrus.FieldLogger, updater uploader.Prog err = cmd.Wait() if err != nil { + exec.LogErrorAsExitCode(err, log) return stdoutBuf.String(), stderrBuf.String(), err } quit <- struct{}{} @@ -229,7 +231,7 @@ func RunRestore(restoreCmd *Command, log logrus.FieldLogger, updater uploader.Pr } }() - stdout, stderr, err := exec.RunCommand(restoreCmd.Cmd()) + stdout, stderr, err := exec.RunCommandWithLog(restoreCmd.Cmd(), log) quit <- struct{}{} // update progress to 100% diff --git a/pkg/util/exec/exec.go b/pkg/util/exec/exec.go index 84bffb257..109118d58 100644 --- a/pkg/util/exec/exec.go +++ b/pkg/util/exec/exec.go @@ -22,6 +22,7 @@ import ( "os/exec" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // RunCommand runs a command and returns its stdout, stderr, and its returned @@ -52,3 +53,25 @@ func RunCommand(cmd *exec.Cmd) (string, string, error) { return stdout, stderr, runErr } + +func RunCommandWithLog(cmd *exec.Cmd, log logrus.FieldLogger) (string, string, error) { + stdout, stderr, err := RunCommand(cmd) + LogErrorAsExitCode(err, log) + return stdout, stderr, err +} + +func LogErrorAsExitCode(err error, log logrus.FieldLogger) { + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + log.Errorf("Restic command fail with ExitCode: %d. Process ID is %d, Exit error is: %s", exitError.ExitCode(), exitError.Pid(), exitError.String()) + // Golang's os.exec -1 ExitCode means signal kill. Usually this is caused + // by CGroup's OOM. Log a warning to notice user. + // https://github.com/golang/go/blob/master/src/os/exec_posix.go#L128-L136 + if exitError.ExitCode() == -1 { + log.Warnf("The ExitCode is -1, which means the process is terminated by signal. Usually this is caused by CGroup kill due to out of memory. Please check whether there is such information in the work nodes' dmesg log.") + } + } else { + log.WithError(err).Info("Error cannot be convert to ExitError format.") + } + } +} From e71ee0cc5f1d9ed1b8354a91914101b1b74a85e7 Mon Sep 17 00:00:00 2001 From: kayrus Date: Wed, 5 Jul 2023 17:23:16 +0200 Subject: [PATCH 3/8] Add support for OpenStack CSI drivers topology keys Signed-off-by: kayrus --- changelogs/unreleased/6464-openstack-csi-topology-keys | 1 + pkg/backup/item_backupper.go | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelogs/unreleased/6464-openstack-csi-topology-keys diff --git a/changelogs/unreleased/6464-openstack-csi-topology-keys b/changelogs/unreleased/6464-openstack-csi-topology-keys new file mode 100644 index 000000000..d4c3c976c --- /dev/null +++ b/changelogs/unreleased/6464-openstack-csi-topology-keys @@ -0,0 +1 @@ +Add support for OpenStack CSI drivers topology keys diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index a9a3ae440..d6cd5ba6b 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -445,6 +445,10 @@ const ( azureCsiZoneKey = "topology.disk.csi.azure.com/zone" gkeCsiZoneKey = "topology.gke.io/zone" gkeZoneSeparator = "__" + + // OpenStack CSI drivers topology keys + cinderCsiZoneKey = "topology.manila.csi.openstack.org/zone" + manilaCsiZoneKey = "topology.cinder.csi.openstack.org/zone" ) // takePVSnapshot triggers a snapshot for the volume/disk underlying a PersistentVolume if the provided @@ -506,7 +510,7 @@ func (ib *itemBackupper) takePVSnapshot(obj runtime.Unstructured, log logrus.Fie if !labelFound { var k string log.Infof("label %q is not present on PersistentVolume", zoneLabelDeprecated) - k, pvFailureDomainZone = zoneFromPVNodeAffinity(pv, awsEbsCsiZoneKey, azureCsiZoneKey, gkeCsiZoneKey, zoneLabel, zoneLabelDeprecated) + k, pvFailureDomainZone = zoneFromPVNodeAffinity(pv, awsEbsCsiZoneKey, azureCsiZoneKey, gkeCsiZoneKey, cinderCsiZoneKey, manilaCsiZoneKey, zoneLabel, zoneLabelDeprecated) if pvFailureDomainZone != "" { log.Infof("zone info from nodeAffinity requirements: %s, key: %s", pvFailureDomainZone, k) } else { From ff83d5e0c99bb299e718655aa87f50b2cde1bbca Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Wed, 5 Jul 2023 12:36:07 -0400 Subject: [PATCH 4/8] typo: s/inokes/invokes Signed-off-by: Tiger Kaovilai --- site/content/docs/main/file-system-backup.md | 4 ++-- site/content/docs/v1.10/file-system-backup.md | 4 ++-- site/content/docs/v1.11/file-system-backup.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/site/content/docs/main/file-system-backup.md b/site/content/docs/main/file-system-backup.md index 89c43ce7b..be32119a4 100644 --- a/site/content/docs/main/file-system-backup.md +++ b/site/content/docs/main/file-system-backup.md @@ -539,7 +539,7 @@ that it's backing up for the volumes to be backed up using FSB. 5. Meanwhile, each `PodVolumeBackup` is handled by the controller on the appropriate node, which: - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data - finds the pod volume's subdirectory within the above volume - - based on the path selection, Velero inokes restic or kopia for backup + - based on the path selection, Velero invokes restic or kopia for backup - updates the status of the custom resource to `Completed` or `Failed` 6. As each `PodVolumeBackup` finishes, the main Velero process adds it to the Velero backup in a file named `-podvolumebackups.json.gz`. This file gets uploaded to object storage alongside the backup tarball. @@ -564,7 +564,7 @@ some reason (i.e. lack of cluster resources), the FSB restore will not be done. - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data - waits for the pod to be running the init container - finds the pod volume's subdirectory within the above volume - - based on the path selection, Velero inokes restic or kopia for restore + - based on the path selection, Velero invokes restic or kopia for restore - on success, writes a file into the pod volume, in a `.velero` subdirectory, whose name is the UID of the Velero restore that this pod volume restore is for - updates the status of the custom resource to `Completed` or `Failed` diff --git a/site/content/docs/v1.10/file-system-backup.md b/site/content/docs/v1.10/file-system-backup.md index 6fcd1a33b..96e861cf3 100644 --- a/site/content/docs/v1.10/file-system-backup.md +++ b/site/content/docs/v1.10/file-system-backup.md @@ -539,7 +539,7 @@ that it's backing up for the volumes to be backed up using FSB. 5. Meanwhile, each `PodVolumeBackup` is handled by the controller on the appropriate node, which: - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data - finds the pod volume's subdirectory within the above volume - - based on the path selection, Velero inokes restic or kopia for backup + - based on the path selection, Velero invokes restic or kopia for backup - updates the status of the custom resource to `Completed` or `Failed` 6. As each `PodVolumeBackup` finishes, the main Velero process adds it to the Velero backup in a file named `-podvolumebackups.json.gz`. This file gets uploaded to object storage alongside the backup tarball. @@ -564,7 +564,7 @@ some reason (i.e. lack of cluster resources), the FSB restore will not be done. - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data - waits for the pod to be running the init container - finds the pod volume's subdirectory within the above volume - - based on the path selection, Velero inokes restic or kopia for restore + - based on the path selection, Velero invokes restic or kopia for restore - on success, writes a file into the pod volume, in a `.velero` subdirectory, whose name is the UID of the Velero restore that this pod volume restore is for - updates the status of the custom resource to `Completed` or `Failed` diff --git a/site/content/docs/v1.11/file-system-backup.md b/site/content/docs/v1.11/file-system-backup.md index 881747895..5022adbcd 100644 --- a/site/content/docs/v1.11/file-system-backup.md +++ b/site/content/docs/v1.11/file-system-backup.md @@ -539,7 +539,7 @@ that it's backing up for the volumes to be backed up using FSB. 5. Meanwhile, each `PodVolumeBackup` is handled by the controller on the appropriate node, which: - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data - finds the pod volume's subdirectory within the above volume - - based on the path selection, Velero inokes restic or kopia for backup + - based on the path selection, Velero invokes restic or kopia for backup - updates the status of the custom resource to `Completed` or `Failed` 6. As each `PodVolumeBackup` finishes, the main Velero process adds it to the Velero backup in a file named `-podvolumebackups.json.gz`. This file gets uploaded to object storage alongside the backup tarball. @@ -564,7 +564,7 @@ some reason (i.e. lack of cluster resources), the FSB restore will not be done. - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data - waits for the pod to be running the init container - finds the pod volume's subdirectory within the above volume - - based on the path selection, Velero inokes restic or kopia for restore + - based on the path selection, Velero invokes restic or kopia for restore - on success, writes a file into the pod volume, in a `.velero` subdirectory, whose name is the UID of the Velero restore that this pod volume restore is for - updates the status of the custom resource to `Completed` or `Failed` From 8a7aa2051ca3f8caa00e3ac39f3f3e73ca8de9de Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Thu, 6 Jul 2023 12:10:57 +0800 Subject: [PATCH 5/8] add node name data mover CR Signed-off-by: Lyndon-Li --- config/crd/v2alpha1/bases/velero.io_datadownloads.yaml | 7 +++++++ config/crd/v2alpha1/bases/velero.io_datauploads.yaml | 7 +++++++ config/crd/v2alpha1/crds/crds.go | 4 ++-- pkg/apis/velero/v2alpha1/data_download_types.go | 5 +++++ pkg/apis/velero/v2alpha1/data_upload_types.go | 5 +++++ pkg/controller/data_download_controller.go | 5 +++-- pkg/controller/data_upload_controller.go | 5 +++-- 7 files changed, 32 insertions(+), 6 deletions(-) diff --git a/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml index 20d604153..8389028f7 100644 --- a/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml +++ b/config/crd/v2alpha1/bases/velero.io_datadownloads.yaml @@ -41,6 +41,10 @@ spec: jsonPath: .metadata.creationTimestamp name: Age type: date + - description: Name of the node where the DataDownload is processed + jsonPath: .status.node + name: Node + type: string name: v2alpha1 schema: openAPIV3Schema: @@ -132,6 +136,9 @@ spec: message: description: Message is a message about the DataDownload's status. type: string + node: + description: Node is name of the node where the DataDownload is processed. + type: string phase: description: Phase is the current state of the DataDownload. enum: diff --git a/config/crd/v2alpha1/bases/velero.io_datauploads.yaml b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml index ea354f1d4..ffc279958 100644 --- a/config/crd/v2alpha1/bases/velero.io_datauploads.yaml +++ b/config/crd/v2alpha1/bases/velero.io_datauploads.yaml @@ -42,6 +42,10 @@ spec: jsonPath: .metadata.creationTimestamp name: Age type: date + - description: Name of the node where the DataUpload is processed + jsonPath: .status.node + name: Node + type: string name: v2alpha1 schema: openAPIV3Schema: @@ -147,6 +151,9 @@ spec: message: description: Message is a message about the DataUpload's status. type: string + node: + description: Node is name of the node where the DataUpload is processed. + type: string path: description: Path is the full path of the snapshot volume being backed up. diff --git a/config/crd/v2alpha1/crds/crds.go b/config/crd/v2alpha1/crds/crds.go index 00f9ec3fd..65981a8b6 100644 --- a/config/crd/v2alpha1/crds/crds.go +++ b/config/crd/v2alpha1/crds/crds.go @@ -29,8 +29,8 @@ import ( ) var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcX\xcdr\xe4\xb8\r\xbe\xfb)P\x93\x83/#yg\x93J\xa5\xfa6\xd3N\xaa\\ٙt\xad\xa7\xfaNI\x90\x9a;\x14\xc9\xf0\xa7\x1d'\x95wO\x81?jI\xcdvۻ\x99\xd5M$\b|\x04\xc0\x0f \xab\xaa\xbaa\x9a\xef\xd1X\xae\xe4\x06\x98\xe6\xf8/\x87\x92\xfel\xfd\xed/\xb6\xe6\xea\xee\xf8\xe1\xe6\x1b\x97\xdd\x06\xb6\xde:5\xfe\x8cVy\xd3\xe2=\xf6\\rǕ\xbc\x19ѱ\x8e9\xb6\xb9\x01`R*\xc7h\xd8\xd2/@\xab\xa43J\b4Հ\xb2\xfe\xe6\x1bl<\x17\x1d\x9a\xa0<\x9b>\xfeP\x7f\xf8\xb1\xfe\xe1\x06@\xb2\x117@\xfa:\xf5$\x85b\x9d\xad\x8f(Ш\x9a\xab\x1b\xab\xb1%ŃQ^o\xe04\x11\x17&\xa3\x11\xf0=s\xec>\xe9\bÂ[\xf7\xf7\xb3\xa9\x9f\xb8uaZ\vo\x98X\xd9\x0e3\x96\xcb\xc1\vf\x96s7\x00\xb6U\x1a7\xf0\x85Lk\xd6\"\x8d\xa5=\x05(\x15\xb0\xae\v^bbg\xb8th\xb6J\xf81{\xa7\x82\x0emk\xb8v\xc1\vsX`\x1dsނ\xf5\xed\x01\x98\x85/\xf8t\xf7 wF\r\x06m\x84\x05\xf0\x8bUr\xc7\xdca\x03u\x14\xaf\xf5\x81YL\xb3ѕ\x8fa\"\r\xb9g\xc2k\x9d\xe1r(!\xf8\xcaG\x84Λ\x10B\xdaw\x8b\xe0\x0e\xdc.\xa1=1K\xf0\x8c\xc3\xee\"\x900O\xea\xacc\xa3^#\x9a-\x8d\x90:\xe6\xb0\x04h\xabF-\xd0a\aͳü\x8d^\x99\x91\xb9\rp\xe9\xfe\xfc\xa7˾HΪ\xc3\xd2{%\x97\x8e\xf9D\xa30\x1b\x8eH(J\x03\x9a\xa2w\x94c\xe2\xb7\x00q\xa4\xe0\xd3l}D\x12\xf5\xceǯB\xa1\x94\x03Ճ; |b\xed7\xaf\xe1\xd1)\xc3\x06\x84\x9fT\x1b\xc3\xf7t@\x83A\xa2\x89\x12\x94\xbd\xc0)v\xca\x14C\xa7\xb1\xad\xa3lR\x96u\xad\xe2\xb74\xf4\x7fϭ\xd6 +\xe6V\xa6\x9a:Hp%\xcb\t\xf6q\xc0rr\xc5\xe9\xe3\x8fL\xe8\x03\xfb\x10\xcfv{\xc0\x91m\x92\xbc\xd2(?\xee\x1e\xf6\x7f|\\\f\x03h\xa34\x1a\xc73\xc5\xc4oƞ\xb3QX\xee\xfb\x96\x14F)\xe8\x886ц\xa0$\xa2\xc0.a\x88\xe1\xe4\x16\fj\x83\x16\xa5\x9b{7\x7f\xaa\a&A5\xbf`\xebjxDCj\xc0\x1e\x94\x17\x1d\xb1\xed\x11\x8d\x03\x83\xad\x1a$\xff\xf7\xa4ۂS\xc1\xa8`\x0e\x13ߝ\xbe@L\x92\t82\xe1\xf1=0\xd9\xc1Ȟ\xc1 Y\x01/g\xfa\x82\x88\xad\xe1\xb32\b\\\xf6j\x03\a\xe7\xb4\xdd\xdc\xdd\r\xdc\xe5\xaaѪq\xf4\x92\xbb\xe7\xbbP\x00x\xe3\x9d2\xf6\xae\xc3#\x8a;ˇ\x8a\x99\xf6\xc0\x1d\xb6\xce\x1b\xbcc\x9aW\x01\xba\f\x95\xa3\x1e\xbb?\x98Tg\xec\xed\x02\xebY\x8e\xc5/\x10\xfe\v\x11 ֧\xc4gii\xdc\xc5\xc9\xd14D\xde\xf9\xf9\xaf\x8f_!\x9b\x0e\xc1X{?\xf8\xfd\xb4ОB@\x0e\xe3\xb2G\x13\x83\xd8\x1b5\x06\x9d(;\xad\xb8t\xe1\xa7\x15\x1c\xe5\xda\xfd\xd67#w\x14\xf7\x7fz\xb4\x8ebU\xc36\x94Rh\x10\xbc\xa6\xfc\xedjx\x90\xb0e#\x8a-\xb3\xf8\xdd\x03@\x9e\xb6\x159\xf6u!\x98w\x01k\xe1\xe8\xb5\xd9D.\xe3\x17\xe25'\x84G\x8d-\x85\x8e\xbcG\xcbx\xcf\x13\xbd\xf5\xca\x00[\xc8\xd6\v\x95\xe5#K_\x91\xe2\xd6B+L\x9fJk209c\xe3ĵ6J\x9e)\x05\x10\x17\xf9٠V\x96;e\x9eO,]\x9fi\xb8\x10\x00\xfaZ&[\x14Wv\xb2\rB\xc0eG\x9e\xc4)\xef\x88\"\xa2\x82\x80I\xc9Aѹ\xb8\xec\xe0\xf8=8ZE\x89j\xd1ўdX\xbe un\x81K8\xb5/0oS\xd6;k\x94\x12\xc8ּG\xb9\xf5Y\x1d\xa9\x81\x92=\x1f\xce\xf78\xef\xb4.\x05\xfe\x8a\xfb\ni83I\xbb\xa0\x9c#$\xd5H\xe3UNH\"ޞ\x0f\xa9\xb6\x15\x8c\xf6\x1cEg/\xc5\xf2\xec|\xe4\r\a+W\xc29\xa1\xcc\xc7#\x95\x97P\xec\x83\x02\n,\xf1\x88\r]\x14M\x16\x10\xc6\x14\xacᡟi\xe4\x16\u07bd\x03e\xe0]\xec\xb4߽\x8f\xe9\xea\xb9p\x15\x973\x1b\x05\x8dO\\\x88l\xf7MYLћ\xaa\xbb\xf2\xee\x8a\x03\xfe\xb1\x12_\xf9\xc1Q\xdb\x11\xf6\xee\x14<1\xee\xa6rW\xc0<\x99\xb6\xef\xa1\xc1\x9e(֠\xf3F\xd2I@c\x88rlP\xa9\xbc{Ӧ\xacd\xda\x1e\x94{\xb8\xbf\xb2\x9d\xc7I0\xb3\xcb\xc3}\xe6\x96}\x88\xc2D1I\x12\x9c*\x05\x94\xa0G\x0e\t\xc5\xe8mhC\x05\x9c\xee5\xd7 /\xa53ne\xf8\xc0\xa9\xad\x90\xd3̉\xf2\x8et\x0f*%\"\xb7a\x7f\u0601\xd7\x118Q\fU\xd7\x06\xa1\xe3}\x8f\x06\xa5\x8b\xf55\x1a\xde\xed\xb7\xb7\xf6d\xa4\xa4\xb3\x9fa\b\x1d\xd6ȴƎ\xdaQ\x8alrԛ\\\xe4\x98\x19\xd0\xed\xc36\xae\xf8\xe7\xebL4;\x87*7\xdd\x1d\xa8\x10\xa4\xe8F\x8d\xb0\xdbo\xa9\x03+lc\xb7?Gx\xb9\xcaAjx/D\xf0\f\xe5Y\xfc\x12\x9e\x97\x1c{\x85N\x01\xf4\xf1\x15\x96w\xfbR!\x9d\xdc\x01\xee\xc0\x1cI\xa4{\x014\xcfE\x9d\x90\xcfG\n\xe7\xaf\xc3۾\n\xf0\xf6E\xc4\xdb5\xe4\vx\x9b\xe7\xdf\f\x99\x8a77\u061d\xa3\xae^\x88\\\x05\xfaX\x1cl__\xa2ʖ\xabrw\xb5\x92YS\xfcj\xfaD\x96\xeb\x89%Ӭf\xe7G\xf2Umh\xb8\x9e\xbf\xb6\x11\x8d\xef1)\xec\xad7\x81\x86\xd2+\r\xdd\xca~U+\xda\xc6\xf7\x8d\xf9U\xf6Z\xfbv\xbe\"\xdc\xf7L7\xabw,'T\xbcO\xe7G\x94R\xffv\xd2\x17\x97\x06z$u\xd8\x01\x1eQ\x02\xb5ڌ\v\xec\xb2N[\xc3W\xea\xc6\xc3\xc5\xe7v}E\n\xfeN\x8aB٥\x9e\xa9\x00\xfa|]~L\xa1\xebNE*\xce$\xa4\x17\x825\x027\xe0\x8c\xbf\xd4?\x16\x0fʈֲ\xe1\x1aQ\x7f\x8eR\U0006a616\x00k\xa8\xa9X\xf7\xb4\xb76\xc5\xfeME#\xf4\xbdW@\xecH\xa6\x94g\x13ϼܒ\xa3\xf4c\x89\r\xbe\xe0Sa\xf4cۢ.1T\x05;\x83\x9a\x99\xe2\xd4\xd9K\xe4|2^0Jd\x95\xe7\x8a:\xa7\xa7\xbe\xc2\xdc\xdfB\x02\xbe\xc9\xd3\t\xdf5g\xe7\x1b\xc9A\x89|\x80\xc2k\x9c\xf4c\x83\x86<\x1e\xde\xfb\xb2\xeb33\x15\x92\x9e\xc9n\x11\xb2\x93\x86\xa9\x0f\v\xaa\xe8\xf4Pe\x88\x97\xa6ܙv\xdcj\xc1J\x85-\xefd\xd12\x9c\x922\x1f\xf4\x89R\xdf\xda#L\xaf\xa3\xe5\xc2Wz\xe2,Ea\xfeX\xb9\x9a\x9f^=\xbf\x8f\x85\x17.Q\xcbW\xe8km\xecB\xf8\x1a\xa9\xa6\a\xf0\x12\xa5\xce\xd9\xf1\x9c\v\x97f~O\x1a,:\xeal0 \xeff\xba\xd3S\xc6|\xc47\xd3\x03\xdd\x06\xfe\xf3ߛ\xff\x05\x00\x00\xff\xff\xd8T?\xb3K\x1a\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcY\xcbsۼ\x11\xbf\xfb\xaf\xd8I\x0f\xb9X\xf4\x97\xaf\x9dNG\xb7Dng4\xfd\x92j\"\xd7w\x88\\\x91\x88A\x80\xc5C\xae\xdb\xe9\xff\xdeY<(> \xd1N\x93ꐉ\x01\xecb\x9f\xbf\xdd\x05W\xab\xd5\r\xeb\xf8#jÕ\\\x03\xeb8\xfeӢ\xa4\xbfL\xf1\xf4'Spuw\xfap\xf3\xc4e\xb5\x86\x8d3V\xb5_\xd1(\xa7K\xbc\xc7#\x97\xdcr%oZ\xb4\xacb\x96\xado\x00\x98\x94\xca2Z6\xf4'@\xa9\xa4\xd5J\bԫ\x1ae\xf1\xe4\x0exp\\T\xa8=\xf3t\xf5\xe9\x97\xe2ï\xc5/7\x00\x92\xb5\xb8\x06\xe2\xe7:\xa1Xe\x8a\x13\nԪ\xe0\xea\xc6tX\x12\xdbZ+\u05ed\xe1\xbc\x11\xc8\xe2\x95A\xdc{f\xd9\xdf=\a\xbf(\xb8\xb1\x7f\x9dl\xfcƍ\xf5\x9b\x9dp\x9a\x89ѭ~\xddpY;\xc1\xf4p\xe7\x06\xc0\x94\xaa\xc35|\xa1+;V\"\xadEM\xbc\b+`U\xe5m\xc3\xc4NsiQo\x94pm\xb2\xc9\n*4\xa5\xe6\x9d\xf5\xba\x9f\x05\x02c\x99u\x06\x8c+\x1b`\x06\xbe\xe0\xf3\xddV\ued2a5\x9a \x12\xc07\xa3\xe4\x8e\xd9f\rE8^t\r3\x18w\x83\xf9\xf6~#.\xd9\x17\x92\xd6X\xcde\x9d\xbb\xff\x81\xb7\b\x95\xd3\xdem\xa4s\x89`\x1bn\x86\x82=3C\xc2i\x8b\xd5E1\xfc>13\x96\xb5\xddT\x9e\x01i\x10\xa8b\x16s\xe2lT\xdb\t\xb4X\xc1\xe1\xc5bR\xe2\xa8t\xcb\xec\x1a\xb8\xb4\x7f\xfc\xc3eKDS\x15\x9e\xf4^ɱY>\xd1*\f\x96\x83$\xe4\xa1\x1au\xd66\xca2\xf1\xbf\bb\x89\xc1\xa7\x01}\x90$\xf0\x1d\xae/\x8aB\xe1\x06\xea\b\xb6A\xf8\xc4\xca'\xd7\xc1\xde*\xcdj\x84\xdfT\x19\x9c\xf7ܠ\x8e\xce;\x84#\xa6QNTpH\x1a\x03\x18\xabt\u058b\x1d\x96E\xa0\x8a|\x13ۉ+\xc7w\xfe\xe0 +5\xb2l\x90%\x94)\xfc\t\xaed>\xd2>֘\x8f\xb2\xb0}\xfa\x95\x89\xaea\x1fBz\x97\r\xb6l\x1dϫ\x0e\xe5\xc7\xdd\xf6\xf1\xf7\xfb\xd12@\xa7U\x87\xda\xf2\x84/\xe17\x00\xce\xc1*\x8c\xb5~O\f\xc3)\xa8\b1\xd1x\xffE\xb4\xc0*\xca\x10\xfc\xca\rh\xec4\x1a\x94vh\xdb\xf4SG`\x12\xd4\xe1\x1b\x96\xb6\x80=jb\x93<\\*yBmAc\xa9j\xc9\xff\xd5\xf36`\x95\xbfT0\x8b\x11\xf0\xce?\x8fN\x92\t81\xe1\xf0\x16\x98\xac\xa0e/\xa0\x91n\x01'\a\xfc\xfc\x11S\xc0g\xa5\x11\xb8<\xaa54\xd6vf}wWs\x9b\nF\xa9\xda\xd6In_\xee<\xf6\xf3\x83\xb3J\x9b\xbb\nO(\xee\f\xafWL\x97\r\xb7XZ\xa7\xf1\x8eu|\xe5E\x97\xbeh\x14m\xf5;\x1dK\x8cy?\x92u\x16a\xe1\xe7\xd1\xfe\x8a\a\b\xf4\x81\x1b`\x914hq64-\x91u\xbe\xfey\xff\x00\xe9j\uf329\xf5\xbd\xddτ\xe6\xec\x022\x18\x97G\xd4\xc1\x89G\xadZ\xcf\x13e\xd5).\xad\xff\xa3\x14\x1c\xe5\xd4\xfc\xc6\x1dZn\xc9\xef\xffph,\xf9\xaa\x80\x8d\xaf\xa2p@p\x1d\xc5oU\xc0V\u0086\xb5(6\xcc\xe0Ow\x00YڬȰ\xafs\xc1\xb0\x01\x98\x1e\x0eV\x1bl\xa4\x1a~\xc1_g8\xd8wX\x92\xe3\xc8vDď<\xa2\xdcQi`\x83\x93ň]>]\xe9\x97\x05\xb7顉<\x9fr4I,9\x80䄷\xe1\xe4\x8c)\x80\x98\x82tO\xa3\xb1S\x86[\xa5_\x88q\xc0\xe7b\xc6\xe1\x82\xf1\xe9W2Y\xa2X\xd0d\xe3\x0f\x01\x97\x15\xd9\x11\xfb\x98#x\b\f\xbcLJ֊r\xe2\x92y\xc3ok\x89\x86BԠ%\x8d\xa4'\x1e\x8097\xc0%\x9c\xbb\x17\x18v)S\xad\x0eJ\tdS\xbc+\r\xdfK֙F\xd9\x05ݶGH'\x1f^:\xa4\xcb7\xfb\xed-\xfd\x93\xd6).N\xbc\x8a\x00L\xc9Cu|\x0e\xb2\x10\x80\x96\x0em\xf6[0\x91|n\x04\xe9\x84`\a\x81k\xb0\xda\xcd\x15\xbb\x1c\x86\xf4Kl7\x82\x99쁉\x821\x00\xfd\xf1\\\xf4%~P\xfa\x13\xb6aS\xa4\xe9\rN出\xd1\x01\x11\xef\xeb.\xab\x13\x8d<\xf2\xc8\xeb\xf9\xdd\xc3\xd9\xe8Z\x8a\\UmV3\x06W\x92ũD\x90$\xab\x96\xd6W\xa9~P\x97t\xe4ulC3\x97\x1e9\x8aʼ9\xd9\x17\xec\xe1\x85X\x80\xb0^\x89T\xec\"R\x11=x\x061 \x9c\xf1\xa3\x11mf\x14\b%\xa5 Ds!ҽo\xaaJ\xe4ܾSWn\t\xc3\xff69>\xb1\x83\xa5\x01\xc2\xebn\x15<3n\xfb\xd65\a\xe0\x89\x97\xb9\x85\x03\x1e\xa9]\xd2h\x9d\x96T\xd9Pkj \x8cg\xa9\\\x06گ(e\x06efA\xa1iE\xf2Z\xd0\xff\xa7\x98=L\xf4\x8c2\xae{\x9b\x84\xbe\x83\xed\x1f'\x96\x84\x1c\x9fNr*\xcdkNc\x81\xecw\xcemK\x00\x87\x8c\xa4q\xec\xf4p\xe5\xf1\xb6\xa0F!\xf5p\x04\x80gv\x94\xa1\xe1r\x02p\x1a;6\xfbm\x86gOQ\xc5\xfc\xcad\xe7\xa25v\x8f\x9bWفD\xc9\xe05-?7\xbcl\xc6~\x9b\x8d\b^\x16\xf6\x84\xbeE}\x83\x98y\xa0^\xe5\x1b\xd6əi\x96M\xb6\x87\xf1:\xdd\x1a\xbb>\xbb\xbb{ܼ\xaa\xa9\xf7\xaf\x1e\xafk\xeb\xc3\x13W\xb4r\xe9\xb4Fi\xd3\xc3\x17M\xb8\xdf\xd1ؗ\xe1\xc9h\xf8(\xb0\xd4\f\xcf)\xfc䬫\x01ڰԠ\xfb\x87\x89\xf4,\x95k\x87\xcf\xec\x02\xa5\x9f\xe4\x89\x1bV\x80'\x94@S\vゐ۳4Ŕ&\x9fN=\x97\x88b\xe1\r2ͬQ\xbc\xf4\"\xf0@\xc1\xe9G\xd2\xf7\xe6\nO\x0f\xa2\x94~\x19#\xcc#:\xbdw\xd1 \xba\xca2}Um\xcc&g\xdf+|E\xe3D\xa6@\xfc\xc4^!\\\x19\xa6-\x93\xed\x15\xae\xcf\b\xcc\x00\x03\x1d\x98D\x98\xb862}\x7f\x03Ѣ1\xac^\xc2\xf1\xcf\xe1Tx\xe9\x88$\xc0\x0eTGǢ\xbd71\xd9\xde\x04\xa3\x1d\xb3͂\x04;f\x9b\x94\xd6G'\x84\xa7\x99պ\xd8\xfa\x1e\x90b\xf8G\x95\x1a\xf5\u0093D\xea\x12+n:\xc1^2\x8c\x93\"\xc3\x1c\x1fdK\xc2\xd5TZ\xe7\xf1p}\xba\xef\xbf>\xe4\xa7\xc5\xdc'\x84\x9c\x0f\x86\x1f\x03&\xfb\xfdW\x85\x9fs\xc3\x158J\x99\xbc\xbd\x7fe\xfb\xbb\xbdOY\xc7+\x94\x96:z\xed\xcbϸ\x97\x92W\a\x9a\xc1\x1b\xd9\xdbڿ\xd17\xa9%\x89G\x87\x17\xfa\x81\xf85,\xd7\r\xec)\xc5\tX\xfcC\xf0f\xfa\xbd\xe2\xb6\xff\xfc\xc1l|\x8d.\x1b&kJ\bI%ŗ\xa4\x1c\xe3Y\x81\x1f\x95\xf3\xb1\xf8\xff\xcfJ\x9e\r\x97٢\x97\xbc\x1a\xf0\x8eo\x10\xc3\x15w\xe8_\xff\xd7\xf0\xef\xff\xdc\xfc7\x00\x00\xff\xffgK\fV\xa3\x1e\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcY_o\xe4\xb8\r\x7fϧ \xb6\x0fyY;\xb7ע(\xe6mw\xd2\x02Ao\xd3\xc1e\x91w٢=\xbaȒ*ɓ\xa6E\xbf{A\xc9\xf2\xf8\x8f&N\xf6z緑(\xf2'\x92\xfa\x91\xd2\x14EqŌxD\xeb\x84V;`F\xe0\xbf<*\xfa\xe5ʧ\xbf\xb8R\xe8\x9bӧ\xab'\xa1\xf8\x0e\xf6\xbd\xf3\xba\xfb\x19\x9d\xeem\x8d\xb7\xd8\b%\xbc\xd0\xea\xaaC\xcf8\xf3lw\x05\xc0\x94ҞѰ\xa3\x9f\x00\xb5V\xdej)\xd1\x16-\xaa\U000a9bf0\xea\x85\xe4h\x83\xf2d\xfa\xf4C\xf9\xe9\xc7\xf2\x87+\x00\xc5:\xdc\x01\xe9\xe3\xfaYI\u0378+O(\xd1\xeaR\xe8+g\xb0&ŭս\xd9\xc1y\".\x1c\x8cF\xc0\xb7̳\xdbAG\x18\x96\xc2\xf9\xbf\xaf\xa6~\x12·i#{\xcb\xe4\xc2v\x98qB\xb5\xbddv>w\x05\xe0jmp\a\xf7dڰ\x1ail\xd8S\x80R\x00\xe3\xdd\x00S\xa0\xab_\xb0\xf6%<\xa0%5\xe0\x8e\xba\x97\x9ch\xff\x84փ\xc5Z\xb7J\xfc{\xd4\xed\xc0\xeb`T2\x8f\x03\xf1\x9e\xbf\xc0\x90\x8aI81\xd9\xe3G`\x8aC\xc7^\xc0\"Y\x81^M\xf4\x05\x11W\xc2Wm\x11\x84j\xf4\x0e\x8e\xde\x1b\xb7\xbb\xb9i\x85O\xe5\xab\xd6]\xd7+\xe1_nB%\x12U\xef\xb5u7\x1cO(o\x9ch\vf\xeb\xa3\xf0X\xfb\xde\xe2\r3\xa2\b\xd0U(ae\xc7\xff`\x87\x82\xe7\xaegXW\x11\x8d_\xa8<\xafD\x80\xca\x0f\xa5\x13\x1b\x96\xc6]\x9c\x1dMC䝟\xff\xfa\xf0\r\x92\xe9\x10\x8c\xa5\xf7\x83\xdf\xcf\v\xdd9\x04\xe40\xa1\x1a\xb41\x88\x8d\xd5]Љ\x8a\x1b-\x94\x0f?j)P-\xdd\xef\xfa\xaa\x13\x9e\xe2\xfe\xcf\x1e\x9d\xa7X\x95\xb0\x0f5\x1d*\x84\xde\xd0A\xe2%\xdc)س\x0e\xe5\x9e9\xfc\xcd\x03@\x9ev\x059\xf6m!\x98\xb6#K\xe1\xe8\xb5\xc9D\xea'.\xc4k\xca\x02\x0f\x06k\n\x1dy\x8f\x96\x89F\f<\xdbh\vl&[\xceT\xe6\x8f,}Y\xae]\n-0}ɭI\xc0Ԅ\xd1\x06\xd2wQr\xa5\x14@^,\x14\x16\x8dv\xc2k\xfbr.\x17\xe5JÅ\x00\xd0W3U\xa3\xdc\xd8\xc9>\b\x81P\x9c<\x89c\xde\x11ED\x05\x01\x93V\xad\xa6sq\xd9\xc1\xf1\xbb\xf3\xb4\x8a\x12ա\xa7=\xa9,\x93\v\x05\xe7>\n\xa6\xfd\xd2rg\x95\xd6\x12ْ\xf7(\xb7\xbe\xea\x13ur\xaa\x11\xedz\x8fӖ\xefR\xe07ܗIÉI\xda\x05\xe5\x1c!):\x1a/RB\x12\xf16\xa2\x1d\x8al\xc6h#Prw)\x96\xab\xf3\x916\x1c\xacl\x84sD\x99\x8e\xc7P^B\xd7\x11\x14P`\x89G\\h\xe7h2\x830\xa6`\tw\xcdD\xa3p\xf0\xe1\x03h\v\x1fb\xcb\xff\xe1cL\xd7^H_\b5\xb1\x91\xd1\xf8,\xa4Lvߕ\xc5\x14\xbd\xb1\xcdн\xdfp\xc0?\x16\xe2\v?x\xea\x7f\xc2\u07bd\x86g&\xfcX\xee2\x98G\xd3\xee#T\xd8\x10\xc5Z\xf4\xbdUt\x12\xd0Z\xa2\x1c\x17T\xea\u07bfkSN1\xe3\x8e\xda\xdf\xddnl\xe7a\x14L\xecrw\x9b\xb8\xe51Da\xa4\x98A\x12\xbc\xce\x05\x94\xa0G\x0e\t\xc5\xe8}hC\x05\x1c/X[\x90\xe7\xd2\t\xb7\xb6\xa2\x15\xd4V\xa8q\xe6Ly'\xba\x90\xe5\x12Q\xb8\xb0?\xe4Л\b\x9c(\x86\xaak\x85\xc0EӠE\xe5c}\x8d\x86\x0f\x8f\xfbkw6\x92\xd3\xd9L0\x84\x0e\xabc\xc6 \xa7\xbe\x98\";8\xea].\xf2̶\xe8\x1f\xc366\xfc\xf3m\"\x9a\x9cC\x95\x9b.1T\b\x86\xe8F\x8dpx\xdcS\a\x96\xd9\xc6\xe1q\x8d\xf0r\x95\x83\xa1\xe1\xbd\x10\xc1\x15\xcaU\xfc\x06<\xaf9v\x83N\x01\xcc\xe9\r\x96\x0f\x8f\xb9B:\xba\x03\xfc\x91y\x92\x18.(P\xbdduB:\x1fC8\xbf\x0fo\xfd&\xc0\xfbW\x11\uf5d0/\xe0\xad^~5d*\xde\xc2\"_\xa3.^\x89\\\x01\xe6\x94\x1d\xac\xdf^\xa2\xf2\x96\x8b|w\xb5\x90YR\xfcb\xfaL\x96ˉ9\xd3,f\xa7G\xf2Mmh\xb8B\xbe\xb5\x11\x8d\x0fCC\xd8\xeb\xde\x06\x1a\x1a\x9e\x8b\xe8V\xf6]\xadh\x1d\x1fZ\xa6w\xea\xad\xf6m\xbd\"\xdc\xf7,\x9f\xd4;\x96\x12*^\xec\xd3kN\xae\x7f;\xeb\x8bK\x03=\x92:\xe4\x80'T@\xad6\x13\x12y\xd2\xe9J\xf8F\xddx\xb8\xf8\\/\xafH\xc1߃\xa2Pv\xa9gʀ^\xafK\xaf:t\xdd)H\xc5JB\xf5R\xb2J\xe2\x0e\xbc\xed/\xf5\x8fكҡs\xac\xdd\"\xea\xafQ*^\x15\x87%\xc0*j*\x96=\xed\xb5\x1bb\xff\xae\xa2\xa14\xdf\xc2p\xafy\x00\xa0\xbe\xe3\x95\xe4]XB\x0f\xbe\x01\xe6@2\xb9\x9c\x1f\xa1\xbd~=@\xd5w9f\xba\xc7\xe7\xcc\xe8\xe7\xbaF\x93c\xcb\x02\x0e\x16\r\xb3٩\xd5\xf3\xect2^vrę\xe6\xb2:\xc7\xf7\xcf\xcc\xdc\xdf\xc2ax\x97\xa7\a|[\xceN\xb7\xa3\xa3\x96\xe90\x87'J\xd5w\x15Z\xf2xx\x04M\xaeO,\x999\x80L\xf1Y\xc8\xce\x1aƞ0\xa8\xa2\x93LU*^\xe0R\x97̅3\x92\xe5\x8al\xdaɬ}9\x1f\x90D:#\xbd\xbf\xb7_\x19\x9f\x8c\xf3E8\xf7\ue6cb\xc2\xf4\x05w1?>\x05\xff6\x16^\xb9\xd0͟\xe6\xb7Z\xea\x99\xf0\x16\xc1\x0f\xff\n\xe4\xe8}\xca\xd4k^\x9e\x9b\xf9=)9\xeb\xa8\xd5`@\xce'\xba\x87g\x95\xe9H_\x8d\x8f\x85;\xf8\xcf\x7f\xaf\xfe\x17\x00\x00\xff\xff\x9f\xc23\x7f`\x1b\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcY\xcbsۼ\x11\xbf\xfb\xaf\xd8I\x0f\xb9X\xf4\x97\xaf\x9dNG\xb7Dng4\xfd\xe2z\xa2\xd4w\x88\\\x91\x88A\x80\xc5C\xae\xdb\xe9\xff\xdeY<(> \xd1J\x93\xf2\xe0\xb1\x00\xecb\x1f\xd8\xdf\xee\x02\xab\xd5\xea\x86u\xfc\t\xb5\xe1J\xae\x81u\x1c\xffiQ\xd2/S<\xff\xc9\x14\\\xdd\x1d?\xdc\xfeR|\xf8\xb5\xf8\xe5\x06@\xb2\x16\xd7@\xfc\\'\x14\xabLqD\x81Z\x15\\ݘ\x0eKb[k\xe5\xba5\x9c&\x02Y\xdc2\x88{\xcf,\xfb\xbb\xe7\xe0\a\x057\xf6\xaf\x93\x89߸\xb1~\xb2\x13N31\xdaՏ\x1b.k'\x98\x1e\xce\xdc\x00\x98Ru\xb8\x86\aڲc%\xd2X\xd4ċ\xb0\x02VU\xde6LH\x12\xf8\x0e\xc7\x17E\xa1\xe3\x06\xea\x00\xb6A\xf8\xc4\xcag\xd7\xc1\xce*\xcdj\x84\xdfT\x19\x9c\xf7Ҡ\x8e\xceۇ%\xa6QNT\xb0O\x1a\x03\x18\xabt\u058b\x1d\x96E\xa0\x8a|\x13ۉ+\xc7{\xfe\xe0CVjd\xd9C\x96P\xa6\xf0+\xb8\x92\xf9\x93\xf6\xb1\xc67\x9d\xb2\xa15\xa5\xaa\xb07\x1d\x0e%\xe2\x06:\xadJ4\xe6¹'\xf2\x91\f\x0f\xa7\x81\x99Y\u008a\xe3\xafLt\r\xfb\x10P\xa6l\xb0e\xebH\xa1:\x94\x1f\x1f\xb7O\xbfߍ\x86\x81\x04\xe9P[\x9e`.|\x03\xfc\x1e\x8c\xc2X\xd9\xf7\xc40\xac\x82\x8a\x80\x1b\x8d\xd74\x82\x16VQ\x86`\x10n@c\xa7Ѡ\xb4C\x17\xa7O\x1d\x80IP\xfboX\xda\x02v\xa8\x89M:h\xa5\x92G\xd4\x164\x96\xaa\x96\xfc_=o\x03V\xf9M\x05\xb3\x18q\xf7\xf4y\x90\x94L\xc0\x91\t\x87\xb7\xc0d\x05-{\x05\x8d\xb4\v89\xe0痘\x02>+\x8d\xc0\xe5A\xad\xa1\xb1\xb63뻻\x9a۔\xb7JնNr\xfbz\xe7S\x10\xdf;\xab\xb4\xb9\xab\xf0\x88\xe2\xce\xf0z\xc5t\xd9p\x8b\xa5u\x1a\xefX\xc7W^t\xe9sW\xd1V\xbf\xd31ә\xf7#Yg\x1e\r\x9fO:\x17<@\xb9\x87\x8e\x13\x8b\xa4A\x8b\x93\xa1i\x88\xac\xf3\xe5ϻ\xaf\x90\xb6\xf6ΘZ\xdf\xdb\xfdDhN. \x83qy@\x1d\x9cxЪ\xf5}%\x93%\x8a\x05M6~\x11pY\x91\x1d\xb1?s\x04\x0f\x81\x81\x97I\xc9ZQL\x9c3o\xf8\xb6\x96h\xe8\x88\x1a\xb4\xa4\x91\xcc 8\x97p*\xa2`X,M\xb5\xda+%\x90M\xf1\xae4|'Yg\x1ae\x17t\xdb\x1e \xad\xfc\xfa\xda!m\xbe\xd9mo\xe9O\x1a\xa7sq\xe4U\x04`\n\x1e*'\xe6 \v\x01hi\xd1f\xb7\x05\x13\xc9\xe7F\x90N\b\xb6\x17\xb8\x06\xab\xdd\\\xb1\xf3ǐ\xbe\xc4v#\x98\xc9.\x98(\x18\x0f\xa0_\x9e;}\x89\x1f\x94~\x85m\xd8\x14iz\x83S\xfa\xa1\xa2x@\xc4\xfb\xf4\x0f/\xdc6Y\xca\v\xc7\x0fbq\x93\x04\xfc\x11\xfa\xc4b'\xa8\xa3\x0e\x17\x94y|\xdax}\x974#T\xfe\x1e\xcd\x02\xcb\xf3\aq\xa6\xdbӈ \xa7\xddD\xcas\xca)\x8a/\xc2\b\xac\xc0u\xd7\xcbN\x01\xce5Vs\x99W#\x7fe\xa6\xc7J\x9f\x89\xda\x19\xb6{S0\xcb>\xab#u^\xf2\xc0\xeb\xf9\xde\xc3\x16\xedR\x88\\Tm\x963\x06[\x92\xc5)E\x90$\xab\x96\xc6W)\x7fP\x95t\xe0u\xac\x863\x9b\x1e8\x8a\xca\\\x1d\xec\v\xf6\xf0B,@X\xafDJv\x11\xa9\x88\x1e<\x83x \x9c\xf1\x1d\x1aMf\x14\b)\xa5 D>m\xdeT\xd4\xfb\x86\xfcme}\xb8i\x8bV.\x9d\xd6(m\xba\x7f\xa3\x0e\xf7;\n\xfb2\xdc\\\r\xef&\x96\x8a\xe19\x85\xef\x9cu5@\x1b\x96\nt\x7f?\x92n\xc7r\xe5\xf0\x89]\xa0\xf4\x9d Date: Fri, 7 Jul 2023 10:34:06 -0400 Subject: [PATCH 6/8] Clarify FSB restore pod status. Signed-off-by: Raghuram Devarakonda --- site/content/docs/main/file-system-backup.md | 6 +++--- site/content/docs/v1.11/file-system-backup.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/site/content/docs/main/file-system-backup.md b/site/content/docs/main/file-system-backup.md index 89c43ce7b..7a9d9199f 100644 --- a/site/content/docs/main/file-system-backup.md +++ b/site/content/docs/main/file-system-backup.md @@ -539,7 +539,7 @@ that it's backing up for the volumes to be backed up using FSB. 5. Meanwhile, each `PodVolumeBackup` is handled by the controller on the appropriate node, which: - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data - finds the pod volume's subdirectory within the above volume - - based on the path selection, Velero inokes restic or kopia for backup + - based on the path selection, Velero invokes restic or kopia for backup - updates the status of the custom resource to `Completed` or `Failed` 6. As each `PodVolumeBackup` finishes, the main Velero process adds it to the Velero backup in a file named `-podvolumebackups.json.gz`. This file gets uploaded to object storage alongside the backup tarball. @@ -556,7 +556,7 @@ It will be used for restores, as seen in the next section. 3. Velero adds an init container to the pod, whose job is to wait for all FSB restores for the pod to complete (more on this shortly) 4. Velero creates the pod, with the added init container, by submitting it to the Kubernetes API. Then, the Kubernetes -scheduler schedules this pod to a worker node, and the pod must be in a running state. If the pod fails to start for +scheduler schedules this pod to a worker node. If the pod fails to be scheduled for some reason (i.e. lack of cluster resources), the FSB restore will not be done. 5. Velero creates a `PodVolumeRestore` custom resource for each volume to be restored in the pod 6. The main Velero process now waits for each `PodVolumeRestore` resource to complete or fail @@ -564,7 +564,7 @@ some reason (i.e. lack of cluster resources), the FSB restore will not be done. - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data - waits for the pod to be running the init container - finds the pod volume's subdirectory within the above volume - - based on the path selection, Velero inokes restic or kopia for restore + - based on the path selection, Velero invokes restic or kopia for restore - on success, writes a file into the pod volume, in a `.velero` subdirectory, whose name is the UID of the Velero restore that this pod volume restore is for - updates the status of the custom resource to `Completed` or `Failed` diff --git a/site/content/docs/v1.11/file-system-backup.md b/site/content/docs/v1.11/file-system-backup.md index 881747895..0fb442e11 100644 --- a/site/content/docs/v1.11/file-system-backup.md +++ b/site/content/docs/v1.11/file-system-backup.md @@ -539,7 +539,7 @@ that it's backing up for the volumes to be backed up using FSB. 5. Meanwhile, each `PodVolumeBackup` is handled by the controller on the appropriate node, which: - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data - finds the pod volume's subdirectory within the above volume - - based on the path selection, Velero inokes restic or kopia for backup + - based on the path selection, Velero invokes restic or kopia for backup - updates the status of the custom resource to `Completed` or `Failed` 6. As each `PodVolumeBackup` finishes, the main Velero process adds it to the Velero backup in a file named `-podvolumebackups.json.gz`. This file gets uploaded to object storage alongside the backup tarball. @@ -556,7 +556,7 @@ It will be used for restores, as seen in the next section. 3. Velero adds an init container to the pod, whose job is to wait for all FSB restores for the pod to complete (more on this shortly) 4. Velero creates the pod, with the added init container, by submitting it to the Kubernetes API. Then, the Kubernetes -scheduler schedules this pod to a worker node, and the pod must be in a running state. If the pod fails to start for +scheduler schedules this pod to a worker node. If the pod fails to be scheduled for some reason (i.e. lack of cluster resources), the FSB restore will not be done. 5. Velero creates a `PodVolumeRestore` custom resource for each volume to be restored in the pod 6. The main Velero process now waits for each `PodVolumeRestore` resource to complete or fail @@ -564,7 +564,7 @@ some reason (i.e. lack of cluster resources), the FSB restore will not be done. - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data - waits for the pod to be running the init container - finds the pod volume's subdirectory within the above volume - - based on the path selection, Velero inokes restic or kopia for restore + - based on the path selection, Velero invokes restic or kopia for restore - on success, writes a file into the pod volume, in a `.velero` subdirectory, whose name is the UID of the Velero restore that this pod volume restore is for - updates the status of the custom resource to `Completed` or `Failed` From bc8742566bfcdfc24758114dee1c51bc6e9033cc Mon Sep 17 00:00:00 2001 From: danfengl Date: Tue, 20 Jun 2023 10:24:11 +0000 Subject: [PATCH 7/8] Install plugin for datamover pipeline Signed-off-by: danfengl --- go.mod | 2 +- test/e2e/Makefile | 9 +- test/e2e/backup/backup.go | 9 +- test/e2e/backups/sync_backups.go | 4 +- test/e2e/backups/ttl.go | 2 +- .../api-group/enable_api_group_extentions.go | 4 +- .../api-group/enable_api_group_versions.go | 2 +- test/e2e/basic/namespace-mapping.go | 2 +- test/e2e/bsl-mgmt/deletion.go | 4 +- test/e2e/e2e_suite_test.go | 4 +- test/e2e/migration/migration.go | 116 ++++++++++++------ test/e2e/types.go | 4 + test/e2e/upgrade/upgrade.go | 4 +- test/e2e/util/k8s/common.go | 1 + test/e2e/util/k8s/statefulset.go | 39 ++++++ test/e2e/util/kibishii/kibishii_utils.go | 19 ++- test/e2e/util/velero/install.go | 38 +++--- test/e2e/util/velero/velero_utils.go | 113 +++++++++++++---- 18 files changed, 276 insertions(+), 100 deletions(-) create mode 100644 test/e2e/util/k8s/statefulset.go diff --git a/go.mod b/go.mod index 61885fe44..ee43d5fa3 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/stretchr/testify v1.8.2 github.com/vmware-tanzu/crash-diagnostics v0.3.7 go.uber.org/zap v1.24.0 + golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f golang.org/x/mod v0.10.0 golang.org/x/net v0.9.0 golang.org/x/oauth2 v0.7.0 @@ -139,7 +140,6 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.8.0 // indirect - golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.7.0 // indirect golang.org/x/term v0.7.0 // indirect diff --git a/test/e2e/Makefile b/test/e2e/Makefile index f2d7594e1..0fbf46e3e 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -73,6 +73,7 @@ BSL_PREFIX ?= BSL_CONFIG ?= VSL_CONFIG ?= CLOUD_PROVIDER ?= +STANDBY_CLUSTER_CLOUD_PROVIDER ?= OBJECT_STORE_PROVIDER ?= INSTALL_VELERO ?= true REGISTRY_CREDENTIAL_FILE ?= @@ -99,6 +100,9 @@ STANDBY_CLUSTER ?= UPLOADER_TYPE ?= +SNAPSHOT_MOVE_DATA ?= false +DATA_MOVER_PLUGIN ?= + .PHONY:ginkgo ginkgo: # Make sure ginkgo is in $GOPATH/bin @@ -143,7 +147,10 @@ run: ginkgo -velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ -default-cluster=$(DEFAULT_CLUSTER) \ -standby-cluster=$(STANDBY_CLUSTER) \ - -uploader-type=$(UPLOADER_TYPE) + -uploader-type=$(UPLOADER_TYPE) \ + -snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \ + -data-mover-plugin=$(DATA_MOVER_plugin) \ + -standby-cluster-cloud-provider=$(STANDBY_CLUSTER_CLOUD_PROVIDER) build: ginkgo mkdir -p $(OUTPUT_DIR) diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 9bf694c00..8a1f2d40e 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -87,7 +87,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { } else { veleroCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots } - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) } backupName = "backup-" + UUIDgen.String() restoreName = "restore-" + UUIDgen.String() @@ -125,12 +125,9 @@ func BackupRestoreTest(useVolumeSnapshots bool) { veleroCfg.DefaultVolumesToFsBackup = useVolumeSnapshots } - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) } - - Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, - veleroCfg.AddBSLPlugins, veleroCfg.Features)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed()) // Create Secret for additional BSL secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen) diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index 1bfea90b2..a12a270c8 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -59,7 +59,7 @@ func BackupsSyncTest() { if VeleroCfg.InstallVelero { veleroCfg := VeleroCfg veleroCfg.UseVolumeSnapshots = false - Expect(VeleroInstall(context.Background(), &VeleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) } }) @@ -109,7 +109,7 @@ func BackupsSyncTest() { By("Install velero", func() { veleroCfg := VeleroCfg veleroCfg.UseVolumeSnapshots = false - Expect(VeleroInstall(ctx, &VeleroCfg)).To(Succeed()) + Expect(VeleroInstall(ctx, &VeleroCfg, false)).To(Succeed()) }) By("Check all backups in object storage are synced to Velero", func() { diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 09afb23e8..a60d95ea5 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -70,7 +70,7 @@ func TTLTest() { // Make sure GCFrequency is shorter than backup TTL veleroCfg.GCFrequency = "4m0s" veleroCfg.UseVolumeSnapshots = useVolumeSnapshots - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) } }) diff --git a/test/e2e/basic/api-group/enable_api_group_extentions.go b/test/e2e/basic/api-group/enable_api_group_extentions.go index a3d6a3b88..002833d55 100644 --- a/test/e2e/basic/api-group/enable_api_group_extentions.go +++ b/test/e2e/basic/api-group/enable_api_group_extentions.go @@ -100,7 +100,7 @@ func APIExtensionsVersionsTest() { Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) veleroCfg.Features = "EnableAPIGroupVersions" veleroCfg.UseVolumeSnapshots = false - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) }) By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultCluster), func() { @@ -129,7 +129,7 @@ func APIExtensionsVersionsTest() { By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() { Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed()) veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) }) By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() { diff --git a/test/e2e/basic/api-group/enable_api_group_versions.go b/test/e2e/basic/api-group/enable_api_group_versions.go index ba1d73f6e..61d7ba08e 100644 --- a/test/e2e/basic/api-group/enable_api_group_versions.go +++ b/test/e2e/basic/api-group/enable_api_group_versions.go @@ -75,7 +75,7 @@ func APIGropuVersionsTest() { if veleroCfg.InstallVelero { veleroCfg.Features = "EnableAPIGroupVersions" veleroCfg.UseVolumeSnapshots = false - err = VeleroInstall(context.Background(), &veleroCfg) + err = VeleroInstall(context.Background(), &veleroCfg, false) Expect(err).NotTo(HaveOccurred()) } testCaseNum = 4 diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index 8ea16d365..ae9c0509e 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -115,7 +115,7 @@ func (n *NamespaceMapping) Verify() error { func (n *NamespaceMapping) Clean() error { if !n.VeleroCfg.Debug { - if err := DeleteStorageClass(context.Background(), n.Client, "kibishii-storage-class"); err != nil { + if err := DeleteStorageClass(context.Background(), n.Client, KibishiiStorageClassName); err != nil { return err } for _, ns := range n.MappedNamespaceList { diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index e15307369..874cf53c7 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -104,9 +104,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { } By(fmt.Sprintf("Add an additional plugin for provider %s", veleroCfg.AdditionalBSLProvider), func() { - Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, - veleroCfg.AddBSLPlugins, veleroCfg.Features)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed()) }) additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 651affb0e..df9d18ea4 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -84,7 +84,9 @@ func init() { flag.StringVar(&VeleroCfg.StandbyCluster, "standby-cluster", "", "Standby cluster context for migration test.") flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "", "Identify persistent volume backup uploader.") flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "Identify persistent volume backup uploader.") - + flag.BoolVar(&VeleroCfg.SnapshotMoveData, "snapshot-move-data", false, "Install default plugin for data mover.") + flag.StringVar(&VeleroCfg.DataMoverPlugin, "data-mover-plugin", "", "Install customized plugin for data mover.") + flag.StringVar(&VeleroCfg.StandbyClusterCloudProvider, "standby-cluster-cloud-provider", "", "Install customized plugin for data mover.") } var _ = Describe("[APIGroup][APIVersion] Velero tests with various CRD API group versions", APIGropuVersionsTest) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 0a133f23e..6b10ba925 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -59,13 +59,14 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) BeforeEach(func() { veleroCfg = VeleroCfg UUIDgen, err = uuid.NewRandom() - migrationNamespace = "migration-workload-" + UUIDgen.String() + migrationNamespace = "migration-" + UUIDgen.String() if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } - if useVolumeSnapshots && veleroCfg.CloudProvider == "aws" { + if useVolumeSnapshots && veleroCfg.CloudProvider == "aws" && !veleroCfg.SnapshotMoveData { Skip("Volume snapshots migration not supported on AWS provisioned by Sheperd public pool") } + if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" { Skip("Migration test needs 2 clusters") } @@ -79,9 +80,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) AfterEach(func() { if !veleroCfg.Debug { - By("Clean backups after test", func() { - DeleteBackups(context.Background(), *veleroCfg.DefaultClient) - }) + // TODO: delete backup created by case self, not all + // By("Clean backups after test", func() { + // DeleteBackups(context.Background(), *veleroCfg.DefaultClient) + // }) if veleroCfg.InstallVelero { By(fmt.Sprintf("Uninstall Velero and delete sample workload namespace %s", migrationNamespace), func() { Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) @@ -104,6 +106,16 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) When("kibishii is the sample workload", func() { It("should be successfully backed up and restored to the default BackupStorageLocation", func() { + + if veleroCfg.SnapshotMoveData { + if !useVolumeSnapshots { + Skip("FSB migration test is not needed in data mover scenario") + } + // TODO: remove this block once Velero version in cluster A is great than V1.11 for all migration path. + if veleroCLI2Version.VeleroVersion != "self" { + Skip(fmt.Sprintf("Only V1.12 support data mover scenario instead of %s", veleroCLI2Version.VeleroVersion)) + } + } oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) defer ctxCancel() flag.Parse() @@ -132,25 +144,20 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots - // TODO: self means 1.10 and upper version - if veleroCLI2Version.VeleroVersion != "self" { + + // self represents v1.12 + if veleroCLI2Version.VeleroVersion == "self" { + if OriginVeleroCfg.SnapshotMoveData { + OriginVeleroCfg.UseNodeAgent = true + } + } else { Expect(err).To(Succeed()) fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion) OriginVeleroCfg.VeleroImage = "" OriginVeleroCfg.RestoreHelperImage = "" OriginVeleroCfg.Plugins = "" - //TODO: Remove this once origin Velero version is 1.10 and upper - OriginVeleroCfg.UploaderType = "" - if supportUploaderType { - OriginVeleroCfg.UseRestic = false - OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots - } else { - OriginVeleroCfg.UseRestic = !useVolumeSnapshots - OriginVeleroCfg.UseNodeAgent = false - } } - - Expect(VeleroInstall(context.Background(), &OriginVeleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, false)).To(Succeed()) if veleroCLI2Version.VeleroVersion != "self" { Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI, OriginVeleroCfg.MigrateFromVeleroVersion)).To(Succeed()) @@ -167,10 +174,15 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", migrationNamespace)) }) + KibishiiData := *DefaultKibishiiData By("Deploy sample workload of Kibishii", func() { + if OriginVeleroCfg.SnapshotMoveData { + KibishiiData.ExpectedNodes = 6 + } + Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider, migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features, - veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed()) + veleroCfg.KibishiiDirectory, useVolumeSnapshots, &KibishiiData)).To(Succeed()) }) By(fmt.Sprintf("Backup namespace %s", migrationNamespace), func() { @@ -178,6 +190,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) BackupStorageClassCfg.BackupName = backupScName BackupStorageClassCfg.IncludeResources = "StorageClass" BackupStorageClassCfg.IncludeClusterResources = true + //TODO Remove UseRestic parameter once minor version is 1.10 or upper BackupStorageClassCfg.UseResticIfFSBackup = !supportUploaderType Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI, @@ -195,6 +208,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots //TODO Remove UseRestic parameter once minor version is 1.10 or upper BackupCfg.UseResticIfFSBackup = !supportUploaderType + BackupCfg.SnapshotMoveData = OriginVeleroCfg.SnapshotMoveData Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI, OriginVeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { @@ -211,21 +225,27 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) migrationNamespace, 2)).To(Succeed()) }) } + var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = migrationNamespace - By("Snapshot should be created in cloud object store", func() { - snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2, - migrationNamespace, backupName, KibishiiPVCNameList) - Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) - }) + + if !OriginVeleroCfg.SnapshotMoveData { + By("Snapshot should be created in cloud object store", func() { + snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2, + migrationNamespace, backupName, KibishiiPVCNameList) + Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") + Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + }) + } else { + //TODO: checkpoint for datamover + } } - if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, "EnableCSI") { - // Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before - // v1.0 have issues to fail upgrade case. + if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && + strings.EqualFold(veleroCfg.Features, "EnableCSI") && + !OriginVeleroCfg.SnapshotMoveData { By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { time.Sleep(5 * time.Minute) }) @@ -233,7 +253,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots { + if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots && !OriginVeleroCfg.SnapshotMoveData { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } @@ -253,7 +273,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient veleroCfg.UseNodeAgent = !useVolumeSnapshots veleroCfg.UseRestic = false - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + if veleroCfg.SnapshotMoveData { + veleroCfg.UseNodeAgent = true + } + Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed()) }) By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() { @@ -262,12 +285,27 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) By(fmt.Sprintf("Restore %s", migrationNamespace), func() { - Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string { - RunDebug(context.Background(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, "", restoreName) - return "Fail to restore workload" - }) + if OriginVeleroCfg.SnapshotMoveData { + By(fmt.Sprintf("Create a storage class %s for restore PV provisioned by storage class %s on different cloud provider", StorageClassName, KibishiiStorageClassName), func() { + Expect(InstallStorageClass(context.Background(), fmt.Sprintf("testdata/storage-class/%s.yaml", veleroCfg.StandbyClusterCloudProvider))).To(Succeed()) + }) + configmaptName := "datamover-storage-class-config" + labels := map[string]string{"velero.io/change-storage-class": "RestoreItemAction", + "velero.io/plugin-config": ""} + data := map[string]string{KibishiiStorageClassName: StorageClassName} + + By(fmt.Sprintf("Create ConfigMap %s in namespace %s", configmaptName, veleroCfg.VeleroNamespace), func() { + _, err := CreateConfigMap(veleroCfg.StandbyClient.ClientGo, veleroCfg.VeleroNamespace, configmaptName, labels, data) + Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", veleroCfg.VeleroNamespace)) + }) + } else { + Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, "", restoreName) + return "Fail to restore workload" + }) + } Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string { RunDebug(context.Background(), veleroCfg.VeleroCLI, @@ -278,7 +316,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() { Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace, - oneHourTimeout, DefaultKibishiiData)).To(Succeed(), "Fail to verify workload after restore") + oneHourTimeout, &KibishiiData)).To(Succeed(), "Fail to verify workload after restore") }) }) }) diff --git a/test/e2e/types.go b/test/e2e/types.go index 99f4c23cb..4e3dcd1cd 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -72,6 +72,9 @@ type VeleroConfig struct { DefaultVolumesToFsBackup bool UseVolumeSnapshots bool VeleroServerDebugMode bool + SnapshotMoveData bool + DataMoverPlugin string + StandbyClusterCloudProvider string } type SnapshotCheckPoint struct { @@ -98,6 +101,7 @@ type BackupConfig struct { OrderedResources string UseResticIfFSBackup bool DefaultVolumesToFsBackup bool + SnapshotMoveData bool } type VeleroCLI2Version struct { diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 1e661d823..58b892255 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -136,7 +136,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.UseNodeAgent = false } - Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, false)).To(Succeed()) Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI, tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion)).To(Succeed()) }) @@ -223,7 +223,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfg.UseNodeAgent = !useVolumeSnapshots Expect(err).To(Succeed()) if supportUploaderType { - Expect(VeleroInstall(context.Background(), &tmpCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &tmpCfg, false)).To(Succeed()) Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI, tmpCfg.VeleroVersion)).To(Succeed()) } else { diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index cf18e7f0a..020de0e28 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -200,6 +200,7 @@ func AddLabelToCRD(ctx context.Context, crd, label string) error { func KubectlApplyByFile(ctx context.Context, file string) error { args := []string{"apply", "-f", file, "--force=true"} + fmt.Println(args) return exec.CommandContext(ctx, "kubectl", args...).Run() } diff --git a/test/e2e/util/k8s/statefulset.go b/test/e2e/util/k8s/statefulset.go new file mode 100644 index 000000000..e9a1e564d --- /dev/null +++ b/test/e2e/util/k8s/statefulset.go @@ -0,0 +1,39 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "fmt" + "os/exec" + + "github.com/pkg/errors" + "golang.org/x/net/context" + + veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" +) + +func ScaleStatefulSet(ctx context.Context, namespace, name string, replicas int) error { + cmd := exec.CommandContext(ctx, "kubectl", "scale", "statefulsets", name, fmt.Sprintf("--replicas=%d", replicas), "-n", namespace) + fmt.Printf("Scale kibishii stateful set in namespace %s with CMD: %s", name, cmd.Args) + + _, stderr, err := veleroexec.RunCommand(cmd) + if err != nil { + return errors.Wrap(err, stderr) + } + + return nil +} diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index 2edb3acc0..ea4e419a7 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -48,8 +48,11 @@ type KibishiiData struct { ExpectedNodes int } -var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2} +var DefaultKibishiiWorkerCounts = 2 +var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, DefaultKibishiiWorkerCounts} + var KibishiiPVCNameList = []string{"kibishii-data-kibishii-deployment-0", "kibishii-data-kibishii-deployment-1"} +var KibishiiStorageClassName = "kibishii-storage-class" // RunKibishiiTests runs kibishii tests on the provider. func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, @@ -196,11 +199,15 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc } func installKibishii(ctx context.Context, namespace string, cloudPlatform, veleroFeatures, - kibishiiDirectory string, useVolumeSnapshots bool) error { + kibishiiDirectory string, useVolumeSnapshots bool, workerReplicas int) error { if strings.EqualFold(cloudPlatform, "azure") && strings.EqualFold(veleroFeatures, "EnableCSI") { cloudPlatform = "azure-csi" } + if strings.EqualFold(cloudPlatform, "aws") && + strings.EqualFold(veleroFeatures, "EnableCSI") { + cloudPlatform = "aws-csi" + } // We use kustomize to generate YAML for Kibishii from the checked-in yaml directories kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k", kibishiiDirectory+cloudPlatform, "--timeout=90s") @@ -216,6 +223,12 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler if err != nil { return errors.Wrapf(err, "failed to label namespace with PSA policy, stderr=%s", stderr) } + if workerReplicas != DefaultKibishiiWorkerCounts { + err = ScaleStatefulSet(ctx, namespace, "kibishii-deployment", workerReplicas) + if err != nil { + return errors.Wrapf(err, "failed to scale statefulset, stderr=%s", err.Error()) + } + } kibishiiSetWaitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "statefulset.apps/kibishii-deployment", "-n", namespace, "-w", "--timeout=30m") @@ -311,7 +324,7 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie } if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, veleroFeatures, - kibishiiDirectory, useVolumeSnapshots); err != nil { + kibishiiDirectory, useVolumeSnapshots, kibishiiData.ExpectedNodes); err != nil { return errors.Wrap(err, "Failed to install Kibishii workload") } // wait for kibishii pod startup diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index be50c95b2..1830b15d3 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -55,8 +55,23 @@ type installOptions struct { VeleroServerDebugMode bool } -func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error { +func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluster bool) error { fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05")) + // veleroCfg struct including a set of BSL params and a set of additional BSL params, + // additional BSL set is for additional BSL test only, so only default BSL set is effective + // for VeleroInstall(). + // + // veleroCfg struct including 2 sets of cluster setting, but VeleroInstall() only read + // default cluster settings, so if E2E test needs install on the standby cluster, default cluster + // setting should be reset to the value of standby cluster's. + // + // Some other setting might not needed by standby cluster installation like "snapshotMoveData", because in + // standby cluster only restore if performed, so CSI plugin is not needed, but it is installed due to + // the only one veleroCfg setting is provided as current design, since it will not introduce any issues as + // we can predict, so keep it intact for now. + if isStandbyCluster { + veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider + } if veleroCfg.CloudProvider != "kind" { fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider") // If ObjectStoreProvider is not provided, then using the value same as CloudProvider @@ -69,7 +84,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error { } } - providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.CloudProvider, veleroCfg.Plugins, veleroCfg.Features) + pluginsTmp, err := getPlugins(ctx, *veleroCfg) if err != nil { return errors.WithMessage(err, "Failed to get provider plugins") } @@ -91,22 +106,19 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error { } } - veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg, providerPluginsTmp) + veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg, pluginsTmp) if err != nil { return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider) } veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots - if !veleroCfg.UseRestic { - veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent - } - veleroInstallOptions.UseRestic = veleroCfg.UseRestic + veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent veleroInstallOptions.Image = veleroCfg.VeleroImage veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace veleroInstallOptions.UploaderType = veleroCfg.UploaderType GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency) veleroInstallOptions.GarbageCollectionFrequency = GCFrequency - err = installVeleroServer(ctx, veleroCfg.VeleroCLI, &installOptions{ + err = installVeleroServer(ctx, veleroCfg.VeleroCLI, veleroCfg.CloudProvider, &installOptions{ Options: veleroInstallOptions, RegistryCredentialFile: veleroCfg.RegistryCredentialFile, RestoreHelperImage: veleroCfg.RestoreHelperImage, @@ -176,7 +188,7 @@ func clearupvSpherePluginConfig(c clientset.Interface, ns, secretName, configMap return nil } -func installVeleroServer(ctx context.Context, cli string, options *installOptions) error { +func installVeleroServer(ctx context.Context, cli, cloudProvider string, options *installOptions) error { args := []string{"install"} namespace := "velero" if len(options.Namespace) > 0 { @@ -192,9 +204,6 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption if options.DefaultVolumesToFsBackup { args = append(args, "--default-volumes-to-fs-backup") } - if options.UseRestic { - args = append(args, "--use-restic") - } if options.UseVolumeSnapshots { args = append(args, "--use-volume-snapshots") } @@ -219,10 +228,11 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption if len(options.Plugins) > 0 { args = append(args, "--plugins", options.Plugins.String()) } + fmt.Println("Start to install Azure VolumeSnapshotClass ...") if len(options.Features) > 0 { args = append(args, "--features", options.Features) if strings.EqualFold(options.Features, "EnableCSI") && options.UseVolumeSnapshots { - if strings.EqualFold(options.ProviderName, "Azure") { + if strings.EqualFold(cloudProvider, "azure") { if err := KubectlApplyByFile(ctx, "util/csi/AzureVolumeSnapshotClass.yaml"); err != nil { return err } @@ -528,7 +538,7 @@ func PrepareVelero(ctx context.Context, caseName string) error { return nil } fmt.Printf("need to install velero for case %s \n", caseName) - return VeleroInstall(context.Background(), &VeleroCfg) + return VeleroInstall(context.Background(), &VeleroCfg, false) } func VeleroUninstall(ctx context.Context, cli, namespace string) error { diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index 7da65f945..c5d1834c1 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -34,6 +34,7 @@ import ( "time" "github.com/pkg/errors" + "golang.org/x/exp/slices" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -70,15 +71,16 @@ var pluginsMatrix = map[string]map[string][]string{ "csi": {"velero/velero-plugin-for-csi:v0.5.0"}, }, "main": { - "aws": {"velero/velero-plugin-for-aws:main"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"}, - "gcp": {"velero/velero-plugin-for-gcp:main"}, - "csi": {"velero/velero-plugin-for-csi:main"}, + "aws": {"velero/velero-plugin-for-aws:main"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"velero/velero-plugin-for-gcp:main"}, + "csi": {"velero/velero-plugin-for-csi:main"}, + "datamover": {"velero/velero-plugin-for-aws:main"}, }, } -func GetProviderPluginsByVersion(version, providerName, feature string) ([]string, error) { +func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature string, needDataMoverPlugin bool) ([]string, error) { var cloudMap map[string][]string arr := strings.Split(version, ".") if len(arr) >= 3 { @@ -92,17 +94,47 @@ func GetProviderPluginsByVersion(version, providerName, feature string) ([]strin } var pluginsForFeature []string - plugins, ok := cloudMap[providerName] - if !ok { - return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName) + if cloudProvider == "kind" { + plugins, ok := cloudMap["aws"] + if !ok { + return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) + } + return plugins, nil } + + plugins, ok := cloudMap[cloudProvider] + if !ok { + return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) + } + + if objectStoreProvider != cloudProvider { + pluginsForObjectStoreProvider, ok := cloudMap[objectStoreProvider] + if !ok { + return nil, errors.Errorf("fail to get plugins by version: %s and object store provider %s", version, objectStoreProvider) + } + plugins = append(plugins, pluginsForObjectStoreProvider...) + } + if strings.EqualFold(feature, "EnableCSI") { pluginsForFeature, ok = cloudMap["csi"] if !ok { - return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName) + return nil, errors.Errorf("fail to get CSI plugins by version: %s ", version) } + plugins = append(plugins, pluginsForFeature...) } - return append(plugins, pluginsForFeature...), nil + if needDataMoverPlugin { + pluginsForDatamover, ok := cloudMap["datamover"] + if !ok { + return nil, errors.Errorf("fail to get plugins by for datamover") + } + for _, p := range pluginsForDatamover { + if !slices.Contains(plugins, p) { + plugins = append(plugins, pluginsForDatamover...) + } + } + + } + return plugins, nil } // getProviderVeleroInstallOptions returns Velero InstallOptions for the provider. @@ -280,6 +312,10 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin args = append(args, "--selector", backupCfg.Selector) } + if backupCfg.SnapshotMoveData { + args = append(args, "--snapshot-move-data") + } + if backupCfg.UseVolumeSnapshots { if backupCfg.ProvideSnapshotsVolumeParam { args = append(args, "--snapshot-volumes") @@ -516,36 +552,67 @@ func VeleroVersion(ctx context.Context, veleroCLI, veleroNamespace string) error return nil } -func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, cloudProvider, providerPlugins, feature string) ([]string, error) { +// getProviderPlugins only provide plugin for specific cloud provider +func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider string) ([]string, error) { + if cloudProvider == "" { + return []string{}, errors.New("CloudProvider should be provided") + } + + version, err := getVeleroVersion(ctx, veleroCLI, true) + if err != nil { + return nil, errors.WithMessage(err, "failed to get velero version") + } + + plugins, err := getPluginsByVersion(version, cloudProvider, cloudProvider, "", false) + if err != nil { + return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", cloudProvider, version) + } + + return plugins, nil +} + +// getPlugins will collect all kinds plugins for VeleroInstall, such as provider +// plugins(cloud provider/object store provider, if object store provider is not +// provided, it should be set to value as cloud provider's), feature plugins (CSI/Datamover) +func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { + veleroCLI := veleroCfg.VeleroCLI + cloudProvider := veleroCfg.CloudProvider + objectStoreProvider := veleroCfg.ObjectStoreProvider + providerPlugins := veleroCfg.Plugins + feature := veleroCfg.Features + needDataMoverPlugin := false + // Fetch the plugins for the provider before checking for the object store provider below. var plugins []string if len(providerPlugins) > 0 { plugins = strings.Split(providerPlugins, ",") } else { + if cloudProvider == "" { + return []string{}, errors.New("CloudProvider should be provided") + } + if objectStoreProvider == "" { + objectStoreProvider = cloudProvider + } version, err := getVeleroVersion(ctx, veleroCLI, true) if err != nil { return nil, errors.WithMessage(err, "failed to get velero version") } - plugins, err = GetProviderPluginsByVersion(version, objectStoreProvider, feature) + + if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" { + needDataMoverPlugin = true + } + plugins, err = getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature, needDataMoverPlugin) if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version) } - if objectStoreProvider != "" && cloudProvider != "kind" && objectStoreProvider != cloudProvider { - pluginsTmp, err := GetProviderPluginsByVersion(version, cloudProvider, feature) - if err != nil { - return nil, errors.WithMessage(err, "failed to get velero version") - } - plugins = append(plugins, pluginsTmp...) - } } return plugins, nil } // VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and // installs them in the current Velero installation, skipping over those that are already installed. -func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins, feature string) error { - plugins, err := getProviderPlugins(ctx, veleroCLI, provider, provider, addPlugins, feature) - fmt.Printf("addPlugins cmd =%v\n", addPlugins) +func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { + plugins, err := getProviderPlugins(ctx, veleroCLI, provider) fmt.Printf("provider cmd = %v\n", provider) fmt.Printf("plugins cmd = %v\n", plugins) if err != nil { From 9f5162ece39cbfdb30af6b586cb79ac3d5e8924b Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Fri, 7 Jul 2023 15:02:36 +0800 Subject: [PATCH 8/8] add wait timeout for expose prepare Signed-off-by: Lyndon-Li --- go.mod | 2 +- pkg/builder/data_download_builder.go | 6 + pkg/builder/data_upload_builder.go | 6 + pkg/cmd/cli/nodeagent/server.go | 18 +- pkg/controller/data_download_controller.go | 95 ++++++++--- .../data_download_controller_test.go | 129 +++++++++++++- pkg/controller/data_upload_controller.go | 109 +++++++++--- pkg/controller/data_upload_controller_test.go | 160 ++++++++++++++++-- 8 files changed, 462 insertions(+), 63 deletions(-) diff --git a/go.mod b/go.mod index 61885fe44..3a2e3bfae 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/vmware-tanzu/velero -go 1.18 +go 1.20 require ( cloud.google.com/go/storage v1.30.1 diff --git a/pkg/builder/data_download_builder.go b/pkg/builder/data_download_builder.go index c564c80cf..9a85c7905 100644 --- a/pkg/builder/data_download_builder.go +++ b/pkg/builder/data_download_builder.go @@ -110,3 +110,9 @@ func (d *DataDownloadBuilder) ObjectMeta(opts ...ObjectMetaOpt) *DataDownloadBui return d } + +// StartTimestamp sets the DataDownload's StartTimestamp. +func (d *DataDownloadBuilder) StartTimestamp(startTime *metav1.Time) *DataDownloadBuilder { + d.object.Status.StartTimestamp = startTime + return d +} diff --git a/pkg/builder/data_upload_builder.go b/pkg/builder/data_upload_builder.go index a844ef6ef..cb5d0b2de 100644 --- a/pkg/builder/data_upload_builder.go +++ b/pkg/builder/data_upload_builder.go @@ -113,3 +113,9 @@ func (d *DataUploadBuilder) CSISnapshot(cSISnapshot *velerov2alpha1api.CSISnapsh d.object.Spec.CSISnapshot = cSISnapshot return d } + +// StartTimestamp sets the DataUpload's StartTimestamp. +func (d *DataUploadBuilder) StartTimestamp(startTime *metav1.Time) *DataUploadBuilder { + d.object.Status.StartTimestamp = startTime + return d +} diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index 3f635602e..276256945 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -71,20 +71,23 @@ const ( // files will be written to defaultCredentialsDirectory = "/tmp/credentials" - defaultResourceTimeout = 10 * time.Minute + defaultResourceTimeout = 10 * time.Minute + defaultDataMoverPrepareTimeout = 30 * time.Minute ) type nodeAgentServerConfig struct { - metricsAddress string - resourceTimeout time.Duration + metricsAddress string + resourceTimeout time.Duration + dataMoverPrepareTimeout time.Duration } func NewServerCommand(f client.Factory) *cobra.Command { logLevelFlag := logging.LogLevelFlag(logrus.InfoLevel) formatFlag := logging.NewFormatFlag() config := nodeAgentServerConfig{ - metricsAddress: defaultMetricsAddress, - resourceTimeout: defaultResourceTimeout, + metricsAddress: defaultMetricsAddress, + resourceTimeout: defaultResourceTimeout, + dataMoverPrepareTimeout: defaultDataMoverPrepareTimeout, } command := &cobra.Command{ @@ -110,6 +113,7 @@ func NewServerCommand(f client.Factory) *cobra.Command { command.Flags().Var(logLevelFlag, "log-level", fmt.Sprintf("The level at which to log. Valid values are %s.", strings.Join(logLevelFlag.AllowedValues(), ", "))) command.Flags().Var(formatFlag, "log-format", fmt.Sprintf("The format for log output. Valid values are %s.", strings.Join(formatFlag.AllowedValues(), ", "))) command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters. Default is 10 minutes.") + command.Flags().DurationVar(&config.dataMoverPrepareTimeout, "data-mover-prepare-timeout", config.dataMoverPrepareTimeout, "How long to wait for preparing a DataUpload/DataDownload. Default is 30 minutes.") return command } @@ -256,11 +260,11 @@ func (s *nodeAgentServer) run() { s.logger.WithError(err).Fatal("Unable to create the pod volume restore controller") } - if err = controller.NewDataUploadReconciler(s.mgr.GetClient(), s.kubeClient, s.csiSnapshotClient.SnapshotV1(), repoEnsurer, clock.RealClock{}, credentialGetter, s.nodeName, s.fileSystem, s.logger).SetupWithManager(s.mgr); err != nil { + if err = controller.NewDataUploadReconciler(s.mgr.GetClient(), s.kubeClient, s.csiSnapshotClient.SnapshotV1(), repoEnsurer, clock.RealClock{}, credentialGetter, s.nodeName, s.fileSystem, s.config.dataMoverPrepareTimeout, s.logger).SetupWithManager(s.mgr); err != nil { s.logger.WithError(err).Fatal("Unable to create the data upload controller") } - if err = controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.kubeClient, repoEnsurer, credentialGetter, s.nodeName, s.logger).SetupWithManager(s.mgr); err != nil { + if err = controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.kubeClient, repoEnsurer, credentialGetter, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger).SetupWithManager(s.mgr); err != nil { s.logger.WithError(err).Fatal("Unable to create the data download controller") } diff --git a/pkg/controller/data_download_controller.go b/pkg/controller/data_download_controller.go index 51be1ab9b..d7e3e3e26 100644 --- a/pkg/controller/data_download_controller.go +++ b/pkg/controller/data_download_controller.go @@ -62,10 +62,11 @@ type DataDownloadReconciler struct { nodeName string repositoryEnsurer *repository.Ensurer dataPathMgr *datapath.Manager + preparingTimeout time.Duration } func NewDataDownloadReconciler(client client.Client, kubeClient kubernetes.Interface, - repoEnsurer *repository.Ensurer, credentialGetter *credentials.CredentialGetter, nodeName string, logger logrus.FieldLogger) *DataDownloadReconciler { + repoEnsurer *repository.Ensurer, credentialGetter *credentials.CredentialGetter, nodeName string, preparingTimeout time.Duration, logger logrus.FieldLogger) *DataDownloadReconciler { return &DataDownloadReconciler{ client: client, kubeClient: kubeClient, @@ -77,6 +78,7 @@ func NewDataDownloadReconciler(client client.Client, kubeClient kubernetes.Inter repositoryEnsurer: repoEnsurer, restoreExposer: exposer.NewGenericRestoreExposer(kubeClient, logger), dataPathMgr: datapath.NewManager(1), + preparingTimeout: preparingTimeout, } } @@ -143,6 +145,14 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request } log.Info("Restore is exposed") + return ctrl.Result{}, nil + } else if dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseAccepted { + if dd.Status.StartTimestamp != nil { + if time.Since(dd.Status.StartTimestamp.Time) >= r.preparingTimeout { + r.onPrepareTimeout(ctx, dd) + } + } + return ctrl.Result{}, nil } else if dd.Status.Phase == velerov2alpha1api.DataDownloadPhasePrepared { log.Info("Data download is prepared") @@ -184,7 +194,6 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request // Update status to InProgress original := dd.DeepCopy() dd.Status.Phase = velerov2alpha1api.DataDownloadPhaseInProgress - dd.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} if err := r.client.Patch(ctx, dd, client.MergeFrom(original)); err != nil { log.WithError(err).Error("Unable to update status to in progress") return ctrl.Result{}, err @@ -345,8 +354,15 @@ func (r *DataDownloadReconciler) OnDataDownloadProgress(ctx context.Context, nam // re-enqueue the previous related request once the related pod is in running status to keep going on the rest logic. and below logic will avoid handling the unwanted // pod status and also avoid block others CR handling func (r *DataDownloadReconciler) SetupWithManager(mgr ctrl.Manager) error { + s := kube.NewPeriodicalEnqueueSource(r.logger, r.client, &velerov2alpha1api.DataDownloadList{}, preparingMonitorFrequency, kube.PeriodicalEnqueueSourceOption{}) + gp := kube.NewGenericEventPredicate(func(object client.Object) bool { + dd := object.(*velerov2alpha1api.DataDownload) + return (dd.Status.Phase == velerov2alpha1api.DataDownloadPhaseAccepted) + }) + return ctrl.NewControllerManagedBy(mgr). For(&velerov2alpha1api.DataDownload{}). + Watches(s, nil, builder.WithPredicates(gp)). Watches(&source.Kind{Type: &v1.Pod{}}, kube.EnqueueRequestsFromMapUpdateFunc(r.findSnapshotRestoreForPod), builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { @@ -400,9 +416,15 @@ func (r *DataDownloadReconciler) findSnapshotRestoreForPod(podObj client.Object) requests := make([]reconcile.Request, 1) r.logger.WithField("Restore pod", pod.Name).Infof("Preparing data download %s", dd.Name) - err = r.patchDataDownload(context.Background(), dd, r.prepareDataDownload) - if err != nil { - r.logger.WithField("Restore pod", pod.Name).WithError(err).Error("unable to patch data download") + + // we don't expect anyone else update the CR during the Prepare process + updated, err := r.exclusiveUpdateDataDownload(context.Background(), dd, r.prepareDataDownload) + if err != nil || !updated { + r.logger.WithFields(logrus.Fields{ + "Datadownload": dd.Name, + "Restore pod": pod.Name, + "updated": updated, + }).WithError(err).Warn("failed to patch datadownload, prepare will halt for this datadownload") return []reconcile.Request{} } @@ -416,16 +438,6 @@ func (r *DataDownloadReconciler) findSnapshotRestoreForPod(podObj client.Object) return requests } -func (r *DataDownloadReconciler) patchDataDownload(ctx context.Context, req *velerov2alpha1api.DataDownload, mutate func(*velerov2alpha1api.DataDownload)) error { - original := req.DeepCopy() - mutate(req) - if err := r.client.Patch(ctx, req, client.MergeFrom(original)); err != nil { - return errors.Wrap(err, "error patching data download") - } - - return nil -} - func (r *DataDownloadReconciler) prepareDataDownload(ssb *velerov2alpha1api.DataDownload) { ssb.Status.Phase = velerov2alpha1api.DataDownloadPhasePrepared ssb.Status.Node = r.nodeName @@ -453,17 +465,62 @@ func (r *DataDownloadReconciler) updateStatusToFailed(ctx context.Context, dd *v } func (r *DataDownloadReconciler) acceptDataDownload(ctx context.Context, dd *velerov2alpha1api.DataDownload) (bool, error) { - updated := dd.DeepCopy() - updated.Status.Phase = velerov2alpha1api.DataDownloadPhaseAccepted + r.logger.Infof("Accepting data download %s", dd.Name) - r.logger.Infof("Accepting snapshot restore %s", dd.Name) // For all data download controller in each node-agent will try to update download CR, and only one controller will success, // and the success one could handle later logic + succeeded, err := r.exclusiveUpdateDataDownload(ctx, dd, func(dd *velerov2alpha1api.DataDownload) { + dd.Status.Phase = velerov2alpha1api.DataDownloadPhaseAccepted + dd.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} + }) + + if err != nil { + return false, err + } + + if succeeded { + r.logger.WithField("DataDownload", dd.Name).Infof("This datadownload has been accepted by %s", r.nodeName) + return true, nil + } + + r.logger.WithField("DataDownload", dd.Name).Info("This datadownload has been accepted by others") + return false, nil +} + +func (r *DataDownloadReconciler) onPrepareTimeout(ctx context.Context, dd *velerov2alpha1api.DataDownload) { + log := r.logger.WithField("DataDownload", dd.Name) + + log.Info("Timeout happened for preparing datadownload") + + succeeded, err := r.exclusiveUpdateDataDownload(ctx, dd, func(dd *velerov2alpha1api.DataDownload) { + dd.Status.Phase = velerov2alpha1api.DataDownloadPhaseFailed + dd.Status.Message = "timeout on preparing data download" + }) + + if err != nil { + log.WithError(err).Warn("Failed to update datadownload") + return + } + + if !succeeded { + log.Warn("Dataupload has been updated by others") + return + } + + r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd)) + + log.Info("Dataupload has been cleaned up") +} + +func (r *DataDownloadReconciler) exclusiveUpdateDataDownload(ctx context.Context, dd *velerov2alpha1api.DataDownload, + updateFunc func(*velerov2alpha1api.DataDownload)) (bool, error) { + updated := dd.DeepCopy() + updateFunc(updated) + err := r.client.Update(ctx, updated) if err == nil { return true, nil } else if apierrors.IsConflict(err) { - r.logger.WithField("DataDownload", dd.Name).Error("This data download restore has been accepted by others") return false, nil } else { return false, err diff --git a/pkg/controller/data_download_controller_test.go b/pkg/controller/data_download_controller_test.go index 773112207..8447a9eb3 100644 --- a/pkg/controller/data_download_controller_test.go +++ b/pkg/controller/data_download_controller_test.go @@ -29,6 +29,7 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" clientgofake "k8s.io/client-go/kubernetes/fake" @@ -65,6 +66,29 @@ func dataDownloadBuilder() *builder.DataDownloadBuilder { } func initDataDownloadReconciler(objects []runtime.Object, needError ...bool) (*DataDownloadReconciler, error) { + var errs []error = make([]error, 4) + if len(needError) == 4 { + if needError[0] { + errs[0] = fmt.Errorf("Get error") + } + + if needError[1] { + errs[1] = fmt.Errorf("Create error") + } + + if needError[2] { + errs[2] = fmt.Errorf("Update error") + } + + if needError[3] { + errs[3] = fmt.Errorf("Patch error") + } + } + + return initDataDownloadReconcilerWithError(objects, errs...) +} + +func initDataDownloadReconcilerWithError(objects []runtime.Object, needError ...error) (*DataDownloadReconciler, error) { scheme := runtime.NewScheme() err := velerov1api.AddToScheme(scheme) if err != nil { @@ -112,7 +136,7 @@ func initDataDownloadReconciler(objects []runtime.Object, needError ...bool) (*D if err != nil { return nil, err } - return NewDataDownloadReconciler(fakeClient, fakeKubeClient, nil, &credentials.CredentialGetter{FromFile: credentialFileStore}, "test_node", velerotest.NewLogger()), nil + return NewDataDownloadReconciler(fakeClient, fakeKubeClient, nil, &credentials.CredentialGetter{FromFile: credentialFileStore}, "test_node", time.Minute*5, velerotest.NewLogger()), nil } func TestDataDownloadReconcile(t *testing.T) { @@ -132,6 +156,7 @@ func TestDataDownloadReconcile(t *testing.T) { notMockCleanUp bool mockCancel bool mockClose bool + expected *velerov2alpha1api.DataDownload expectedStatusMsg string expectedResult *ctrl.Result }{ @@ -215,7 +240,7 @@ func TestDataDownloadReconcile(t *testing.T) { dd: builder.ForDataDownload("test-ns", dataDownloadName).Result(), targetPVC: builder.ForPersistentVolumeClaim("test-ns", "test-pvc").Result(), needErrs: []bool{true, false, false, false}, - expectedStatusMsg: "Create error", + expectedStatusMsg: "Get error", }, { name: "Unsupported dataDownload type", @@ -246,6 +271,11 @@ func TestDataDownloadReconcile(t *testing.T) { expectedStatusMsg: "Error to expose restore exposer", isExposeErr: true, }, + { + name: "prepare timeout", + dd: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseAccepted).StartTimestamp(&metav1.Time{Time: time.Now().Add(-time.Minute * 5)}).Result(), + expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseFailed).Result(), + }, } for _, test := range tests { @@ -345,6 +375,11 @@ func TestDataDownloadReconcile(t *testing.T) { Namespace: test.dd.Namespace, }, &dd) + if test.expected != nil { + require.NoError(t, err) + assert.Equal(t, dd.Status.Phase, test.expected.Status.Phase) + } + if test.isGetExposeErr { assert.Contains(t, dd.Status.Message, test.expectedStatusMsg) } @@ -580,3 +615,93 @@ func TestFindDataDownloadForPod(t *testing.T) { } } } + +func TestAcceptDataDownload(t *testing.T) { + tests := []struct { + name string + dd *velerov2alpha1api.DataDownload + needErrs []error + succeeded bool + expectedErr string + }{ + { + name: "update fail", + dd: dataDownloadBuilder().Result(), + needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, + expectedErr: "fake-update-error", + }, + { + name: "accepted by others", + dd: dataDownloadBuilder().Result(), + needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, + }, + { + name: "succeed", + dd: dataDownloadBuilder().Result(), + needErrs: []error{nil, nil, nil, nil}, + succeeded: true, + }, + } + for _, test := range tests { + ctx := context.Background() + r, err := initDataDownloadReconcilerWithError(nil, test.needErrs...) + require.NoError(t, err) + + err = r.client.Create(ctx, test.dd) + require.NoError(t, err) + + succeeded, err := r.acceptDataDownload(ctx, test.dd) + assert.Equal(t, test.succeeded, succeeded) + if test.expectedErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, test.expectedErr) + } + } +} + +func TestOnDdPrepareTimeout(t *testing.T) { + tests := []struct { + name string + dd *velerov2alpha1api.DataDownload + needErrs []error + expected *velerov2alpha1api.DataDownload + }{ + { + name: "update fail", + dd: dataDownloadBuilder().Result(), + needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, + expected: dataDownloadBuilder().Result(), + }, + { + name: "update interrupted", + dd: dataDownloadBuilder().Result(), + needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, + expected: dataDownloadBuilder().Result(), + }, + { + name: "succeed", + dd: dataDownloadBuilder().Result(), + needErrs: []error{nil, nil, nil, nil}, + expected: dataDownloadBuilder().Phase(velerov2alpha1api.DataDownloadPhaseFailed).Result(), + }, + } + for _, test := range tests { + ctx := context.Background() + r, err := initDataDownloadReconcilerWithError(nil, test.needErrs...) + require.NoError(t, err) + + err = r.client.Create(ctx, test.dd) + require.NoError(t, err) + + r.onPrepareTimeout(ctx, test.dd) + + dd := velerov2alpha1api.DataDownload{} + _ = r.client.Get(ctx, kbclient.ObjectKey{ + Name: test.dd.Name, + Namespace: test.dd.Namespace, + }, &dd) + + assert.Equal(t, test.expected.Status.Phase, dd.Status.Phase) + } +} diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 101c08370..49ca428e1 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -55,6 +55,8 @@ import ( const dataMoverType string = "velero" const dataUploadDownloadRequestor string = "snapshot-data-upload-download" +const preparingMonitorFrequency time.Duration = time.Minute + // DataUploadReconciler reconciles a DataUpload object type DataUploadReconciler struct { client client.Client @@ -68,11 +70,12 @@ type DataUploadReconciler struct { logger logrus.FieldLogger snapshotExposerList map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer dataPathMgr *datapath.Manager + preparingTimeout time.Duration } func NewDataUploadReconciler(client client.Client, kubeClient kubernetes.Interface, csiSnapshotClient snapshotter.SnapshotV1Interface, repoEnsurer *repository.Ensurer, clock clocks.WithTickerAndDelayedExecution, - cred *credentials.CredentialGetter, nodeName string, fs filesystem.Interface, log logrus.FieldLogger) *DataUploadReconciler { + cred *credentials.CredentialGetter, nodeName string, fs filesystem.Interface, preparingTimeout time.Duration, log logrus.FieldLogger) *DataUploadReconciler { return &DataUploadReconciler{ client: client, kubeClient: kubeClient, @@ -85,6 +88,7 @@ func NewDataUploadReconciler(client client.Client, kubeClient kubernetes.Interfa repoEnsurer: repoEnsurer, snapshotExposerList: map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{velerov2alpha1api.SnapshotTypeCSI: exposer.NewCSISnapshotExposer(kubeClient, csiSnapshotClient, log)}, dataPathMgr: datapath.NewManager(1), + preparingTimeout: preparingTimeout, } } @@ -143,6 +147,14 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) // ep.Expose() will trigger to create one pod whose volume is restored by a given volume snapshot, // but the pod maybe is not in the same node of the current controller, so we need to return it here. // And then only the controller who is in the same node could do the rest work. + return ctrl.Result{}, nil + } else if du.Status.Phase == velerov2alpha1api.DataUploadPhaseAccepted { + if du.Status.StartTimestamp != nil { + if time.Since(du.Status.StartTimestamp.Time) >= r.preparingTimeout { + r.onPrepareTimeout(ctx, &du) + } + } + return ctrl.Result{}, nil } else if du.Status.Phase == velerov2alpha1api.DataUploadPhasePrepared { log.Info("Data upload is prepared") @@ -183,7 +195,6 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Update status to InProgress original := du.DeepCopy() du.Status.Phase = velerov2alpha1api.DataUploadPhaseInProgress - du.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} if err := r.client.Patch(ctx, &du, client.MergeFrom(original)); err != nil { return r.errorOut(ctx, &du, err, "error updating dataupload status", log) } @@ -363,8 +374,15 @@ func (r *DataUploadReconciler) OnDataUploadProgress(ctx context.Context, namespa // re-enqueue the previous related request once the related pod is in running status to keep going on the rest logic. and below logic will avoid handling the unwanted // pod status and also avoid block others CR handling func (r *DataUploadReconciler) SetupWithManager(mgr ctrl.Manager) error { + s := kube.NewPeriodicalEnqueueSource(r.logger, r.client, &velerov2alpha1api.DataUploadList{}, preparingMonitorFrequency, kube.PeriodicalEnqueueSourceOption{}) + gp := kube.NewGenericEventPredicate(func(object client.Object) bool { + du := object.(*velerov2alpha1api.DataUpload) + return (du.Status.Phase == velerov2alpha1api.DataUploadPhaseAccepted) + }) + return ctrl.NewControllerManagedBy(mgr). For(&velerov2alpha1api.DataUpload{}). + Watches(s, nil, builder.WithPredicates(gp)). Watches(&source.Kind{Type: &corev1.Pod{}}, kube.EnqueueRequestsFromMapUpdateFunc(r.findDataUploadForPod), builder.WithPredicates(predicate.Funcs{ UpdateFunc: func(ue event.UpdateEvent) bool { @@ -416,8 +434,15 @@ func (r *DataUploadReconciler) findDataUploadForPod(podObj client.Object) []reco } r.logger.WithField("Backup pod", pod.Name).Infof("Preparing dataupload %s", du.Name) - if err := r.patchDataUpload(context.Background(), du, r.prepareDataUpload); err != nil { - r.logger.WithField("Backup pod", pod.Name).WithError(err).Error("failed to patch dataupload") + + // we don't expect anyone else update the CR during the Prepare process + updated, err := r.exclusiveUpdateDataUpload(context.Background(), du, r.prepareDataUpload) + if err != nil || !updated { + r.logger.WithFields(logrus.Fields{ + "Dataupload": du.Name, + "Backup pod": pod.Name, + "updated": updated, + }).WithError(err).Warn("failed to patch dataupload, prepare will halt for this dataupload") return []reconcile.Request{} } @@ -430,16 +455,6 @@ func (r *DataUploadReconciler) findDataUploadForPod(podObj client.Object) []reco return []reconcile.Request{requests} } -func (r *DataUploadReconciler) patchDataUpload(ctx context.Context, req *velerov2alpha1api.DataUpload, mutate func(*velerov2alpha1api.DataUpload)) error { - original := req.DeepCopy() - mutate(req) - if err := r.client.Patch(ctx, req, client.MergeFrom(original)); err != nil { - return errors.Wrap(err, "error patching DataUpload") - } - - return nil -} - func (r *DataUploadReconciler) prepareDataUpload(du *velerov2alpha1api.DataUpload) { du.Status.Phase = velerov2alpha1api.DataUploadPhasePrepared du.Status.Node = r.nodeName @@ -475,19 +490,73 @@ func (r *DataUploadReconciler) updateStatusToFailed(ctx context.Context, du *vel } func (r *DataUploadReconciler) acceptDataUpload(ctx context.Context, du *velerov2alpha1api.DataUpload) (bool, error) { - updated := du.DeepCopy() - updated.Status.Phase = velerov2alpha1api.DataUploadPhaseAccepted - - r.logger.Infof("Accepting snapshot backup %s", du.Name) + r.logger.Infof("Accepting data upload %s", du.Name) // For all data upload controller in each node-agent will try to update dataupload CR, and only one controller will success, // and the success one could handle later logic + succeeded, err := r.exclusiveUpdateDataUpload(ctx, du, func(du *velerov2alpha1api.DataUpload) { + du.Status.Phase = velerov2alpha1api.DataUploadPhaseAccepted + du.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} + }) + + if err != nil { + return false, err + } + + if succeeded { + r.logger.WithField("Dataupload", du.Name).Infof("This datauplod has been accepted by %s", r.nodeName) + return true, nil + } + + r.logger.WithField("Dataupload", du.Name).Info("This datauplod has been accepted by others") + return false, nil +} + +func (r *DataUploadReconciler) onPrepareTimeout(ctx context.Context, du *velerov2alpha1api.DataUpload) { + log := r.logger.WithField("Dataupload", du.Name) + + log.Info("Timeout happened for preparing dataupload") + + succeeded, err := r.exclusiveUpdateDataUpload(ctx, du, func(du *velerov2alpha1api.DataUpload) { + du.Status.Phase = velerov2alpha1api.DataUploadPhaseFailed + du.Status.Message = "timeout on preparing data upload" + }) + + if err != nil { + log.WithError(err).Warn("Failed to update dataupload") + return + } + + if !succeeded { + log.Warn("Dataupload has been updated by others") + return + } + + ep, ok := r.snapshotExposerList[du.Spec.SnapshotType] + if !ok { + log.WithError(fmt.Errorf("%v type of snapshot exposer is not exist", du.Spec.SnapshotType)). + Warn("Failed to clean up resources on canceled") + } else { + var volumeSnapshotName string + if du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { // Other exposer should have another condition + volumeSnapshotName = du.Spec.CSISnapshot.VolumeSnapshot + } + + ep.CleanUp(ctx, getOwnerObject(du), volumeSnapshotName, du.Spec.SourceNamespace) + + log.Info("Dataupload has been cleaned up") + } +} + +func (r *DataUploadReconciler) exclusiveUpdateDataUpload(ctx context.Context, du *velerov2alpha1api.DataUpload, + updateFunc func(*velerov2alpha1api.DataUpload)) (bool, error) { + updated := du.DeepCopy() + updateFunc(updated) + err := r.client.Update(ctx, updated) if err == nil { - r.logger.WithField("Dataupload", du.Name).Infof("This datauplod backup has been accepted by %s", r.nodeName) return true, nil } else if apierrors.IsConflict(err) { - r.logger.WithField("Dataupload", du.Name).Info("This datauplod backup has been accepted by others") return false, nil } else { return false, err diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index 654e07531..e7a3b476f 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -58,45 +58,68 @@ const fakeSnapshotType velerov2alpha1api.SnapshotType = "fake-snapshot" type FakeClient struct { kbclient.Client - getError bool - createError bool - updateError bool - patchError bool + getError error + createError error + updateError error + patchError error } func (c *FakeClient) Get(ctx context.Context, key kbclient.ObjectKey, obj kbclient.Object) error { - if c.getError { - return fmt.Errorf("Create error") + if c.getError != nil { + return c.getError } return c.Client.Get(ctx, key, obj) } func (c *FakeClient) Create(ctx context.Context, obj kbclient.Object, opts ...kbclient.CreateOption) error { - if c.createError { - return fmt.Errorf("Create error") + if c.createError != nil { + return c.createError } return c.Client.Create(ctx, obj, opts...) } func (c *FakeClient) Update(ctx context.Context, obj kbclient.Object, opts ...kbclient.UpdateOption) error { - if c.updateError { - return fmt.Errorf("Update error") + if c.updateError != nil { + return c.updateError } return c.Client.Update(ctx, obj, opts...) } func (c *FakeClient) Patch(ctx context.Context, obj kbclient.Object, patch kbclient.Patch, opts ...kbclient.PatchOption) error { - if c.patchError { - return fmt.Errorf("Patch error") + if c.patchError != nil { + return c.patchError } return c.Client.Patch(ctx, obj, patch, opts...) } func initDataUploaderReconciler(needError ...bool) (*DataUploadReconciler, error) { + var errs []error = make([]error, 4) + if len(needError) == 4 { + if needError[0] { + errs[0] = fmt.Errorf("Get error") + } + + if needError[1] { + errs[1] = fmt.Errorf("Create error") + } + + if needError[2] { + errs[2] = fmt.Errorf("Update error") + } + + if needError[3] { + errs[3] = fmt.Errorf("Patch error") + } + } + + return initDataUploaderReconcilerWithError(errs...) +} + +func initDataUploaderReconcilerWithError(needError ...error) (*DataUploadReconciler, error) { vscName := "fake-vsc" vsObject := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ @@ -170,7 +193,7 @@ func initDataUploaderReconciler(needError ...bool) (*DataUploadReconciler, error return nil, err } return NewDataUploadReconciler(fakeClient, fakeKubeClient, fakeSnapshotClient.SnapshotV1(), nil, - testclocks.NewFakeClock(now), &credentials.CredentialGetter{FromFile: credentialFileStore}, "test_node", fakeFS, velerotest.NewLogger()), nil + testclocks.NewFakeClock(now), &credentials.CredentialGetter{FromFile: credentialFileStore}, "test_node", fakeFS, time.Minute*5, velerotest.NewLogger()), nil } func dataUploadBuilder() *builder.DataUploadBuilder { @@ -277,7 +300,7 @@ func TestReconcile(t *testing.T) { expectedProcessed: false, expected: nil, expectedRequeue: ctrl.Result{}, - expectedErrMsg: "getting DataUpload: Create error", + expectedErrMsg: "getting DataUpload: Get error", needErrs: []bool{true, false, false, false}, }, { name: "Unsupported data mover type", @@ -339,6 +362,11 @@ func TestReconcile(t *testing.T) { expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhasePrepared).Result(), expectedRequeue: ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, }, + { + name: "prepare timeout", + du: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).SnapshotType(fakeSnapshotType).StartTimestamp(&metav1.Time{Time: time.Now().Add(-time.Minute * 5)}).Result(), + expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), + }, } for _, test := range tests { @@ -599,3 +627,107 @@ func TestFindDataUploadForPod(t *testing.T) { } } } + +type fakeAPIStatus struct { + reason metav1.StatusReason +} + +func (f *fakeAPIStatus) Status() metav1.Status { + return metav1.Status{ + Reason: f.reason, + } +} + +func (f *fakeAPIStatus) Error() string { + return string(f.reason) +} + +func TestAcceptDataUpload(t *testing.T) { + tests := []struct { + name string + du *velerov2alpha1api.DataUpload + needErrs []error + succeeded bool + expectedErr string + }{ + { + name: "update fail", + du: dataUploadBuilder().Result(), + needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, + expectedErr: "fake-update-error", + }, + { + name: "accepted by others", + du: dataUploadBuilder().Result(), + needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, + }, + { + name: "succeed", + du: dataUploadBuilder().Result(), + needErrs: []error{nil, nil, nil, nil}, + succeeded: true, + }, + } + for _, test := range tests { + ctx := context.Background() + r, err := initDataUploaderReconcilerWithError(test.needErrs...) + require.NoError(t, err) + + err = r.client.Create(ctx, test.du) + require.NoError(t, err) + + succeeded, err := r.acceptDataUpload(ctx, test.du) + assert.Equal(t, test.succeeded, succeeded) + if test.expectedErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, test.expectedErr) + } + } +} + +func TestOnDuPrepareTimeout(t *testing.T) { + tests := []struct { + name string + du *velerov2alpha1api.DataUpload + needErrs []error + expected *velerov2alpha1api.DataUpload + }{ + { + name: "update fail", + du: dataUploadBuilder().Result(), + needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, + expected: dataUploadBuilder().Result(), + }, + { + name: "update interrupted", + du: dataUploadBuilder().Result(), + needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, + expected: dataUploadBuilder().Result(), + }, + { + name: "succeed", + du: dataUploadBuilder().Result(), + needErrs: []error{nil, nil, nil, nil}, + expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), + }, + } + for _, test := range tests { + ctx := context.Background() + r, err := initDataUploaderReconcilerWithError(test.needErrs...) + require.NoError(t, err) + + err = r.client.Create(ctx, test.du) + require.NoError(t, err) + + r.onPrepareTimeout(ctx, test.du) + + du := velerov2alpha1api.DataUpload{} + _ = r.client.Get(ctx, kbclient.ObjectKey{ + Name: test.du.Name, + Namespace: test.du.Namespace, + }, &du) + + assert.Equal(t, test.expected.Status.Phase, du.Status.Phase) + } +}