Use velero debug command to dump the logs when case failed

Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com>
This commit is contained in:
Wenkai Yin(尹文开)
2021-09-14 10:15:23 +08:00
parent b9eb6fdb33
commit 64229ed57c
5 changed files with 28 additions and 24 deletions

View File

@@ -251,7 +251,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
err = veleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false)
if err != nil {
veleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup)
runDebug(context.Background(), veleroCLI, veleroNamespace, backup, "")
return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr)
}
@@ -290,7 +290,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
if tc.want != nil {
if err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil {
veleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restore)
runDebug(context.Background(), veleroCLI, veleroNamespace, "", restore)
return errors.Wrapf(err, "restore %s namespaces on target cluster", namespacesStr)
}

View File

@@ -42,7 +42,7 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
}
defer func() {
if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
if err := deleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace))
}
}()
@@ -51,7 +51,7 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac
}
if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil {
veleroBackupLogs(oneHourTimeout, veleroCLI, veleroNamespace, backupName)
runDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
}
@@ -69,7 +69,7 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac
}
if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
veleroRestoreLogs(oneHourTimeout, veleroCLI, veleroNamespace, restoreName)
runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
}

View File

@@ -109,7 +109,7 @@ func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName
}
}
if err := veleroBackupExcludeNamespaces(ctx, veleroCLI, veleroNamespace, backupName, excludeNamespaces); err != nil {
veleroBackupLogs(ctx, veleroCLI, veleroNamespace, backupName)
runDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
return errors.Wrapf(err, "Failed to backup backup namespaces %s-*", nsBaseName)
}
@@ -120,7 +120,7 @@ func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName
err = veleroRestore(ctx, veleroCLI, veleroNamespace, restoreName, backupName)
if err != nil {
veleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restoreName)
runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
return errors.Wrap(err, "Restore failed")
}

View File

@@ -107,7 +107,7 @@ func runUpgradeTests(client testClient, upgradeToVeleroImage, upgradeToVeleroVer
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", upgradeNamespace)
}
defer func() {
if err := deleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil {
if err := deleteNamespace(context.Background(), client, upgradeNamespace, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", upgradeNamespace))
}
}()
@@ -116,7 +116,9 @@ func runUpgradeTests(client testClient, upgradeToVeleroImage, upgradeToVeleroVer
}
if err := veleroBackupNamespace(oneHourTimeout, upgradeFromVeleroCLI, veleroNamespace, backupName, upgradeNamespace, backupLocation, useVolumeSnapshots); err != nil {
veleroBackupLogs(oneHourTimeout, upgradeFromVeleroCLI, veleroNamespace, backupName)
// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
// TODO move to "runDebug" after we bump up to 1.7 in the upgrade case
veleroBackupLogs(context.Background(), upgradeFromVeleroCLI, veleroNamespace, backupName)
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", upgradeNamespace)
}
@@ -140,7 +142,7 @@ func runUpgradeTests(client testClient, upgradeToVeleroImage, upgradeToVeleroVer
return errors.Wrapf(err, "Velero install version mismatch.")
}
if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
veleroRestoreLogs(oneHourTimeout, veleroCLI, veleroNamespace, restoreName)
runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
}

View File

@@ -269,22 +269,24 @@ func veleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace str
return nil
}
func veleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error {
describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "describe", restoreName)
describeCmd.Stdout = os.Stdout
describeCmd.Stderr = os.Stderr
err := describeCmd.Run()
if err != nil {
return err
func runDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore string) {
output := fmt.Sprintf("debug-bundle-%d.tar.gz", time.Now().UnixNano())
args := []string{"debug", "--namespace", veleroNamespace, "--output", output, "--verbose"}
if len(backup) > 0 {
args = append(args, "--backup", backup)
}
logCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "logs", restoreName)
logCmd.Stdout = os.Stdout
logCmd.Stderr = os.Stderr
err = logCmd.Run()
if err != nil {
return err
if len(restore) > 0 {
args = append(args, "--restore", restore)
}
cmd := exec.CommandContext(ctx, veleroCLI, args...)
cmd.Stdout = os.Stdout
cmd.Stdin = os.Stdin
fmt.Printf("debug cmd=%s\n", cmd.String())
fmt.Printf("Generating the debug tarball at %s\n", output)
if err := cmd.Run(); err != nil {
fmt.Println(errors.Wrapf(err, "failed to run the debug command"))
}
return nil
}
func veleroCreateBackupLocation(ctx context.Context,