testing: Recover describe/log functions from panic when backup fails (#733)

* handle panic for describe functions when backup fails

* reduce velero logs panic recover repeated code

Author: kaovilai

tests/e2e/lib/velero_helpers.go

@@ -3,6 +3,7 @@ package lib
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"log"
 	"regexp"
 	"strings"
@@ -33,7 +34,7 @@ func GetVeleroClient() (veleroClientset.Interface, error) {
 }
 
 // https://github.com/vmware-tanzu/velero/blob/11bfe82342c9f54c63f40d3e97313ce763b446f2/pkg/cmd/cli/backup/describe.go#L77-L111
-func DescribeBackup(ocClient client.Client, backup velero.Backup) string {
+func DescribeBackup(ocClient client.Client, backup velero.Backup) (backupDescription string) {
 	err := ocClient.Get(context.Background(), client.ObjectKey{
 		Namespace: backup.Namespace,
 		Name:      backup.Name,
@@ -75,7 +76,15 @@ func DescribeBackup(ocClient client.Client, backup velero.Backup) string {
 			log.Printf("error getting VolumeSnapshotContent objects for backup %s: %v\n", backup.Name, err)
 		}
 	}
-
+	// output.DescribeBackup is a helper function from velero CLI that attempts to download logs for a backup.
+	// if a backup failed, this function may panic. Recover from the panic and return string of backup object
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("Recovered from panic in DescribeBackup: %v\n", r)
+			log.Print("returning backup object instead")
+			backupDescription = fmt.Sprint(backup)
+		}
+	}()
 	return output.DescribeBackup(context.Background(), ocClient, &backup, deleteRequestList.Items, podVolumeBackupList.Items, vscList.Items, details, veleroClient, insecureSkipTLSVerify, caCertFile)
 }
 
@@ -104,25 +113,36 @@ func DescribeRestore(ocClient client.Client, restore velero.Restore) string {
 	return output.DescribeRestore(context.Background(), ocClient, &restore, podvolumeRestoreList.Items, details, veleroClient, insecureSkipTLSVerify, caCertFile)
 }
 
-func BackupLogs(ocClient client.Client, backup velero.Backup) string {
+func BackupLogs(ocClient client.Client, backup velero.Backup) (backupLogs string) {
 	insecureSkipTLSVerify := true
 	caCertFile := ""
 	// new io.Writer that store the logs in a string
 	logs := &bytes.Buffer{}
 	// new io.Writer that store the logs in a string
-
+	// if a backup failed, this function may panic. Recover from the panic and return container logs
+	defer func() {
+		if r := recover(); r != nil {
+			backupLogs = recoverFromPanicLogs(backup.Namespace, r, "BackupLogs")
+			
+		}
+	}()
 	downloadrequest.Stream(context.Background(), ocClient, backup.Namespace, backup.Name, velero.DownloadTargetKindBackupLog, logs, time.Minute, insecureSkipTLSVerify, caCertFile)
 
 	return logs.String()
 }
 
-func RestoreLogs(ocClient client.Client, restore velero.Restore) string {
+func RestoreLogs(ocClient client.Client, restore velero.Restore) (restoreLogs string) {
 	insecureSkipTLSVerify := true
 	caCertFile := ""
 	// new io.Writer that store the logs in a string
 	logs := &bytes.Buffer{}
 	// new io.Writer that store the logs in a string
-
+	// if a backup failed, this function may panic. Recover from the panic and return container logs
+	defer func() {
+		if r := recover(); r != nil {
+			restoreLogs = recoverFromPanicLogs(restore.Namespace, r, "RestoreLogs")
+		}
+	}()
 	downloadrequest.Stream(context.Background(), ocClient, restore.Namespace, restore.Name, velero.DownloadTargetKindRestoreLog, logs, time.Minute, insecureSkipTLSVerify, caCertFile)
 
 	return logs.String()
@@ -136,6 +156,16 @@ var errorIgnorePatterns = []string{
 	"blob unknown",
 }
 
+func recoverFromPanicLogs(veleroNamespace string, panicReason interface{}, panicFrom string) string {
+	log.Printf("Recovered from panic in %s: %v\n", panicFrom, panicReason)
+	log.Print("returning container logs instead")
+	containerLogs, err := GetVeleroContainerLogs(veleroNamespace)
+	if err != nil {
+		log.Printf("error getting container logs: %v\n", err)
+	}
+	return containerLogs
+}
+
 func BackupErrorLogs(ocClient client.Client, backup velero.Backup) []string {
 	bl := BackupLogs(ocClient, backup)
 	errorRegex, err := regexp.Compile("error|Error")