From 1f7876863c7248d3532ce827fac1524fac9a880d Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 29 Apr 2022 19:40:10 -0400 Subject: [PATCH 01/34] Print namespace events when installing apps fail This is useful for debugging pods not starting up or stuck. Example event that would come up before #650 is merged ``` Event: Error: couldn't find key access_key in Secret openshift-adp/oadp-ts-example-velero-1-aws-registry-secret, Src: kubelet, Reason: Failed ``` --- tests/e2e/backup_restore_suite_test.go | 18 +++++++++++++++--- tests/e2e/dpa_deployment_suite_test.go | 12 ++++++++++++ tests/e2e/lib/apps.go | 20 ++++++++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 2b405650dc..eb638e6238 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -33,6 +33,16 @@ var _ = Describe("AWS backup restore tests", func() { Expect(err).ToNot(HaveOccurred()) }) + var lastInstallingApplicationNamespace string + var lastInstallTime time.Time + var _ = ReportAfterEach(func(report SpecReport){ + if report.Failed() { + // print namespace error events for app namespace + if lastInstallingApplicationNamespace != "" { + PrintNamespaceEventsAfterTime(lastInstallingApplicationNamespace, lastInstallTime) + } + } + }) type BackupRestoreCase struct { ApplicationTemplate string @@ -67,10 +77,11 @@ var _ = Describe("AWS backup restore tests", func() { DescribeTable("backup and restore applications", func(brCase BackupRestoreCase, expectedErr error) { - err := dpaCR.Build(brCase.BackupRestoreType) Expect(err).NotTo(HaveOccurred()) + lastInstallingApplicationNamespace = dpaCR.Namespace + lastInstallTime = time.Now() err = dpaCR.CreateOrUpdate(&dpaCR.CustomResource.Spec) Expect(err).NotTo(HaveOccurred()) @@ -81,9 +92,8 @@ var _ = Describe("AWS backup restore tests", func() { log.Printf("Waiting for restic pods to be running") Eventually(AreResticPodsRunning(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) } - if brCase.BackupRestoreType == CSI { - log.Printf("Creating VolumeSnapshot for CSI backuprestore of %s", brCase.Name) + log.Printf("Creating VolumeSnapshotClass for CSI backuprestore of %s", brCase.Name) err = InstallApplication(dpaCR.Client, "./sample-applications/gp2-csi/volumeSnapshotClass.yaml") Expect(err).ToNot(HaveOccurred()) } @@ -101,6 +111,8 @@ var _ = Describe("AWS backup restore tests", func() { restoreName := fmt.Sprintf("%s-%s", brCase.Name, restoreUid.String()) // install app + lastInstallingApplicationNamespace = brCase.ApplicationNamespace + lastInstallTime = time.Now() log.Printf("Installing application for case %s", brCase.Name) err = InstallApplication(dpaCR.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) diff --git a/tests/e2e/dpa_deployment_suite_test.go b/tests/e2e/dpa_deployment_suite_test.go index 4c7a88f9a5..dd83bf4292 100644 --- a/tests/e2e/dpa_deployment_suite_test.go +++ b/tests/e2e/dpa_deployment_suite_test.go @@ -485,6 +485,16 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() { if provider == "aws" { genericTests = append(genericTests, awsTests...) } + var lastInstallingApplicationNamespace string + var lastInstallTime time.Time + var _ = ReportAfterEach(func(report SpecReport){ + if report.Failed() { + // print namespace error events for app namespace + if lastInstallingApplicationNamespace != "" { + PrintNamespaceEventsAfterTime(lastInstallingApplicationNamespace, lastInstallTime) + } + } + }) DescribeTable("Updating custom resource with new configuration", func(installCase InstallCase, expectedErr error) { @@ -498,6 +508,8 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() { installCase.DpaSpec.BackupLocations[0].Velero.Config["credentialsFile"] = "bsl-cloud-credentials-" + dpaCR.Provider + "/cloud" } } + lastInstallingApplicationNamespace = dpaCR.Namespace + lastInstallTime = time.Now() err = dpaCR.CreateOrUpdate(installCase.DpaSpec) Expect(err).ToNot(HaveOccurred()) if installCase.WantError { diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 9261db2b99..677ece2d0d 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -191,6 +191,26 @@ func AreApplicationPodsRunning(namespace string) wait.ConditionFunc { } } +func PrintNamespaceEventsAfterTime(namespace string, startTime time.Time) { + log.Println("Printing events for namespace: ", namespace) + clientset, err := setUpClient() + if err != nil { + ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("Error getting client: %v\n", err))) + return + } + events, err := clientset.CoreV1().Events(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("Error getting events: %v\n", err))) + return + } + for _, event := range events.Items { + // only get events before 10 seconds ago + if event.LastTimestamp.Before(startTime) { + ginkgo.GinkgoWriter.Println(fmt.Sprintf("Event: %v, Src: %v, Reason: %v", event.Message, event.Source.Component, event.Reason)) + } + } +} + func RunMustGather(oc_cli string, artifact_dir string) error { ocClient := oc_cli ocAdmin := "adm" From dc47683cdd11f297a45ec1ed796db22bcfd517a2 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 29 Apr 2022 19:46:59 -0400 Subject: [PATCH 02/34] use metav1 Time --- tests/e2e/lib/apps.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 677ece2d0d..76a2637312 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -205,7 +205,7 @@ func PrintNamespaceEventsAfterTime(namespace string, startTime time.Time) { } for _, event := range events.Items { // only get events before 10 seconds ago - if event.LastTimestamp.Before(startTime) { + if event.LastTimestamp.Before(&metav1.Time{Time: startTime}) { ginkgo.GinkgoWriter.Println(fmt.Sprintf("Event: %v, Src: %v, Reason: %v", event.Message, event.Source.Component, event.Reason)) } } From c27cb1329e0bf9686fa9105321a23ba2600946e3 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 29 Apr 2022 19:52:11 -0400 Subject: [PATCH 03/34] format --- tests/e2e/backup_restore_suite_test.go | 2 +- tests/e2e/dpa_deployment_suite_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index eb638e6238..33e0c48451 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -35,7 +35,7 @@ var _ = Describe("AWS backup restore tests", func() { }) var lastInstallingApplicationNamespace string var lastInstallTime time.Time - var _ = ReportAfterEach(func(report SpecReport){ + var _ = ReportAfterEach(func(report SpecReport) { if report.Failed() { // print namespace error events for app namespace if lastInstallingApplicationNamespace != "" { diff --git a/tests/e2e/dpa_deployment_suite_test.go b/tests/e2e/dpa_deployment_suite_test.go index dd83bf4292..5617cadc62 100644 --- a/tests/e2e/dpa_deployment_suite_test.go +++ b/tests/e2e/dpa_deployment_suite_test.go @@ -487,7 +487,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() { } var lastInstallingApplicationNamespace string var lastInstallTime time.Time - var _ = ReportAfterEach(func(report SpecReport){ + var _ = ReportAfterEach(func(report SpecReport) { if report.Failed() { // print namespace error events for app namespace if lastInstallingApplicationNamespace != "" { From 520c78f44cb77d2d89dcf46b2dbd3a50bb2493d4 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 29 Apr 2022 19:52:22 -0400 Subject: [PATCH 04/34] Get events after time --- tests/e2e/lib/apps.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 76a2637312..cf4f456d35 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -204,8 +204,8 @@ func PrintNamespaceEventsAfterTime(namespace string, startTime time.Time) { return } for _, event := range events.Items { - // only get events before 10 seconds ago - if event.LastTimestamp.Before(&metav1.Time{Time: startTime}) { + // only get events after time + if event.LastTimestamp.After(startTime) { ginkgo.GinkgoWriter.Println(fmt.Sprintf("Event: %v, Src: %v, Reason: %v", event.Message, event.Source.Component, event.Reason)) } } From a3875861b9622a215bf5a700e5544778dc73256f Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 29 Apr 2022 22:09:13 -0400 Subject: [PATCH 05/34] add sort and count to event --- tests/e2e/lib/apps.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index cf4f456d35..ad68965c0b 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "os/exec" + "sort" "time" "github.com/onsi/ginkgo/v2" @@ -203,10 +204,15 @@ func PrintNamespaceEventsAfterTime(namespace string, startTime time.Time) { ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("Error getting events: %v\n", err))) return } - for _, event := range events.Items { + eventItems := events.Items + // sort events by timestamp + sort.Slice(eventItems, func(i, j int) bool { + return eventItems[i].LastTimestamp.Before(&eventItems[j].LastTimestamp) + }) + for _, event := range eventItems { // only get events after time if event.LastTimestamp.After(startTime) { - ginkgo.GinkgoWriter.Println(fmt.Sprintf("Event: %v, Src: %v, Reason: %v", event.Message, event.Source.Component, event.Reason)) + ginkgo.GinkgoWriter.Println(fmt.Sprintf("Event: %v, Type: %v, Count: %v, Src: %v, Reason: %v", event.Message, event.Type, event.Count, event.InvolvedObject, event.Reason)) } } } From 9aa908181f940503c6cd97723d11a6c17b365956 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 29 Apr 2022 22:52:14 -0400 Subject: [PATCH 06/34] Update lastInstallTime and lastInstallApplicationNamespace for app restore --- tests/e2e/backup_restore_suite_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 33e0c48451..24b33d186f 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -148,6 +148,8 @@ var _ = Describe("AWS backup restore tests", func() { Eventually(IsNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) // run restore + lastInstallingApplicationNamespace = brCase.ApplicationNamespace + lastInstallTime = time.Now() log.Printf("Creating restore %s for case %s", restoreName, brCase.Name) err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName) Expect(err).ToNot(HaveOccurred()) From 33a62c8697957c4ee0150a55c3bdd7a2193a7b62 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 29 Apr 2022 23:30:08 -0400 Subject: [PATCH 07/34] Add restic restore workaround to e2e. Add opts for CreateRestoreFromBackup --- tests/e2e/backup_restore_suite_test.go | 56 ++++++++++++++++++++------ tests/e2e/lib/apps.go | 11 +++++ tests/e2e/lib/restore.go | 31 +++++++++++++- 3 files changed, 85 insertions(+), 13 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 24b33d186f..9c3f8fe8ae 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -124,6 +124,8 @@ var _ = Describe("AWS backup restore tests", func() { err = brCase.PreBackupVerify(dpaCR.Client, brCase.ApplicationNamespace) Expect(err).ToNot(HaveOccurred()) + hasDCsInNamespace, err := HasDCsInNamespace(dpaCR.Client, brCase.ApplicationNamespace) + Expect(err).ToNot(HaveOccurred()) // create backup log.Printf("Creating backup %s for case %s", backupName, brCase.Name) err = CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace}) @@ -147,19 +149,49 @@ var _ = Describe("AWS backup restore tests", func() { // Wait for namespace to be deleted Eventually(IsNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) - // run restore - lastInstallingApplicationNamespace = brCase.ApplicationNamespace - lastInstallTime = time.Now() - log.Printf("Creating restore %s for case %s", restoreName, brCase.Name) - err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName) - Expect(err).ToNot(HaveOccurred()) - Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) - Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) + // Check if backup needs restic deploymentconfig workaround. https://github.com/openshift/oadp-operator/blob/master/docs/TROUBLESHOOTING.md#deployconfig + if brCase.BackupRestoreType == RESTIC && hasDCsInNamespace { + log.Printf("DC found in backup namespace, using DC restic workaround") + var dcWorkaroundResources = []string{"replicationcontroller","deploymentconfig","templateinstances.template.openshift.io"} + // run restore + lastInstallingApplicationNamespace = brCase.ApplicationNamespace + lastInstallTime = time.Now() + log.Printf("Creating restore %s excluding DC workaround resources for case %s", restoreName, brCase.Name) + err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName, WithExcludedResources(dcWorkaroundResources)) + Expect(err).ToNot(HaveOccurred()) + Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) + + // Check if restore succeeded + succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName) + Expect(err).ToNot(HaveOccurred()) + Expect(succeeded).To(Equal(true)) - // Check if restore succeeded - succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName) - Expect(err).ToNot(HaveOccurred()) - Expect(succeeded).To(Equal(true)) + // run restore + log.Printf("Creating restore %s including DC workaround resources for case %s", restoreName, brCase.Name) + err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName, WithIncludedResources(dcWorkaroundResources)) + Expect(err).ToNot(HaveOccurred()) + Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) + + // Check if restore succeeded + succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName) + Expect(err).ToNot(HaveOccurred()) + Expect(succeeded).To(Equal(true)) + + } else { + // run restore + log.Printf("Creating restore %s for case %s", restoreName, brCase.Name) + err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName) + Expect(err).ToNot(HaveOccurred()) + Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) + + // Check if restore succeeded + succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName) + Expect(err).ToNot(HaveOccurred()) + Expect(succeeded).To(Equal(true)) + } // verify app is running Eventually(AreApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue()) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index ad68965c0b..4fd361d9e3 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -116,6 +116,17 @@ func UninstallApplication(ocClient client.Client, file string) error { // return true, err // } // } +func HasDCsInNamespace(ocClient client.Client, namespace string) (bool, error) { + dcList := &ocpappsv1.DeploymentConfigList{} + err := ocClient.List(context.Background(), dcList, client.InNamespace(namespace)) + if err!= nil { + return false, err + } + if len(dcList.Items) == 0 { + return false, nil + } + return true, nil +} func IsDCReady(ocClient client.Client, namespace, dcName string) wait.ConditionFunc { return func() (bool, error) { diff --git a/tests/e2e/lib/restore.go b/tests/e2e/lib/restore.go index 6f945973ab..903ad01a09 100755 --- a/tests/e2e/lib/restore.go +++ b/tests/e2e/lib/restore.go @@ -11,7 +11,33 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func CreateRestoreFromBackup(ocClient client.Client, veleroNamespace, backupName, restoreName string) error { +type restoreOpts func(*velero.Restore) + +func WithIncludedResources(resources []string) restoreOpts { + return func(restore *velero.Restore) { + restore.Spec.IncludedResources = resources + } +} + +func WithExcludedResources(resources []string) restoreOpts { + return func(restore *velero.Restore) { + restore.Spec.ExcludedResources = resources + } +} + +func WithIncludedNamespaces(namespaces []string) restoreOpts { + return func(restore *velero.Restore) { + restore.Spec.IncludedNamespaces = namespaces + } +} + +func WithExcludedNamespaces(namespaces []string) restoreOpts { + return func(restore *velero.Restore) { + restore.Spec.ExcludedNamespaces = namespaces + } +} + +func CreateRestoreFromBackup(ocClient client.Client, veleroNamespace, backupName, restoreName string, opts ...restoreOpts) error { restore := velero.Restore{ ObjectMeta: metav1.ObjectMeta{ Name: restoreName, @@ -21,6 +47,9 @@ func CreateRestoreFromBackup(ocClient client.Client, veleroNamespace, backupName BackupName: backupName, }, } + for _, opt := range opts { + opt(&restore) + } err := ocClient.Create(context.Background(), &restore) return err } From 0653ae3e8d41dfdaaefab3f27cf611c901efe3ff Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Fri, 29 Apr 2022 23:41:21 -0400 Subject: [PATCH 08/34] Implementing DC workaround in e2e https://github.com/openshift/oadp-operator/blob/master/docs/TROUBLESHOOTING.md#deployconfig --- tests/e2e/backup_restore_suite_test.go | 4 +-- tests/e2e/lib/apps.go | 41 ++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 9c3f8fe8ae..02f0b723c8 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -124,7 +124,7 @@ var _ = Describe("AWS backup restore tests", func() { err = brCase.PreBackupVerify(dpaCR.Client, brCase.ApplicationNamespace) Expect(err).ToNot(HaveOccurred()) - hasDCsInNamespace, err := HasDCsInNamespace(dpaCR.Client, brCase.ApplicationNamespace) + nsRequiresResticDCWorkaround, err := NamespaceRequiresResticDCWorkaround(dpaCR.Client, brCase.ApplicationNamespace) Expect(err).ToNot(HaveOccurred()) // create backup log.Printf("Creating backup %s for case %s", backupName, brCase.Name) @@ -150,7 +150,7 @@ var _ = Describe("AWS backup restore tests", func() { Eventually(IsNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) // Check if backup needs restic deploymentconfig workaround. https://github.com/openshift/oadp-operator/blob/master/docs/TROUBLESHOOTING.md#deployconfig - if brCase.BackupRestoreType == RESTIC && hasDCsInNamespace { + if brCase.BackupRestoreType == RESTIC && nsRequiresResticDCWorkaround { log.Printf("DC found in backup namespace, using DC restic workaround") var dcWorkaroundResources = []string{"replicationcontroller","deploymentconfig","templateinstances.template.openshift.io"} // run restore diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 4fd361d9e3..723b2b7f41 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -17,6 +17,7 @@ import ( ocpappsv1 "github.com/openshift/api/apps/v1" routev1 "github.com/openshift/api/route/v1" security "github.com/openshift/api/security/v1" + templatev1 "github.com/openshift/api/template/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -128,6 +129,46 @@ func HasDCsInNamespace(ocClient client.Client, namespace string) (bool, error) { return true, nil } +func HasReplicationControllersInNamespace(ocClient client.Client, namespace string) (bool, error) { + rcList := &corev1.ReplicationControllerList{} + err := ocClient.List(context.Background(), rcList, client.InNamespace(namespace)) + if err!= nil { + return false, err + } + if len(rcList.Items) == 0 { + return false, nil + } + return true, nil +} + +func HasTemplateInstancesInNamespace(ocClient client.Client, namespace string) (bool, error) { + tiList := &templatev1.TemplateInstanceList{} + err := ocClient.List(context.Background(), tiList, client.InNamespace(namespace)) + if err!= nil { + return false, err + } + if len(tiList.Items) == 0 { + return false, nil + } + return true, nil +} + +func NamespaceRequiresResticDCWorkaround(ocClient client.Client, namespace string) (bool, error) { + hasDC ,err := HasDCsInNamespace(ocClient, namespace) + if err != nil { + return false, err + } + hasRC, err := HasReplicationControllersInNamespace(ocClient, namespace) + if err!= nil { + return false, err + } + hasTI, err := HasTemplateInstancesInNamespace(ocClient, namespace) + if err!= nil { + return false, err + } + return hasDC || hasRC || hasTI, nil +} + func IsDCReady(ocClient client.Client, namespace, dcName string) wait.ConditionFunc { return func() (bool, error) { dc := ocpappsv1.DeploymentConfig{} From bd53a9ee7b38a337e59516c873656b47abd00ef9 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sat, 30 Apr 2022 02:05:55 -0400 Subject: [PATCH 09/34] Handle IsNotRegisteredError --- tests/e2e/lib/apps.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 723b2b7f41..03fc29a6f9 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -23,6 +23,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" @@ -120,7 +121,7 @@ func UninstallApplication(ocClient client.Client, file string) error { func HasDCsInNamespace(ocClient client.Client, namespace string) (bool, error) { dcList := &ocpappsv1.DeploymentConfigList{} err := ocClient.List(context.Background(), dcList, client.InNamespace(namespace)) - if err!= nil { + if err != nil { return false, err } if len(dcList.Items) == 0 { @@ -132,7 +133,7 @@ func HasDCsInNamespace(ocClient client.Client, namespace string) (bool, error) { func HasReplicationControllersInNamespace(ocClient client.Client, namespace string) (bool, error) { rcList := &corev1.ReplicationControllerList{} err := ocClient.List(context.Background(), rcList, client.InNamespace(namespace)) - if err!= nil { + if err != nil { return false, err } if len(rcList.Items) == 0 { @@ -144,7 +145,10 @@ func HasReplicationControllersInNamespace(ocClient client.Client, namespace stri func HasTemplateInstancesInNamespace(ocClient client.Client, namespace string) (bool, error) { tiList := &templatev1.TemplateInstanceList{} err := ocClient.List(context.Background(), tiList, client.InNamespace(namespace)) - if err!= nil { + if err != nil { + if runtime.IsNotRegisteredError(err) { + return false, nil + } return false, err } if len(tiList.Items) == 0 { @@ -159,11 +163,11 @@ func NamespaceRequiresResticDCWorkaround(ocClient client.Client, namespace strin return false, err } hasRC, err := HasReplicationControllersInNamespace(ocClient, namespace) - if err!= nil { + if err != nil { return false, err } hasTI, err := HasTemplateInstancesInNamespace(ocClient, namespace) - if err!= nil { + if err != nil { return false, err } return hasDC || hasRC || hasTI, nil From 20bc2dc849327c19990a24095f82f08b8a506e93 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sat, 30 Apr 2022 10:22:39 -0400 Subject: [PATCH 10/34] different restore name for workaround --- tests/e2e/backup_restore_suite_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 02f0b723c8..55726d9ae3 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -157,25 +157,27 @@ var _ = Describe("AWS backup restore tests", func() { lastInstallingApplicationNamespace = brCase.ApplicationNamespace lastInstallTime = time.Now() log.Printf("Creating restore %s excluding DC workaround resources for case %s", restoreName, brCase.Name) - err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName, WithExcludedResources(dcWorkaroundResources)) + noDcDrestoreName := fmt.Sprintf("%s-no-dc-workaround", restoreName) + err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, noDcDrestoreName , WithExcludedResources(dcWorkaroundResources)) Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // Check if restore succeeded - succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName) + succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, noDcDrestoreName) Expect(err).ToNot(HaveOccurred()) Expect(succeeded).To(Equal(true)) // run restore log.Printf("Creating restore %s including DC workaround resources for case %s", restoreName, brCase.Name) - err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName, WithIncludedResources(dcWorkaroundResources)) + withDcRestoreName := fmt.Sprintf("%s-with-dc-workaround", restoreName) + err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, withDcRestoreName, WithIncludedResources(dcWorkaroundResources)) Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // Check if restore succeeded - succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName) + succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, withDcRestoreName) Expect(err).ToNot(HaveOccurred()) Expect(succeeded).To(Equal(true)) From 2af9cef629af0734b1c0754ea0f75b1d77617fef Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sat, 30 Apr 2022 11:00:58 -0400 Subject: [PATCH 11/34] try add templatev1 to scheme --- tests/e2e/lib/apps.go | 7 +++---- tests/e2e/lib/dpa_helpers.go | 3 +++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 03fc29a6f9..7f09ce5675 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -23,7 +23,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" @@ -146,9 +145,9 @@ func HasTemplateInstancesInNamespace(ocClient client.Client, namespace string) ( tiList := &templatev1.TemplateInstanceList{} err := ocClient.List(context.Background(), tiList, client.InNamespace(namespace)) if err != nil { - if runtime.IsNotRegisteredError(err) { - return false, nil - } + // if runtime.IsNotRegisteredError(err) { + // return false, nil + // } return false, err } if len(tiList.Items) == 0 { diff --git a/tests/e2e/lib/dpa_helpers.go b/tests/e2e/lib/dpa_helpers.go index 6f0c690fde..b1a27e164c 100755 --- a/tests/e2e/lib/dpa_helpers.go +++ b/tests/e2e/lib/dpa_helpers.go @@ -17,6 +17,7 @@ import ( appsv1 "github.com/openshift/api/apps/v1" security "github.com/openshift/api/security/v1" + templatev1 "github.com/openshift/api/template/v1" oadpv1alpha1 "github.com/openshift/oadp-operator/api/v1alpha1" operators "github.com/operator-framework/api/pkg/operators/v1alpha1" velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -172,6 +173,8 @@ func (v *DpaCustomResource) SetClient() error { oadpv1alpha1.AddToScheme(client.Scheme()) velero.AddToScheme(client.Scheme()) appsv1.AddToScheme(client.Scheme()) + corev1.AddToScheme(client.Scheme()) + templatev1.AddToScheme(client.Scheme()) security.AddToScheme(client.Scheme()) operators.AddToScheme(client.Scheme()) From 154ce5dc1b87acdbec1360f24ab90f8cd35d92cb Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 11:11:14 -0400 Subject: [PATCH 12/34] updateLastInstallingNamespace func --- tests/e2e/backup_restore_suite_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 55726d9ae3..b66d7d9f17 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -75,13 +75,17 @@ var _ = Describe("AWS backup restore tests", func() { return err }) + updateLastInstallingNamespace := func (namespace string) { + lastInstallingApplicationNamespace = namespace + lastInstallTime = time.Now() + } + DescribeTable("backup and restore applications", func(brCase BackupRestoreCase, expectedErr error) { err := dpaCR.Build(brCase.BackupRestoreType) Expect(err).NotTo(HaveOccurred()) - lastInstallingApplicationNamespace = dpaCR.Namespace - lastInstallTime = time.Now() + updateLastInstallingNamespace(dpaCR.Namespace) err = dpaCR.CreateOrUpdate(&dpaCR.CustomResource.Spec) Expect(err).NotTo(HaveOccurred()) @@ -111,8 +115,7 @@ var _ = Describe("AWS backup restore tests", func() { restoreName := fmt.Sprintf("%s-%s", brCase.Name, restoreUid.String()) // install app - lastInstallingApplicationNamespace = brCase.ApplicationNamespace - lastInstallTime = time.Now() + updateLastInstallingNamespace(brCase.ApplicationNamespace) log.Printf("Installing application for case %s", brCase.Name) err = InstallApplication(dpaCR.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) @@ -149,13 +152,12 @@ var _ = Describe("AWS backup restore tests", func() { // Wait for namespace to be deleted Eventually(IsNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) + updateLastInstallingNamespace(brCase.ApplicationNamespace) // Check if backup needs restic deploymentconfig workaround. https://github.com/openshift/oadp-operator/blob/master/docs/TROUBLESHOOTING.md#deployconfig if brCase.BackupRestoreType == RESTIC && nsRequiresResticDCWorkaround { log.Printf("DC found in backup namespace, using DC restic workaround") var dcWorkaroundResources = []string{"replicationcontroller","deploymentconfig","templateinstances.template.openshift.io"} // run restore - lastInstallingApplicationNamespace = brCase.ApplicationNamespace - lastInstallTime = time.Now() log.Printf("Creating restore %s excluding DC workaround resources for case %s", restoreName, brCase.Name) noDcDrestoreName := fmt.Sprintf("%s-no-dc-workaround", restoreName) err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, noDcDrestoreName , WithExcludedResources(dcWorkaroundResources)) From cb569bfdf625dc333cd07727b2ca6c2e54c5746b Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sat, 30 Apr 2022 01:24:49 -0400 Subject: [PATCH 13/34] E2E Wait for snapshot to be ready for CSI B/R case Fix ``` Event: failed to provision volume with StorageClass "gp2-csi": error getting handle for DataSource Type VolumeSnapshot by Name velero-mysql-l5sz5: snapshot velero-mysql-l5sz5 is not Ready, Type: Warning, Count: 10, Src: {PersistentVolumeClaim mysql-persistent mysql 72d98c8b-10ff-4a32-a319-88bed69e1227 v1 34225 }, Reason: ProvisioningFailed ``` seen at https://prow.ci.openshift.org/view/gs/origin-ci-test/pr-logs/pull/openshift_oadp-operator/654/pull-ci-openshift-oadp-operator-master-4.9-operator-e2e-aws/1520234660829859840#1:build-log.txt%3A375 --- go.mod | 1 + go.sum | 2 ++ tests/e2e/backup_restore_suite_test.go | 5 +++++ tests/e2e/lib/apps.go | 20 ++++++++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/go.mod b/go.mod index 54ad3c293e..8fc0f32dec 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/aws/aws-sdk-go v1.28.2 github.com/go-logr/logr v0.4.0 github.com/google/uuid v1.1.2 + github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 github.com/onsi/ginkgo v1.16.4 github.com/onsi/ginkgo/v2 v2.1.1 github.com/onsi/gomega v1.17.0 diff --git a/go.sum b/go.sum index 823f891d24..3715bd24e0 100644 --- a/go.sum +++ b/go.sum @@ -488,6 +488,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= +github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA= +github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index b66d7d9f17..5dd5922ab2 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -144,6 +144,11 @@ var _ = Describe("AWS backup restore tests", func() { Expect(succeeded).To(Equal(true)) log.Printf("Backup for case %s succeeded", brCase.Name) + if brCase.BackupRestoreType == CSI { + // wait for volume snapshot to be Ready + Eventually(IsVolumeSnapshotsReady(dpaCR.Client, namespace), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + } + // uninstall app log.Printf("Uninstalling app for case %s", brCase.Name) err = UninstallApplication(dpaCR.Client, brCase.ApplicationTemplate) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 7f09ce5675..9bbd326fab 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -13,6 +13,7 @@ import ( "sort" "time" + volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" "github.com/onsi/ginkgo/v2" ocpappsv1 "github.com/openshift/api/apps/v1" routev1 "github.com/openshift/api/route/v1" @@ -172,6 +173,25 @@ func NamespaceRequiresResticDCWorkaround(ocClient client.Client, namespace strin return hasDC || hasRC || hasTI, nil } +func IsVolumeSnapshotsReady(ocClient client.Client, namespace string) wait.ConditionFunc { + return func() (bool, error) { + vList := &volumesnapshotv1.VolumeSnapshotList{} + err := ocClient.List(context.Background(), vList, client.InNamespace(namespace)) + if err!= nil { + return false, err + } + if len(vList.Items) == 0 { + return false, nil + } + for _, v := range vList.Items { + if v.Status.ReadyToUse == nil || *v.Status.ReadyToUse == false { + return false, nil + } + } + return true, nil + } +} + func IsDCReady(ocClient client.Client, namespace, dcName string) wait.ConditionFunc { return func() (bool, error) { dc := ocpappsv1.DeploymentConfig{} From 505e95446ab6db8eea0c0627d15410faf4c98509 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sat, 30 Apr 2022 10:57:54 -0400 Subject: [PATCH 14/34] addToScheme volumesnapshotv1 --- tests/e2e/lib/apps.go | 29 +++++++++++++++++++++++++++-- tests/e2e/lib/dpa_helpers.go | 2 ++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 9bbd326fab..b0abfad9d7 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -176,18 +176,43 @@ func NamespaceRequiresResticDCWorkaround(ocClient client.Client, namespace strin func IsVolumeSnapshotsReady(ocClient client.Client, namespace string) wait.ConditionFunc { return func() (bool, error) { vList := &volumesnapshotv1.VolumeSnapshotList{} + // vListBeta := &volumesnapshotv1beta1.VolumeSnapshotList{} err := ocClient.List(context.Background(), vList, client.InNamespace(namespace)) - if err!= nil { + if err != nil { + // try beta version + // if runtime.IsNotRegisteredError(err) { + // // try v1beta1 + // vList = nil // reset + // err = ocClient.List(context.Background(), vList, client.InNamespace(namespace)) + // if err!= nil { + // if runtime.IsNotRegisteredError(err) { + // return false, nil + // } + // } + // } return false, err } + // if vList != nil { if len(vList.Items) == 0 { return false, nil } for _, v := range vList.Items { - if v.Status.ReadyToUse == nil || *v.Status.ReadyToUse == false { + if v.Status.ReadyToUse == nil || !*v.Status.ReadyToUse { return false, nil } } + // } + // } else { + // if len(vListBeta.Items) == 0 { + // return false, nil + // } + // for _, v := range vListBeta.Items { + // if v.Status.ReadyToUse == nil || !*v.Status.ReadyToUse { + // return false, nil + // } + // } + // } + return true, nil } } diff --git a/tests/e2e/lib/dpa_helpers.go b/tests/e2e/lib/dpa_helpers.go index b1a27e164c..03d5b6f680 100755 --- a/tests/e2e/lib/dpa_helpers.go +++ b/tests/e2e/lib/dpa_helpers.go @@ -15,6 +15,7 @@ import ( utils "github.com/openshift/oadp-operator/tests/e2e/utils" + volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" appsv1 "github.com/openshift/api/apps/v1" security "github.com/openshift/api/security/v1" templatev1 "github.com/openshift/api/template/v1" @@ -177,6 +178,7 @@ func (v *DpaCustomResource) SetClient() error { templatev1.AddToScheme(client.Scheme()) security.AddToScheme(client.Scheme()) operators.AddToScheme(client.Scheme()) + volumesnapshotv1.AddToScheme(client.Scheme()) v.Client = client return nil From a077d2f3f7c93e7ef0ac081503845d15115396c1 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sat, 30 Apr 2022 23:24:23 -0400 Subject: [PATCH 15/34] check volumesnapshots by label --- tests/e2e/backup_restore_suite_test.go | 2 +- tests/e2e/lib/apps.go | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 5dd5922ab2..5565fa910f 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -146,7 +146,7 @@ var _ = Describe("AWS backup restore tests", func() { if brCase.BackupRestoreType == CSI { // wait for volume snapshot to be Ready - Eventually(IsVolumeSnapshotsReady(dpaCR.Client, namespace), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + Eventually(IsVolumeSnapshotsReady(dpaCR.Client, backupName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) } // uninstall app diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index b0abfad9d7..bc85b61857 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -19,6 +19,7 @@ import ( routev1 "github.com/openshift/api/route/v1" security "github.com/openshift/api/security/v1" templatev1 "github.com/openshift/api/template/v1" + "github.com/vmware-tanzu/velero/pkg/label" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -173,11 +174,11 @@ func NamespaceRequiresResticDCWorkaround(ocClient client.Client, namespace strin return hasDC || hasRC || hasTI, nil } -func IsVolumeSnapshotsReady(ocClient client.Client, namespace string) wait.ConditionFunc { +func IsVolumeSnapshotsReady(ocClient client.Client, backupName string) wait.ConditionFunc { return func() (bool, error) { vList := &volumesnapshotv1.VolumeSnapshotList{} // vListBeta := &volumesnapshotv1beta1.VolumeSnapshotList{} - err := ocClient.List(context.Background(), vList, client.InNamespace(namespace)) + err := ocClient.List(context.Background(), vList, &client.ListOptions{LabelSelector: label.NewSelectorForBackup(backupName)}) if err != nil { // try beta version // if runtime.IsNotRegisteredError(err) { @@ -194,10 +195,17 @@ func IsVolumeSnapshotsReady(ocClient client.Client, namespace string) wait.Condi } // if vList != nil { if len(vList.Items) == 0 { + ginkgo.GinkgoWriter.Println("No VolumeSnapshots found") return false, nil } for _, v := range vList.Items { - if v.Status.ReadyToUse == nil || !*v.Status.ReadyToUse { + if v.Status.ReadyToUse == nil { + ginkgo.GinkgoWriter.Println("VolumeSnapshots Ready status not found for " + v.Name) + ginkgo.GinkgoWriter.Println(fmt.Sprintf("status: %v", v.Status)) + return false, nil + } + if !*v.Status.ReadyToUse { + ginkgo.GinkgoWriter.Println("VolumeSnapshots Ready status is false " + v.Name) return false, nil } } From 83c3572778a581687dd71eb2e0b819242976a320 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 02:32:43 -0400 Subject: [PATCH 16/34] Describe backup and restore in backup restore suite test --- go.sum | 19 +++++ tests/e2e/backup_restore_suite_test.go | 11 +-- tests/e2e/lib/backup.go | 14 ++-- tests/e2e/lib/restore.go | 2 +- tests/e2e/lib/velero_helpers.go | 99 ++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 12 deletions(-) create mode 100644 tests/e2e/lib/velero_helpers.go diff --git a/go.sum b/go.sum index 3715bd24e0..5ca861e20a 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,7 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v42.0.0+incompatible h1:yz6sFf5bHZ+gEOQVuK5JhPqTTAmv+OvSLSaqgzqaCwY= github.com/Azure/azure-sdk-for-go v42.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -60,7 +61,9 @@ github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQW github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 h1:iM6UAvjR97ZIeR93qTcwpKNMpV+/FTWjwEbuPD495Tk= github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM= +github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 h1:LXl088ZQlP0SBppGFsRZonW6hSvwgL5gRByMbvUbx8U= github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= @@ -72,7 +75,9 @@ github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.3.0 h1:zebkZaadz7+wIQYgC7GXaz3Wb28yKYfVkkBKwc38VF8= github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= +github.com/Azure/go-autorest/autorest/validation v0.2.0 h1:15vMO4y76dehZSq7pAaOLQxC6dZYsSrj2GQpflyM/L4= github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= @@ -178,6 +183,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= @@ -189,6 +195,7 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/drone/envsubst v1.0.3-0.20200709223903-efdb65b94e5a/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -212,6 +219,7 @@ github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwo github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -299,9 +307,11 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= github.com/gobuffalo/flect v0.2.3/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-yaml v1.8.1/go.mod h1:wS4gNoLalDSJxo/SpngzPQ2BN4uuZVLCmbM4S3vd4+Y= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -453,6 +463,7 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= @@ -487,12 +498,14 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 h1:ipLtV9ubLEYx42YvwDa12eVPQvjuGZoPdbCozGzVNRc= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lucas-clemente/aes12 v0.0.0-20171027163421-cd47fb39b79f/go.mod h1:JpH9J1c9oX6otFSgdUHwUBUizmKlrMjxWnIAjff4m04= @@ -513,12 +526,14 @@ github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwm github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -530,6 +545,7 @@ github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nr github.com/mikefarah/yq/v3 v3.0.0-20201202084205-8846255d1c37/go.mod h1:dYWq+UWoFCDY1TndvFUQuhBbIYmZpjreC8adEAx93zE= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= @@ -540,6 +556,7 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= @@ -1297,6 +1314,7 @@ k8s.io/apiserver v0.21.3/go.mod h1:eDPWlZG6/cCCMj/JBcEpDoK+I+6i3r9GsChYBHSbAzU= k8s.io/apiserver v0.22.1/go.mod h1:2mcM6dzSt+XndzVQJX21Gx0/Klo7Aen7i0Ai6tIa400= k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= k8s.io/cli-runtime v0.19.2/go.mod h1:CMynmJM4Yf02TlkbhKxoSzi4Zf518PukJ5xep/NaNeY= +k8s.io/cli-runtime v0.20.9 h1:KJ/z9I9r0Oyb7mjfsO4ZdasGCihIiTO7llXz+oZ0rgA= k8s.io/cli-runtime v0.20.9/go.mod h1:MjEpWNIcmJL+gANS1+SVfvWpRSQRxOvTcx+vzvDTtTY= k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw= k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU= @@ -1341,6 +1359,7 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-aggregator v0.19.12 h1:OwyNUe/7/gxzEnaLd3sC9Yrpx0fZAERzvFslX5Qq5g8= k8s.io/kube-aggregator v0.19.12/go.mod h1:K76wPd03pSHEmS1FgJOcpryac5C3va4cbCvSu+4EmE0= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 5565fa910f..7d198b3b9e 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -131,15 +131,16 @@ var _ = Describe("AWS backup restore tests", func() { Expect(err).ToNot(HaveOccurred()) // create backup log.Printf("Creating backup %s for case %s", backupName, brCase.Name) - err = CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace}) + backup, err := CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace}) Expect(err).ToNot(HaveOccurred()) // wait for backup to not be running Eventually(IsBackupDone(dpaCR.Client, namespace, backupName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + log.Printf(DescribeBackup(dpaCR.Client, backup)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) - + // check if backup succeeded - succeeded, err := IsBackupCompletedSuccessfully(dpaCR.Client, namespace, backupName) + succeeded, err := IsBackupCompletedSuccessfully(dpaCR.Client, backup) Expect(err).ToNot(HaveOccurred()) Expect(succeeded).To(Equal(true)) log.Printf("Backup for case %s succeeded", brCase.Name) @@ -153,7 +154,7 @@ var _ = Describe("AWS backup restore tests", func() { log.Printf("Uninstalling app for case %s", brCase.Name) err = UninstallApplication(dpaCR.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) - + // Wait for namespace to be deleted Eventually(IsNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) @@ -168,6 +169,7 @@ var _ = Describe("AWS backup restore tests", func() { err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, noDcDrestoreName , WithExcludedResources(dcWorkaroundResources)) Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + log.Printf(DescribeRestore(dpaCR.Client, restore)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // Check if restore succeeded @@ -181,6 +183,7 @@ var _ = Describe("AWS backup restore tests", func() { err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, withDcRestoreName, WithIncludedResources(dcWorkaroundResources)) Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + log.Printf(DescribeRestore(dpaCR.Client, restore)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // Check if restore succeeded diff --git a/tests/e2e/lib/backup.go b/tests/e2e/lib/backup.go index 44b8756a0f..4060d4c0ff 100755 --- a/tests/e2e/lib/backup.go +++ b/tests/e2e/lib/backup.go @@ -11,7 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupName string, namespaces []string) error { +func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupName string, namespaces []string) (velero.Backup, error) { backup := velero.Backup{ ObjectMeta: metav1.ObjectMeta{ @@ -23,7 +23,7 @@ func CreateBackupForNamespaces(ocClient client.Client, veleroNamespace, backupNa }, } err := ocClient.Create(context.Background(), &backup) - return err + return backup, err } func IsBackupDone(ocClient client.Client, veleroNamespace, name string) wait.ConditionFunc { @@ -46,17 +46,17 @@ func IsBackupDone(ocClient client.Client, veleroNamespace, name string) wait.Con } } -func IsBackupCompletedSuccessfully(ocClient client.Client, veleroNamespace, name string) (bool, error) { - backup := velero.Backup{} +func IsBackupCompletedSuccessfully(ocClient client.Client, backup velero.Backup) (bool, error) { err := ocClient.Get(context.Background(), client.ObjectKey{ - Namespace: veleroNamespace, - Name: name, + Namespace: backup.Namespace, + Name: backup.Name, }, &backup) if err != nil { return false, err } + if backup.Status.Phase == velero.BackupPhaseCompleted { return true, nil } - return false, fmt.Errorf("backup phase is: %s; expected: %s\nvalidation errors: %v\nvelero failure logs: %v", backup.Status.Phase, velero.BackupPhaseCompleted, backup.Status.ValidationErrors, GetVeleroContainerFailureLogs(veleroNamespace)) + return false, fmt.Errorf("backup phase is: %s; expected: %s\nvalidation errors: %v\nvelero failure logs: %v", backup.Status.Phase, velero.BackupPhaseCompleted, backup.Status.ValidationErrors, GetVeleroContainerFailureLogs(backup.Namespace)) } diff --git a/tests/e2e/lib/restore.go b/tests/e2e/lib/restore.go index 903ad01a09..32bcc768a2 100755 --- a/tests/e2e/lib/restore.go +++ b/tests/e2e/lib/restore.go @@ -51,7 +51,7 @@ func CreateRestoreFromBackup(ocClient client.Client, veleroNamespace, backupName opt(&restore) } err := ocClient.Create(context.Background(), &restore) - return err + return restore, err } func IsRestoreDone(ocClient client.Client, veleroNamespace, name string) wait.ConditionFunc { diff --git a/tests/e2e/lib/velero_helpers.go b/tests/e2e/lib/velero_helpers.go new file mode 100644 index 0000000000..853bef4cfc --- /dev/null +++ b/tests/e2e/lib/velero_helpers.go @@ -0,0 +1,99 @@ +package lib + +import ( + "context" + "fmt" + "os" + + snapshotv1beta1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1" + snapshotv1beta1client "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" + velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" + "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/util/output" + "github.com/vmware-tanzu/velero/pkg/features" + veleroClientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" + "github.com/vmware-tanzu/velero/pkg/label" + "github.com/vmware-tanzu/velero/pkg/restic" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func GetVeleroClient() (veleroClientset.Interface, error) { + if vc, err := veleroClientset.NewForConfig(getKubeConfig()); err == nil { + return vc, nil + } else { + return nil, err + } +} + +// https://github.com/vmware-tanzu/velero/blob/11bfe82342c9f54c63f40d3e97313ce763b446f2/pkg/cmd/cli/backup/describe.go#L77-L111 +func DescribeBackup(ocClient client.Client, backup velero.Backup) string { + err := ocClient.Get(context.Background(), client.ObjectKey{ + Namespace: backup.Namespace, + Name: backup.Name, + }, &backup) + if err != nil { + return "could not get provided backup: " + err.Error() + } + veleroClient, err := GetVeleroClient() + if err!= nil { + return err.Error() + } + details := true + insecureSkipTLSVerify := true + caCertFile := "" + + deleteRequestListOptions := pkgbackup.NewDeleteBackupRequestListOptions(backup.Name, string(backup.UID)) + deleteRequestList, err := veleroClient.VeleroV1().DeleteBackupRequests(backup.Namespace).List(context.TODO(), deleteRequestListOptions) + if err != nil { + fmt.Fprintf(os.Stderr, "error getting DeleteBackupRequests for backup %s: %v\n", backup.Name, err) + } + + opts := label.NewListOptionsForBackup(backup.Name) + podVolumeBackupList, err := veleroClient.VeleroV1().PodVolumeBackups(backup.Namespace).List(context.TODO(), opts) + if err != nil { + fmt.Fprintf(os.Stderr, "error getting PodVolumeBackups for backup %s: %v\n", backup.Name, err) + } + + var csiClient *snapshotv1beta1client.Clientset + // declare vscList up here since it may be empty and we'll pass the empty Items field into DescribeBackup + vscList := new(snapshotv1beta1api.VolumeSnapshotContentList) + if features.IsEnabled(velero.CSIFeatureFlag) { + clientConfig := getKubeConfig() + + csiClient, err = snapshotv1beta1client.NewForConfig(clientConfig) + cmd.CheckError(err) + + vscList, err = csiClient.SnapshotV1beta1().VolumeSnapshotContents().List(context.TODO(), opts) + if err != nil { + fmt.Fprintf(os.Stderr, "error getting VolumeSnapshotContent objects for backup %s: %v\n", backup.Name, err) + } + } + + return output.DescribeBackup(context.Background(), ocClient, &backup, deleteRequestList.Items, podVolumeBackupList.Items, vscList.Items, details, veleroClient, insecureSkipTLSVerify, caCertFile) +} + +// https://github.com/vmware-tanzu/velero/blob/11bfe82342c9f54c63f40d3e97313ce763b446f2/pkg/cmd/cli/restore/describe.go#L72-L78 +func DescribeRestore(ocClient client.Client, restore velero.Restore) string { + err := ocClient.Get(context.Background(), client.ObjectKey{ + Namespace: restore.Namespace, + Name: restore.Name, + }, &restore) + if err != nil { + return "could not get provided backup: " + err.Error() + } + veleroClient, err := GetVeleroClient() + if err!= nil { + return err.Error() + } + details := true + insecureSkipTLSVerify := true + caCertFile := "" + opts := restic.NewPodVolumeRestoreListOptions(restore.Name) + podvolumeRestoreList, err := veleroClient.VeleroV1().PodVolumeRestores(restore.Namespace).List(context.TODO(), opts) + if err != nil { + fmt.Fprintf(os.Stderr, "error getting PodVolumeRestores for restore %s: %v\n", restore.Name, err) + } + + return output.DescribeRestore(context.Background(), ocClient, &restore, podvolumeRestoreList.Items, details, veleroClient, insecureSkipTLSVerify, caCertFile) +} \ No newline at end of file From 43430aa8d2daf382fdc7e0b0648da5fed1a045a0 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 11:34:49 -0400 Subject: [PATCH 17/34] format --- tests/e2e/backup_restore_suite_test.go | 19 +++++++++---------- tests/e2e/lib/apps.go | 4 ++-- tests/e2e/lib/backup.go | 2 +- tests/e2e/lib/restore.go | 2 +- tests/e2e/lib/velero_helpers.go | 8 ++++---- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 7d198b3b9e..04424bb854 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -75,7 +75,7 @@ var _ = Describe("AWS backup restore tests", func() { return err }) - updateLastInstallingNamespace := func (namespace string) { + updateLastInstallingNamespace := func(namespace string) { lastInstallingApplicationNamespace = namespace lastInstallTime = time.Now() } @@ -138,7 +138,7 @@ var _ = Describe("AWS backup restore tests", func() { Eventually(IsBackupDone(dpaCR.Client, namespace, backupName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) log.Printf(DescribeBackup(dpaCR.Client, backup)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) - + // check if backup succeeded succeeded, err := IsBackupCompletedSuccessfully(dpaCR.Client, backup) Expect(err).ToNot(HaveOccurred()) @@ -154,7 +154,7 @@ var _ = Describe("AWS backup restore tests", func() { log.Printf("Uninstalling app for case %s", brCase.Name) err = UninstallApplication(dpaCR.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) - + // Wait for namespace to be deleted Eventually(IsNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue()) @@ -162,16 +162,15 @@ var _ = Describe("AWS backup restore tests", func() { // Check if backup needs restic deploymentconfig workaround. https://github.com/openshift/oadp-operator/blob/master/docs/TROUBLESHOOTING.md#deployconfig if brCase.BackupRestoreType == RESTIC && nsRequiresResticDCWorkaround { log.Printf("DC found in backup namespace, using DC restic workaround") - var dcWorkaroundResources = []string{"replicationcontroller","deploymentconfig","templateinstances.template.openshift.io"} + var dcWorkaroundResources = []string{"replicationcontroller", "deploymentconfig", "templateinstances.template.openshift.io"} // run restore log.Printf("Creating restore %s excluding DC workaround resources for case %s", restoreName, brCase.Name) noDcDrestoreName := fmt.Sprintf("%s-no-dc-workaround", restoreName) - err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, noDcDrestoreName , WithExcludedResources(dcWorkaroundResources)) + restore, err := CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, noDcDrestoreName, WithExcludedResources(dcWorkaroundResources)) Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) log.Printf(DescribeRestore(dpaCR.Client, restore)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) - // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, noDcDrestoreName) Expect(err).ToNot(HaveOccurred()) @@ -180,12 +179,11 @@ var _ = Describe("AWS backup restore tests", func() { // run restore log.Printf("Creating restore %s including DC workaround resources for case %s", restoreName, brCase.Name) withDcRestoreName := fmt.Sprintf("%s-with-dc-workaround", restoreName) - err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, withDcRestoreName, WithIncludedResources(dcWorkaroundResources)) + restore, err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, withDcRestoreName, WithIncludedResources(dcWorkaroundResources)) Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) log.Printf(DescribeRestore(dpaCR.Client, restore)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) - // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, withDcRestoreName) Expect(err).ToNot(HaveOccurred()) @@ -194,11 +192,12 @@ var _ = Describe("AWS backup restore tests", func() { } else { // run restore log.Printf("Creating restore %s for case %s", restoreName, brCase.Name) - err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName) + restore, err := CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName) Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + log.Printf(DescribeRestore(dpaCR.Client, restore)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) - + // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName) Expect(err).ToNot(HaveOccurred()) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index bc85b61857..8d471800e0 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -159,7 +159,7 @@ func HasTemplateInstancesInNamespace(ocClient client.Client, namespace string) ( } func NamespaceRequiresResticDCWorkaround(ocClient client.Client, namespace string) (bool, error) { - hasDC ,err := HasDCsInNamespace(ocClient, namespace) + hasDC, err := HasDCsInNamespace(ocClient, namespace) if err != nil { return false, err } @@ -209,7 +209,7 @@ func IsVolumeSnapshotsReady(ocClient client.Client, backupName string) wait.Cond return false, nil } } - // } + // } // } else { // if len(vListBeta.Items) == 0 { // return false, nil diff --git a/tests/e2e/lib/backup.go b/tests/e2e/lib/backup.go index 4060d4c0ff..cb959a4a5d 100755 --- a/tests/e2e/lib/backup.go +++ b/tests/e2e/lib/backup.go @@ -54,7 +54,7 @@ func IsBackupCompletedSuccessfully(ocClient client.Client, backup velero.Backup) if err != nil { return false, err } - + if backup.Status.Phase == velero.BackupPhaseCompleted { return true, nil } diff --git a/tests/e2e/lib/restore.go b/tests/e2e/lib/restore.go index 32bcc768a2..48b35580f3 100755 --- a/tests/e2e/lib/restore.go +++ b/tests/e2e/lib/restore.go @@ -37,7 +37,7 @@ func WithExcludedNamespaces(namespaces []string) restoreOpts { } } -func CreateRestoreFromBackup(ocClient client.Client, veleroNamespace, backupName, restoreName string, opts ...restoreOpts) error { +func CreateRestoreFromBackup(ocClient client.Client, veleroNamespace, backupName, restoreName string, opts ...restoreOpts) (velero.Restore, error) { restore := velero.Restore{ ObjectMeta: metav1.ObjectMeta{ Name: restoreName, diff --git a/tests/e2e/lib/velero_helpers.go b/tests/e2e/lib/velero_helpers.go index 853bef4cfc..84ee855195 100644 --- a/tests/e2e/lib/velero_helpers.go +++ b/tests/e2e/lib/velero_helpers.go @@ -36,13 +36,13 @@ func DescribeBackup(ocClient client.Client, backup velero.Backup) string { return "could not get provided backup: " + err.Error() } veleroClient, err := GetVeleroClient() - if err!= nil { + if err != nil { return err.Error() } details := true insecureSkipTLSVerify := true caCertFile := "" - + deleteRequestListOptions := pkgbackup.NewDeleteBackupRequestListOptions(backup.Name, string(backup.UID)) deleteRequestList, err := veleroClient.VeleroV1().DeleteBackupRequests(backup.Namespace).List(context.TODO(), deleteRequestListOptions) if err != nil { @@ -83,7 +83,7 @@ func DescribeRestore(ocClient client.Client, restore velero.Restore) string { return "could not get provided backup: " + err.Error() } veleroClient, err := GetVeleroClient() - if err!= nil { + if err != nil { return err.Error() } details := true @@ -96,4 +96,4 @@ func DescribeRestore(ocClient client.Client, restore velero.Restore) string { } return output.DescribeRestore(context.Background(), ocClient, &restore, podvolumeRestoreList.Items, details, veleroClient, insecureSkipTLSVerify, caCertFile) -} \ No newline at end of file +} From 68e727f288d545a1437796314add112f472434a9 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 11:37:25 -0400 Subject: [PATCH 18/34] fix make test, install.AllCRDs param removal --- controllers/velero.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/velero.go b/controllers/velero.go index d5d64dd8ea..ca9766cea9 100644 --- a/controllers/velero.go +++ b/controllers/velero.go @@ -124,7 +124,7 @@ func (r *DPAReconciler) ReconcileVeleroCRDs(log logr.Logger) (bool, error) { func (r *DPAReconciler) InstallVeleroCRDs(log logr.Logger) error { var err error // Install CRDs - for _, unstructuredCrd := range install.AllCRDs("v1").Items { + for _, unstructuredCrd := range install.AllCRDs().Items { foundCrd := &v1.CustomResourceDefinition{} crd := &v1.CustomResourceDefinition{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredCrd.Object, crd); err != nil { From bf6142e5309ef73ba1dd28ae6fcd2dc344b1ea6c Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 15:17:53 -0400 Subject: [PATCH 19/34] log so we know we are waiting on volumesnapshot --- tests/e2e/lib/apps.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 8d471800e0..25d04bd0af 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -185,7 +185,7 @@ func IsVolumeSnapshotsReady(ocClient client.Client, backupName string) wait.Cond // // try v1beta1 // vList = nil // reset // err = ocClient.List(context.Background(), vList, client.InNamespace(namespace)) - // if err!= nil { + // if err != nil { // if runtime.IsNotRegisteredError(err) { // return false, nil // } @@ -199,6 +199,7 @@ func IsVolumeSnapshotsReady(ocClient client.Client, backupName string) wait.Cond return false, nil } for _, v := range vList.Items { + log.Println(fmt.Sprintf("waiting for volume snapshot %s to be ready", v.Name)) if v.Status.ReadyToUse == nil { ginkgo.GinkgoWriter.Println("VolumeSnapshots Ready status not found for " + v.Name) ginkgo.GinkgoWriter.Println(fmt.Sprintf("status: %v", v.Status)) From 7a9fe013794ad192dc550a51110ce7b12708e8f3 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 15:19:50 -0400 Subject: [PATCH 20/34] Wait for builds before considering app ready --- tests/e2e/backup_restore_suite_test.go | 2 + tests/e2e/lib/apps.go | 63 +++++++++++++++++++++++++- tests/e2e/lib/dpa_helpers.go | 5 +- 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 04424bb854..de432474a0 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -120,6 +120,7 @@ var _ = Describe("AWS backup restore tests", func() { err = InstallApplication(dpaCR.Client, brCase.ApplicationTemplate) Expect(err).ToNot(HaveOccurred()) // wait for pods to be running + Eventually(AreAppBuildsReady(dpaCR.Client, brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) Eventually(AreApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue()) // Run optional custom verification @@ -205,6 +206,7 @@ var _ = Describe("AWS backup restore tests", func() { } // verify app is running + Eventually(AreAppBuildsReady(dpaCR.Client, brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) Eventually(AreApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue()) // Run optional custom verification diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 25d04bd0af..1d06261244 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -16,6 +16,7 @@ import ( volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" "github.com/onsi/ginkgo/v2" ocpappsv1 "github.com/openshift/api/apps/v1" + buildv1 "github.com/openshift/api/build/v1" routev1 "github.com/openshift/api/route/v1" security "github.com/openshift/api/security/v1" templatev1 "github.com/openshift/api/template/v1" @@ -25,12 +26,23 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" ) +const e2eAppLabelKey = "e2e-app" +const e2eAppLabelValue = "true" + +var ( + e2eAppLabelRequirement, _ = labels.NewRequirement(e2eAppLabelKey, selection.Equals, []string{e2eAppLabelValue}) + e2eAppLabelSelector = labels.NewSelector().Add(*e2eAppLabelRequirement) +) + + func InstallApplication(ocClient client.Client, file string) error { template, err := os.ReadFile(file) if err != nil { @@ -44,6 +56,12 @@ func InstallApplication(ocClient client.Client, file string) error { return err } for _, resource := range obj.Items { + labels := resource.GetLabels() + if labels == nil { + labels = make(map[string]string) + } + labels[e2eAppLabelKey] = "true" + resource.SetLabels(labels) err = ocClient.Create(context.Background(), &resource) if apierrors.IsAlreadyExists(err) { continue @@ -236,6 +254,12 @@ func IsDCReady(ocClient client.Client, namespace, dcName string) wait.ConditionF if err != nil { return false, err } + // check dc for false availability condition which occurs when a new replication controller is created (after a new build completed) even if there are satisfactory available replicas + for _, condition := range dc.Status.Conditions { + if condition.Type == ocpappsv1.DeploymentAvailable && condition.Status == corev1.ConditionFalse { + return false, nil + } + } if dc.Status.AvailableReplicas != dc.Status.Replicas || dc.Status.Replicas == 0 { for _, condition := range dc.Status.Conditions { if len(condition.Message) > 0 { @@ -244,6 +268,13 @@ func IsDCReady(ocClient client.Client, namespace, dcName string) wait.ConditionF } return false, errors.New("DC is not in a ready state") } + for _, trigger := range dc.Spec.Triggers { + if trigger.Type == ocpappsv1.DeploymentTriggerOnImageChange { + if trigger.ImageChangeParams.Automatic { + return areAppBuildsReady(ocClient, namespace) + } + } + } return true, nil } } @@ -270,6 +301,36 @@ func IsDeploymentReady(ocClient client.Client, namespace, dName string) wait.Con } } +func AreAppBuildsReady(ocClient client.Client, namespace string) wait.ConditionFunc { + return func() (bool, error) { + return areAppBuildsReady(ocClient, namespace) + } +} + +func areAppBuildsReady(ocClient client.Client, namespace string) (bool, error) { + buildList := &buildv1.BuildList{} + err := ocClient.List(context.Background(), buildList, &client.ListOptions{Namespace: namespace, LabelSelector: e2eAppLabelSelector}) + if err != nil && !apierrors.IsNotFound(err) { + return false, err + } + if buildList.Items != nil{ + for _, build := range buildList.Items { + if build.Status.Phase == buildv1.BuildPhaseNew || + build.Status.Phase == buildv1.BuildPhasePending || + build.Status.Phase == buildv1.BuildPhaseRunning { + log.Println("Build is not ready: " + build.Name) + return false, nil + } + if build.Status.Phase == buildv1.BuildPhaseFailed || build.Status.Phase == buildv1.BuildPhaseError { + ginkgo.GinkgoWriter.Println("Build failed/error: " + build.Name) + ginkgo.GinkgoWriter.Println(fmt.Sprintf("status: %v", build.Status)) + return false, errors.New("found build failed or error") + } + } + } + return true, nil +} + func AreApplicationPodsRunning(namespace string) wait.ConditionFunc { return func() (bool, error) { clientset, err := setUpClient() @@ -278,7 +339,7 @@ func AreApplicationPodsRunning(namespace string) wait.ConditionFunc { } // select Velero pod with this label veleroOptions := metav1.ListOptions{ - LabelSelector: "e2e-app=true", + LabelSelector: e2eAppLabelSelector.String(), } // get pods in test namespace with labelSelector podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), veleroOptions) diff --git a/tests/e2e/lib/dpa_helpers.go b/tests/e2e/lib/dpa_helpers.go index 03d5b6f680..959af74907 100755 --- a/tests/e2e/lib/dpa_helpers.go +++ b/tests/e2e/lib/dpa_helpers.go @@ -11,15 +11,15 @@ import ( "reflect" "strings" + buildv1 "github.com/openshift/api/build/v1" "github.com/openshift/oadp-operator/pkg/common" - utils "github.com/openshift/oadp-operator/tests/e2e/utils" - volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" appsv1 "github.com/openshift/api/apps/v1" security "github.com/openshift/api/security/v1" templatev1 "github.com/openshift/api/template/v1" oadpv1alpha1 "github.com/openshift/oadp-operator/api/v1alpha1" + utils "github.com/openshift/oadp-operator/tests/e2e/utils" operators "github.com/operator-framework/api/pkg/operators/v1alpha1" velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" corev1 "k8s.io/api/core/v1" @@ -179,6 +179,7 @@ func (v *DpaCustomResource) SetClient() error { security.AddToScheme(client.Scheme()) operators.AddToScheme(client.Scheme()) volumesnapshotv1.AddToScheme(client.Scheme()) + buildv1.AddToScheme(client.Scheme()) v.Client = client return nil From 8ea62d7994a60cf8eee5d00352223aee2f4ae00e Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 15:22:28 -0400 Subject: [PATCH 21/34] move B/R describe to ginkgowriter --- tests/e2e/backup_restore_suite_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index de432474a0..609a0c89c5 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -137,7 +137,7 @@ var _ = Describe("AWS backup restore tests", func() { // wait for backup to not be running Eventually(IsBackupDone(dpaCR.Client, namespace, backupName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) - log.Printf(DescribeBackup(dpaCR.Client, backup)) + GinkgoWriter.Println(DescribeBackup(dpaCR.Client, backup)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // check if backup succeeded @@ -196,7 +196,7 @@ var _ = Describe("AWS backup restore tests", func() { restore, err := CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName) Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) - log.Printf(DescribeRestore(dpaCR.Client, restore)) + GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // Check if restore succeeded From 97346f3758ac4cdcf6db11744c40f5c0e985ee89 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 15:31:18 -0400 Subject: [PATCH 22/34] add builds ready check to after restic DC restore workaround first step --- tests/e2e/backup_restore_suite_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 609a0c89c5..a846e6aecc 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -176,6 +176,7 @@ var _ = Describe("AWS backup restore tests", func() { succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, noDcDrestoreName) Expect(err).ToNot(HaveOccurred()) Expect(succeeded).To(Equal(true)) + Eventually(AreAppBuildsReady(dpaCR.Client, brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) // run restore log.Printf("Creating restore %s including DC workaround resources for case %s", restoreName, brCase.Name) From 203c31e48e46cc49daef5bcec5e247836d5528f5 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 15:45:15 -0400 Subject: [PATCH 23/34] IsVolumeSnapshotsReady ->AreVolumeSnapshotsReady --- tests/e2e/backup_restore_suite_test.go | 2 +- tests/e2e/lib/apps.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index a846e6aecc..9c8d106f33 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -148,7 +148,7 @@ var _ = Describe("AWS backup restore tests", func() { if brCase.BackupRestoreType == CSI { // wait for volume snapshot to be Ready - Eventually(IsVolumeSnapshotsReady(dpaCR.Client, backupName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + Eventually(AreVolumeSnapshotsReady(dpaCR.Client, backupName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) } // uninstall app diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 1d06261244..d9bca15b4e 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -192,7 +192,7 @@ func NamespaceRequiresResticDCWorkaround(ocClient client.Client, namespace strin return hasDC || hasRC || hasTI, nil } -func IsVolumeSnapshotsReady(ocClient client.Client, backupName string) wait.ConditionFunc { +func AreVolumeSnapshotsReady(ocClient client.Client, backupName string) wait.ConditionFunc { return func() (bool, error) { vList := &volumesnapshotv1.VolumeSnapshotList{} // vListBeta := &volumesnapshotv1beta1.VolumeSnapshotList{} From 900c9657c2bc03f27c27415c20f986cabf89f0a9 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 16:07:27 -0400 Subject: [PATCH 24/34] use correct restore name to check for done --- tests/e2e/backup_restore_suite_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 9c8d106f33..e1651855e9 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -169,8 +169,8 @@ var _ = Describe("AWS backup restore tests", func() { noDcDrestoreName := fmt.Sprintf("%s-no-dc-workaround", restoreName) restore, err := CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, noDcDrestoreName, WithExcludedResources(dcWorkaroundResources)) Expect(err).ToNot(HaveOccurred()) - Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) - log.Printf(DescribeRestore(dpaCR.Client, restore)) + Eventually(IsRestoreDone(dpaCR.Client, namespace, noDcDrestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + log.Print(DescribeRestore(dpaCR.Client, restore)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, noDcDrestoreName) @@ -183,8 +183,8 @@ var _ = Describe("AWS backup restore tests", func() { withDcRestoreName := fmt.Sprintf("%s-with-dc-workaround", restoreName) restore, err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, withDcRestoreName, WithIncludedResources(dcWorkaroundResources)) Expect(err).ToNot(HaveOccurred()) - Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) - log.Printf(DescribeRestore(dpaCR.Client, restore)) + Eventually(IsRestoreDone(dpaCR.Client, namespace, withDcRestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) + log.Print(DescribeRestore(dpaCR.Client, restore)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, withDcRestoreName) From d2c0fc291049d4a3e2980b9ffeaf73797ab56640 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 16:15:59 -0400 Subject: [PATCH 25/34] Use GinkgoWriter for describe --- tests/e2e/backup_restore_suite_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index e1651855e9..e7c85f661e 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -170,7 +170,7 @@ var _ = Describe("AWS backup restore tests", func() { restore, err := CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, noDcDrestoreName, WithExcludedResources(dcWorkaroundResources)) Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, noDcDrestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) - log.Print(DescribeRestore(dpaCR.Client, restore)) + GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, noDcDrestoreName) @@ -184,7 +184,7 @@ var _ = Describe("AWS backup restore tests", func() { restore, err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, withDcRestoreName, WithIncludedResources(dcWorkaroundResources)) Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, withDcRestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) - log.Print(DescribeRestore(dpaCR.Client, restore)) + GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore)) Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, withDcRestoreName) From a3cd1ece260d4031ca6677289a9e489436cc2958 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 18:35:22 -0400 Subject: [PATCH 26/34] fix getVeleroPods labels --- tests/e2e/lib/dpa_helpers.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/e2e/lib/dpa_helpers.go b/tests/e2e/lib/dpa_helpers.go index 959af74907..1c00686833 100755 --- a/tests/e2e/lib/dpa_helpers.go +++ b/tests/e2e/lib/dpa_helpers.go @@ -11,6 +11,7 @@ import ( "reflect" "strings" + . "github.com/onsi/ginkgo/v2" buildv1 "github.com/openshift/api/build/v1" "github.com/openshift/oadp-operator/pkg/common" @@ -192,13 +193,22 @@ func GetVeleroPods(namespace string) (*corev1.PodList, error) { } // select Velero pod with this label veleroOptions := metav1.ListOptions{ + LabelSelector: "component=velero", + } + veleroOptionsDeploy := metav1.ListOptions{ LabelSelector: "deploy=velero", } // get pods in test namespace with labelSelector - podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), veleroOptions) - if err != nil { + var podList *corev1.PodList + if podList, err = clientset.CoreV1().Pods(namespace).List(context.TODO(), veleroOptions); err != nil { return nil, err } + if len(podList.Items) == 0 { + // handle some oadp versions where label was deploy=velero + if podList, err = clientset.CoreV1().Pods(namespace).List(context.TODO(), veleroOptionsDeploy); err!= nil { + return nil, err + } + } return podList, nil } @@ -208,6 +218,10 @@ func AreVeleroPodsRunning(namespace string) wait.ConditionFunc { if err != nil { return false, err } + if podList.Items == nil || len(podList.Items) == 0 { + GinkgoWriter.Println("velero pods not found") + return false, nil + } for _, podInfo := range (*podList).Items { if podInfo.Status.Phase != corev1.PodRunning { log.Printf("pod: %s is not yet running with status: %v", podInfo.Name, podInfo.Status) From f20f536033c60cda914fe7b823821c6d2e3b25bf Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 18:39:45 -0400 Subject: [PATCH 27/34] B/R logs request via downloadrequest --- tests/e2e/backup_restore_suite_test.go | 10 +++-- tests/e2e/lib/velero_helpers.go | 61 +++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index e7c85f661e..da7368ceca 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -138,7 +138,7 @@ var _ = Describe("AWS backup restore tests", func() { // wait for backup to not be running Eventually(IsBackupDone(dpaCR.Client, namespace, backupName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) GinkgoWriter.Println(DescribeBackup(dpaCR.Client, backup)) - Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) + Expect(BackupErrorLogs(dpaCR.Client,backup)).To(Equal([]string{})) // check if backup succeeded succeeded, err := IsBackupCompletedSuccessfully(dpaCR.Client, backup) @@ -171,7 +171,8 @@ var _ = Describe("AWS backup restore tests", func() { Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, noDcDrestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore)) - Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) + Expect(RestoreErrorLogs(dpaCR.Client,restore)).To(Equal([]string{})) + // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, noDcDrestoreName) Expect(err).ToNot(HaveOccurred()) @@ -185,7 +186,8 @@ var _ = Describe("AWS backup restore tests", func() { Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, withDcRestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore)) - Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) + Expect(RestoreErrorLogs(dpaCR.Client,restore)).To(Equal([]string{})) + // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, withDcRestoreName) Expect(err).ToNot(HaveOccurred()) @@ -198,7 +200,7 @@ var _ = Describe("AWS backup restore tests", func() { Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore)) - Expect(GetVeleroContainerFailureLogs(dpaCR.Namespace)).To(Equal([]string{})) + Expect(RestoreErrorLogs(dpaCR.Client,restore)).To(Equal([]string{})) // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName) diff --git a/tests/e2e/lib/velero_helpers.go b/tests/e2e/lib/velero_helpers.go index 84ee855195..039f216cdd 100644 --- a/tests/e2e/lib/velero_helpers.go +++ b/tests/e2e/lib/velero_helpers.go @@ -1,15 +1,20 @@ package lib import ( + "bytes" "context" "fmt" "os" + "regexp" + "strings" + "time" snapshotv1beta1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1beta1" snapshotv1beta1client "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/features" veleroClientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" @@ -94,6 +99,60 @@ func DescribeRestore(ocClient client.Client, restore velero.Restore) string { if err != nil { fmt.Fprintf(os.Stderr, "error getting PodVolumeRestores for restore %s: %v\n", restore.Name, err) } - + return output.DescribeRestore(context.Background(), ocClient, &restore, podvolumeRestoreList.Items, details, veleroClient, insecureSkipTLSVerify, caCertFile) } + +func BackupLogs(ocClient client.Client, backup velero.Backup) string { + insecureSkipTLSVerify := true + caCertFile := "" + // new io.Writer that store the logs in a string + logs := &bytes.Buffer{} + // new io.Writer that store the logs in a string + + downloadrequest.Stream(context.Background(), ocClient, backup.Namespace, backup.Name, velero.DownloadTargetKindBackupLog, logs, time.Minute, insecureSkipTLSVerify, caCertFile) + + return logs.String() +} + +func RestoreLogs(ocClient client.Client, restore velero.Restore) string { + insecureSkipTLSVerify := true + caCertFile := "" + // new io.Writer that store the logs in a string + logs := &bytes.Buffer{} + // new io.Writer that store the logs in a string + + downloadrequest.Stream(context.Background(), ocClient, restore.Namespace, restore.Name, velero.DownloadTargetKindRestoreLog, logs, time.Minute, insecureSkipTLSVerify, caCertFile) + + return logs.String() +} + +func BackupErrorLogs(ocClient client.Client, backup velero.Backup) []string { + bl := BackupLogs(ocClient, backup) + errorRegex, err := regexp.Compile("error|Error") + if err != nil { + return []string{"could not compile regex: ", err.Error()} + } + logLines := []string{} + for _, line := range strings.Split(bl, "\n") { + if errorRegex.MatchString(line) { + logLines = append(logLines, line) + } + } + return logLines +} + +func RestoreErrorLogs(ocClient client.Client, restore velero.Restore) []string { + rl := RestoreLogs(ocClient, restore) + errorRegex, err := regexp.Compile("error|Error") + if err != nil { + return []string{"could not compile regex: ", err.Error()} + } + logLines := []string{} + for _, line := range strings.Split(rl, "\n") { + if errorRegex.MatchString(line) { + logLines = append(logLines, line) + } + } + return logLines +} From e314324d36d695a2133ff44f47168f23beb767e6 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 19:30:04 -0400 Subject: [PATCH 28/34] Check cluster version skip before building dpa --- tests/e2e/backup_restore_suite_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index da7368ceca..6c7eb39359 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -82,6 +82,10 @@ var _ = Describe("AWS backup restore tests", func() { DescribeTable("backup and restore applications", func(brCase BackupRestoreCase, expectedErr error) { + if notVersionTarget, reason := NotServerVersionTarget(brCase.MinK8SVersion, brCase.MaxK8SVersion); notVersionTarget { + Skip(reason) + } + err := dpaCR.Build(brCase.BackupRestoreType) Expect(err).NotTo(HaveOccurred()) @@ -106,9 +110,7 @@ var _ = Describe("AWS backup restore tests", func() { log.Printf("Waiting for registry pods to be running") Eventually(AreRegistryDeploymentsAvailable(namespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue()) } - if notVersionTarget, reason := NotServerVersionTarget(brCase.MinK8SVersion, brCase.MaxK8SVersion); notVersionTarget { - Skip(reason) - } + backupUid, _ := uuid.NewUUID() restoreUid, _ := uuid.NewUUID() backupName := fmt.Sprintf("%s-%s", brCase.Name, backupUid.String()) From 92c32cbc550cbb452bf0bc83904c577b5b90277c Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Sun, 1 May 2022 21:04:58 -0400 Subject: [PATCH 29/34] Azure resource group (#45) * fix awk for azure resource group * newline proof awk for resourcegroup --- tests/e2e/scripts/azure_settings.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/scripts/azure_settings.sh b/tests/e2e/scripts/azure_settings.sh index 3bcb31bc09..ee4d7d1d4c 100644 --- a/tests/e2e/scripts/azure_settings.sh +++ b/tests/e2e/scripts/azure_settings.sh @@ -4,7 +4,7 @@ CI_AZURE_SUBSCRIPTION_ID=$(cat $CI_JSON_CRED_FILE | awk '/subscriptionId/ { gsub CI_AZURE_CLIENT_ID=$(cat $CI_JSON_CRED_FILE | awk '/clientId/ { gsub(/["]|,.*/,""); print $2}') CI_AZURE_CLIENT_SECRET=$(cat $CI_JSON_CRED_FILE | awk '/clientSecret/ { gsub(/["]|,.*/,""); print $2}') CI_AZURE_TENANT_ID=$(cat $CI_JSON_CRED_FILE | awk '/tenantId/ { gsub(/["]|,.*/,""); print $2}') -CI_AZURE_RESOURCE_GROUP=$(cat $AZURE_RESOURCE_FILE | awk '/infraID/ { gsub(/["]|,.*/,""); print $2}') +CI_AZURE_RESOURCE_GROUP=$(cat $AZURE_RESOURCE_FILE | awk '{ gsub(/["\n \{\}:]|.*infraID/,"");gsub(/,/," "); print $1}') if [ "$OPENSHIFT_CI" == "true" ]; then\ CI_AZURE_RESOURCE_GROUP="${CI_AZURE_RESOURCE_GROUP}-rg"; \ From 8440ffee803b1ac0e5876d59ad920707fa3366f0 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Mon, 2 May 2022 11:38:37 -0400 Subject: [PATCH 30/34] remove unused commented codes --- tests/e2e/lib/apps.go | 57 ------------------------------------------- 1 file changed, 57 deletions(-) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index d9bca15b4e..79698123f6 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -107,36 +107,6 @@ func UninstallApplication(ocClient client.Client, file string) error { return nil } -// func AreApplicationPodsRunning(namespace string) wait.ConditionFunc { -// return func() (bool, error) { -// clientset, err := setUpClient() -// if err != nil { -// return false, err -// } -// // select Velero pod with this label -// veleroOptions := metav1.ListOptions{ -// LabelSelector: "e2e-app=true", -// } -// // get pods in test namespace with labelSelector -// podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), veleroOptions) -// if err != nil { -// return false, nil -// } -// if len(podList.Items) == 0 { -// return false, nil -// } -// // get pod name and status with specified label selector -// for _, podInfo := range podList.Items { -// phase := podInfo.Status.Phase -// if phase != corev1.PodRunning && phase != corev1.PodSucceeded { -// ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("Pod %v not yet succeeded", podInfo.Name))) -// ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("status: %v", podInfo.Status))) -// return false, nil -// } -// } -// return true, err -// } -// } func HasDCsInNamespace(ocClient client.Client, namespace string) (bool, error) { dcList := &ocpappsv1.DeploymentConfigList{} err := ocClient.List(context.Background(), dcList, client.InNamespace(namespace)) @@ -165,9 +135,6 @@ func HasTemplateInstancesInNamespace(ocClient client.Client, namespace string) ( tiList := &templatev1.TemplateInstanceList{} err := ocClient.List(context.Background(), tiList, client.InNamespace(namespace)) if err != nil { - // if runtime.IsNotRegisteredError(err) { - // return false, nil - // } return false, err } if len(tiList.Items) == 0 { @@ -198,20 +165,8 @@ func AreVolumeSnapshotsReady(ocClient client.Client, backupName string) wait.Con // vListBeta := &volumesnapshotv1beta1.VolumeSnapshotList{} err := ocClient.List(context.Background(), vList, &client.ListOptions{LabelSelector: label.NewSelectorForBackup(backupName)}) if err != nil { - // try beta version - // if runtime.IsNotRegisteredError(err) { - // // try v1beta1 - // vList = nil // reset - // err = ocClient.List(context.Background(), vList, client.InNamespace(namespace)) - // if err != nil { - // if runtime.IsNotRegisteredError(err) { - // return false, nil - // } - // } - // } return false, err } - // if vList != nil { if len(vList.Items) == 0 { ginkgo.GinkgoWriter.Println("No VolumeSnapshots found") return false, nil @@ -228,18 +183,6 @@ func AreVolumeSnapshotsReady(ocClient client.Client, backupName string) wait.Con return false, nil } } - // } - // } else { - // if len(vListBeta.Items) == 0 { - // return false, nil - // } - // for _, v := range vListBeta.Items { - // if v.Status.ReadyToUse == nil || !*v.Status.ReadyToUse { - // return false, nil - // } - // } - // } - return true, nil } } From 641bd1f738166a094b68dbd61ffec95374816594 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Mon, 2 May 2022 11:42:56 -0400 Subject: [PATCH 31/34] break loop after found DC availability condition --- tests/e2e/lib/apps.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 79698123f6..e89dd84ebe 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -199,8 +199,11 @@ func IsDCReady(ocClient client.Client, namespace, dcName string) wait.ConditionF } // check dc for false availability condition which occurs when a new replication controller is created (after a new build completed) even if there are satisfactory available replicas for _, condition := range dc.Status.Conditions { - if condition.Type == ocpappsv1.DeploymentAvailable && condition.Status == corev1.ConditionFalse { - return false, nil + if condition.Type == ocpappsv1.DeploymentAvailable { + if condition.Status == corev1.ConditionFalse { + return false, nil + } + break } } if dc.Status.AvailableReplicas != dc.Status.Replicas || dc.Status.Replicas == 0 { From e1467b2474051b5fcd1215776782f37a143c92b1 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Mon, 2 May 2022 12:06:21 -0400 Subject: [PATCH 32/34] rebase reverted go.mod velero update --- controllers/velero.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/velero.go b/controllers/velero.go index ca9766cea9..d5d64dd8ea 100644 --- a/controllers/velero.go +++ b/controllers/velero.go @@ -124,7 +124,7 @@ func (r *DPAReconciler) ReconcileVeleroCRDs(log logr.Logger) (bool, error) { func (r *DPAReconciler) InstallVeleroCRDs(log logr.Logger) error { var err error // Install CRDs - for _, unstructuredCrd := range install.AllCRDs().Items { + for _, unstructuredCrd := range install.AllCRDs("v1").Items { foundCrd := &v1.CustomResourceDefinition{} crd := &v1.CustomResourceDefinition{} if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredCrd.Object, crd); err != nil { From d370624cc02a86ae74f411509f493f4e94a71fbc Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Mon, 2 May 2022 12:07:24 -0400 Subject: [PATCH 33/34] make test go fmt --- tests/e2e/backup_restore_suite_test.go | 8 ++++---- tests/e2e/lib/apps.go | 7 +++---- tests/e2e/lib/dpa_helpers.go | 2 +- tests/e2e/lib/velero_helpers.go | 8 ++++---- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 6c7eb39359..907edc6bd0 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -140,7 +140,7 @@ var _ = Describe("AWS backup restore tests", func() { // wait for backup to not be running Eventually(IsBackupDone(dpaCR.Client, namespace, backupName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) GinkgoWriter.Println(DescribeBackup(dpaCR.Client, backup)) - Expect(BackupErrorLogs(dpaCR.Client,backup)).To(Equal([]string{})) + Expect(BackupErrorLogs(dpaCR.Client, backup)).To(Equal([]string{})) // check if backup succeeded succeeded, err := IsBackupCompletedSuccessfully(dpaCR.Client, backup) @@ -173,7 +173,7 @@ var _ = Describe("AWS backup restore tests", func() { Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, noDcDrestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore)) - Expect(RestoreErrorLogs(dpaCR.Client,restore)).To(Equal([]string{})) + Expect(RestoreErrorLogs(dpaCR.Client, restore)).To(Equal([]string{})) // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, noDcDrestoreName) @@ -188,7 +188,7 @@ var _ = Describe("AWS backup restore tests", func() { Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, withDcRestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore)) - Expect(RestoreErrorLogs(dpaCR.Client,restore)).To(Equal([]string{})) + Expect(RestoreErrorLogs(dpaCR.Client, restore)).To(Equal([]string{})) // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, withDcRestoreName) @@ -202,7 +202,7 @@ var _ = Describe("AWS backup restore tests", func() { Expect(err).ToNot(HaveOccurred()) Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue()) GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore)) - Expect(RestoreErrorLogs(dpaCR.Client,restore)).To(Equal([]string{})) + Expect(RestoreErrorLogs(dpaCR.Client, restore)).To(Equal([]string{})) // Check if restore succeeded succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index e89dd84ebe..2ceb0979e6 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -39,10 +39,9 @@ const e2eAppLabelValue = "true" var ( e2eAppLabelRequirement, _ = labels.NewRequirement(e2eAppLabelKey, selection.Equals, []string{e2eAppLabelValue}) - e2eAppLabelSelector = labels.NewSelector().Add(*e2eAppLabelRequirement) + e2eAppLabelSelector = labels.NewSelector().Add(*e2eAppLabelRequirement) ) - func InstallApplication(ocClient client.Client, file string) error { template, err := os.ReadFile(file) if err != nil { @@ -259,10 +258,10 @@ func areAppBuildsReady(ocClient client.Client, namespace string) (bool, error) { if err != nil && !apierrors.IsNotFound(err) { return false, err } - if buildList.Items != nil{ + if buildList.Items != nil { for _, build := range buildList.Items { if build.Status.Phase == buildv1.BuildPhaseNew || - build.Status.Phase == buildv1.BuildPhasePending || + build.Status.Phase == buildv1.BuildPhasePending || build.Status.Phase == buildv1.BuildPhaseRunning { log.Println("Build is not ready: " + build.Name) return false, nil diff --git a/tests/e2e/lib/dpa_helpers.go b/tests/e2e/lib/dpa_helpers.go index 1c00686833..2db9849399 100755 --- a/tests/e2e/lib/dpa_helpers.go +++ b/tests/e2e/lib/dpa_helpers.go @@ -205,7 +205,7 @@ func GetVeleroPods(namespace string) (*corev1.PodList, error) { } if len(podList.Items) == 0 { // handle some oadp versions where label was deploy=velero - if podList, err = clientset.CoreV1().Pods(namespace).List(context.TODO(), veleroOptionsDeploy); err!= nil { + if podList, err = clientset.CoreV1().Pods(namespace).List(context.TODO(), veleroOptionsDeploy); err != nil { return nil, err } } diff --git a/tests/e2e/lib/velero_helpers.go b/tests/e2e/lib/velero_helpers.go index 039f216cdd..5da62e8210 100644 --- a/tests/e2e/lib/velero_helpers.go +++ b/tests/e2e/lib/velero_helpers.go @@ -99,7 +99,7 @@ func DescribeRestore(ocClient client.Client, restore velero.Restore) string { if err != nil { fmt.Fprintf(os.Stderr, "error getting PodVolumeRestores for restore %s: %v\n", restore.Name, err) } - + return output.DescribeRestore(context.Background(), ocClient, &restore, podvolumeRestoreList.Items, details, veleroClient, insecureSkipTLSVerify, caCertFile) } @@ -109,9 +109,9 @@ func BackupLogs(ocClient client.Client, backup velero.Backup) string { // new io.Writer that store the logs in a string logs := &bytes.Buffer{} // new io.Writer that store the logs in a string - + downloadrequest.Stream(context.Background(), ocClient, backup.Namespace, backup.Name, velero.DownloadTargetKindBackupLog, logs, time.Minute, insecureSkipTLSVerify, caCertFile) - + return logs.String() } @@ -121,7 +121,7 @@ func RestoreLogs(ocClient client.Client, restore velero.Restore) string { // new io.Writer that store the logs in a string logs := &bytes.Buffer{} // new io.Writer that store the logs in a string - + downloadrequest.Stream(context.Background(), ocClient, restore.Namespace, restore.Name, velero.DownloadTargetKindRestoreLog, logs, time.Minute, insecureSkipTLSVerify, caCertFile) return logs.String() From 61a649efaf916f5367ea9b98a599144e63984805 Mon Sep 17 00:00:00 2001 From: Tiger Kaovilai Date: Mon, 2 May 2022 12:59:51 -0400 Subject: [PATCH 34/34] use log.Printf --- tests/e2e/lib/velero_helpers.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/e2e/lib/velero_helpers.go b/tests/e2e/lib/velero_helpers.go index 5da62e8210..d5960c4014 100644 --- a/tests/e2e/lib/velero_helpers.go +++ b/tests/e2e/lib/velero_helpers.go @@ -3,8 +3,7 @@ package lib import ( "bytes" "context" - "fmt" - "os" + "log" "regexp" "strings" "time" @@ -51,13 +50,13 @@ func DescribeBackup(ocClient client.Client, backup velero.Backup) string { deleteRequestListOptions := pkgbackup.NewDeleteBackupRequestListOptions(backup.Name, string(backup.UID)) deleteRequestList, err := veleroClient.VeleroV1().DeleteBackupRequests(backup.Namespace).List(context.TODO(), deleteRequestListOptions) if err != nil { - fmt.Fprintf(os.Stderr, "error getting DeleteBackupRequests for backup %s: %v\n", backup.Name, err) + log.Printf("error getting DeleteBackupRequests for backup %s: %v\n", backup.Name, err) } opts := label.NewListOptionsForBackup(backup.Name) podVolumeBackupList, err := veleroClient.VeleroV1().PodVolumeBackups(backup.Namespace).List(context.TODO(), opts) if err != nil { - fmt.Fprintf(os.Stderr, "error getting PodVolumeBackups for backup %s: %v\n", backup.Name, err) + log.Printf("error getting PodVolumeBackups for backup %s: %v\n", backup.Name, err) } var csiClient *snapshotv1beta1client.Clientset @@ -71,7 +70,7 @@ func DescribeBackup(ocClient client.Client, backup velero.Backup) string { vscList, err = csiClient.SnapshotV1beta1().VolumeSnapshotContents().List(context.TODO(), opts) if err != nil { - fmt.Fprintf(os.Stderr, "error getting VolumeSnapshotContent objects for backup %s: %v\n", backup.Name, err) + log.Printf("error getting VolumeSnapshotContent objects for backup %s: %v\n", backup.Name, err) } } @@ -97,7 +96,7 @@ func DescribeRestore(ocClient client.Client, restore velero.Restore) string { opts := restic.NewPodVolumeRestoreListOptions(restore.Name) podvolumeRestoreList, err := veleroClient.VeleroV1().PodVolumeRestores(restore.Namespace).List(context.TODO(), opts) if err != nil { - fmt.Fprintf(os.Stderr, "error getting PodVolumeRestores for restore %s: %v\n", restore.Name, err) + log.Printf("error getting PodVolumeRestores for restore %s: %v\n", restore.Name, err) } return output.DescribeRestore(context.Background(), ocClient, &restore, podvolumeRestoreList.Items, details, veleroClient, insecureSkipTLSVerify, caCertFile)