Skip to content

Commit 2b11fe1

Browse files
kaovilaikaovilai
authored
e2e: Cherry-pick #790, #801 to oadp-1.1 - enable restic data verification, tuning timeouts, reliability (#806)
* cherry-pick #790 set defaultVolumesToRestic in e2e for restic backup fix restic prebackup verify delete backupFile if it exists Print command output, fix err handling. ignore log `num errors=0` merge fix fix cutset variable renames, DpaCustomResource struct cleanup eliminate custom "credentialsFile", prefering bsl.Credentials trim restore name, run post restore script keep 43 chars for restore name with uuid restore name no longer needs trimming Fix creating restore name detect sha256sum is working else use shasum update checksum_cmd to uppercase and actually use var remove two-phase restore. Use single restore with post restore script. refactor DoesBSLExist, DoesVSLExist print diff Make nill velero config map equal to empty map. cherry-pick of #801 e2e: Increase app build timeouts to 5 minutes Create PVC for CSI before installing app. Add mongo pvc for ibm cloud InstallApplication Update if already exists. Resolves issue if existing is outdated Revert "Create PVC for CSI before installing app." This reverts commit efdb994. e2e: resolve pvc create failure if previous test fail to uninstall app namespace Change backup timeout to 12 minutes Example of test failing because it didn't wait long enough https://prow.ci.openshift.org/view/gs/origin-ci-test/pr-logs/pull/openshift_oadp-operator/801/pull-ci-openshift-oadp-operator-master-4.11-operator-e2e-aws/1561223345897213952#1:build-log.txt%3A7247 Default velero csi snapshot timeout is 10 minutes. Resolve azure 4.8, 4.9 fallback namespace deletion failure due to empty resource name. Revert "cherry-pick of #801" This reverts commit b9bc43e. * e2e: print container logs on failure only, increase app build timeouts (#801) * e2e: do not print container logs on server version skip * e2e: Increase app build timeouts to 5 minutes * Create PVC for CSI before installing app. * Add mongo pvc for ibm cloud * InstallApplication Update if already exists. Resolves issue if existing is outdated * Revert "Create PVC for CSI before installing app." This reverts commit efdb994. * e2e: resolve pvc create failure if previous test fail to uninstall app namespace * Change backup timeout to 12 minutes Example of test failing because it didn't wait long enough https://prow.ci.openshift.org/view/gs/origin-ci-test/pr-logs/pull/openshift_oadp-operator/801/pull-ci-openshift-oadp-operator-master-4.11-operator-e2e-aws/1561223345897213952#1:build-log.txt%3A7247 Default velero csi snapshot timeout is 10 minutes. * Resolve azure 4.8, 4.9 fallback namespace deletion failure due to empty resource name. * DeepEqual check resource fields before making update call Signed-off-by: Tiger Kaovilai <[email protected]> Signed-off-by: Tiger Kaovilai <[email protected]> Signed-off-by: Tiger Kaovilai <[email protected]>
1 parent 2cde554 commit 2b11fe1

File tree

9 files changed

+223
-141
lines changed

9 files changed

+223
-141
lines changed

‎tests/e2e/backup_restore_suite_test.go

Lines changed: 46 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
package e2e_test
22

33
import (
4+
"context"
45
"errors"
56
"fmt"
67
"log"
78
"time"
89

910
"github.com/google/uuid"
1011
. "github.com/onsi/ginkgo/v2"
12+
"github.com/onsi/ginkgo/v2/types"
1113
. "github.com/onsi/gomega"
1214
. "github.com/openshift/oadp-operator/tests/e2e/lib"
13-
utils "github.com/openshift/oadp-operator/tests/e2e/utils"
15+
corev1 "k8s.io/api/core/v1"
16+
k8serror "k8s.io/apimachinery/pkg/api/errors"
17+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1418
"sigs.k8s.io/controller-runtime/pkg/client"
1519
)
1620

@@ -53,32 +57,38 @@ var _ = Describe("AWS backup restore tests", func() {
5357
var _ = BeforeEach(func() {
5458
testSuiteInstanceName := "ts-" + instanceName
5559
dpaCR.Name = testSuiteInstanceName
56-
57-
credData, err := utils.ReadFile(credFile)
58-
Expect(err).NotTo(HaveOccurred())
59-
err = CreateCredentialsSecret(credData, namespace, GetSecretRef(credSecretRef))
60-
Expect(err).NotTo(HaveOccurred())
6160
})
6261

63-
var _ = AfterEach(func() {
64-
GinkgoWriter.Println("Printing velero deployment pod logs")
65-
logs, err := GetVeleroContainerLogs(namespace)
66-
Expect(err).NotTo(HaveOccurred())
67-
GinkgoWriter.Println(logs)
68-
GinkgoWriter.Println("End of velero deployment pod logs")
69-
err = dpaCR.Delete()
70-
Expect(err).ToNot(HaveOccurred())
71-
72-
})
7362
var lastInstallingApplicationNamespace string
7463
var lastInstallTime time.Time
7564
var _ = ReportAfterEach(func(report SpecReport) {
65+
if report.State == types.SpecStateSkipped {
66+
// do not run if the test is skipped
67+
return
68+
}
69+
GinkgoWriter.Println("Report after each: state: ", report.State.String())
7670
if report.Failed() {
7771
// print namespace error events for app namespace
7872
if lastInstallingApplicationNamespace != "" {
7973
PrintNamespaceEventsAfterTime(lastInstallingApplicationNamespace, lastInstallTime)
8074
}
75+
GinkgoWriter.Println("Printing velero deployment pod logs")
76+
logs, err := GetVeleroContainerLogs(namespace)
77+
Expect(err).NotTo(HaveOccurred())
78+
GinkgoWriter.Println(logs)
79+
GinkgoWriter.Println("End of velero deployment pod logs")
8180
}
81+
// remove app namespace if leftover (likely previously failed before reaching uninstall applications) to clear items such as PVCs which are immutable so that next test can create new ones
82+
err := dpaCR.Client.Delete(context.Background(), &corev1.Namespace{ObjectMeta: v1.ObjectMeta{
83+
Name: lastInstallingApplicationNamespace,
84+
Namespace: lastInstallingApplicationNamespace,
85+
}}, &client.DeleteOptions{})
86+
if k8serror.IsNotFound(err) {
87+
err = nil
88+
}
89+
Expect(err).ToNot(HaveOccurred())
90+
err = dpaCR.Delete()
91+
Expect(err).ToNot(HaveOccurred())
8292
})
8393

8494
type BackupRestoreCase struct {
@@ -160,7 +170,7 @@ var _ = Describe("AWS backup restore tests", func() {
160170
}
161171

162172
// wait for pods to be running
163-
Eventually(AreAppBuildsReady(dpaCR.Client, brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue())
173+
Eventually(AreAppBuildsReady(dpaCR.Client, brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*5, time.Second*5).Should(BeTrue())
164174
Eventually(AreApplicationPodsRunning(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*9, time.Second*5).Should(BeTrue())
165175

166176
// Run optional custom verification
@@ -172,11 +182,11 @@ var _ = Describe("AWS backup restore tests", func() {
172182
Expect(err).ToNot(HaveOccurred())
173183
// create backup
174184
log.Printf("Creating backup %s for case %s", backupName, brCase.Name)
175-
backup, err := CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace})
185+
backup, err := CreateBackupForNamespaces(dpaCR.Client, namespace, backupName, []string{brCase.ApplicationNamespace}, brCase.BackupRestoreType == RESTIC)
176186
Expect(err).ToNot(HaveOccurred())
177187

178188
// wait for backup to not be running
179-
Eventually(IsBackupDone(dpaCR.Client, namespace, backupName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue())
189+
Eventually(IsBackupDone(dpaCR.Client, namespace, backupName), timeoutMultiplier*time.Minute*12, time.Second*10).Should(BeTrue())
180190
GinkgoWriter.Println(DescribeBackup(dpaCR.Client, backup))
181191
Expect(BackupErrorLogs(dpaCR.Client, backup)).To(Equal([]string{}))
182192

@@ -200,52 +210,24 @@ var _ = Describe("AWS backup restore tests", func() {
200210
Eventually(IsNamespaceDeleted(brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*2, time.Second*5).Should(BeTrue())
201211

202212
updateLastInstallingNamespace(brCase.ApplicationNamespace)
203-
// Check if backup needs restic deploymentconfig workaround. https://github.com/openshift/oadp-operator/blob/master/docs/TROUBLESHOOTING.md#deployconfig
204-
if brCase.BackupRestoreType == RESTIC && nsRequiresResticDCWorkaround {
205-
log.Printf("DC found in backup namespace, using DC restic workaround")
206-
var dcWorkaroundResources = []string{"replicationcontroller", "deploymentconfig", "templateinstances.template.openshift.io"}
207-
// run restore
208-
log.Printf("Creating restore %s excluding DC workaround resources for case %s", restoreName, brCase.Name)
209-
noDcDrestoreName := fmt.Sprintf("%s-no-dc-workaround", restoreName)
210-
restore, err := CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, noDcDrestoreName, WithExcludedResources(dcWorkaroundResources))
211-
Expect(err).ToNot(HaveOccurred())
212-
Eventually(IsRestoreDone(dpaCR.Client, namespace, noDcDrestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue())
213-
GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore))
214-
Expect(RestoreErrorLogs(dpaCR.Client, restore)).To(Equal([]string{}))
215-
216-
// Check if restore succeeded
217-
succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, noDcDrestoreName)
218-
Expect(err).ToNot(HaveOccurred())
219-
Expect(succeeded).To(Equal(true))
220-
Eventually(AreAppBuildsReady(dpaCR.Client, brCase.ApplicationNamespace), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue())
221-
222-
// run restore
223-
log.Printf("Creating restore %s including DC workaround resources for case %s", restoreName, brCase.Name)
224-
withDcRestoreName := fmt.Sprintf("%s-with-dc-workaround", restoreName)
225-
restore, err = CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, withDcRestoreName, WithIncludedResources(dcWorkaroundResources))
226-
Expect(err).ToNot(HaveOccurred())
227-
Eventually(IsRestoreDone(dpaCR.Client, namespace, withDcRestoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue())
228-
GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore))
229-
Expect(RestoreErrorLogs(dpaCR.Client, restore)).To(Equal([]string{}))
230-
231-
// Check if restore succeeded
232-
succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, withDcRestoreName)
233-
Expect(err).ToNot(HaveOccurred())
234-
Expect(succeeded).To(Equal(true))
213+
// run restore
214+
log.Printf("Creating restore %s for case %s", restoreName, brCase.Name)
215+
restore, err := CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName)
216+
Expect(err).ToNot(HaveOccurred())
217+
Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue())
218+
GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore))
219+
Expect(RestoreErrorLogs(dpaCR.Client, restore)).To(Equal([]string{}))
235220

236-
} else {
237-
// run restore
238-
log.Printf("Creating restore %s for case %s", restoreName, brCase.Name)
239-
restore, err := CreateRestoreFromBackup(dpaCR.Client, namespace, backupName, restoreName)
240-
Expect(err).ToNot(HaveOccurred())
241-
Eventually(IsRestoreDone(dpaCR.Client, namespace, restoreName), timeoutMultiplier*time.Minute*4, time.Second*10).Should(BeTrue())
242-
GinkgoWriter.Println(DescribeRestore(dpaCR.Client, restore))
243-
Expect(RestoreErrorLogs(dpaCR.Client, restore)).To(Equal([]string{}))
221+
// Check if restore succeeded
222+
succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName)
223+
Expect(err).ToNot(HaveOccurred())
224+
Expect(succeeded).To(Equal(true))
244225

245-
// Check if restore succeeded
246-
succeeded, err = IsRestoreCompletedSuccessfully(dpaCR.Client, namespace, restoreName)
226+
if brCase.BackupRestoreType == RESTIC && nsRequiresResticDCWorkaround {
227+
// run the restic post restore script if restore type is RESTIC
228+
log.Printf("Running restic post restore script for case %s", brCase.Name)
229+
err = RunResticPostRestoreScript(restoreName)
247230
Expect(err).ToNot(HaveOccurred())
248-
Expect(succeeded).To(Equal(true))
249231
}
250232

251233
// verify app is running
@@ -294,15 +276,15 @@ var _ = Describe("AWS backup restore tests", func() {
294276
ApplicationNamespace: "mongo-persistent",
295277
Name: "mongo-restic-e2e",
296278
BackupRestoreType: RESTIC,
297-
PreBackupVerify: mongoready(false, RESTIC),
279+
PreBackupVerify: mongoready(true, RESTIC),
298280
PostRestoreVerify: mongoready(false, RESTIC),
299281
}, nil),
300282
Entry("MySQL application RESTIC", BackupRestoreCase{
301283
ApplicationTemplate: "./sample-applications/mysql-persistent/mysql-persistent.yaml",
302284
ApplicationNamespace: "mysql-persistent",
303285
Name: "mysql-restic-e2e",
304286
BackupRestoreType: RESTIC,
305-
PreBackupVerify: mysqlReady(false, RESTIC),
287+
PreBackupVerify: mysqlReady(true, RESTIC),
306288
PostRestoreVerify: mysqlReady(false, RESTIC),
307289
}, nil),
308290
)

‎tests/e2e/dpa_deployment_suite_test.go

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
2121
provider := Dpa.Spec.BackupLocations[0].Velero.Provider
2222
bucket := Dpa.Spec.BackupLocations[0].Velero.ObjectStorage.Bucket
2323
bslConfig := Dpa.Spec.BackupLocations[0].Velero.Config
24+
bslCredential := corev1.SecretKeySelector{
25+
LocalObjectReference: corev1.LocalObjectReference{
26+
Name: "bsl-cloud-credentials-" + provider,
27+
},
28+
Key: "cloud",
29+
}
2430

2531
type InstallCase struct {
2632
Name string
@@ -57,6 +63,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
5763
Prefix: VeleroPrefix,
5864
},
5965
},
66+
Credential: &bslCredential,
6067
},
6168
},
6269
},
@@ -90,6 +97,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
9097
Prefix: VeleroPrefix,
9198
},
9299
},
100+
Credential: &bslCredential,
93101
},
94102
},
95103
},
@@ -131,6 +139,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
131139
Prefix: VeleroPrefix,
132140
},
133141
},
142+
Credential: &bslCredential,
134143
},
135144
},
136145
},
@@ -177,6 +186,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
177186
Prefix: VeleroPrefix,
178187
},
179188
},
189+
Credential: &bslCredential,
180190
},
181191
},
182192
},
@@ -212,6 +222,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
212222
Prefix: VeleroPrefix,
213223
},
214224
},
225+
Credential: &bslCredential,
215226
},
216227
},
217228
},
@@ -245,6 +256,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
245256
Prefix: VeleroPrefix,
246257
},
247258
},
259+
Credential: &bslCredential,
248260
},
249261
},
250262
},
@@ -277,6 +289,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
277289
Prefix: VeleroPrefix,
278290
},
279291
},
292+
Credential: &bslCredential,
280293
},
281294
},
282295
},
@@ -311,6 +324,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
311324
Prefix: VeleroPrefix,
312325
},
313326
},
327+
Credential: &bslCredential,
314328
},
315329
},
316330
},
@@ -333,6 +347,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
333347
Prefix: VeleroPrefix,
334348
},
335349
},
350+
Credential: &bslCredential,
336351
},
337352
},
338353
},
@@ -369,6 +384,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
369384
Prefix: VeleroPrefix,
370385
},
371386
},
387+
Credential: &bslCredential,
372388
},
373389
},
374390
},
@@ -431,6 +447,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
431447
Prefix: VeleroPrefix,
432448
},
433449
},
450+
Credential: &bslCredential,
434451
},
435452
},
436453
},
@@ -466,6 +483,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
466483
Prefix: VeleroPrefix,
467484
},
468485
},
486+
Credential: &bslCredential,
469487
},
470488
},
471489
},
@@ -499,6 +517,7 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
499517
Prefix: VeleroPrefix,
500518
},
501519
},
520+
Credential: &bslCredential,
502521
},
503522
},
504523
},
@@ -536,10 +555,15 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
536555
err := dpaCR.Build(installCase.BRestoreType)
537556
Expect(err).NotTo(HaveOccurred())
538557
if len(installCase.DpaSpec.BackupLocations) > 0 {
539-
if installCase.DpaSpec.BackupLocations[0].Velero.Config != nil {
540-
installCase.DpaSpec.BackupLocations[0].Velero.Config["credentialsFile"] = "bsl-cloud-credentials-" + dpaCR.Provider + "/cloud"
541-
if installCase.TestCarriageReturn {
542-
installCase.DpaSpec.BackupLocations[0].Velero.Config["credentialsFile"] = "bsl-cloud-credentials-" + dpaCR.Provider + "-with-carriage-return/cloud"
558+
if installCase.DpaSpec.BackupLocations[0].Velero.Credential == nil {
559+
installCase.DpaSpec.BackupLocations[0].Velero.Credential = &bslCredential
560+
}
561+
if installCase.TestCarriageReturn {
562+
installCase.DpaSpec.BackupLocations[0].Velero.Credential = &corev1.SecretKeySelector{
563+
LocalObjectReference: corev1.LocalObjectReference{
564+
Name: "bsl-cloud-credentials-" + dpaCR.Provider + "-with-carriage-return",
565+
},
566+
Key: bslCredential.Key,
543567
}
544568
}
545569
}
@@ -564,13 +588,13 @@ var _ = Describe("Configuration testing for DPA Custom Resource", func() {
564588
log.Printf("Checking for bsl spec")
565589
for _, bsl := range dpa.Spec.BackupLocations {
566590
// Check if bsl matches the spec
567-
Eventually(DoesBSLExist(namespace, *bsl.Velero, installCase.DpaSpec), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue())
591+
Expect(DoesBSLSpecMatchesDpa(namespace, *bsl.Velero, installCase.DpaSpec)).To(BeTrue())
568592
}
569593
}
570594
if len(dpa.Spec.SnapshotLocations) > 0 {
571595
log.Printf("Checking for vsl spec")
572596
for _, vsl := range dpa.Spec.SnapshotLocations {
573-
Eventually(DoesVSLExist(namespace, *vsl.Velero, installCase.DpaSpec), timeoutMultiplier*time.Minute*3, time.Second*5).Should(BeTrue())
597+
Expect(DoesVSLSpecMatchesDpa(namespace, *vsl.Velero, installCase.DpaSpec)).To(BeTrue())
574598
}
575599
}
576600

0 commit comments

Comments
 (0)