e2e test support: Store test pod logs and events

test/e2e/mnist_pytorch_mcad_job_test.go

@@ -145,9 +145,6 @@ func TestMNISTPyTorchMCAD(t *testing.T) {
 	test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium).
 		Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
 
-	// Retrieving the job logs once it has completed or timed out
-	defer WriteJobLogs(test, job.Namespace, job.Name)
-
 	test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
 	test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
 		Or(

test/e2e/mnist_raycluster_sdk_test.go

@@ -192,9 +192,6 @@ func TestMNISTRayClusterSDK(t *testing.T) {
 	test.Expect(err).NotTo(HaveOccurred())
 	test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name)
 
-	// Retrieving the job logs once it has completed or timed out
-	defer WriteJobLogs(test, job.Namespace, job.Name)
-
 	test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
 	test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
 		Or(

test/e2e/setup.sh

@@ -18,11 +18,9 @@ set -euo pipefail
 : "${KUBERAY_VERSION}"
 
 echo Deploying KubeRay "${KUBERAY_VERSION}"
-kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=90s"
+kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=180s"
 
-kubectl create ns codeflare-system --dry-run=client -o yaml | kubectl apply -f -
-
-cat <<EOF | kubectl apply -n codeflare-system -f -
+cat <<EOF | kubectl apply -f -
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -44,7 +42,7 @@ rules:
       - delete
 EOF
 
-cat <<EOF | kubectl apply -n codeflare-system -f -
+cat <<EOF | kubectl apply -f -
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:

test/support/batch.go

@@ -20,9 +20,7 @@ import (
 	"github.com/onsi/gomega"
 
 	batchv1 "k8s.io/api/batch/v1"
-	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 )
 
 func Job(t Test, namespace, name string) func(g gomega.Gomega) *batchv1.Job {
@@ -37,22 +35,3 @@ func GetJob(t Test, namespace, name string) *batchv1.Job {
 	t.T().Helper()
 	return Job(t, namespace, name)(t)
 }
-
-func WriteJobLogs(t Test, namespace, name string) {
-	t.T().Helper()
-
-	job := GetJob(t, namespace, name)
-
-	pods := GetPods(t, job.Namespace, metav1.ListOptions{
-		LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)},
-	)
-
-	if len(pods) == 0 {
-		t.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name)
-	} else {
-		for i, pod := range pods {
-			t.T().Logf("Retrieving Pod %s/%s logs", pod.Namespace, pod.Name)
-			WriteToOutputDir(t, pod.Name, Log, GetPodLogs(t, &pods[i], corev1.PodLogOptions{}))
-		}
-	}
-}

test/support/core.go

@@ -57,3 +57,35 @@ func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) []byte {
 
 	return bytes
 }
+
+func storeAllPodLogs(t Test, namespace *corev1.Namespace) {
+	t.T().Helper()
+
+	pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), metav1.ListOptions{})
+	t.Expect(err).NotTo(gomega.HaveOccurred())
+
+	for _, pod := range pods.Items {
+		for _, container := range pod.Spec.Containers {
+			t.T().Logf("Retrieving Pod Container %s/%s/%s logs", pod.Namespace, pod.Name, container.Name)
+			storeContainerLog(t, namespace, pod.Name, container.Name)
+		}
+	}
+}
+
+func storeContainerLog(t Test, namespace *corev1.Namespace, podName, containerName string) {
+	t.T().Helper()
+
+	options := corev1.PodLogOptions{Container: containerName}
+	stream, err := t.Client().Core().CoreV1().Pods(namespace.Name).GetLogs(podName, &options).Stream(t.Ctx())
+	t.Expect(err).NotTo(gomega.HaveOccurred())
+
+	defer func() {
+		t.Expect(stream.Close()).To(gomega.Succeed())
+	}()
+
+	bytes, err := io.ReadAll(stream)
+	t.Expect(err).NotTo(gomega.HaveOccurred())
+
+	containerLogFileName := "pod-" + podName + "-" + containerName
+	WriteToOutputDir(t, containerLogFileName, Log, bytes)
+}

test/support/events.go

@@ -0,0 +1,148 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package support
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	eventsv1 "k8s.io/api/events/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// Based on https://github.com/apache/incubator-kie-kogito-operator/blob/28b2d3dc945e48659b199cca33723568b848f72e/test/pkg/framework/logging.go
+
+const (
+	eventLastSeenKey  = "LAST_SEEN"
+	eventFirstSeenKey = "FIRST_SEEN"
+	eventNameKey      = "NAME"
+	eventSubObjectKey = "SUBOBJECT"
+	eventTypeKey      = "TYPE"
+	eventReasonKey    = "REASON"
+	eventMessageKey   = "MESSAGE"
+
+	eventLogFileName = "events"
+)
+
+var eventKeys = []string{
+	eventLastSeenKey,
+	eventFirstSeenKey,
+	eventNameKey,
+	eventSubObjectKey,
+	eventTypeKey,
+	eventReasonKey,
+	eventMessageKey,
+}
+
+func storeEvents(t Test, namespace *corev1.Namespace) {
+	t.T().Helper()
+
+	events, err := t.Client().Core().EventsV1().Events(namespace.Name).List(t.Ctx(), metav1.ListOptions{})
+	t.Expect(err).NotTo(gomega.HaveOccurred())
+
+	bytes, err := renderEventContent(eventKeys, mapEventsToKeys(events))
+	t.Expect(err).NotTo(gomega.HaveOccurred())
+
+	WriteToOutputDir(t, eventLogFileName, Log, bytes)
+}
+
+func mapEventsToKeys(eventList *eventsv1.EventList) []map[string]string {
+	eventMaps := []map[string]string{}
+
+	for _, event := range eventList.Items {
+		eventMap := make(map[string]string)
+		eventMap[eventLastSeenKey] = getDefaultEventValueIfNull(event.DeprecatedLastTimestamp.Format("2006-01-02 15:04:05"))
+		eventMap[eventFirstSeenKey] = getDefaultEventValueIfNull(event.DeprecatedFirstTimestamp.Format("2006-01-02 15:04:05"))
+		eventMap[eventNameKey] = getDefaultEventValueIfNull(event.GetName())
+		eventMap[eventSubObjectKey] = getDefaultEventValueIfNull(event.Regarding.FieldPath)
+		eventMap[eventTypeKey] = getDefaultEventValueIfNull(event.Type)
+		eventMap[eventReasonKey] = getDefaultEventValueIfNull(event.Reason)
+		eventMap[eventMessageKey] = getDefaultEventValueIfNull(event.Note)
+
+		eventMaps = append(eventMaps, eventMap)
+	}
+	return eventMaps
+}
+
+func getDefaultEventValueIfNull(value string) string {
+	if len(value) <= 0 {
+		return "-"
+	}
+	return value
+}
+
+func renderEventContent(keys []string, dataMaps []map[string]string) ([]byte, error) {
+	var content bytes.Buffer
+	// Get size of strings to be written, to be able to format correctly
+	maxStringSizeMap := make(map[string]int)
+	for _, key := range keys {
+		maxSize := len(key)
+		for _, dataMap := range dataMaps {
+			if len(dataMap[key]) > maxSize {
+				maxSize = len(dataMap[key])
+			}
+		}
+		maxStringSizeMap[key] = maxSize
+	}
+
+	// Write headers
+	for _, header := range keys {
+		if _, err := content.WriteString(header); err != nil {
+			return nil, fmt.Errorf("error in writing the header: %v", err)
+		}
+		if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[header] - len(header) + 1)); err != nil {
+			return nil, fmt.Errorf("error in writing headers: %v", err)
+		}
+		if _, err := content.WriteString(" | "); err != nil {
+			return nil, fmt.Errorf("error in writing headers : %v", err)
+		}
+	}
+	if _, err := content.WriteString("\n"); err != nil {
+		return nil, fmt.Errorf("error in writing headers '|': %v", err)
+
+	}
+
+	// Write events
+	for _, dataMap := range dataMaps {
+		for _, key := range keys {
+			if _, err := content.WriteString(dataMap[key]); err != nil {
+				return nil, fmt.Errorf("error in writing events: %v", err)
+			}
+			if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[key] - len(dataMap[key]) + 1)); err != nil {
+				return nil, fmt.Errorf("error in writing events: %v", err)
+			}
+			if _, err := content.WriteString(" | "); err != nil {
+				return nil, fmt.Errorf("error in writing events: %v", err)
+			}
+		}
+		if _, err := content.WriteString("\n"); err != nil {
+			return nil, fmt.Errorf("error in writing events: %v", err)
+		}
+	}
+	return content.Bytes(), nil
+}
+
+func getWhitespaceStr(size int) string {
+	whiteSpaceStr := ""
+	for i := 0; i < size; i++ {
+		whiteSpaceStr += " "
+	}
+	return whiteSpaceStr
+}

test/support/ray_api.go

@@ -29,7 +29,7 @@ func WriteRayJobAPILogs(t Test, rayClient RayClusterClient, jobID string) {
 	t.T().Helper()
 	logs, err := rayClient.GetJobLogs(jobID)
 	t.Expect(err).NotTo(gomega.HaveOccurred())
-	WriteToOutputDir(t, jobID, Log, []byte(logs))
+	WriteToOutputDir(t, "ray-job-log-"+jobID, Log, []byte(logs))
 }
 
 func RayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) func(g gomega.Gomega) *RayJobDetailsResponse {

test/support/test.go

@@ -131,6 +131,8 @@ func (t *T) NewTestNamespace(options ...Option[*corev1.Namespace]) *corev1.Names
 	t.T().Helper()
 	namespace := createTestNamespace(t, options...)
 	t.T().Cleanup(func() {
+		storeAllPodLogs(t, namespace)
+		storeEvents(t, namespace)
 		deleteTestNamespace(t, namespace)
 	})
 	return namespace