Skip to content

Commit c274ced

Browse files
committed
e2e test support: Store test pod logs and events
1 parent 0abc4c8 commit c274ced

File tree

8 files changed

+185
-33
lines changed

8 files changed

+185
-33
lines changed

test/e2e/mnist_pytorch_mcad_job_test.go

-3
Original file line numberDiff line numberDiff line change
@@ -145,9 +145,6 @@ func TestMNISTPyTorchMCAD(t *testing.T) {
145145
test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium).
146146
Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
147147

148-
// Retrieving the job logs once it has completed or timed out
149-
defer WriteJobLogs(test, job.Namespace, job.Name)
150-
151148
test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
152149
test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
153150
Or(

test/e2e/mnist_raycluster_sdk_test.go

-3
Original file line numberDiff line numberDiff line change
@@ -192,9 +192,6 @@ func TestMNISTRayClusterSDK(t *testing.T) {
192192
test.Expect(err).NotTo(HaveOccurred())
193193
test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name)
194194

195-
// Retrieving the job logs once it has completed or timed out
196-
defer WriteJobLogs(test, job.Namespace, job.Name)
197-
198195
test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
199196
test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
200197
Or(

test/e2e/setup.sh

+3-5
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,9 @@ set -euo pipefail
1818
: "${KUBERAY_VERSION}"
1919

2020
echo Deploying KubeRay "${KUBERAY_VERSION}"
21-
kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=90s"
21+
kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=180s"
2222

23-
kubectl create ns codeflare-system --dry-run=client -o yaml | kubectl apply -f -
24-
25-
cat <<EOF | kubectl apply -n codeflare-system -f -
23+
cat <<EOF | kubectl apply -f -
2624
apiVersion: rbac.authorization.k8s.io/v1
2725
kind: ClusterRole
2826
metadata:
@@ -44,7 +42,7 @@ rules:
4442
- delete
4543
EOF
4644

47-
cat <<EOF | kubectl apply -n codeflare-system -f -
45+
cat <<EOF | kubectl apply -f -
4846
kind: ClusterRoleBinding
4947
apiVersion: rbac.authorization.k8s.io/v1
5048
metadata:

test/support/batch.go

-21
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@ import (
2020
"github.com/onsi/gomega"
2121

2222
batchv1 "k8s.io/api/batch/v1"
23-
corev1 "k8s.io/api/core/v1"
2423
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
25-
"k8s.io/apimachinery/pkg/labels"
2624
)
2725

2826
func Job(t Test, namespace, name string) func(g gomega.Gomega) *batchv1.Job {
@@ -37,22 +35,3 @@ func GetJob(t Test, namespace, name string) *batchv1.Job {
3735
t.T().Helper()
3836
return Job(t, namespace, name)(t)
3937
}
40-
41-
func WriteJobLogs(t Test, namespace, name string) {
42-
t.T().Helper()
43-
44-
job := GetJob(t, namespace, name)
45-
46-
pods := GetPods(t, job.Namespace, metav1.ListOptions{
47-
LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)},
48-
)
49-
50-
if len(pods) == 0 {
51-
t.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name)
52-
} else {
53-
for i, pod := range pods {
54-
t.T().Logf("Retrieving Pod %s/%s logs", pod.Namespace, pod.Name)
55-
WriteToOutputDir(t, pod.Name, Log, GetPodLogs(t, &pods[i], corev1.PodLogOptions{}))
56-
}
57-
}
58-
}

test/support/core.go

+32
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,35 @@ func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) []byte {
5757

5858
return bytes
5959
}
60+
61+
func StoreAllPodLogs(t Test, namespace *corev1.Namespace) {
62+
t.T().Helper()
63+
64+
pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), metav1.ListOptions{})
65+
t.Expect(err).NotTo(gomega.HaveOccurred())
66+
67+
for _, pod := range pods.Items {
68+
for _, container := range pod.Spec.Containers {
69+
t.T().Logf("Retrieving Pod Container %s/%s/%s logs", pod.Namespace, pod.Name, container.Name)
70+
storeContainerLog(t, namespace, pod.Name, container.Name)
71+
}
72+
}
73+
}
74+
75+
func storeContainerLog(t Test, namespace *corev1.Namespace, podName, containerName string) {
76+
t.T().Helper()
77+
78+
options := corev1.PodLogOptions{Container: containerName}
79+
stream, err := t.Client().Core().CoreV1().Pods(namespace.Name).GetLogs(podName, &options).Stream(t.Ctx())
80+
t.Expect(err).NotTo(gomega.HaveOccurred())
81+
82+
defer func() {
83+
t.Expect(stream.Close()).To(gomega.Succeed())
84+
}()
85+
86+
bytes, err := io.ReadAll(stream)
87+
t.Expect(err).NotTo(gomega.HaveOccurred())
88+
89+
containerLogFileName := "pod-" + podName + "-" + containerName
90+
WriteToOutputDir(t, containerLogFileName, Log, bytes)
91+
}

test/support/events.go

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/*
2+
Copyright 2023.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package support
18+
19+
import (
20+
"bytes"
21+
"fmt"
22+
23+
"github.com/onsi/gomega"
24+
corev1 "k8s.io/api/core/v1"
25+
eventsv1 "k8s.io/api/events/v1"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
)
28+
29+
// Based on https://github.com/apache/incubator-kie-kogito-operator/blob/28b2d3dc945e48659b199cca33723568b848f72e/test/pkg/framework/logging.go
30+
31+
const (
32+
eventLastSeenKey = "LAST_SEEN"
33+
eventFirstSeenKey = "FIRST_SEEN"
34+
eventNameKey = "NAME"
35+
eventSubObjectKey = "SUBOBJECT"
36+
eventTypeKey = "TYPE"
37+
eventReasonKey = "REASON"
38+
eventMessageKey = "MESSAGE"
39+
40+
eventLogFileName = "events"
41+
)
42+
43+
var eventKeys = []string{
44+
eventLastSeenKey,
45+
eventFirstSeenKey,
46+
eventNameKey,
47+
eventSubObjectKey,
48+
eventTypeKey,
49+
eventReasonKey,
50+
eventMessageKey,
51+
}
52+
53+
func StoreEvents(t Test, namespace *corev1.Namespace) {
54+
t.T().Helper()
55+
56+
events, err := t.Client().Core().EventsV1().Events(namespace.Name).List(t.Ctx(), metav1.ListOptions{})
57+
t.Expect(err).NotTo(gomega.HaveOccurred())
58+
59+
bytes, err := renderEventContent(eventKeys, mapEventsToKeys(events))
60+
t.Expect(err).NotTo(gomega.HaveOccurred())
61+
62+
WriteToOutputDir(t, eventLogFileName, Log, bytes)
63+
}
64+
65+
func mapEventsToKeys(eventList *eventsv1.EventList) []map[string]string {
66+
eventMaps := []map[string]string{}
67+
68+
for _, event := range eventList.Items {
69+
eventMap := make(map[string]string)
70+
eventMap[eventLastSeenKey] = getDefaultEventValueIfNull(event.DeprecatedLastTimestamp.Format("2006-01-02 15:04:05"))
71+
eventMap[eventFirstSeenKey] = getDefaultEventValueIfNull(event.DeprecatedFirstTimestamp.Format("2006-01-02 15:04:05"))
72+
eventMap[eventNameKey] = getDefaultEventValueIfNull(event.GetName())
73+
eventMap[eventSubObjectKey] = getDefaultEventValueIfNull(event.Regarding.FieldPath)
74+
eventMap[eventTypeKey] = getDefaultEventValueIfNull(event.Type)
75+
eventMap[eventReasonKey] = getDefaultEventValueIfNull(event.Reason)
76+
eventMap[eventMessageKey] = getDefaultEventValueIfNull(event.Note)
77+
78+
eventMaps = append(eventMaps, eventMap)
79+
}
80+
return eventMaps
81+
}
82+
83+
func getDefaultEventValueIfNull(value string) string {
84+
if len(value) <= 0 {
85+
return "-"
86+
}
87+
return value
88+
}
89+
90+
func renderEventContent(keys []string, dataMaps []map[string]string) ([]byte, error) {
91+
var content bytes.Buffer
92+
// Get size of strings to be written, to be able to format correctly
93+
maxStringSizeMap := make(map[string]int)
94+
for _, key := range keys {
95+
maxSize := len(key)
96+
for _, dataMap := range dataMaps {
97+
if len(dataMap[key]) > maxSize {
98+
maxSize = len(dataMap[key])
99+
}
100+
}
101+
maxStringSizeMap[key] = maxSize
102+
}
103+
104+
// Write headers
105+
for _, header := range keys {
106+
if _, err := content.WriteString(header); err != nil {
107+
return nil, fmt.Errorf("error in writing the header: %v", err)
108+
}
109+
if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[header] - len(header) + 1)); err != nil {
110+
return nil, fmt.Errorf("error in writing headers: %v", err)
111+
}
112+
if _, err := content.WriteString(" | "); err != nil {
113+
return nil, fmt.Errorf("error in writing headers : %v", err)
114+
}
115+
}
116+
if _, err := content.WriteString("\n"); err != nil {
117+
return nil, fmt.Errorf("error in writing headers '|': %v", err)
118+
119+
}
120+
121+
// Write events
122+
for _, dataMap := range dataMaps {
123+
for _, key := range keys {
124+
if _, err := content.WriteString(dataMap[key]); err != nil {
125+
return nil, fmt.Errorf("error in writing events: %v", err)
126+
}
127+
if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[key] - len(dataMap[key]) + 1)); err != nil {
128+
return nil, fmt.Errorf("error in writing events: %v", err)
129+
}
130+
if _, err := content.WriteString(" | "); err != nil {
131+
return nil, fmt.Errorf("error in writing events: %v", err)
132+
}
133+
}
134+
if _, err := content.WriteString("\n"); err != nil {
135+
return nil, fmt.Errorf("error in writing events: %v", err)
136+
}
137+
}
138+
return content.Bytes(), nil
139+
}
140+
141+
func getWhitespaceStr(size int) string {
142+
whiteSpaceStr := ""
143+
for i := 0; i < size; i++ {
144+
whiteSpaceStr += " "
145+
}
146+
return whiteSpaceStr
147+
}

test/support/ray_api.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ func WriteRayJobAPILogs(t Test, rayClient RayClusterClient, jobID string) {
2929
t.T().Helper()
3030
logs, err := rayClient.GetJobLogs(jobID)
3131
t.Expect(err).NotTo(gomega.HaveOccurred())
32-
WriteToOutputDir(t, jobID, Log, []byte(logs))
32+
WriteToOutputDir(t, "ray-job-log-"+jobID, Log, []byte(logs))
3333
}
3434

3535
func RayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) func(g gomega.Gomega) *RayJobDetailsResponse {

test/support/test.go

+2
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,8 @@ func (t *T) NewTestNamespace(options ...Option[*corev1.Namespace]) *corev1.Names
131131
t.T().Helper()
132132
namespace := createTestNamespace(t, options...)
133133
t.T().Cleanup(func() {
134+
StoreAllPodLogs(t, namespace)
135+
StoreEvents(t, namespace)
134136
deleteTestNamespace(t, namespace)
135137
})
136138
return namespace

0 commit comments

Comments
 (0)