Skip to content

Commit 5adc074

Browse files
authored
feat: add e2e test to verify service is avaliable (#310)
* feat: add e2e test to verify service is avaliable * delete ollama test case * fix comments * move port forward outof eventually
1 parent 568e6e4 commit 5adc074

File tree

4 files changed

+129
-1
lines changed

4 files changed

+129
-1
lines changed

go.mod

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,16 @@ require (
4242
github.com/google/gofuzz v1.2.0 // indirect
4343
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
4444
github.com/google/uuid v1.6.0 // indirect
45+
github.com/gorilla/websocket v1.5.0 // indirect
4546
github.com/josharian/intern v1.0.0 // indirect
4647
github.com/json-iterator/go v1.1.12 // indirect
4748
github.com/klauspost/compress v1.17.9 // indirect
4849
github.com/mailru/easyjson v0.7.7 // indirect
50+
github.com/moby/spdystream v0.5.0 // indirect
4951
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
5052
github.com/modern-go/reflect2 v1.0.2 // indirect
5153
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
54+
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
5255
github.com/pkg/errors v0.9.1 // indirect
5356
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
5457
github.com/prometheus/client_golang v1.20.2 // indirect

go.sum

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
2+
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
13
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
24
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
35
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -46,6 +48,8 @@ github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/Z
4648
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
4749
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
4850
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
51+
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
52+
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
4953
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
5054
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
5155
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -62,13 +66,17 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
6266
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
6367
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
6468
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
69+
github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU=
70+
github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
6571
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
6672
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
6773
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
6874
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
6975
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
7076
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
7177
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
78+
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
79+
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
7280
github.com/onsi/ginkgo/v2 v2.23.0 h1:FA1xjp8ieYDzlgS5ABTpdUDB7wtngggONc8a7ku2NqQ=
7381
github.com/onsi/ginkgo/v2 v2.23.0/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGmaSRvxnM=
7482
github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8=

test/e2e/playground_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ var _ = ginkgo.Describe("playground e2e tests", func() {
6363
defer func() {
6464
gomega.Expect(k8sClient.Delete(ctx, model)).To(gomega.Succeed())
6565
}()
66-
playground := wrapper.MakePlayground("qwen2-0--5b", ns.Name).ModelClaim("qwen2-0--5b").BackendRuntime("llmaz-ollama").Replicas(1).Obj()
66+
playground := wrapper.MakePlayground("qwen2-0--5b", ns.Name).ModelClaim("qwen2-0--5b").BackendRuntime("llmaz-ollama").BackendRuntimeEnv("OLLAMA_HOST", "0.0.0.0:8080").Replicas(1).Obj()
6767
gomega.Expect(k8sClient.Create(ctx, playground)).To(gomega.Succeed())
6868
validation.ValidatePlayground(ctx, k8sClient, playground)
6969
validation.ValidatePlaygroundStatusEqualTo(ctx, k8sClient, playground, inferenceapi.PlaygroundAvailable, "PlaygroundReady", metav1.ConditionTrue)
@@ -91,6 +91,7 @@ var _ = ginkgo.Describe("playground e2e tests", func() {
9191
validation.ValidateService(ctx, k8sClient, service)
9292
validation.ValidateServiceStatusEqualTo(ctx, k8sClient, service, inferenceapi.ServiceAvailable, "ServiceReady", metav1.ConditionTrue)
9393
validation.ValidateServicePods(ctx, k8sClient, service)
94+
gomega.Expect(validation.ValidateServiceAvaliable(ctx, k8sClient, cfg, service, validation.CheckServiceAvaliable)).To(gomega.Succeed())
9495
})
9596
ginkgo.It("Deploy a huggingface model with customized backendRuntime", func() {
9697
backendRuntime := wrapper.MakeBackendRuntime("llmaz-llamacpp").
@@ -116,6 +117,7 @@ var _ = ginkgo.Describe("playground e2e tests", func() {
116117
validation.ValidateService(ctx, k8sClient, service)
117118
validation.ValidateServiceStatusEqualTo(ctx, k8sClient, service, inferenceapi.ServiceAvailable, "ServiceReady", metav1.ConditionTrue)
118119
validation.ValidateServicePods(ctx, k8sClient, service)
120+
gomega.Expect(validation.ValidateServiceAvaliable(ctx, k8sClient, cfg, service, validation.CheckServiceAvaliable)).To(gomega.Succeed())
119121
})
120122
ginkgo.It("Deploy a huggingface model with llama.cpp, HPA enabled", func() {
121123
model := wrapper.MakeModel("qwen2-0-5b-gguf").FamilyName("qwen2").ModelSourceWithModelHub("Huggingface").ModelSourceWithModelID("Qwen/Qwen2-0.5B-Instruct-GGUF", "qwen2-0_5b-instruct-q5_k_m.gguf", "", nil, nil).Obj()

test/util/validation/validate_service.go

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,23 @@ import (
2020
"context"
2121
"errors"
2222
"fmt"
23+
"io"
24+
"net/http"
25+
"os"
26+
"os/signal"
2327
"strconv"
28+
"strings"
29+
"syscall"
2430

2531
"github.com/onsi/gomega"
2632
corev1 "k8s.io/api/core/v1"
2733
apimeta "k8s.io/apimachinery/pkg/api/meta"
2834
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2935
"k8s.io/apimachinery/pkg/types"
36+
"k8s.io/client-go/kubernetes"
37+
"k8s.io/client-go/rest"
38+
"k8s.io/client-go/tools/portforward"
39+
"k8s.io/client-go/transport/spdy"
3040
"sigs.k8s.io/controller-runtime/pkg/client"
3141
lws "sigs.k8s.io/lws/api/leaderworkerset/v1"
3242

@@ -233,3 +243,108 @@ func ValidateServicePods(ctx context.Context, k8sClient client.Client, service *
233243
return nil
234244
}).Should(gomega.Succeed())
235245
}
246+
247+
type CheckServiceAvailableFunc func() error
248+
249+
func ValidateServiceAvaliable(ctx context.Context, k8sClient client.Client, cfg *rest.Config, service *inferenceapi.Service, check CheckServiceAvailableFunc) error {
250+
pods := corev1.PodList{}
251+
podSelector := client.MatchingLabels(map[string]string{
252+
lws.SetNameLabelKey: service.Name,
253+
})
254+
if err := k8sClient.List(ctx, &pods, podSelector, client.InNamespace(service.Namespace)); err != nil {
255+
return err
256+
}
257+
if len(pods.Items) != int(*service.Spec.Replicas)*int(*service.Spec.WorkloadTemplate.Size) {
258+
return fmt.Errorf("pods number not right, want: %d, got: %d", int(*service.Spec.Replicas)*int(*service.Spec.WorkloadTemplate.Size), len(pods.Items))
259+
}
260+
261+
var targetPod *corev1.Pod
262+
for i := range pods.Items {
263+
if pods.Items[i].Status.Phase == corev1.PodRunning {
264+
targetPod = &pods.Items[i]
265+
break
266+
}
267+
}
268+
269+
if targetPod == nil {
270+
return fmt.Errorf("no running pods found for service %s", service.Name)
271+
}
272+
273+
portForwardK8sClient, err := kubernetes.NewForConfig(cfg)
274+
if err != nil {
275+
return fmt.Errorf("init port forward client failed: %w", err)
276+
}
277+
278+
targetPort := targetPod.Spec.Containers[0].Ports[0].ContainerPort
279+
stopChan, readyChan := make(chan struct{}, 1), make(chan struct{}, 1)
280+
req := portForwardK8sClient.CoreV1().RESTClient().Post().
281+
Resource("pods").
282+
Namespace(service.Namespace).
283+
Name(targetPod.Name).
284+
SubResource("portforward")
285+
286+
transport, upgrader, err := spdy.RoundTripperFor(cfg)
287+
if err != nil {
288+
return fmt.Errorf("creating round tripper failed: %v", err)
289+
}
290+
291+
dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", req.URL())
292+
// create port forwarder
293+
fw, err := portforward.New(dialer, []string{fmt.Sprintf("%d:%d", modelSource.DEFAULT_BACKEND_PORT, targetPort)}, stopChan, readyChan, os.Stdout, os.Stderr)
294+
if err != nil {
295+
return fmt.Errorf("creating port forwarder failed: %v", err)
296+
}
297+
// stop port forward when done
298+
defer fw.Close()
299+
signals := make(chan os.Signal, 1)
300+
signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
301+
302+
go func() {
303+
<-signals
304+
fmt.Println("Received termination signal, shutting down port forward...")
305+
close(stopChan)
306+
}()
307+
308+
// wait for port forward to be ready
309+
go func() {
310+
if err = fw.ForwardPorts(); err != nil {
311+
fmt.Printf("Error forwarding ports: %v\n", err)
312+
close(stopChan)
313+
}
314+
}()
315+
<-readyChan
316+
gomega.Eventually(check()).Should(gomega.Succeed())
317+
return nil
318+
}
319+
320+
func CheckServiceAvaliable() error {
321+
url := fmt.Sprintf("http://localhost:%d/completions", modelSource.DEFAULT_BACKEND_PORT)
322+
reqBody := `{"prompt":"What is the capital city of China?","stream":false}`
323+
324+
req, err := http.NewRequest("POST", url, strings.NewReader(reqBody))
325+
if err != nil {
326+
return err
327+
}
328+
client := &http.Client{}
329+
resp, err := client.Do(req)
330+
if err != nil {
331+
return err
332+
}
333+
defer func() {
334+
_ = resp.Body.Close()
335+
}()
336+
337+
if resp.StatusCode != http.StatusOK {
338+
return fmt.Errorf("error HTTP status code %d", resp.StatusCode)
339+
}
340+
341+
body, err := io.ReadAll(resp.Body)
342+
if err != nil {
343+
return fmt.Errorf("error reading response: %v", err)
344+
}
345+
346+
if !strings.Contains(strings.ToLower(string(body)), "beijing") {
347+
return fmt.Errorf("error response body: %s", string(body))
348+
}
349+
return nil
350+
}

0 commit comments

Comments
 (0)