Skip to content

Commit c2661d8

Browse files
committed
feat: add e2e test to verify service is avaliable
1 parent 7bca83f commit c2661d8

File tree

4 files changed

+171
-1
lines changed

4 files changed

+171
-1
lines changed

go.mod

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,16 @@ require (
4242
github.com/google/gofuzz v1.2.0 // indirect
4343
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
4444
github.com/google/uuid v1.6.0 // indirect
45+
github.com/gorilla/websocket v1.5.0 // indirect
4546
github.com/josharian/intern v1.0.0 // indirect
4647
github.com/json-iterator/go v1.1.12 // indirect
4748
github.com/klauspost/compress v1.17.9 // indirect
4849
github.com/mailru/easyjson v0.7.7 // indirect
50+
github.com/moby/spdystream v0.5.0 // indirect
4951
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
5052
github.com/modern-go/reflect2 v1.0.2 // indirect
5153
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
54+
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
5255
github.com/pkg/errors v0.9.1 // indirect
5356
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
5457
github.com/prometheus/client_golang v1.20.2 // indirect

go.sum

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
2+
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
13
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
24
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
35
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -46,6 +48,8 @@ github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/Z
4648
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
4749
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
4850
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
51+
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
52+
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
4953
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
5054
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
5155
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -62,13 +66,17 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
6266
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
6367
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
6468
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
69+
github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU=
70+
github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
6571
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
6672
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
6773
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
6874
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
6975
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
7076
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
7177
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
78+
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
79+
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
7280
github.com/onsi/ginkgo/v2 v2.23.0 h1:FA1xjp8ieYDzlgS5ABTpdUDB7wtngggONc8a7ku2NqQ=
7381
github.com/onsi/ginkgo/v2 v2.23.0/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGmaSRvxnM=
7482
github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8=

test/e2e/playground_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ var _ = ginkgo.Describe("playground e2e tests", func() {
6363
defer func() {
6464
gomega.Expect(k8sClient.Delete(ctx, model)).To(gomega.Succeed())
6565
}()
66-
playground := wrapper.MakePlayground("qwen2-0--5b", ns.Name).ModelClaim("qwen2-0--5b").BackendRuntime("llmaz-ollama").Replicas(1).Obj()
66+
playground := wrapper.MakePlayground("qwen2-0--5b", ns.Name).ModelClaim("qwen2-0--5b").BackendRuntime("llmaz-ollama").BackendRuntimeEnv("OLLAMA_HOST", "0.0.0.0:8080").Replicas(1).Obj()
6767
gomega.Expect(k8sClient.Create(ctx, playground)).To(gomega.Succeed())
6868
validation.ValidatePlayground(ctx, k8sClient, playground)
6969
validation.ValidatePlaygroundStatusEqualTo(ctx, k8sClient, playground, inferenceapi.PlaygroundAvailable, "PlaygroundReady", metav1.ConditionTrue)
@@ -73,6 +73,7 @@ var _ = ginkgo.Describe("playground e2e tests", func() {
7373
validation.ValidateService(ctx, k8sClient, service)
7474
validation.ValidateServiceStatusEqualTo(ctx, k8sClient, service, inferenceapi.ServiceAvailable, "ServiceReady", metav1.ConditionTrue)
7575
validation.ValidateServicePods(ctx, k8sClient, service)
76+
validation.ValidateServiceAvaliable(ctx, k8sClient, cfg, service, validation.CheckOllamaServeAvaliable)
7677
})
7778
ginkgo.It("Deploy a huggingface model with llama.cpp", func() {
7879
model := wrapper.MakeModel("qwen2-0-5b-gguf").FamilyName("qwen2").ModelSourceWithModelHub("Huggingface").ModelSourceWithModelID("Qwen/Qwen2-0.5B-Instruct-GGUF", "qwen2-0_5b-instruct-q5_k_m.gguf", "", nil, nil).Obj()
@@ -91,6 +92,7 @@ var _ = ginkgo.Describe("playground e2e tests", func() {
9192
validation.ValidateService(ctx, k8sClient, service)
9293
validation.ValidateServiceStatusEqualTo(ctx, k8sClient, service, inferenceapi.ServiceAvailable, "ServiceReady", metav1.ConditionTrue)
9394
validation.ValidateServicePods(ctx, k8sClient, service)
95+
validation.ValidateServiceAvaliable(ctx, k8sClient, cfg, service, validation.CheckLlamacppServeAvaliable)
9496
})
9597
ginkgo.It("Deploy a huggingface model with customized backendRuntime", func() {
9698
backendRuntime := wrapper.MakeBackendRuntime("llmaz-llamacpp").

test/util/validation/validate_service.go

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,24 @@ import (
2020
"context"
2121
"errors"
2222
"fmt"
23+
"io"
24+
"net/http"
25+
"os"
26+
"os/signal"
2327
"strconv"
28+
"strings"
29+
"syscall"
30+
"time"
2431

2532
"github.com/onsi/gomega"
2633
corev1 "k8s.io/api/core/v1"
2734
apimeta "k8s.io/apimachinery/pkg/api/meta"
2835
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2936
"k8s.io/apimachinery/pkg/types"
37+
"k8s.io/client-go/kubernetes"
38+
"k8s.io/client-go/rest"
39+
"k8s.io/client-go/tools/portforward"
40+
"k8s.io/client-go/transport/spdy"
3041
"sigs.k8s.io/controller-runtime/pkg/client"
3142
lws "sigs.k8s.io/lws/api/leaderworkerset/v1"
3243

@@ -233,3 +244,149 @@ func ValidateServicePods(ctx context.Context, k8sClient client.Client, service *
233244
return nil
234245
}).Should(gomega.Succeed())
235246
}
247+
248+
type CheckServiceAvailableFunc func(int) error
249+
250+
func ValidateServiceAvaliable(ctx context.Context, k8sClient client.Client, cfg *rest.Config, service *inferenceapi.Service, check CheckServiceAvailableFunc) {
251+
gomega.Eventually(func() error {
252+
pods := corev1.PodList{}
253+
podSelector := client.MatchingLabels(map[string]string{
254+
lws.SetNameLabelKey: service.Name,
255+
})
256+
if err := k8sClient.List(ctx, &pods, podSelector, client.InNamespace(service.Namespace)); err != nil {
257+
return err
258+
}
259+
if len(pods.Items) != int(*service.Spec.Replicas)*int(*service.Spec.WorkloadTemplate.Size) {
260+
return fmt.Errorf("pods number not right, want: %d, got: %d", int(*service.Spec.Replicas)*int(*service.Spec.WorkloadTemplate.Size), len(pods.Items))
261+
}
262+
263+
var targetPod *corev1.Pod
264+
for i := range pods.Items {
265+
if pods.Items[i].Status.Phase == corev1.PodRunning {
266+
targetPod = &pods.Items[i]
267+
break
268+
}
269+
}
270+
271+
if targetPod == nil {
272+
return fmt.Errorf("no running pods found for service %s", service.Name)
273+
}
274+
275+
portForwardK8sClient, err := kubernetes.NewForConfig(cfg)
276+
if err != nil {
277+
return fmt.Errorf("init port forward client failed: %w", err)
278+
}
279+
280+
targetPort := targetPod.Spec.Containers[0].Ports[0].ContainerPort
281+
localPort := 8080
282+
283+
stopChan, readyChan := make(chan struct{}, 1), make(chan struct{}, 1)
284+
req := portForwardK8sClient.CoreV1().RESTClient().Post().
285+
Resource("pods").
286+
Namespace(service.Namespace).
287+
Name(targetPod.Name).
288+
SubResource("portforward")
289+
290+
transport, upgrader, err := spdy.RoundTripperFor(cfg)
291+
if err != nil {
292+
return fmt.Errorf("creating round tripper failed: %v", err)
293+
}
294+
295+
dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", req.URL())
296+
297+
// create port forwarder
298+
fw, err := portforward.New(dialer, []string{fmt.Sprintf("%d:%d", localPort, targetPort)}, stopChan, readyChan, os.Stdout, os.Stderr)
299+
if err != nil {
300+
return fmt.Errorf("creating port forwarder failed: %v", err)
301+
}
302+
303+
signals := make(chan os.Signal, 1)
304+
signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
305+
306+
go func() {
307+
<-signals
308+
fmt.Println("Received termination signal, shutting down port forward...")
309+
close(stopChan)
310+
}()
311+
312+
// wait for port forward to be ready
313+
go func() {
314+
if err = fw.ForwardPorts(); err != nil {
315+
fmt.Printf("Error forwarding ports: %v\n", err)
316+
close(stopChan)
317+
}
318+
}()
319+
<-readyChan
320+
fmt.Printf("Port forwarding is ready. Local port has been forwarded to service %s (via pod %s) on port %d\n",
321+
service.Name, targetPod.Name, targetPort)
322+
323+
time.Sleep(60 * time.Second)
324+
325+
return check(localPort)
326+
}).Should(gomega.Succeed())
327+
}
328+
329+
func CheckOllamaServeAvaliable(localPort int) error {
330+
url := fmt.Sprintf("http://localhost:%d/api/generate", localPort)
331+
reqBody := `{"model":"qwen2:0.5b","prompt":"What is the capital city of China?","stream":false}`
332+
333+
// wait for ollama serve to download the model
334+
time.Sleep(60 * time.Second)
335+
336+
fmt.Printf("url: %s, req body: %s\n", url, reqBody)
337+
req, err := http.NewRequest("POST", url, strings.NewReader(reqBody))
338+
if err != nil {
339+
return err
340+
}
341+
client := &http.Client{}
342+
resp, err := client.Do(req)
343+
if err != nil {
344+
return err
345+
}
346+
defer resp.Body.Close()
347+
348+
body, err := io.ReadAll(resp.Body)
349+
if err != nil {
350+
return fmt.Errorf("error reading response: %v", err)
351+
}
352+
353+
if resp.StatusCode != http.StatusOK {
354+
return fmt.Errorf("error HTTP status code %d", resp.StatusCode)
355+
}
356+
357+
if strings.Contains(strings.ToLower(string(body)), "beijing") {
358+
return fmt.Errorf("error response body: %s", string(body))
359+
}
360+
return nil
361+
}
362+
363+
func CheckLlamacppServeAvaliable(localPort int) error {
364+
url := fmt.Sprintf("http://localhost:%d/completions", localPort)
365+
reqBody := `{"prompt":"What is the capital city of China?","stream":false}`
366+
367+
fmt.Printf("url: %s, req body: %s\n", url, reqBody)
368+
req, err := http.NewRequest("POST", url, strings.NewReader(reqBody))
369+
if err != nil {
370+
return err
371+
}
372+
client := &http.Client{}
373+
resp, err := client.Do(req)
374+
if err != nil {
375+
return err
376+
}
377+
defer resp.Body.Close()
378+
379+
body, err := io.ReadAll(resp.Body)
380+
if err != nil {
381+
return fmt.Errorf("error reading response: %v", err)
382+
}
383+
384+
if resp.StatusCode != http.StatusOK {
385+
return fmt.Errorf("error HTTP status code %d", resp.StatusCode)
386+
}
387+
388+
if strings.Contains(strings.ToLower(string(body)), "beijing") {
389+
return fmt.Errorf("error response body: %s", string(body))
390+
}
391+
return nil
392+
}

0 commit comments

Comments
 (0)