Skip to content

Commit 080aa64

Browse files
pacoxupacoxu
committed
add basic update logic for ai gateway
1 parent 032b899 commit 080aa64

File tree

2 files changed

+195
-0
lines changed

2 files changed

+195
-0
lines changed

‎pkg/controller/inference/gateway.go‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
/*
2+
Copyright 2025 The InftyAI Team.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package inference
18+
19+
import (
20+
"context"
21+
22+
aigv1a1 "github.com/envoyproxy/ai-gateway/api/v1alpha1"
23+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
"k8s.io/apimachinery/pkg/types"
25+
"sigs.k8s.io/controller-runtime/pkg/client"
26+
)
27+
28+
// gateway related utils: currently only support envoy ai gateway
29+
30+
// IsAIGatewayRouteExist check if the AIGatewayRoute exist
31+
func IsAIGatewayRouteExist(ctx context.Context, client client.Client) (bool, error) {
32+
var route aigv1a1.AIGatewayRoute
33+
err := client.Get(ctx, types.NamespacedName{
34+
Name: "envoy-ai-gateway-basic",
35+
Namespace: "default",
36+
}, &route)
37+
if err != nil {
38+
return false, err
39+
}
40+
return true, nil
41+
}
42+
43+
// create a AIServiceBackend using playground model name
44+
// example like below:
45+
// apiVersion: aigateway.envoyproxy.io/v1alpha1
46+
// kind: AIServiceBackend
47+
// metadata:
48+
// name: envoy-ai-gateway-llmaz-model-1 # backendRef
49+
// namespace: default
50+
// spec:
51+
// schema:
52+
// name: OpenAI
53+
// backendRef:
54+
// name: qwen2-0--5b-lb # model name
55+
// kind: Service
56+
// port: 8080
57+
func CreateAIServiceBackend(ctx context.Context, client client.Client, backendRefName, namespace string, port int, schemaName string) error {
58+
if schemaName == "" {
59+
schemaName = "OpenAI"
60+
}
61+
// create the AIServiceBackend
62+
backend := &aigv1a1.AIServiceBackend{
63+
ObjectMeta: metav1.ObjectMeta{
64+
Name: backendRefName,
65+
Namespace: namespace,
66+
},
67+
Spec: aigv1a1.AIServiceBackendSpec{
68+
Schema: aigv1a1.AIServiceBackendSchema{
69+
Name: schemaName,
70+
},
71+
BackendRef: aigv1a1.AIServiceBackendRef{
72+
Name: backendRefName,
73+
Kind: "Service",
74+
Port: port,
75+
},
76+
},
77+
}
78+
return client.Create(ctx, backend)
79+
}
80+
81+
// update aigateway.envoyproxy.io/v1alpha1 AIGatewayRoute default `envoy-ai-gateway-basic` spec.rules list
82+
// example like below:
83+
// - matches:
84+
// - headers:
85+
// - type: Exact
86+
// name: x-ai-eg-model
87+
// value: qwen2-0.5b # model name
88+
// backendRefs:
89+
// - name: envoy-ai-gateway-llmaz-model-1 # backendRef
90+
func UpdateAIGatewayRoute(ctx context.Context, client client.Client, backendRefName, namespace, modelName string) error {
91+
// get the AIGatewayRoute
92+
var route aigv1a1.AIGatewayRoute
93+
if err := client.Get(ctx, types.NamespacedName{
94+
Name: "envoy-ai-gateway-basic",
95+
Namespace: namespace,
96+
}, &route); err != nil {
97+
return err
98+
}
99+
// update the spec.rules list if the rule does not exist
100+
for _, r := range route.Spec.Rules {
101+
if len(r.Matches) == 0 {
102+
continue
103+
}
104+
if len(r.BackendRefs) == 0 {
105+
continue
106+
}
107+
if r.Matches[0].Headers[0].Value == modelName && r.BackendRefs[0].Name == backendRefName {
108+
return nil
109+
}
110+
}
111+
// if the rule does not exist, append it to the spec.rules list
112+
rule := aigv1a1.AIGatewayRouteRule{
113+
Matches: []aigv1a1.AIGatewayRouteMatch{
114+
{
115+
Headers: []aigv1a1.AIGatewayRouteHeaderMatch{
116+
{
117+
Type: aigv1a1.HeaderMatchTypeExact,
118+
Name: "x-ai-eg-model",
119+
Value: modelName,
120+
},
121+
},
122+
},
123+
},
124+
BackendRefs: []aigv1a1.AIGatewayRouteBackendRef{
125+
{
126+
Name: backendRefName,
127+
},
128+
},
129+
}
130+
route.Spec.Rules = append(route.Spec.Rules, rule)
131+
// update the AIGatewayRoute
132+
return client.Update(ctx, &route)
133+
}
134+
135+
// delete a rule by modelName/backendRefname from the AIGatewayRoute
136+
func deleteAIGatewayRoute(ctx context.Context, client client.Client, modelName, backendRefName string) error {
137+
// get the AIGatewayRoute
138+
var route aigv1a1.AIGatewayRoute
139+
if err := client.Get(ctx, types.NamespacedName{
140+
Name: "envoy-ai-gateway-basic",
141+
Namespace: "default",
142+
}, &route); err != nil {
143+
return err
144+
}
145+
// delete the rule by modelName/backendRefname
146+
var newRules []aigv1a1.AIGatewayRouteRule
147+
for _, rule := range route.Spec.Rules {
148+
if len(rule.Matches) == 0 {
149+
continue
150+
}
151+
if len(rule.BackendRefs) == 0 {
152+
continue
153+
}
154+
if rule.Matches[0].Headers[0].Value == modelName && rule.BackendRefs[0].Name == backendRefName {
155+
continue
156+
}
157+
newRules = append(newRules, rule)
158+
}
159+
route.Spec.Rules = newRules
160+
161+
// update the AIGatewayRoute
162+
return client.Update(ctx, &route)
163+
}

‎pkg/controller/inference/playground_controller.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import (
4242
"sigs.k8s.io/controller-runtime/pkg/predicate"
4343
lws "sigs.k8s.io/lws/api/leaderworkerset/v1"
4444

45+
aigv1a1 "github.com/envoyproxy/ai-gateway/api/v1alpha1"
4546
coreapi "github.com/inftyai/llmaz/api/core/v1alpha1"
4647
inferenceapi "github.com/inftyai/llmaz/api/inference/v1alpha1"
4748
coreclientgo "github.com/inftyai/llmaz/client-go/applyconfiguration/core/v1alpha1"
@@ -149,6 +150,37 @@ func (r *PlaygroundReconciler) Reconcile(ctx context.Context, req ctrl.Request)
149150
return ctrl.Result{}, err
150151
}
151152

153+
// Handle ai gateway route and backend runtime.
154+
gatewayExist, err := IsAIGatewayRouteExist(ctx, r.Client)
155+
if err != nil {
156+
logger.Error(err, "failed to check ai gateway route")
157+
return ctrl.Result{}, err
158+
}
159+
if gatewayExist {
160+
var aiServiceBackend aigv1a1.AIServiceBackend
161+
if err := r.Get(ctx, types.NamespacedName{Name: playground.Name, Namespace: playground.Namespace}, &aiServiceBackend); err != nil {
162+
if apierrors.IsNotFound(err) {
163+
err = CreateAIServiceBackend(ctx, r.Client, playground.Name, playground.Namespace, modelSource.DEFAULT_BACKEND_PORT, "")
164+
if err != nil {
165+
logger.Error(err, "failed to create aiServiceBackend", "Playground", klog.KObj(playground))
166+
return ctrl.Result{}, err
167+
}
168+
logger.Info("created aiServiceBackend", "Playground", klog.KObj(playground))
169+
} else {
170+
logger.Error(err, "failed to get aiServiceBackend", "Playground", klog.KObj(playground))
171+
return ctrl.Result{}, err
172+
}
173+
} else {
174+
// Update the aiServiceBackend if it exists.
175+
// todo
176+
}
177+
err = UpdateAIGatewayRoute(ctx, r.Client, playground.Name, playground.Namespace, string(playground.Spec.ModelClaim.ModelName))
178+
if err != nil {
179+
logger.Error(err, "failed to update ai gateway route", "Playground", klog.KObj(playground))
180+
return ctrl.Result{}, err
181+
}
182+
}
183+
152184
return ctrl.Result{}, nil
153185
}
154186

0 commit comments

Comments
 (0)