Skip to content

Commit b22eb51

Browse files
JeffwanJeffwan
committed
Support watcher pod rbac in kv cache controller
update hpkv key in redis Signed-off-by: Jiaxin Shan <[email protected]>
1 parent 5595870 commit b22eb51

File tree

9 files changed

+246
-125
lines changed

9 files changed

+246
-125
lines changed

‎cmd/kvcache-watcher/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ import (
5252
const KVCacheLabelKeyIdentifier = "kvcache.orchestration.aibrix.ai/name"
5353
const KVCacheLabelKeyRole = "kvcache.orchestration.aibrix.ai/role"
5454
const KVCacheLabelValueRoleCache = "cache"
55-
const HPKVRedisNodeMemberKey = "hpkv_nodes"
55+
const HPKVRedisNodeMemberKey = "hpkv_cluster_metadata"
5656
const InfiniStoreRedisNodeMemberKey = "kvcache_nodes"
5757

5858
const networkStatusAnnotation = "k8s.volcengine.com/network-status"

‎config/rbac/controller-manager/role.yaml‎

Lines changed: 42 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,37 @@ rules:
1212
- create
1313
- patch
1414
- update
15+
- apiGroups:
16+
- ""
17+
resources:
18+
- secrets
19+
verbs:
20+
- get
21+
- list
22+
- update
23+
- watch
24+
- apiGroups:
25+
- admissionregistration.k8s.io
26+
resources:
27+
- mutatingwebhookconfigurations
28+
- validatingwebhookconfigurations
29+
verbs:
30+
- get
31+
- list
32+
- update
33+
- watch
34+
- apiGroups:
35+
- apiextensions.k8s.io
36+
resources:
37+
- customresourcedefinitions
38+
verbs:
39+
- get
40+
- list
1541
- apiGroups:
1642
- apps
1743
resources:
1844
- deployments
45+
- statefulsets
1946
verbs:
2047
- create
2148
- delete
@@ -28,30 +55,11 @@ rules:
2855
- apps
2956
resources:
3057
- deployments/status
58+
- statefulsets/status
3159
verbs:
3260
- get
3361
- patch
3462
- update
35-
- apiGroups:
36-
- apps
37-
resources:
38-
- statefulsets
39-
verbs:
40-
- create
41-
- delete
42-
- get
43-
- list
44-
- patch
45-
- update
46-
- watch
47-
- apiGroups:
48-
- apps
49-
resources:
50-
- statefulsets/status
51-
verbs:
52-
- get
53-
- patch
54-
- update
5563
- apiGroups:
5664
- autoscaling
5765
resources:
@@ -107,6 +115,7 @@ rules:
107115
- ""
108116
resources:
109117
- pods/status
118+
- services
110119
verbs:
111120
- create
112121
- delete
@@ -118,14 +127,12 @@ rules:
118127
- apiGroups:
119128
- ""
120129
resources:
121-
- services
130+
- serviceaccounts
122131
verbs:
123132
- create
124133
- delete
125134
- get
126135
- list
127-
- patch
128-
- update
129136
- watch
130137
- apiGroups:
131138
- ""
@@ -159,17 +166,6 @@ rules:
159166
- gateway.networking.k8s.io
160167
resources:
161168
- httproutes
162-
verbs:
163-
- create
164-
- delete
165-
- get
166-
- list
167-
- patch
168-
- update
169-
- watch
170-
- apiGroups:
171-
- gateway.networking.k8s.io
172-
resources:
173169
- referencegrants
174170
verbs:
175171
- create
@@ -209,57 +205,7 @@ rules:
209205
- orchestration.aibrix.ai
210206
resources:
211207
- kvcaches
212-
verbs:
213-
- create
214-
- delete
215-
- get
216-
- list
217-
- patch
218-
- update
219-
- watch
220-
- apiGroups:
221-
- orchestration.aibrix.ai
222-
resources:
223-
- kvcaches/finalizers
224-
verbs:
225-
- update
226-
- apiGroups:
227-
- orchestration.aibrix.ai
228-
resources:
229-
- kvcaches/status
230-
verbs:
231-
- get
232-
- patch
233-
- update
234-
- apiGroups:
235-
- orchestration.aibrix.ai
236-
resources:
237208
- rayclusterfleets
238-
verbs:
239-
- create
240-
- delete
241-
- get
242-
- list
243-
- patch
244-
- update
245-
- watch
246-
- apiGroups:
247-
- orchestration.aibrix.ai
248-
resources:
249-
- rayclusterfleets/finalizers
250-
verbs:
251-
- update
252-
- apiGroups:
253-
- orchestration.aibrix.ai
254-
resources:
255-
- rayclusterfleets/status
256-
verbs:
257-
- get
258-
- patch
259-
- update
260-
- apiGroups:
261-
- orchestration.aibrix.ai
262-
resources:
263209
- rayclusterreplicasets
264210
verbs:
265211
- create
@@ -272,12 +218,16 @@ rules:
272218
- apiGroups:
273219
- orchestration.aibrix.ai
274220
resources:
221+
- kvcaches/finalizers
222+
- rayclusterfleets/finalizers
275223
- rayclusterreplicasets/finalizers
276224
verbs:
277225
- update
278226
- apiGroups:
279227
- orchestration.aibrix.ai
280228
resources:
229+
- kvcaches/status
230+
- rayclusterfleets/status
281231
- rayclusterreplicasets/status
282232
verbs:
283233
- get
@@ -310,28 +260,23 @@ rules:
310260
- patch
311261
- update
312262
- apiGroups:
313-
- ""
263+
- rbac.authorization.k8s.io
314264
resources:
315-
- secrets
265+
- rolebindings
316266
verbs:
267+
- create
268+
- delete
317269
- get
318270
- list
319-
- update
320271
- watch
321272
- apiGroups:
322-
- admissionregistration.k8s.io
273+
- rbac.authorization.k8s.io
323274
resources:
324-
- mutatingwebhookconfigurations
325-
- validatingwebhookconfigurations
275+
- roles
326276
verbs:
277+
- create
278+
- delete
327279
- get
328280
- list
329281
- update
330282
- watch
331-
- apiGroups:
332-
- apiextensions.k8s.io
333-
resources:
334-
- customresourcedefinitions
335-
verbs:
336-
- get
337-
- list

‎pkg/controller/kvcache/backends/common.go‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
orchestrationv1alpha1 "github.com/vllm-project/aibrix/api/orchestration/v1alpha1"
2323
"github.com/vllm-project/aibrix/pkg/constants"
2424
corev1 "k8s.io/api/core/v1"
25+
rbacv1 "k8s.io/api/rbac/v1"
2526
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2627
"k8s.io/apimachinery/pkg/util/intstr"
2728
)
@@ -113,3 +114,75 @@ func buildRedisService(kvCache *orchestrationv1alpha1.KVCache) *corev1.Service {
113114

114115
return svc
115116
}
117+
118+
// buildServiceAccount creates a new ServiceAccount for Distributed kv cache solution.
119+
func buildServiceAccount(kvCache *orchestrationv1alpha1.KVCache) *corev1.ServiceAccount {
120+
sa := &corev1.ServiceAccount{
121+
ObjectMeta: metav1.ObjectMeta{
122+
Name: kvCache.Name,
123+
Namespace: kvCache.Namespace,
124+
Labels: map[string]string{
125+
constants.KVCacheLabelKeyIdentifier: kvCache.Name,
126+
constants.KVCacheLabelKeyRole: constants.KVCacheLabelValueRoleCache,
127+
},
128+
},
129+
}
130+
131+
return sa
132+
}
133+
134+
// buildRole creates a new Role for a KVCache resource.
135+
func buildRole(kvCache *orchestrationv1alpha1.KVCache) *rbacv1.Role {
136+
role := &rbacv1.Role{
137+
ObjectMeta: metav1.ObjectMeta{
138+
Name: kvCache.Name,
139+
Namespace: kvCache.Namespace,
140+
Labels: map[string]string{
141+
constants.KVCacheLabelKeyIdentifier: kvCache.Name,
142+
constants.KVCacheLabelKeyRole: constants.KVCacheLabelValueRoleCache,
143+
},
144+
},
145+
Rules: []rbacv1.PolicyRule{
146+
{
147+
APIGroups: []string{""},
148+
Resources: []string{"pods"},
149+
Verbs: []string{"get", "list", "watch"},
150+
},
151+
{
152+
APIGroups: []string{""},
153+
Resources: []string{"pods/exec"},
154+
Verbs: []string{"create"},
155+
},
156+
},
157+
}
158+
159+
return role
160+
}
161+
162+
// buildRoleBinding creates rolebinding for a kvCache object
163+
func buildRoleBinding(kvCache *orchestrationv1alpha1.KVCache) *rbacv1.RoleBinding {
164+
rb := &rbacv1.RoleBinding{
165+
ObjectMeta: metav1.ObjectMeta{
166+
Name: kvCache.Name,
167+
Namespace: kvCache.Namespace,
168+
Labels: map[string]string{
169+
constants.KVCacheLabelKeyIdentifier: kvCache.Name,
170+
constants.KVCacheLabelKeyRole: constants.KVCacheLabelValueRoleCache,
171+
},
172+
},
173+
Subjects: []rbacv1.Subject{
174+
{
175+
Kind: rbacv1.ServiceAccountKind,
176+
Name: kvCache.Name,
177+
Namespace: kvCache.Namespace,
178+
},
179+
},
180+
RoleRef: rbacv1.RoleRef{
181+
APIGroup: rbacv1.GroupName,
182+
Kind: "Role",
183+
Name: kvCache.Name,
184+
},
185+
}
186+
187+
return rb
188+
}

‎pkg/controller/kvcache/backends/distributed.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,18 @@ func (r *DistributedReconciler) Reconcile(ctx context.Context, kvCache *orchestr
6060
return reconcile.Result{}, err
6161
}
6262

63+
if err := r.reconcileWatcherPodServiceAccount(ctx, r.Backend.BuildWatcherPodServiceAccount(kvCache)); err != nil {
64+
return reconcile.Result{}, err
65+
}
66+
67+
if err := r.reconcileWatcherPodRole(ctx, r.Backend.BuildWatcherPodRole(kvCache)); err != nil {
68+
return reconcile.Result{}, err
69+
}
70+
71+
if err := r.reconcileWatcherPodRoleBinding(ctx, r.Backend.BuildWatcherPodRoleBinding(kvCache)); err != nil {
72+
return reconcile.Result{}, err
73+
}
74+
6375
// Handle infinistore kvCache Deployment
6476
if err := r.ReconcileStatefulsetObject(ctx, r.Backend.BuildCacheStatefulSet(kvCache)); err != nil {
6577
return ctrl.Result{}, err

‎pkg/controller/kvcache/backends/distributed_test.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222

2323
appsv1 "k8s.io/api/apps/v1"
2424
corev1 "k8s.io/api/core/v1"
25+
rbacv1 "k8s.io/api/rbac/v1"
2526

2627
"github.com/stretchr/testify/assert"
2728
"github.com/vllm-project/aibrix/api/orchestration/v1alpha1"
@@ -181,6 +182,9 @@ type mockBackend struct {
181182
watcher *corev1.Pod
182183
svc *corev1.Service
183184
sts *appsv1.StatefulSet
185+
sa *corev1.ServiceAccount
186+
role *rbacv1.Role
187+
rb *rbacv1.RoleBinding
184188
}
185189

186190
func (m mockBackend) Name() string {
@@ -199,6 +203,18 @@ func (m mockBackend) BuildMetadataService(*v1alpha1.KVCache) *corev1.Service {
199203
return m.svc
200204
}
201205

206+
func (m mockBackend) BuildWatcherPodServiceAccount(*v1alpha1.KVCache) *corev1.ServiceAccount {
207+
return m.sa
208+
}
209+
210+
func (m mockBackend) BuildWatcherPodRole(*v1alpha1.KVCache) *rbacv1.Role {
211+
return m.role
212+
}
213+
214+
func (m mockBackend) BuildWatcherPodRoleBinding(*v1alpha1.KVCache) *rbacv1.RoleBinding {
215+
return m.rb
216+
}
217+
202218
func (m mockBackend) BuildWatcherPod(*v1alpha1.KVCache) *corev1.Pod {
203219
return m.watcher
204220
}

0 commit comments

Comments
 (0)