Skip to content

Commit 2d23602

Browse files
zwzhang0107zwzhang0107
authored
use unified internal protocol for running hooks plugin (#283)
Signed-off-by: zwzhang0107 <zuoweizhang@outlook.com>
1 parent 5da05b3 commit 2d23602

File tree

20 files changed

+864
-218
lines changed

20 files changed

+864
-218
lines changed

‎docs/design-archive/koordlet-runtime-hooks.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
* [Rule](#rule)
1212
* [Working Mode](#working-mode)
1313
* [Proxy](#proxy)
14-
* [Bypass](#bypass)
14+
* [Standalone](#standalone)
1515
* [Examples for hooks extensions](#examples-for-hooks-extensions)
1616

1717
## Summary
@@ -52,7 +52,7 @@ of injection policies. For each policy update, the `Rule` module also provides a
5252
parameters of pods and containers.
5353

5454
### Working Mode
55-
Runtime Hooks supports two working modes for different scenarios: `Proxy` and `Bypass`.
55+
Runtime Hooks supports two working modes for different scenarios: `Proxy` and `Standalone`.
5656

5757
![image](../images/runtime-hooks-working-mode.svg)
5858

@@ -63,8 +63,8 @@ of Kubelet to `Koordinator Runtime Manager`. For each cri request, `Runtime Mana
6363
`Runtime Hooks` to execute the injection synchronously. `Proxy` running mode has a better performance which can
6464
guarantee the related parameters set timely and appropriately.
6565

66-
#### Bypass
67-
`Bypass` working mode simplifies the runtime configuration and architecture. `Runtime Manager` is not needed in this
66+
#### Standalone
67+
`Standalone` working mode simplifies the runtime configuration and architecture. `Runtime Manager` is not needed in this
6868
working mode, so that `Runtime Hooks` works as an independent module. Without the proxy intercepts requests between
6969
Kubelet and Runtime, `Runtime Hooks` can only set related parameters asynchronously, which limits the using scenarios.
7070

‎pkg/koordlet/runtimehooks/hooks/groupidentity/bvt.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,15 @@ type bvtPlugin struct {
4343

4444
func (b *bvtPlugin) Register() {
4545
klog.V(5).Infof("register hook %v", name)
46-
hooks.Register(rmconfig.PreRunPodSandbox, name, description, b.PreRunPodSandbox)
46+
hooks.Register(rmconfig.PreRunPodSandbox, name, description, b.SetPodBvtValue)
4747
rule.Register(name, description,
4848
rule.WithParseFunc(b.parseRule),
4949
rule.WithUpdateCallback(b.ruleUpdateCb),
5050
rule.WithSystemSupported(b.SystemSupported))
51+
//reconciler.RegisterCgroupReconciler(reconciler.PodLevel, sysutil.CPUBVTWarpNs, b.SetPodBvtValue,
52+
// "reconcile pod level cpu bvt value")
53+
//reconciler.RegisterCgroupReconciler(reconciler.KubeQOSLevel, sysutil.CPUBVTWarpNs, b.SetKubeQOSBvtValue,
54+
// "reconcile kubeqos level cpu bvt value")
5155
}
5256

5357
func (b *bvtPlugin) SystemSupported() bool {

‎pkg/koordlet/runtimehooks/hooks/groupidentity/bvt_test.go‎

Lines changed: 8 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,8 @@ import (
2020
"strconv"
2121
"testing"
2222

23-
"github.com/stretchr/testify/assert"
2423
corev1 "k8s.io/api/core/v1"
25-
"k8s.io/utils/pointer"
2624

27-
ext "github.com/koordinator-sh/koordinator/apis/extension"
28-
runtimeapi "github.com/koordinator-sh/koordinator/apis/runtime/v1alpha1"
2925
"github.com/koordinator-sh/koordinator/pkg/util"
3026
"github.com/koordinator-sh/koordinator/pkg/util/system"
3127
)
@@ -34,101 +30,12 @@ func initCPUBvt(dirWithKube string, value int64, helper *system.FileTestUtil) {
3430
helper.WriteCgroupFileContents(dirWithKube, system.CPUBVTWarpNs, strconv.FormatInt(value, 10))
3531
}
3632

37-
func getPodCPUBurst(podDirWithKube string, helper *system.FileTestUtil) int64 {
33+
func getPodCPUBvt(podDirWithKube string, helper *system.FileTestUtil) int64 {
3834
valueStr := helper.ReadCgroupFileContents(podDirWithKube, system.CPUBVTWarpNs)
3935
value, _ := strconv.ParseInt(valueStr, 10, 64)
4036
return value
4137
}
4238

43-
func Test_bvtPlugin_PreRunPodSandbox(t *testing.T) {
44-
defaultRule := &bvtRule{
45-
podQOSParams: map[ext.QoSClass]int64{
46-
ext.QoSLSR: 2,
47-
ext.QoSLS: 2,
48-
ext.QoSBE: -1,
49-
},
50-
kubeQOSDirParams: map[corev1.PodQOSClass]int64{
51-
corev1.PodQOSGuaranteed: 0,
52-
corev1.PodQOSBurstable: 2,
53-
corev1.PodQOSBestEffort: -1,
54-
},
55-
kubeQOSPodParams: map[corev1.PodQOSClass]int64{
56-
corev1.PodQOSGuaranteed: 2,
57-
corev1.PodQOSBurstable: 2,
58-
corev1.PodQOSBestEffort: -1,
59-
},
60-
}
61-
type fields struct {
62-
systemSupported *bool
63-
}
64-
type args struct {
65-
request *runtimeapi.PodSandboxHookRequest
66-
response *runtimeapi.PodSandboxHookResponse
67-
}
68-
type want struct {
69-
bvtValue int64
70-
}
71-
tests := []struct {
72-
name string
73-
fields fields
74-
args args
75-
want want
76-
}{
77-
{
78-
name: "set ls pod bvt",
79-
fields: fields{
80-
systemSupported: pointer.Bool(true),
81-
},
82-
args: args{
83-
request: &runtimeapi.PodSandboxHookRequest{
84-
Labels: map[string]string{
85-
ext.LabelPodQoS: string(ext.QoSLS),
86-
},
87-
CgroupParent: "kubepods/pod-guaranteed-test-uid/",
88-
},
89-
response: &runtimeapi.PodSandboxHookResponse{},
90-
},
91-
want: want{
92-
bvtValue: 2,
93-
},
94-
},
95-
{
96-
name: "set be pod bvt",
97-
fields: fields{
98-
systemSupported: pointer.Bool(true),
99-
},
100-
args: args{
101-
request: &runtimeapi.PodSandboxHookRequest{
102-
Labels: map[string]string{
103-
ext.LabelPodQoS: string(ext.QoSBE),
104-
},
105-
CgroupParent: "kubepods/besteffort/pod-besteffort-test-uid/",
106-
},
107-
response: &runtimeapi.PodSandboxHookResponse{},
108-
},
109-
want: want{
110-
bvtValue: -1,
111-
},
112-
},
113-
}
114-
for _, tt := range tests {
115-
t.Run(tt.name, func(t *testing.T) {
116-
testHelper := system.NewFileTestUtil(t)
117-
initCPUBvt(tt.args.request.CgroupParent, 0, testHelper)
118-
119-
b := &bvtPlugin{
120-
rule: defaultRule,
121-
sysSupported: tt.fields.systemSupported,
122-
}
123-
err := b.PreRunPodSandbox(tt.args.request, tt.args.response)
124-
assert.NoError(t, err)
125-
126-
gotBvt := getPodCPUBurst(tt.args.request.CgroupParent, testHelper)
127-
assert.Equal(t, tt.want.bvtValue, gotBvt, "pod bvt should equal")
128-
})
129-
}
130-
}
131-
13239
func Test_bvtPlugin_systemSupported(t *testing.T) {
13340
kubeRootDir := util.GetKubeQosRelativePath(corev1.PodQOSGuaranteed)
13441
type fields struct {
@@ -165,3 +72,10 @@ func Test_bvtPlugin_systemSupported(t *testing.T) {
16572
})
16673
}
16774
}
75+
76+
func Test_bvtPlugin_Register(t *testing.T) {
77+
t.Run("register bvt plugin", func(t *testing.T) {
78+
b := &bvtPlugin{}
79+
b.Register()
80+
})
81+
}

‎pkg/koordlet/runtimehooks/hooks/groupidentity/interceptor.go‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,31 +17,38 @@ limitations under the License.
1717
package groupidentity
1818

1919
import (
20-
"strconv"
21-
2220
"k8s.io/klog/v2"
21+
"k8s.io/utils/pointer"
2322

2423
ext "github.com/koordinator-sh/koordinator/apis/extension"
25-
runtimeapi "github.com/koordinator-sh/koordinator/apis/runtime/v1alpha1"
26-
"github.com/koordinator-sh/koordinator/pkg/koordlet/audit"
24+
"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol"
2725
"github.com/koordinator-sh/koordinator/pkg/util"
28-
sysutil "github.com/koordinator-sh/koordinator/pkg/util/system"
2926
)
3027

31-
func (b *bvtPlugin) PreRunPodSandbox(requestIf, responseIf interface{}) error {
28+
func (b *bvtPlugin) SetPodBvtValue(p protocol.HooksProtocol) error {
3229
if !b.SystemSupported() {
3330
klog.V(5).Infof("plugin %s is not supported by system", name)
3431
return nil
3532
}
3633
r := b.getRule()
37-
req := requestIf.(*runtimeapi.PodSandboxHookRequest)
34+
podCtx := p.(*protocol.PodContext)
35+
req := podCtx.Request
3836
podQoS := ext.GetQoSClassByLabels(req.Labels)
3937
podKubeQoS := util.GetKubeQoSByCgroupParent(req.CgroupParent)
4038
podBvt := r.getPodBvtValue(podQoS, podKubeQoS)
41-
// CgroupParent e.g. kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod586c1b35_63de_4ee0_9da3_2cebdca672c8.slice
42-
klog.V(5).Infof("set pod bvt on cgroup parent %v", req.CgroupParent)
43-
if req.PodMeta != nil {
44-
audit.V(2).Pod(req.PodMeta.Namespace, req.PodMeta.Name).Reason(name).Message("set bvt to %v", podBvt).Do()
39+
podCtx.Response.Resources.CPUBvt = pointer.Int64(podBvt)
40+
return nil
41+
}
42+
43+
func (b *bvtPlugin) SetKubeQOSBvtValue(p protocol.HooksProtocol) error {
44+
if !b.SystemSupported() {
45+
klog.V(5).Infof("plugin %s is not supported by system", name)
46+
return nil
4547
}
46-
return sysutil.CgroupFileWrite(req.CgroupParent, sysutil.CPUBVTWarpNs, strconv.FormatInt(podBvt, 10))
48+
r := b.getRule()
49+
kubeQOSCtx := p.(*protocol.KubeQOSContext)
50+
req := kubeQOSCtx.Request
51+
bvtValue := r.getKubeQoSDirBvtValue(req.KubeQOSClass)
52+
kubeQOSCtx.Response.Resources.CPUBvt = pointer.Int64(bvtValue)
53+
return nil
4754
}

0 commit comments

Comments
 (0)