Skip to content

Commit 89e9e5c

Browse files
ZiMengShengwangjianyu.wjy
ZiMengSheng
and
wangjianyu.wjy
authored
scheduler: support pod request exact match reservation (#2121)
Signed-off-by: wangjianyu.wjy <wangjianyu.wjy@alibaba-inc.com> Co-authored-by: wangjianyu.wjy <wangjianyu.wjy@alibaba-inc.com>
1 parent 61e47f0 commit 89e9e5c

File tree

5 files changed

+215
-5
lines changed

5 files changed

+215
-5
lines changed

‎apis/extension/reservation.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,3 +161,57 @@ func SetReservationRestrictedOptions(obj metav1.Object, options *ReservationRest
161161
obj.SetAnnotations(annotations)
162162
return nil
163163
}
164+
165+
const (
166+
AnnotationExactMatchReservationSpec = SchedulingDomainPrefix + "/exact-match-reservation"
167+
)
168+
169+
type ExactMatchReservationSpec struct {
170+
ResourceNames []corev1.ResourceName `json:"resourceNames,omitempty"`
171+
}
172+
173+
func SetExactMatchReservationSpec(obj metav1.Object, spec *ExactMatchReservationSpec) error {
174+
data, err := json.Marshal(spec)
175+
if err != nil {
176+
return err
177+
}
178+
annotations := obj.GetAnnotations()
179+
if annotations == nil {
180+
annotations = map[string]string{}
181+
}
182+
annotations[AnnotationExactMatchReservationSpec] = string(data)
183+
obj.SetAnnotations(annotations)
184+
return nil
185+
}
186+
187+
func GetExactMatchReservationSpec(annotations map[string]string) (*ExactMatchReservationSpec, error) {
188+
if s := annotations[AnnotationExactMatchReservationSpec]; s != "" {
189+
var exactMatchReservationSpec ExactMatchReservationSpec
190+
if err := json.Unmarshal([]byte(s), &exactMatchReservationSpec); err != nil {
191+
return nil, err
192+
}
193+
return &exactMatchReservationSpec, nil
194+
}
195+
return nil, nil
196+
}
197+
198+
func ExactMatchReservation(podRequests, reservationAllocatable corev1.ResourceList, spec *ExactMatchReservationSpec) bool {
199+
if spec == nil || len(spec.ResourceNames) == 0 {
200+
return true
201+
}
202+
for _, resourceName := range spec.ResourceNames {
203+
allocatable, existsInReservation := reservationAllocatable[resourceName]
204+
request, existsInPod := podRequests[resourceName]
205+
if !existsInReservation || !existsInPod {
206+
if !existsInReservation && !existsInPod {
207+
return true
208+
}
209+
return false
210+
}
211+
212+
if allocatable.Cmp(request) != 0 {
213+
return false
214+
}
215+
}
216+
return true
217+
}

‎apis/extension/reservation_test.go‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
"github.com/stretchr/testify/assert"
2323
corev1 "k8s.io/api/core/v1"
24+
"k8s.io/apimachinery/pkg/api/resource"
2425
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2526
"k8s.io/apimachinery/pkg/util/uuid"
2627

@@ -44,3 +45,85 @@ func TestSetReservationAllocated(t *testing.T) {
4445
}
4546
assert.Equal(t, expectReservationAllocated, reservationAllocated)
4647
}
48+
49+
func TestExactMatchReservation(t *testing.T) {
50+
tests := []struct {
51+
name string
52+
podRequests corev1.ResourceList
53+
reservationAllocatable corev1.ResourceList
54+
spec *ExactMatchReservationSpec
55+
want bool
56+
}{
57+
{
58+
name: "exact matched cpu",
59+
spec: &ExactMatchReservationSpec{
60+
ResourceNames: []corev1.ResourceName{
61+
corev1.ResourceCPU,
62+
},
63+
},
64+
podRequests: corev1.ResourceList{
65+
corev1.ResourceCPU: resource.MustParse("1"),
66+
},
67+
reservationAllocatable: corev1.ResourceList{
68+
corev1.ResourceCPU: resource.MustParse("1"),
69+
},
70+
want: true,
71+
},
72+
{
73+
name: "exact matched cpu",
74+
spec: &ExactMatchReservationSpec{
75+
ResourceNames: []corev1.ResourceName{
76+
corev1.ResourceCPU,
77+
},
78+
},
79+
podRequests: corev1.ResourceList{
80+
corev1.ResourceCPU: resource.MustParse("1"),
81+
corev1.ResourceMemory: resource.MustParse("1Gi"),
82+
},
83+
reservationAllocatable: corev1.ResourceList{
84+
corev1.ResourceCPU: resource.MustParse("1"),
85+
corev1.ResourceMemory: resource.MustParse("2Gi"),
86+
},
87+
want: true,
88+
},
89+
{
90+
name: "exact matched cpu, memory not exact matched",
91+
spec: &ExactMatchReservationSpec{
92+
ResourceNames: []corev1.ResourceName{
93+
corev1.ResourceCPU,
94+
corev1.ResourceMemory,
95+
},
96+
},
97+
podRequests: corev1.ResourceList{
98+
corev1.ResourceCPU: resource.MustParse("1"),
99+
corev1.ResourceMemory: resource.MustParse("1Gi"),
100+
},
101+
reservationAllocatable: corev1.ResourceList{
102+
corev1.ResourceCPU: resource.MustParse("1"),
103+
corev1.ResourceMemory: resource.MustParse("2Gi"),
104+
},
105+
want: false,
106+
},
107+
{
108+
name: "exact matched cpu, memory exact match spec doesn't matter",
109+
spec: &ExactMatchReservationSpec{
110+
ResourceNames: []corev1.ResourceName{
111+
corev1.ResourceCPU,
112+
corev1.ResourceMemory,
113+
},
114+
},
115+
podRequests: corev1.ResourceList{
116+
corev1.ResourceCPU: resource.MustParse("1"),
117+
},
118+
reservationAllocatable: corev1.ResourceList{
119+
corev1.ResourceCPU: resource.MustParse("1"),
120+
},
121+
want: true,
122+
},
123+
}
124+
for _, tt := range tests {
125+
t.Run(tt.name, func(t *testing.T) {
126+
assert.Equal(t, tt.want, ExactMatchReservation(tt.podRequests, tt.reservationAllocatable, tt.spec))
127+
})
128+
}
129+
}

‎pkg/scheduler/plugins/reservation/plugin.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,7 @@ type nodeDiagnosisState struct {
178178
ownerMatched int // owner matched
179179
isUnschedulableUnmatched int // owner matched but BeforePreFilter unmatched due to unschedulable
180180
affinityUnmatched int // owner matched but BeforePreFilter unmatched due to affinity
181+
notExactMatched int // owner matched but BeforePreFilter unmatched due to not exact match
181182
}
182183

183184
func (s *stateData) Clone() framework.StateData {
@@ -540,15 +541,17 @@ func (pl *Plugin) PostFilter(_ context.Context, cycleState *framework.CycleState
540541
}
541542

542543
func (pl *Plugin) makePostFilterReasons(state *stateData, filteredNodeStatusMap framework.NodeToStatusMap) []string {
543-
ownerMatched, affinityUnmatched, isUnSchedulableUnmatched := 0, 0, 0
544+
ownerMatched, affinityUnmatched, isUnSchedulableUnmatched, notExactMatched := 0, 0, 0, 0
544545
// failure reasons and counts for the nodes which have not been handled by the Reservation's Filter
545546
reasonsByNode := map[string]int{}
546547
for nodeName, diagnosisState := range state.nodeReservationDiagnosis {
547548
isUnSchedulableUnmatched += diagnosisState.isUnschedulableUnmatched
548549
affinityUnmatched += diagnosisState.affinityUnmatched
549550
ownerMatched += diagnosisState.ownerMatched
551+
notExactMatched += diagnosisState.notExactMatched
552+
550553
// calculate the remaining unmatched which is owner-matched and Reservation BeforePreFilter matched
551-
remainUnmatched := diagnosisState.ownerMatched - diagnosisState.affinityUnmatched - diagnosisState.isUnschedulableUnmatched
554+
remainUnmatched := diagnosisState.ownerMatched - diagnosisState.affinityUnmatched - diagnosisState.isUnschedulableUnmatched - diagnosisState.notExactMatched
552555
if remainUnmatched <= 0 { // no need to check other reasons
553556
continue
554557
}
@@ -591,6 +594,12 @@ func (pl *Plugin) makePostFilterReasons(state *stateData, filteredNodeStatusMap
591594
reasons = append(reasons, b.String())
592595
b.Reset()
593596
}
597+
if notExactMatched > 0 {
598+
b.WriteString(strconv.Itoa(notExactMatched))
599+
b.WriteString(" Reservation(s) is not exact matched")
600+
reasons = append(reasons, b.String())
601+
b.Reset()
602+
}
594603
for nodeReason, count := range reasonsByNode { // node reason Filter failed
595604
b.WriteString(strconv.Itoa(count))
596605
b.WriteString(" Reservation(s) for node reason that ")

‎pkg/scheduler/plugins/reservation/plugin_test.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1899,6 +1899,60 @@ func TestPostFilter(t *testing.T) {
18991899
"4 Reservation(s) is unschedulable",
19001900
"4 Reservation(s) matched owner total"),
19011901
},
1902+
{
1903+
name: "show reservation matched owner, unschedulable and exact matched",
1904+
args: args{
1905+
hasStateData: true,
1906+
nodeReservationDiagnosis: map[string]*nodeDiagnosisState{
1907+
"test-node-0": {
1908+
ownerMatched: 3,
1909+
isUnschedulableUnmatched: 0,
1910+
notExactMatched: 3,
1911+
},
1912+
"test-node-1": {
1913+
ownerMatched: 2,
1914+
isUnschedulableUnmatched: 1,
1915+
notExactMatched: 1,
1916+
},
1917+
},
1918+
filteredNodeStatusMap: framework.NodeToStatusMap{
1919+
"test-node-0": {},
1920+
"test-node-1": {},
1921+
},
1922+
},
1923+
want: nil,
1924+
want1: framework.NewStatus(framework.Unschedulable,
1925+
"1 Reservation(s) is unschedulable",
1926+
"4 Reservation(s) is not exact matched",
1927+
"5 Reservation(s) matched owner total"),
1928+
},
1929+
{
1930+
name: "show reservation matched owner, unschedulable and exact matched",
1931+
args: args{
1932+
hasStateData: true,
1933+
nodeReservationDiagnosis: map[string]*nodeDiagnosisState{
1934+
"test-node-0": {
1935+
ownerMatched: 3,
1936+
isUnschedulableUnmatched: 0,
1937+
notExactMatched: 3,
1938+
},
1939+
"test-node-1": {
1940+
ownerMatched: 2,
1941+
isUnschedulableUnmatched: 1,
1942+
notExactMatched: 1,
1943+
},
1944+
},
1945+
filteredNodeStatusMap: framework.NodeToStatusMap{
1946+
"test-node-0": {},
1947+
"test-node-1": {},
1948+
},
1949+
},
1950+
want: nil,
1951+
want1: framework.NewStatus(framework.Unschedulable,
1952+
"1 Reservation(s) is unschedulable",
1953+
"4 Reservation(s) is not exact matched",
1954+
"5 Reservation(s) matched owner total"),
1955+
},
19021956
{
19031957
name: "show reservation matched owner, unschedulable and affinity unmatched",
19041958
args: args{

‎pkg/scheduler/plugins/reservation/transformer.go‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"k8s.io/kubernetes/pkg/scheduler/framework/parallelize"
3434
schedutil "k8s.io/kubernetes/pkg/scheduler/util"
3535

36+
"github.com/koordinator-sh/koordinator/apis/extension"
3637
"github.com/koordinator-sh/koordinator/pkg/scheduler/frameworkext"
3738
"github.com/koordinator-sh/koordinator/pkg/util"
3839
reservationutil "github.com/koordinator-sh/koordinator/pkg/util/reservation"
@@ -61,6 +62,13 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState *
6162
}
6263
requiredNodeAffinity := nodeaffinity.GetRequiredNodeAffinity(pod)
6364

65+
podRequests := resourceapi.PodRequests(pod, resourceapi.PodResourcesOptions{})
66+
exactMatchReservationSpec, err := extension.GetExactMatchReservationSpec(pod.Annotations)
67+
if err != nil {
68+
klog.ErrorS(err, "Failed to parse exact match reservation spec", "pod", klog.KObj(pod))
69+
return nil, false, framework.AsStatus(err)
70+
}
71+
6472
var stateIndex, diagnosisIndex int32
6573
allNodes := pl.reservationCache.listAllNodes()
6674
allNodeReservationStates := make([]*nodeReservationState, len(allNodes))
@@ -106,6 +114,7 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState *
106114
ownerMatched: 0,
107115
isUnschedulableUnmatched: 0,
108116
affinityUnmatched: 0,
117+
notExactMatched: 0,
109118
}
110119
status := pl.reservationCache.forEachAvailableReservationOnNode(node.Name, func(rInfo *frameworkext.ReservationInfo) (bool, *framework.Status) {
111120
if !rInfo.IsAvailable() || rInfo.ParseError != nil {
@@ -121,7 +130,8 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState *
121130
isOwnerMatched := rInfo.Match(pod)
122131
isUnschedulable := rInfo.IsUnschedulable()
123132
isMatchReservationAffinity := matchReservationAffinity(node, rInfo, reservationAffinity)
124-
if !isReservedPod && !isUnschedulable && isOwnerMatched && isMatchReservationAffinity {
133+
isExactMatched := extension.ExactMatchReservation(podRequests, rInfo.Allocatable, exactMatchReservationSpec)
134+
if !isReservedPod && !isUnschedulable && isOwnerMatched && isMatchReservationAffinity && isExactMatched {
125135
matched = append(matched, rInfo.Clone())
126136

127137
} else if len(rInfo.AssignedPods) > 0 {
@@ -133,6 +143,8 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState *
133143
diagnosisState.isUnschedulableUnmatched++
134144
} else if !isMatchReservationAffinity {
135145
diagnosisState.affinityUnmatched++
146+
} else if !isExactMatched {
147+
diagnosisState.notExactMatched++
136148
}
137149
}
138150

@@ -224,8 +236,6 @@ func (pl *Plugin) prepareMatchReservationState(ctx context.Context, cycleState *
224236
allNodeReservationStates = allNodeReservationStates[:stateIndex]
225237
allPluginToRestoreState = allPluginToRestoreState[:stateIndex]
226238
allNodeDiagnosisStates = allNodeDiagnosisStates[:diagnosisIndex]
227-
228-
podRequests := resourceapi.PodRequests(pod, resourceapi.PodResourcesOptions{})
229239
podRequestResources := framework.NewResource(podRequests)
230240
state := &stateData{
231241
hasAffinity: reservationAffinity != nil,

0 commit comments

Comments
 (0)