Skip to content

scheduler: add unschedulable resource for root quota #1925

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
koordinator-bot merged 1 commit into from
Feb 28, 2024
Merged

scheduler: add unschedulable resource for root quota #1925

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions apis/extension/elastic_quota.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ const (
AnnotationChildRequest = QuotaKoordinatorPrefix + "/child-request"
AnnotationResourceKeys = QuotaKoordinatorPrefix + "/resource-keys"
AnnotationTotalResource = QuotaKoordinatorPrefix + "/total-resource"
AnnotationUnschedulableResource = QuotaKoordinatorPrefix + "/unschedulable-resource"
AnnotationQuotaNamespaces = QuotaKoordinatorPrefix + "/namespaces"
AnnotationGuaranteed = QuotaKoordinatorPrefix + "/guaranteed"
AnnotationAllocated = QuotaKoordinatorPrefix + "/allocated"
Expand Down Expand Up @@ -195,3 +196,13 @@ func GetChildRequest(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error)
}
return request, nil
}

func GetUnschedulableResource(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error) {
unschedulable := corev1.ResourceList{}
if quota.Annotations[AnnotationUnschedulableResource] != "" {
if err := json.Unmarshal([]byte(quota.Annotations[AnnotationUnschedulableResource]), &unschedulable); err != nil {
return unschedulable, err
}
}
return unschedulable, nil
}
14 changes: 14 additions & 0 deletions pkg/quota-controller/profile/profile_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
quotav1 "k8s.io/apiserver/pkg/quota/v1"
"k8s.io/client-go/tools/record"
"k8s.io/klog/v2"
nodeutil "k8s.io/kubernetes/pkg/util/node"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand Down Expand Up @@ -138,11 +139,16 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request

// TODO: consider node status.
totalResource := corev1.ResourceList{}
unschedulableResource := corev1.ResourceList{}
for _, node := range nodeList.Items {
totalResource = quotav1.Add(totalResource, GetNodeAllocatable(node))
if node.Spec.Unschedulable || !nodeutil.IsNodeReady(&node) {
unschedulableResource = quotav1.Add(unschedulableResource, GetNodeAllocatable(node))
}
}

decorateTotalResource(profile, totalResource)
decorateTotalResource(profile, unschedulableResource)

resourceKeys := []string{"cpu", "memory"}
raw, ok := profile.Annotations[extension.AnnotationResourceKeys]
Expand Down Expand Up @@ -197,6 +203,14 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request
}
quota.Annotations[extension.AnnotationTotalResource] = string(data)

// update unschedulable resource
data, err = json.Marshal(unschedulableResource)
if err != nil {
klog.Errorf("failed marshal unschedulable resources, err: %v", err)
return ctrl.Result{Requeue: true}, err
}
quota.Annotations[extension.AnnotationUnschedulableResource] = string(data)

if !quotaExist {
err = r.Client.Create(context.TODO(), quota)
if err != nil {
Expand Down
90 changes: 63 additions & 27 deletions pkg/quota-controller/profile/profile_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,30 @@ func defaultCreateNode(nodeName string, labels map[string]string, capacity corev
},
Status: corev1.NodeStatus{
Allocatable: capacity,
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionTrue,
},
},
},
}
}

func defaultCreateUnreadyNode(nodeName string, labels map[string]string, capacity corev1.ResourceList) *corev1.Node {
return &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeName,
Labels: labels,
},
Status: corev1.NodeStatus{
Allocatable: capacity,
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionFalse,
},
},
},
}
}
Expand All @@ -77,7 +101,7 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {

nodes := []*corev1.Node{
defaultCreateNode("node1", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)),
defaultCreateNode("node2", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)),
defaultCreateUnreadyNode("node2", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-a"}, createResourceListWithStorage(10, 1000, 1000)),
defaultCreateNode("node3", map[string]string{"topology.kubernetes.io/zone": "cn-hangzhou-b"}, createResourceListWithStorage(10, 1000, 1000)),
}

Expand All @@ -87,12 +111,13 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
resourceRatio := "0.9"

tests := []struct {
name string
profile *quotav1alpha1.ElasticQuotaProfile
oriQuota *schedv1alpha1.ElasticQuota
expectQuotaMin corev1.ResourceList
expectTotalResource corev1.ResourceList
expectQuotaLabels map[string]string
name string
profile *quotav1alpha1.ElasticQuotaProfile
oriQuota *schedv1alpha1.ElasticQuota
expectQuotaMin corev1.ResourceList
expectTotalResource corev1.ResourceList
expectUnschedulableResource corev1.ResourceList
expectQuotaLabels map[string]string
}{
{
name: "cn-hangzhou-a profile",
Expand All @@ -107,9 +132,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectUnschedulableResource: createResourceListWithStorage(10, 1000, 1000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
Expand All @@ -129,9 +155,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(10, 1000),
expectTotalResource: createResourceListWithStorage(10, 1000, 1000),
oriQuota: nil,
expectQuotaMin: createResourceList(10, 1000),
expectTotalResource: createResourceListWithStorage(10, 1000, 1000),
expectUnschedulableResource: corev1.ResourceList{},
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile2",
extension.LabelQuotaTreeID: treeID2,
Expand All @@ -154,9 +181,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
oriQuota: nil,
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectUnschedulableResource: createResourceListWithStorage(10, 1000, 1000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
Expand Down Expand Up @@ -189,8 +217,9 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
Min: createResourceList(5, 50),
},
},
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectQuotaMin: createResourceList(20, 2000),
expectTotalResource: createResourceListWithStorage(20, 2000, 2000),
expectUnschedulableResource: createResourceListWithStorage(10, 1000, 1000),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
Expand All @@ -213,9 +242,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
expectUnschedulableResource: createResourceListWithStorage(9, 900, 900),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: treeID1,
Expand All @@ -239,9 +269,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
oriQuota: nil,
expectQuotaMin: createResourceList(18, 1800),
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
expectUnschedulableResource: createResourceListWithStorage(9, 900, 900),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: "tree1",
Expand All @@ -268,9 +299,10 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
},
},
},
oriQuota: nil,
expectQuotaMin: corev1.ResourceList{corev1.ResourceCPU: *resource.NewMilliQuantity(18*1000, resource.DecimalSI)},
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
oriQuota: nil,
expectQuotaMin: corev1.ResourceList{corev1.ResourceCPU: *resource.NewMilliQuantity(18*1000, resource.DecimalSI)},
expectTotalResource: createResourceListWithStorage(18, 1800, 1800),
expectUnschedulableResource: createResourceListWithStorage(9, 900, 900),
expectQuotaLabels: map[string]string{
extension.LabelQuotaProfile: "profile1",
extension.LabelQuotaTreeID: "tree1",
Expand Down Expand Up @@ -310,8 +342,12 @@ func TestQuotaProfileReconciler_Reconciler_CreateQuota(t *testing.T) {
err = json.Unmarshal([]byte(quota.Annotations[extension.AnnotationTotalResource]), &total)
assert.NoError(t, err)

unschedulable, err := extension.GetUnschedulableResource(quota)
assert.NoError(t, err)

assert.True(t, quotav1.Equals(tc.expectQuotaMin, quota.Spec.Min))
assert.True(t, quotav1.Equals(tc.expectTotalResource, total))
assert.True(t, quotav1.Equals(tc.expectUnschedulableResource, unschedulable))
assert.Equal(t, tc.expectQuotaLabels, quota.Labels)
})
}
Expand Down
9 changes: 9 additions & 0 deletions pkg/scheduler/plugins/elasticquota/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,15 @@ func syncElasticQuotaMetrics(eq *v1alpha1.ElasticQuota, summary *core.QuotaInfoS
extension.AnnotationNonPreemptibleRequest: summary.NonPreemptibleRequest,
extension.AnnotationNonPreemptibleUsed: summary.NonPreemptibleUsed,
}

// record the unschedulable resource
if extension.IsTreeRootQuota(eq) {
unschedulable, err := extension.GetUnschedulableResource(eq)
if err == nil {
m[extension.AnnotationUnschedulableResource] = unschedulable
}
}

for k, v := range m {
decorateResource(eq, v)

Expand Down
10 changes: 10 additions & 0 deletions pkg/scheduler/plugins/elasticquota/controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,14 @@ func Test_updateElasticQuotaStatusIfChanged(t *testing.T) {

func Test_syncElasticQuotaMetrics(t *testing.T) {
eq := &v1alpha1.ElasticQuota{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"quota.scheduling.koordinator.sh/is-root": "true",
},
Annotations: map[string]string{
"quota.scheduling.koordinator.sh/unschedulable-resource": `{"cpu":"4","memory":"8"}`,
},
},
Spec: v1alpha1.ElasticQuotaSpec{
Max: MakeResourceList().CPU(4).Mem(200).Obj(),
Min: MakeResourceList().CPU(2).Mem(100).Obj(),
Expand Down Expand Up @@ -435,6 +443,8 @@ loopChan:
`label:<name:"field" value:"request" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"memory" > label:<name:"tree" value:"tree-1" > gauge:<value:50 > `,
`label:<name:"field" value:"runtime" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"cpu" > label:<name:"tree" value:"tree-1" > gauge:<value:5000 > `,
`label:<name:"field" value:"runtime" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"memory" > label:<name:"tree" value:"tree-1" > gauge:<value:50 > `,
`label:<name:"field" value:"unschedulable-resource" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"cpu" > label:<name:"tree" value:"tree-1" > gauge:<value:4000 > `,
`label:<name:"field" value:"unschedulable-resource" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"memory" > label:<name:"tree" value:"tree-1" > gauge:<value:8 > `,
`label:<name:"field" value:"used" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"cpu" > label:<name:"tree" value:"tree-1" > gauge:<value:1000 > `,
`label:<name:"field" value:"used" > label:<name:"is_parent" value:"true" > label:<name:"name" value:"test-eq" > label:<name:"parent" value:"root" > label:<name:"resource" value:"memory" > label:<name:"tree" value:"tree-1" > gauge:<value:50 > `,
}
Expand Down