koordlet: support eviction trigger by allocatable

apis/extension/constants.go

@@ -49,6 +49,10 @@ const (
 	// be able to evict.
 	LabelPodEvictEnabled = DomainPrefix + "eviction-enabled"
 
+	// AnnotationPodEvictPolicy are used to set restricted eviction policies for Pod.
+	// When this annotation is missing, there are no policy restrictions
+	AnnotationPodEvictPolicy = DomainPrefix + "eviction-policy"
+
 	// LabelPodSkipEnhancedValidation is the pod label key used to opt out a pod from enhanced validation.
 	LabelPodSkipEnhancedValidation = PodDomainPrefix + "/skip-enhanced-validation"
 )

apis/extension/resource.go

@@ -53,6 +53,12 @@ var (
 			corev1.ResourceMemory: MidMemory,
 		},
 	}
+	ReverseResourceNameMap = map[corev1.ResourceName]PriorityClass{
+		BatchCPU:    PriorityBatch,
+		BatchMemory: PriorityBatch,
+		MidCPU:      PriorityMid,
+		MidMemory:   PriorityMid,
+	}
 )
 
 // TranslateResourceNameByPriorityClass translates defaultResourceName to extend resourceName by PriorityClass

apis/slo/v1alpha1/nodeslo_types.go

@@ -344,6 +344,15 @@ type ResourceThresholdStrategy struct {
 	// +kubebuilder:validation:Minimum=0
 	MemoryEvictLowerPercent *int64 `json:"memoryEvictLowerPercent,omitempty" validate:"omitempty,min=0,max=100,ltfield=MemoryEvictThresholdPercent"`
 
+	// upper: memory ledger evict threshold percentage (0,), default = 110
+	// +kubebuilder:validation:Minimum=0
+	// if memory.requestPercent > MemoryAllocatableEvictThresholdPercent, then start to evict.
+	MemoryAllocatableEvictThresholdPercent *int64 `json:"memoryAllocatableEvictThresholdPercent,omitempty" validate:"omitempty,min=0,gtfield=MemoryAllocatableEvictLowerPercent"`
+	// lower: memory release util requestPercent under MemoryAllocatableEvictLowerPercent, default = 100
+	// +kubebuilder:validation:Minimum=0
+	// evict until memory.requestPercent <= MemoryAllocatableEvictLowerPercent, then stop .
+	MemoryAllocatableEvictLowerPercent *int64 `json:"memoryAllocatableEvictLowerPercent,omitempty" validate:"omitempty,min=0,ltfield=MemoryAllocatableEvictThresholdPercent"`
+
 	// be.satisfactionRate = be.CPURealLimit/be.CPURequest
 	// if be.satisfactionRate > CPUEvictBESatisfactionUpperPercent/100, then stop to evict.
 	CPUEvictBESatisfactionUpperPercent *int64 `json:"cpuEvictBESatisfactionUpperPercent,omitempty" validate:"omitempty,min=0,max=100,gtfield=CPUEvictBESatisfactionLowerPercent"`
@@ -361,18 +370,29 @@ type ResourceThresholdStrategy struct {
 	// upper: CPU evict threshold percentage (0,100)
 	// +kubebuilder:validation:Maximum=100
 	// +kubebuilder:validation:Minimum=0
-	CPUEvictThresholdPercent *int64 `json:"cpuEvictThresholdPercent,omitempty" validate:"omitempty,min=0,max=100"`
+	CPUEvictThresholdPercent *int64 `json:"cpuEvictThresholdPercent,omitempty" validate:"omitempty,min=0,max=100,gtfield=CPUEvictLowerPercent"`
 	// lower: CPU release util usage under CPUEvictLowerPercent, default = CPUEvictThresholdPercent - 2
 	// +kubebuilder:validation:Maximum=100
 	// +kubebuilder:validation:Minimum=0
-	CPUEvictLowerPercent *int64 `json:"cpuEvictLowerPercent,omitempty" validate:"omitempty,min=0,max=100"`
+	CPUEvictLowerPercent *int64 `json:"cpuEvictLowerPercent,omitempty" validate:"omitempty,min=0,max=100,ltfield=CPUEvictThresholdPercent"`
+
+	// upper: cpu ledger evict threshold percentage (0,), default = 110
+	// +kubebuilder:validation:Minimum=0
+	// if cpu.requestPercent > CPUAllocatableEvictThresholdPercent, then start to evict.
+	CPUAllocatableEvictThresholdPercent *int64 `json:"cpuAllocatableEvictThresholdPercent,omitempty" validate:"omitempty,min=0,gtfield=CPUAllocatableEvictLowerPercent"`
+	// lower: cpu release util requestPercent under CPUAllocatableEvictLowerPercent, default = 100
+	// +kubebuilder:validation:Minimum=0
+	// evict until cpu.requestPercent <= CPUAllocatableEvictLowerPercent, then stop .
+	CPUAllocatableEvictLowerPercent *int64 `json:"cpuAllocatableEvictLowerPercent,omitempty" validate:"omitempty,min=0,ltfield=CPUAllocatableEvictThresholdPercent"`
 
 	// CPUEvictPolicy defines the policy for the BECPUEvict feature.
 	// Default: `evictByRealLimit`.
 	CPUEvictPolicy CPUEvictPolicy `json:"cpuEvictPolicy,omitempty"`
 
 	// EvictEnabledPriorityThreshold defines the highest priority for the xxxEvict feature.
 	EvictEnabledPriorityThreshold *int32 `json:"evictEnabledPriorityThreshold,omitempty"`
+	// AllocatableEvictPriorityThreshold defines the highest priority for the xxxAllocatableEvict feature. must less than koord-prod
+	AllocatableEvictPriorityThreshold *int32 `json:"allocatableEvictPriorityThreshold,omitempty" validate:"omitempty,min=0,max=7999"`
 }
 
 // ResctrlQOSCfg stores node-level config of resctrl qos

config/crd/bases/slo.koordinator.sh_nodeslos.yaml

@@ -1665,6 +1665,26 @@ spec:
                   BE pods will be limited if node resource usage overload
                   TODO: ResourceUsedThresholdWithBE need to rename
                 properties:
+                  allocatableEvictPriorityThreshold:
+                    description: AllocatableEvictPriorityThreshold defines the highest
+                      priority for the xxxAllocatableEvict feature. must less than
+                      koord-prod
+                    format: int32
+                    type: integer
+                  cpuAllocatableEvictLowerPercent:
+                    description: |-
+                      lower: cpu release util requestPercent under CPUAllocatableEvictLowerPercent, default = 100
+                      evict until cpu.requestPercent <= CPUAllocatableEvictLowerPercent, then stop .
+                    format: int64
+                    minimum: 0
+                    type: integer
+                  cpuAllocatableEvictThresholdPercent:
+                    description: |-
+                      upper: cpu ledger evict threshold percentage (0,), default = 110
+                      if cpu.requestPercent > CPUAllocatableEvictThresholdPercent, then start to evict.
+                    format: int64
+                    minimum: 0
+                    type: integer
                   cpuEvictBESatisfactionLowerPercent:
                     description: |-
                       be.satisfactionRate = be.CPURealLimit/be.CPURequest; be.cpuUsage = be.CPUUsed/be.CPURealLimit
@@ -1733,6 +1753,20 @@ spec:
                       priority for the xxxEvict feature.
                     format: int32
                     type: integer
+                  memoryAllocatableEvictLowerPercent:
+                    description: |-
+                      lower: memory release util requestPercent under MemoryAllocatableEvictLowerPercent, default = 100
+                      evict until memory.requestPercent <= MemoryAllocatableEvictLowerPercent, then stop .
+                    format: int64
+                    minimum: 0
+                    type: integer
+                  memoryAllocatableEvictThresholdPercent:
+                    description: |-
+                      upper: memory ledger evict threshold percentage (0,), default = 110
+                      if memory.requestPercent > MemoryAllocatableEvictThresholdPercent, then start to evict.
+                    format: int64
+                    minimum: 0
+                    type: integer
                   memoryEvictLowerPercent:
                     description: 'lower: memory release util usage under MemoryEvictLowerPercent,
                       default = MemoryEvictThresholdPercent - 2'

pkg/features/koordlet_features.go

@@ -76,6 +76,18 @@ const (
 	// MemoryEvict evicts those configured priority pods when node lack of resource.
 	MemoryEvict featuregate.Feature = "MemoryEvict"
 
+	// owner: @lijunxin559
+	// alpha: v1.8
+	//
+	// CPUAllocatableEvict evicts those configured priority pods when node lack of allocatable cpu.
+	CPUAllocatableEvict featuregate.Feature = "CPUAllocatableEvict"
+
+	// owner: @lijunxin559
+	// alpha: v1.8
+	//
+	// MemoryAllocatableEvict evicts those configured priority pods when node lack of allocatable memory.
+	MemoryAllocatableEvict featuregate.Feature = "MemoryAllocatableEvict"
+
 	// owner: @saintube @zwzhang0107
 	// alpha: v0.2
 	// beta: v1.1
@@ -195,8 +207,10 @@ var (
 		BECPUManager:           {Default: false, PreRelease: featuregate.Alpha},
 		BECPUEvict:             {Default: false, PreRelease: featuregate.Alpha},
 		CPUEvict:               {Default: false, PreRelease: featuregate.Alpha},
+		CPUAllocatableEvict:    {Default: false, PreRelease: featuregate.Alpha},
 		BEMemoryEvict:          {Default: false, PreRelease: featuregate.Alpha},
 		MemoryEvict:            {Default: false, PreRelease: featuregate.Alpha},
+		MemoryAllocatableEvict: {Default: false, PreRelease: featuregate.Alpha},
 		CPUBurst:               {Default: true, PreRelease: featuregate.Beta},
 		SystemConfig:           {Default: false, PreRelease: featuregate.Alpha},
 		RdtResctrl:             {Default: true, PreRelease: featuregate.Beta},
@@ -224,7 +238,7 @@ func IsFeatureDisabled(nodeSLO *slov1alpha1.NodeSLO, feature featuregate.Feature
 
 	spec := nodeSLO.Spec
 	switch feature {
-	case BECPUSuppress, BEMemoryEvict, BECPUEvict, CPUEvict, MemoryEvict:
+	case BECPUSuppress, BEMemoryEvict, BECPUEvict, CPUEvict, MemoryEvict, CPUAllocatableEvict, MemoryAllocatableEvict:
 		if spec.ResourceUsedThresholdWithBE == nil || spec.ResourceUsedThresholdWithBE.Enable == nil {
 			return true, fmt.Errorf("cannot parse feature config for invalid nodeSLO %v", nodeSLO)
 		}