feat: warmpool selection strategy

updated based on copilot

added test to sandbox controller

lint fixed

fixed e2e tests

Author: vicentefb

api/v1beta1/sandbox_types.go

@@ -230,6 +230,10 @@ type SandboxStatus struct {
 	// A pod may have multiple IPs in dual-stack clusters.
 	// +optional
 	PodIPs []string `json:"podIPs,omitempty"`
+
+	// nodeName is the name of the node where the underlying pod is scheduled.
+	// +optional
+	NodeName string `json:"nodeName,omitempty"`
 }
 
 // +genclient

controllers/sandbox_controller.go

@@ -237,9 +237,11 @@ func (r *SandboxReconciler) reconcileChildResources(ctx context.Context, sandbox
 	allErrors = errors.Join(allErrors, err)
 	if pod == nil {
 		sandbox.Status.PodIPs = nil
+		sandbox.Status.NodeName = ""
 	} else {
 		sandbox.Status.LabelSelector = fmt.Sprintf("%s=%s", sandboxLabel, NameHash(sandbox.Name))
 		sandbox.Status.PodIPs = podIPsFromStatus(pod.Status.PodIPs)
+		sandbox.Status.NodeName = pod.Spec.NodeName
 	}
 
 	// Reconcile Service

controllers/sandbox_controller_test.go

@@ -565,6 +565,7 @@ func TestReconcile(t *testing.T) {
 					},
 					Spec: corev1.PodSpec{
 						Containers: []corev1.Container{{Name: "test-container"}},
+						NodeName:   "node-1",
 					},
 					Status: corev1.PodStatus{
 						PodIPs: []corev1.PodIP{{IP: "10.244.0.5"}, {IP: "fd00::5"}},
@@ -588,6 +589,7 @@ func TestReconcile(t *testing.T) {
 				ServiceFQDN:   "sandbox-name.sandbox-ns.svc.cluster.local",
 				LabelSelector: "agents.x-k8s.io/sandbox-name-hash=ab179450",
 				PodIPs:        []string{"10.244.0.5", "fd00::5"},
+				NodeName:      "node-1",
 				Conditions: []metav1.Condition{
 					{
 						Type:               "Ready",
@@ -633,6 +635,7 @@ func TestReconcile(t *testing.T) {
 					},
 					Spec: corev1.PodSpec{
 						Containers: []corev1.Container{{Name: "test-container"}},
+						NodeName:   "node-2",
 					},
 					Status: corev1.PodStatus{
 						PodIPs: []corev1.PodIP{{IP: "10.244.0.5"}},
@@ -653,6 +656,7 @@ func TestReconcile(t *testing.T) {
 			wantStatus: sandboxv1beta1.SandboxStatus{
 				LabelSelector: "agents.x-k8s.io/sandbox-name-hash=ab179450",
 				PodIPs:        []string{"10.244.0.5"},
+				NodeName:      "node-2",
 				Conditions: []metav1.Condition{
 					{
 						Type:               "Ready",

docs/api.md

@@ -185,6 +185,7 @@ _Appears in:_
 | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | conditions defines the status conditions array |  | Optional: \{\} <br /> |
 | `selector` _string_ | selector is the label selector for pods. |  | Optional: \{\} <br /> |
 | `podIPs` _string array_ | podIPs are the IP addresses of the underlying pod.<br />A pod may have multiple IPs in dual-stack clusters. |  | Optional: \{\} <br /> |
+| `nodeName` _string_ | nodeName is the name of the node where the underlying pod is scheduled. |  | Optional: \{\} <br /> |
 
 
 #### ShutdownPolicy
@@ -220,6 +221,23 @@ Package v1beta1 contains API Schema definitions for the agents v1beta1 API group
 
 
 
+#### AdoptionStrategyType
+
+_Underlying type:_ _string_
+
+AdoptionStrategyType defines the strategy used to pick a Sandbox from a Warm Pool.
+
+
+
+_Appears in:_
+- [SandboxTemplateSpec](#sandboxtemplatespec)
+
+| Field | Description |
+| --- | --- |
+| `OldestReady` | AdoptionStrategyOldestReady selects the oldest ready Sandbox first.<br />If no ready sandboxes are available, it falls back to oldest matching sandboxes.<br /> |
+| `NodeSpread` | AdoptionStrategyNodeSpread spreads adopted Sandboxes across different nodes to prevent resource contention.<br />It prioritizes ready sandboxes first, then selects from the node with the fewest active sandboxes,<br />and uses creation timestamp as a tie-breaker among candidates with the same readiness and occupancy.<br /> |
+
+
 #### EnvVar
 
 
@@ -438,6 +456,7 @@ _Appears in:_
 | `networkPolicyManagement` _[NetworkPolicyManagement](#networkpolicymanagement)_ | networkPolicyManagement defines whether the controller manages the NetworkPolicy.<br />Valid values are "Managed" (default) or "Unmanaged". | Managed | Enum: [Managed Unmanaged] <br />Optional: \{\} <br /> |
 | `envVarsInjectionPolicy` _[EnvVarsInjectionPolicy](#envvarsinjectionpolicy)_ | envVarsInjectionPolicy allows a SandboxClaim to inject or override environment variables defined in the template.<br />If set to Disallowed, the SandboxClaim will be rejected if it specifies any environment variables. | Disallowed | Enum: [Allowed Overrides Disallowed] <br />Optional: \{\} <br /> |
 | `service` _boolean_ | service controls whether the controller should automatically create a<br />headless Service for Sandboxes created from this template.<br />When unset, the controller preserves existing Services for backward<br />compatibility but does not create new ones. Set to true to enable or false<br />to explicitly disable and remove the Service. |  | Optional: \{\} <br /> |
+| `adoptionStrategy` _[AdoptionStrategyType](#adoptionstrategytype)_ | adoptionStrategy defines the strategy used to pick a Sandbox from a Warm Pool during adoption.<br />"OldestReady" selects the oldest ready Sandbox first.<br />"NodeSpread" selects a Sandbox to maximize spread across nodes (preferring ready sandboxes first, then fewest active workloads, then creation-time tie breakers). | OldestReady | Enum: [OldestReady NodeSpread] <br />Optional: \{\} <br /> |
 
 
 #### SandboxWarmPool

extensions/api/v1beta1/sandboxtemplate_types.go

@@ -136,8 +136,30 @@ type SandboxTemplateSpec struct {
 	//nolint:nobools // Enum not used to avoid duplicating the Service API; field is not expected to extend (issue #746).
 	// +optional
 	Service *bool `json:"service,omitempty"`
+
+	// adoptionStrategy defines the strategy used to pick a Sandbox from a Warm Pool during adoption.
+	// "OldestReady" selects the oldest ready Sandbox first.
+	// "NodeSpread" selects a Sandbox to maximize spread across nodes (preferring ready sandboxes first, then fewest active workloads, then creation-time tie breakers).
+	// +kubebuilder:validation:Enum=OldestReady;NodeSpread
+	// +kubebuilder:default=OldestReady
+	// +optional
+	AdoptionStrategy AdoptionStrategyType `json:"adoptionStrategy,omitempty"`
 }
 
+// AdoptionStrategyType defines the strategy used to pick a Sandbox from a Warm Pool.
+type AdoptionStrategyType string
+
+const (
+	// AdoptionStrategyOldestReady selects the oldest ready Sandbox first.
+	// If no ready sandboxes are available, it falls back to oldest matching sandboxes.
+	AdoptionStrategyOldestReady AdoptionStrategyType = "OldestReady"
+
+	// AdoptionStrategyNodeSpread spreads adopted Sandboxes across different nodes to prevent resource contention.
+	// It prioritizes ready sandboxes first, then selects from the node with the fewest active sandboxes,
+	// and uses creation timestamp as a tie-breaker among candidates with the same readiness and occupancy.
+	AdoptionStrategyNodeSpread AdoptionStrategyType = "NodeSpread"
+)
+
 // +genclient
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:scope=Namespaced,shortName=sandboxtemplate

extensions/controllers/queue/simple_sandbox_queue.go

@@ -28,6 +28,7 @@ type SandboxKey types.NamespacedName
 type SandboxQueue interface {
 	Add(warmPoolName string, item SandboxKey)
 	Get(warmPoolName string) (SandboxKey, bool)
+	GetWithStrategy(warmPoolName string, pick func([]SandboxKey) (SandboxKey, bool)) (SandboxKey, bool)
 	RemoveQueue(warmPoolName string)
 	RemoveItem(warmPoolName string, item SandboxKey)
 }
@@ -58,6 +59,15 @@ func (s *SimpleSandboxQueue) Get(warmPoolName string) (SandboxKey, bool) {
 	return q.(*synchronizedQueue).Pop()
 }
 
+// GetWithStrategy pops an item from the specific warm pool's queue using a custom strategy.
+func (s *SimpleSandboxQueue) GetWithStrategy(warmPoolName string, pick func([]SandboxKey) (SandboxKey, bool)) (SandboxKey, bool) {
+	q, ok := s.queues.Load(warmPoolName)
+	if !ok {
+		return SandboxKey{}, false
+	}
+	return q.(*synchronizedQueue).PopWithStrategy(pick)
+}
+
 // RemoveItem deletes a specific sandbox from a warm pool's queue.
 func (s *SimpleSandboxQueue) RemoveItem(warmPoolName string, item SandboxKey) {
 	if q, ok := s.queues.Load(warmPoolName); ok {
@@ -139,8 +149,51 @@ func (q *synchronizedQueue) Pop() (SandboxKey, bool) {
 	return item, true
 }
 
+// PopWithStrategy applies the strategy function to pick an item from the queue,
+// removes it thread-safely, and returns it.
+func (q *synchronizedQueue) PopWithStrategy(pick func([]SandboxKey) (SandboxKey, bool)) (SandboxKey, bool) {
+	q.mu.Lock()
+	if len(q.items) == 0 {
+		q.mu.Unlock()
+		return SandboxKey{}, false
+	}
+
+	// Snapshot the queue items
+	snapshot := make([]SandboxKey, len(q.items))
+	copy(snapshot, q.items)
+	q.mu.Unlock()
+
+	key, ok := pick(snapshot)
+	if !ok {
+		return SandboxKey{}, false
+	}
+
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	// Verify the key is still present in the queue
+	if _, exists := q.set[key]; !exists {
+		return SandboxKey{}, false
+	}
+
+	// Find the picked key in q.items and remove it
+	for i, k := range q.items {
+		if k == key {
+			// Shift left and clear the tail slot
+			last := len(q.items) - 1
+			copy(q.items[i:], q.items[i+1:])
+			q.items[last] = SandboxKey{}
+			q.items = q.items[:last]
+			break
+		}
+	}
+	delete(q.set, key)
+
+	return key, true
+}
+
 // RemoveQueue completely deletes a warm pool's queue from the sync.Map
-// to prevent memory leaks when WarmPools are deleted.
+// to prevent memory leaks when SandboxTemplates or WarmPools are deleted.
 func (s *SimpleSandboxQueue) RemoveQueue(warmPoolName string) {
 	s.queues.Delete(warmPoolName)
 }

extensions/controllers/queue/simple_sandbox_queue_test.go

@@ -131,3 +131,50 @@ func TestSimpleSandboxQueue_RemoveQueue_MemoryLeakFix(t *testing.T) {
 		t.Errorf("Expected queue to be completely removed, but it still existed")
 	}
 }
+
+func TestSimpleSandboxQueue_GetWithStrategy(t *testing.T) {
+	q := NewSimpleSandboxQueue()
+	hash := "template-hash-1"
+
+	key1 := SandboxKey{Namespace: "default", Name: "sb-1"}
+	key2 := SandboxKey{Namespace: "default", Name: "sb-2"}
+	key3 := SandboxKey{Namespace: "default", Name: "sb-3"}
+
+	q.Add(hash, key1)
+	q.Add(hash, key2)
+	q.Add(hash, key3)
+
+	// Custom strategy to pick key2 specifically
+	pickKey2 := func(items []SandboxKey) (SandboxKey, bool) {
+		for _, item := range items {
+			if item.Name == "sb-2" {
+				return item, true
+			}
+		}
+		return SandboxKey{}, false
+	}
+
+	// Pop with strategy
+	got, ok := q.GetWithStrategy(hash, pickKey2)
+	if !ok || got != key2 {
+		t.Errorf("Expected to pick %v, got %v (ok: %v)", key2, got, ok)
+	}
+
+	// First standard pop should be key1 (since key2 was removed)
+	got1, _ := q.Get(hash)
+	if got1 != key1 {
+		t.Errorf("Expected first remaining item to be %v, got %v", key1, got1)
+	}
+
+	// Second standard pop should be key3
+	got3, _ := q.Get(hash)
+	if got3 != key3 {
+		t.Errorf("Expected second remaining item to be %v, got %v", key3, got3)
+	}
+
+	// Queue should now be empty
+	_, ok3 := q.Get(hash)
+	if ok3 {
+		t.Errorf("Expected queue to be empty, but got an item")
+	}
+}