Add new conditions

Signed-off-by: kerthcet <[email protected]>

Author: kerthcet

config/manager/kustomization.yaml

@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: inftyai/llmaz-test
-  newTag: 0901-04
+  newName: inftyai/llmaz
+  newTag: main

docs/examples/llamacpp/playground.yaml

@@ -1,11 +1,11 @@
 apiVersion: inference.llmaz.io/v1alpha1
 kind: Playground
 metadata:
-  name: qwen2-0-5b
+  name: qwen2-0--5b
 spec:
   replicas: 1
   modelClaim:
-    modelName: qwen2-0-5b-gguf
+    modelName: qwen2-0--5b-gguf
   backendConfig:
     name: llamacpp
     args:

pkg/controller/inference/playground_controller.go

@@ -22,6 +22,7 @@ import (
 	"reflect"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -84,19 +85,8 @@ func (r *PlaygroundReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 	service := &inferenceapi.Service{}
 	if err := r.Get(ctx, types.NamespacedName{Name: playground.Name, Namespace: playground.Namespace}, service); err == nil {
-		if !controllerutil.HasControllerReference(service) {
-			condition := metav1.Condition{
-				Type:    inferenceapi.PlaygroundProgressing,
-				Status:  metav1.ConditionFalse,
-				Reason:  "AbortProcessing",
-				Message: "Playground owns the same name with an existing Service",
-			}
-			apimeta.SetStatusCondition(&playground.Status.Conditions, condition)
-			if err := r.Client.Status().Update(ctx, playground); err != nil {
-				return ctrl.Result{}, err
-			}
-			// No need to requeue. Once the Service is deleted, the Playground will be reconciled.
-			return ctrl.Result{}, nil
+		if changed := handleUnexpectedCondition(playground, true, controllerutil.HasControllerReference(service)); changed {
+			return ctrl.Result{}, r.Client.Status().Update(ctx, playground)
 		}
 	}
 
@@ -106,13 +96,19 @@ func (r *PlaygroundReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	if playground.Spec.ModelClaim != nil {
 		model := &coreapi.OpenModel{}
 		if err := r.Get(ctx, types.NamespacedName{Name: string(playground.Spec.ModelClaim.ModelName)}, model); err != nil {
+			if apierrors.IsNotFound(err) && handleUnexpectedCondition(playground, false, true) {
+				return ctrl.Result{}, r.Client.Status().Update(ctx, playground)
+			}
 			return ctrl.Result{}, err
 		}
 		models = append(models, model)
 	} else if playground.Spec.MultiModelsClaim != nil {
 		for _, modelName := range playground.Spec.MultiModelsClaim.ModelNames {
 			model := &coreapi.OpenModel{}
 			if err := r.Get(ctx, types.NamespacedName{Name: string(modelName)}, model); err != nil {
+				if apierrors.IsNotFound(err) && handleUnexpectedCondition(playground, false, true) {
+					return ctrl.Result{}, r.Client.Status().Update(ctx, playground)
+				}
 				return ctrl.Result{}, err
 			}
 			models = append(models, model)
@@ -130,11 +126,6 @@ func (r *PlaygroundReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 
 	// Handle status.
-	service = &inferenceapi.Service{}
-	if err := r.Get(ctx, types.NamespacedName{Name: playground.Name, Namespace: playground.Namespace}, service); err != nil {
-		return ctrl.Result{}, err
-	}
-
 	setPlaygroundCondition(playground, service)
 	if err := r.Client.Status().Update(ctx, playground); err != nil {
 		return ctrl.Result{}, err
@@ -308,17 +299,40 @@ func buildWorkerTemplate(models []*coreapi.OpenModel, playground *inferenceapi.P
 	return template
 }
 
-func setPlaygroundCondition(playground *inferenceapi.Playground, service *inferenceapi.Service) {
-	// For the start up.
-	if len(playground.Status.Conditions) == 0 || !apimeta.IsStatusConditionTrue(service.Status.Conditions, inferenceapi.PlaygroundProgressing) {
+func handleUnexpectedCondition(playground *inferenceapi.Playground, modelExists bool, serviceHasOwner bool) (changed bool) {
+	// Put it in the first place as more serious.
+	if !serviceHasOwner {
+		condition := metav1.Condition{
+			Type:    inferenceapi.PlaygroundProgressing,
+			Status:  metav1.ConditionFalse,
+			Reason:  "AbortProcessing",
+			Message: "Playground owns the same name with an existing Service",
+		}
+		return apimeta.SetStatusCondition(&playground.Status.Conditions, condition)
+	}
+
+	if !modelExists {
+		condition := metav1.Condition{
+			Type:    inferenceapi.PlaygroundProgressing,
+			Status:  metav1.ConditionFalse,
+			Reason:  "AbortProcessing",
+			Message: "Waiting for model creation",
+		}
+		return apimeta.SetStatusCondition(&playground.Status.Conditions, condition)
+	}
+	return false
+}
+
+func setPlaygroundCondition(playground *inferenceapi.Playground, service *inferenceapi.Service) (changed bool) {
+	// For the start up or Playground is recovered from AbortProcessing.
+	if len(playground.Status.Conditions) == 0 || apimeta.IsStatusConditionFalse(playground.Status.Conditions, inferenceapi.PlaygroundProgressing) {
 		condition := metav1.Condition{
 			Type:    inferenceapi.PlaygroundProgressing,
 			Status:  metav1.ConditionTrue,
 			Reason:  "Pending",
 			Message: "Waiting for inferenceService ready",
 		}
-		apimeta.SetStatusCondition(&playground.Status.Conditions, condition)
-		return
+		return apimeta.SetStatusCondition(&playground.Status.Conditions, condition)
 	}
 
 	if apimeta.IsStatusConditionTrue(service.Status.Conditions, inferenceapi.ServiceAvailable) {
@@ -328,11 +342,11 @@ func setPlaygroundCondition(playground *inferenceapi.Playground, service *infere
 			Reason:  "PlaygroundReady",
 			Message: "Playground is ready",
 		}
-		apimeta.SetStatusCondition(&playground.Status.Conditions, condition)
+		return apimeta.SetStatusCondition(&playground.Status.Conditions, condition)
 	} else {
 		// Still in starting up, no need to populate the condition.
 		if apimeta.FindStatusCondition(playground.Status.Conditions, inferenceapi.PlaygroundAvailable) == nil {
-			return
+			return false
 		}
 
 		condition := metav1.Condition{
@@ -341,14 +355,16 @@ func setPlaygroundCondition(playground *inferenceapi.Playground, service *infere
 			Reason:  "PlaygroundInProgress",
 			Message: "Playground is progressing",
 		}
-		apimeta.SetStatusCondition(&playground.Status.Conditions, condition)
+		changed = apimeta.SetStatusCondition(&playground.Status.Conditions, condition) || changed
 
 		// Set the available to false
 		new_condition := metav1.Condition{
 			Type:   inferenceapi.PlaygroundAvailable,
 			Status: metav1.ConditionFalse,
+			Reason: "PlaygroundNotReady",
 		}
-		apimeta.SetStatusCondition(&playground.Status.Conditions, new_condition)
+		changed = apimeta.SetStatusCondition(&playground.Status.Conditions, new_condition) || changed
+		return changed
 	}
 }
 

test/e2e/suit_test.go

@@ -17,6 +17,7 @@ package e2e
 
 import (
 	"context"
+	"os"
 	"testing"
 	"time"
 
@@ -81,6 +82,7 @@ var _ = BeforeSuite(func() {
 	Expect(k8sClient).NotTo(BeNil())
 
 	readyForTesting(k8sClient)
+	os.Setenv("TEST_TYPE", "E2E")
 })
 
 var _ = AfterSuite(func() {