Rename request to limits in Flavor

Signed-off-by: kerthcet <[email protected]>

Author: kerthcet

api/core/v1alpha1/model_types.go

@@ -101,16 +101,16 @@ type FlavorName string
 type Flavor struct {
 	// Name represents the flavor name, which will be used in model claim.
 	Name FlavorName `json:"name"`
-	// Requests defines the required accelerators to serve the model for each replica,
-	// like <nvidia.com/gpu: 8>. For multi-hosts cases, the requests here indicates
+	// Limits defines the required accelerators to serve the model for each replica,
+	// like <nvidia.com/gpu: 8>. For multi-hosts cases, the limits here indicates
 	// the resource requirements for each replica, usually equals to the TP size.
 	// Not recommended to set the cpu and memory usage here:
 	// - if using playground, you can define the cpu/mem usage at backendConfig.
 	// - if using inference service, you can define the cpu/mem at the container resources.
-	// However, if you define the same accelerator requests at playground/service as well,
-	// the requests will be overwritten by the flavor requests.
+	// However, if you define the same accelerator resources at playground/service as well,
+	// the resources will be overwritten by the flavor limit here.
 	// +optional
-	Requests v1.ResourceList `json:"requests,omitempty"`
+	Limits v1.ResourceList `json:"limits,omitempty"`
 	// NodeSelector represents the node candidates for Pod placements, if a node doesn't
 	// meet the nodeSelector, it will be filtered out in the resourceFungibility scheduler plugin.
 	// If nodeSelector is empty, it means every node is a candidate.
@@ -129,11 +129,14 @@ type Flavor struct {
 type InferenceConfig struct {
 	// Flavors represents the accelerator requirements to serve the model.
 	// Flavors are fungible following the priority represented by the slice order.
+	// This is used both in Playground and Inference Service.
 	// +kubebuilder:validation:MaxItems=8
 	// +optional
 	Flavors []Flavor `json:"flavors,omitempty"`
 	// SharedMemorySize represents the size of /dev/shm required in the runtime of
 	// inference workload.
+	// This is only used in Playground.
+	// +optional
 	SharedMemorySize *resource.Quantity `json:"sharedMemorySize,omitempty"`
 }
 

api/core/v1alpha1/zz_generated.deepcopy.go

@@ -54,13 +54,31 @@ spec:
                     description: |-
                       Flavors represents the accelerator requirements to serve the model.
                       Flavors are fungible following the priority represented by the slice order.
+                      This is used both in Playground ans Inference Service.
                     items:
                       description: |-
                         Flavor defines the accelerator requirements for a model and the necessary parameters
                         in autoscaling. Right now, it will be used in two places:
                         - Pod scheduling with node selectors specified.
                         - Cluster autoscaling with essential parameters provided.
                       properties:
+                        limits:
+                          additionalProperties:
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
+                          description: |-
+                            Limits defines the required accelerators to serve the model for each replica,
+                            like <nvidia.com/gpu: 8>. For multi-hosts cases, the limits here indicates
+                            the resource requirements for each replica, usually equals to the TP size.
+                            Not recommended to set the cpu and memory usage here:
+                            - if using playground, you can define the cpu/mem usage at backendConfig.
+                            - if using inference service, you can define the cpu/mem at the container resources.
+                            However, if you define the same accelerator resources at playground/service as well,
+                            the resources will be overwritten by the flavor limit here.
+                          type: object
                         name:
                           description: Name represents the flavor name, which will
                             be used in model claim.
@@ -83,23 +101,6 @@ spec:
                             with <INSTANCE-TYPE: p4d.24xlarge> for AWS.
                             Preset parameters: TP, PP, INSTANCE-TYPE.
                           type: object
-                        requests:
-                          additionalProperties:
-                            anyOf:
-                            - type: integer
-                            - type: string
-                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
-                            x-kubernetes-int-or-string: true
-                          description: |-
-                            Requests defines the required accelerators to serve the model for each replica,
-                            like <nvidia.com/gpu: 8>. For multi-hosts cases, the requests here indicates
-                            the resource requirements for each replica, usually equals to the TP size.
-                            Not recommended to set the cpu and memory usage here:
-                            - if using playground, you can define the cpu/mem usage at backendConfig.
-                            - if using inference service, you can define the cpu/mem at the container resources.
-                            However, if you define the same accelerator requests at playground/service as well,
-                            the requests will be overwritten by the flavor requests.
-                          type: object
                       required:
                       - name
                       type: object
@@ -112,6 +113,7 @@ spec:
                     description: |-
                       SharedMemorySize represents the size of /dev/shm required in the runtime of
                       inference workload.
+                      This is only used in Playground.
                     pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                     x-kubernetes-int-or-string: true
                 type: object

client-go/applyconfiguration/core/v1alpha1/flavor.go

@@ -10,5 +10,5 @@ spec:
   inferenceConfig:
     flavors:
       - name: t4 # GPU type
-        requests:
+        limits:
           nvidia.com/gpu: 1

client-go/applyconfiguration/core/v1alpha1/inferenceconfig.go

@@ -10,5 +10,5 @@ spec:
   inferenceConfig:
     flavors:
       - name: t4 # GPU type
-        requests:
+        limits:
           nvidia.com/gpu: 1

config/crd/bases/llmaz.io_openmodels.yaml

@@ -11,5 +11,5 @@ spec:
   inferenceConfig:
     flavors:
       - name: t4 # GPU type
-        requests:
+        limits:
           nvidia.com/gpu: 1

docs/examples/hostpath/model.yaml

@@ -10,13 +10,13 @@ spec:
   inferenceConfig:
     flavors:
       - name: a100-80gb
-        requests:
+        limits:
           nvidia.com/gpu: 8 # single node request
         params:
           TP: "8" # 8 GPUs per node, equal to nvidia.com/gpu
           PP: "2" # 2 nodes
   # - name: h100
-  #   requests:
+  #   limits:
   #     nvidia.com/gpu: 8 # single node request
   #   params:
   #     TP: "8"

docs/examples/huggingface/model.yaml

@@ -11,5 +11,5 @@ spec:
   inferenceConfig:
     flavors:
       - name: t4 # GPU type
-        requests:
+        limits:
           nvidia.com/gpu: 1