diff --git a/docs/cluster/auth.html b/docs/cluster/auth.html index 1beb27c77..0389fbf71 100644 --- a/docs/cluster/auth.html +++ b/docs/cluster/auth.html @@ -55,6 +55,7 @@

Module codeflare_sdk.cluster.auth

import abc from kubernetes import client, config import os +import urllib3 from ..utils.kube_api_helpers import _kube_api_error_handling global api_client @@ -141,7 +142,10 @@

Module codeflare_sdk.cluster.auth

elif self.skip_tls == False: configuration.ssl_ca_cert = self.ca_cert_path else: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + print("Insecure request warnings have been disabled") configuration.verify_ssl = False + api_client = client.ApiClient(configuration) client.AuthenticationApi(api_client).get_api_group() config_path = None @@ -566,7 +570,10 @@

Methods

elif self.skip_tls == False: configuration.ssl_ca_cert = self.ca_cert_path else: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + print("Insecure request warnings have been disabled") configuration.verify_ssl = False + api_client = client.ApiClient(configuration) client.AuthenticationApi(api_client).get_api_group() config_path = None @@ -620,7 +627,10 @@

Methods

elif self.skip_tls == False: configuration.ssl_ca_cert = self.ca_cert_path else: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + print("Insecure request warnings have been disabled") configuration.verify_ssl = False + api_client = client.ApiClient(configuration) client.AuthenticationApi(api_client).get_api_group() config_path = None diff --git a/docs/cluster/awload.html b/docs/cluster/awload.html index d2afb7ea9..57b407e80 100644 --- a/docs/cluster/awload.html +++ b/docs/cluster/awload.html @@ -92,7 +92,7 @@

Module codeflare_sdk.cluster.awload

config_check() api_instance = client.CustomObjectsApi(api_config_handler()) api_instance.create_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=self.namespace, plural="appwrappers", @@ -117,7 +117,7 @@

Module codeflare_sdk.cluster.awload

config_check() api_instance = client.CustomObjectsApi(api_config_handler()) api_instance.delete_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=self.namespace, plural="appwrappers", @@ -186,7 +186,7 @@

Classes

config_check() api_instance = client.CustomObjectsApi(api_config_handler()) api_instance.create_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=self.namespace, plural="appwrappers", @@ -211,7 +211,7 @@

Classes

config_check() api_instance = client.CustomObjectsApi(api_config_handler()) api_instance.delete_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=self.namespace, plural="appwrappers", @@ -248,7 +248,7 @@

Methods

config_check() api_instance = client.CustomObjectsApi(api_config_handler()) api_instance.delete_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=self.namespace, plural="appwrappers", @@ -278,7 +278,7 @@

Methods

config_check() api_instance = client.CustomObjectsApi(api_config_handler()) api_instance.create_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=self.namespace, plural="appwrappers", diff --git a/docs/cluster/cluster.html b/docs/cluster/cluster.html index 11306f306..dbb6cec5e 100644 --- a/docs/cluster/cluster.html +++ b/docs/cluster/cluster.html @@ -70,6 +70,7 @@

Module codeflare_sdk.cluster.cluster

from kubernetes import client, config import yaml import os +import requests class Cluster: @@ -93,38 +94,25 @@

Module codeflare_sdk.cluster.cluster

self.app_wrapper_yaml = self.create_app_wrapper() self.app_wrapper_name = self.app_wrapper_yaml.split(".")[0] - def evaluate_config(self): - if not self.evaluate_dispatch_priority(): - return False - else: - return True - def evaluate_dispatch_priority(self): priority_class = self.config.dispatch_priority - if priority_class is None: - return True - else: - try: - config_check() - api_instance = client.CustomObjectsApi(api_config_handler()) - priority_classes = api_instance.list_cluster_custom_object( - group="scheduling.k8s.io", - version="v1", - plural="priorityclasses", - ) - available_priority_classes = [ - i["metadata"]["name"] for i in priority_classes["items"] - ] - except Exception as e: # pragma: no cover - return _kube_api_error_handling(e) - - if priority_class in available_priority_classes: - return True - else: - print( - f"Priority class {priority_class} is not available in the cluster" - ) - return False + + try: + config_check() + api_instance = client.CustomObjectsApi(api_config_handler()) + priority_classes = api_instance.list_cluster_custom_object( + group="scheduling.k8s.io", + version="v1", + plural="priorityclasses", + ) + except Exception as e: # pragma: no cover + return _kube_api_error_handling(e) + + for pc in priority_classes["items"]: + if pc["metadata"]["name"] == priority_class: + return pc["value"] + print(f"Priority class {priority_class} is not available in the cluster") + return None def create_app_wrapper(self): """ @@ -141,6 +129,16 @@

Module codeflare_sdk.cluster.cluster

f"Namespace {self.config.namespace} is of type {type(self.config.namespace)}. Check your Kubernetes Authentication." ) + # Before attempting to create the cluster AW, let's evaluate the ClusterConfig + if self.config.dispatch_priority: + priority_val = self.evaluate_dispatch_priority() + if priority_val == None: + raise ValueError( + "Invalid Cluster Configuration, AppWrapper not generated" + ) + else: + priority_val = None + name = self.config.name namespace = self.config.namespace min_cpu = self.config.min_cpus @@ -174,6 +172,7 @@

Module codeflare_sdk.cluster.cluster

local_interactive=local_interactive, image_pull_secrets=image_pull_secrets, dispatch_priority=dispatch_priority, + priority_val=priority_val, ) # creates a new cluster with the provided or default spec @@ -182,12 +181,6 @@

Module codeflare_sdk.cluster.cluster

Applies the AppWrapper yaml, pushing the resource request onto the MCAD queue. """ - - # Before attempting to bring up the cluster let's evaluate the ClusterConfig - if not self.evaluate_config(): - print("Invalid Cluster Configuration") - return False - namespace = self.config.namespace try: config_check() @@ -195,7 +188,7 @@

Module codeflare_sdk.cluster.cluster

with open(self.app_wrapper_yaml) as f: aw = yaml.load(f, Loader=yaml.FullLoader) api_instance.create_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=namespace, plural="appwrappers", @@ -214,7 +207,7 @@

Module codeflare_sdk.cluster.cluster

config_check() api_instance = client.CustomObjectsApi(api_config_handler()) api_instance.delete_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=namespace, plural="appwrappers", @@ -290,6 +283,13 @@

Module codeflare_sdk.cluster.cluster

return status, ready + def is_dashboard_ready(self) -> bool: + response = requests.get(self.cluster_dashboard_uri(), timeout=5) + if response.status_code == 200: + return True + else: + return False + def wait_ready(self, timeout: Optional[int] = None): """ Waits for requested cluster to be ready, up to an optional timeout (s). @@ -297,20 +297,22 @@

Module codeflare_sdk.cluster.cluster

""" print("Waiting for requested resources to be set up...") ready = False + dashboard_ready = False status = None time = 0 - while not ready: + while not ready or not dashboard_ready: status, ready = self.status(print_to_console=False) + dashboard_ready = self.is_dashboard_ready() if status == CodeFlareClusterStatus.UNKNOWN: print( "WARNING: Current cluster status is unknown, have you run cluster.up yet?" ) - if not ready: + if not ready or not dashboard_ready: if timeout and time >= timeout: raise TimeoutError(f"wait() timed out after waiting {timeout}s") sleep(5) time += 5 - print("Requested cluster up and running!") + print("Requested cluster and dashboard are up and running!") def details(self, print_to_console: bool = True) -> RayCluster: cluster = _copy_to_ray(self) @@ -342,7 +344,8 @@

Module codeflare_sdk.cluster.cluster

for route in routes["items"]: if route["metadata"]["name"] == f"ray-dashboard-{self.config.name}": - return f"http://{route['spec']['host']}" + protocol = "https" if route["spec"].get("tls") else "http" + return f"{protocol}://{route['spec']['host']}" return "Dashboard route not available yet, have you run cluster.up()?" def list_jobs(self) -> List: @@ -521,7 +524,7 @@

Module codeflare_sdk.cluster.cluster

config_check() api_instance = client.CustomObjectsApi(api_config_handler()) aws = api_instance.list_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=namespace, plural="appwrappers", @@ -582,7 +585,7 @@

Module codeflare_sdk.cluster.cluster

config_check() api_instance = client.CustomObjectsApi(api_config_handler()) aws = api_instance.list_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=namespace, plural="appwrappers", @@ -617,7 +620,8 @@

Module codeflare_sdk.cluster.cluster

ray_route = None for route in routes["items"]: if route["metadata"]["name"] == f"ray-dashboard-{rc['metadata']['name']}": - ray_route = route["spec"]["host"] + protocol = "https" if route["spec"].get("tls") else "http" + ray_route = f"{protocol}://{route['spec']['host']}" return RayCluster( name=rc["metadata"]["name"], @@ -828,38 +832,25 @@

Classes

self.app_wrapper_yaml = self.create_app_wrapper() self.app_wrapper_name = self.app_wrapper_yaml.split(".")[0] - def evaluate_config(self): - if not self.evaluate_dispatch_priority(): - return False - else: - return True - def evaluate_dispatch_priority(self): priority_class = self.config.dispatch_priority - if priority_class is None: - return True - else: - try: - config_check() - api_instance = client.CustomObjectsApi(api_config_handler()) - priority_classes = api_instance.list_cluster_custom_object( - group="scheduling.k8s.io", - version="v1", - plural="priorityclasses", - ) - available_priority_classes = [ - i["metadata"]["name"] for i in priority_classes["items"] - ] - except Exception as e: # pragma: no cover - return _kube_api_error_handling(e) - - if priority_class in available_priority_classes: - return True - else: - print( - f"Priority class {priority_class} is not available in the cluster" - ) - return False + + try: + config_check() + api_instance = client.CustomObjectsApi(api_config_handler()) + priority_classes = api_instance.list_cluster_custom_object( + group="scheduling.k8s.io", + version="v1", + plural="priorityclasses", + ) + except Exception as e: # pragma: no cover + return _kube_api_error_handling(e) + + for pc in priority_classes["items"]: + if pc["metadata"]["name"] == priority_class: + return pc["value"] + print(f"Priority class {priority_class} is not available in the cluster") + return None def create_app_wrapper(self): """ @@ -876,6 +867,16 @@

Classes

f"Namespace {self.config.namespace} is of type {type(self.config.namespace)}. Check your Kubernetes Authentication." ) + # Before attempting to create the cluster AW, let's evaluate the ClusterConfig + if self.config.dispatch_priority: + priority_val = self.evaluate_dispatch_priority() + if priority_val == None: + raise ValueError( + "Invalid Cluster Configuration, AppWrapper not generated" + ) + else: + priority_val = None + name = self.config.name namespace = self.config.namespace min_cpu = self.config.min_cpus @@ -909,6 +910,7 @@

Classes

local_interactive=local_interactive, image_pull_secrets=image_pull_secrets, dispatch_priority=dispatch_priority, + priority_val=priority_val, ) # creates a new cluster with the provided or default spec @@ -917,12 +919,6 @@

Classes

Applies the AppWrapper yaml, pushing the resource request onto the MCAD queue. """ - - # Before attempting to bring up the cluster let's evaluate the ClusterConfig - if not self.evaluate_config(): - print("Invalid Cluster Configuration") - return False - namespace = self.config.namespace try: config_check() @@ -930,7 +926,7 @@

Classes

with open(self.app_wrapper_yaml) as f: aw = yaml.load(f, Loader=yaml.FullLoader) api_instance.create_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=namespace, plural="appwrappers", @@ -949,7 +945,7 @@

Classes

config_check() api_instance = client.CustomObjectsApi(api_config_handler()) api_instance.delete_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=namespace, plural="appwrappers", @@ -1025,6 +1021,13 @@

Classes

return status, ready + def is_dashboard_ready(self) -> bool: + response = requests.get(self.cluster_dashboard_uri(), timeout=5) + if response.status_code == 200: + return True + else: + return False + def wait_ready(self, timeout: Optional[int] = None): """ Waits for requested cluster to be ready, up to an optional timeout (s). @@ -1032,20 +1035,22 @@

Classes

""" print("Waiting for requested resources to be set up...") ready = False + dashboard_ready = False status = None time = 0 - while not ready: + while not ready or not dashboard_ready: status, ready = self.status(print_to_console=False) + dashboard_ready = self.is_dashboard_ready() if status == CodeFlareClusterStatus.UNKNOWN: print( "WARNING: Current cluster status is unknown, have you run cluster.up yet?" ) - if not ready: + if not ready or not dashboard_ready: if timeout and time >= timeout: raise TimeoutError(f"wait() timed out after waiting {timeout}s") sleep(5) time += 5 - print("Requested cluster up and running!") + print("Requested cluster and dashboard are up and running!") def details(self, print_to_console: bool = True) -> RayCluster: cluster = _copy_to_ray(self) @@ -1077,7 +1082,8 @@

Classes

for route in routes["items"]: if route["metadata"]["name"] == f"ray-dashboard-{self.config.name}": - return f"http://{route['spec']['host']}" + protocol = "https" if route["spec"].get("tls") else "http" + return f"{protocol}://{route['spec']['host']}" return "Dashboard route not available yet, have you run cluster.up()?" def list_jobs(self) -> List: @@ -1203,7 +1209,8 @@

Methods

for route in routes["items"]: if route["metadata"]["name"] == f"ray-dashboard-{self.config.name}": - return f"http://{route['spec']['host']}" + protocol = "https" if route["spec"].get("tls") else "http" + return f"{protocol}://{route['spec']['host']}" return "Dashboard route not available yet, have you run cluster.up()?" @@ -1248,6 +1255,16 @@

Methods

f"Namespace {self.config.namespace} is of type {type(self.config.namespace)}. Check your Kubernetes Authentication." ) + # Before attempting to create the cluster AW, let's evaluate the ClusterConfig + if self.config.dispatch_priority: + priority_val = self.evaluate_dispatch_priority() + if priority_val == None: + raise ValueError( + "Invalid Cluster Configuration, AppWrapper not generated" + ) + else: + priority_val = None + name = self.config.name namespace = self.config.namespace min_cpu = self.config.min_cpus @@ -1281,6 +1298,7 @@

Methods

local_interactive=local_interactive, image_pull_secrets=image_pull_secrets, dispatch_priority=dispatch_priority, + priority_val=priority_val, ) @@ -1320,7 +1338,7 @@

Methods

config_check() api_instance = client.CustomObjectsApi(api_config_handler()) api_instance.delete_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=namespace, plural="appwrappers", @@ -1330,22 +1348,6 @@

Methods

return _kube_api_error_handling(e) -
-def evaluate_config(self) -
-
-
-
- -Expand source code - -
def evaluate_config(self):
-    if not self.evaluate_dispatch_priority():
-        return False
-    else:
-        return True
-
-
def evaluate_dispatch_priority(self)
@@ -1357,30 +1359,23 @@

Methods

def evaluate_dispatch_priority(self):
     priority_class = self.config.dispatch_priority
-    if priority_class is None:
-        return True
-    else:
-        try:
-            config_check()
-            api_instance = client.CustomObjectsApi(api_config_handler())
-            priority_classes = api_instance.list_cluster_custom_object(
-                group="scheduling.k8s.io",
-                version="v1",
-                plural="priorityclasses",
-            )
-            available_priority_classes = [
-                i["metadata"]["name"] for i in priority_classes["items"]
-            ]
-        except Exception as e:  # pragma: no cover
-            return _kube_api_error_handling(e)
 
-        if priority_class in available_priority_classes:
-            return True
-        else:
-            print(
-                f"Priority class {priority_class} is not available in the cluster"
-            )
-            return False
+ try: + config_check() + api_instance = client.CustomObjectsApi(api_config_handler()) + priority_classes = api_instance.list_cluster_custom_object( + group="scheduling.k8s.io", + version="v1", + plural="priorityclasses", + ) + except Exception as e: # pragma: no cover + return _kube_api_error_handling(e) + + for pc in priority_classes["items"]: + if pc["metadata"]["name"] == priority_class: + return pc["value"] + print(f"Priority class {priority_class} is not available in the cluster") + return None
@@ -1435,6 +1430,23 @@

Methods

return Cluster(cluster_config) +
+def is_dashboard_ready(self) ‑> bool +
+
+
+
+ +Expand source code + +
def is_dashboard_ready(self) -> bool:
+    response = requests.get(self.cluster_dashboard_uri(), timeout=5)
+    if response.status_code == 200:
+        return True
+    else:
+        return False
+
+
def job_logs(self, job_id: str) ‑> str
@@ -1623,12 +1635,6 @@

Methods

Applies the AppWrapper yaml, pushing the resource request onto the MCAD queue. """ - - # Before attempting to bring up the cluster let's evaluate the ClusterConfig - if not self.evaluate_config(): - print("Invalid Cluster Configuration") - return False - namespace = self.config.namespace try: config_check() @@ -1636,7 +1642,7 @@

Methods

with open(self.app_wrapper_yaml) as f: aw = yaml.load(f, Loader=yaml.FullLoader) api_instance.create_namespaced_custom_object( - group="mcad.ibm.com", + group="workload.codeflare.dev", version="v1beta1", namespace=namespace, plural="appwrappers", @@ -1663,20 +1669,22 @@

Methods

""" print("Waiting for requested resources to be set up...") ready = False + dashboard_ready = False status = None time = 0 - while not ready: + while not ready or not dashboard_ready: status, ready = self.status(print_to_console=False) + dashboard_ready = self.is_dashboard_ready() if status == CodeFlareClusterStatus.UNKNOWN: print( "WARNING: Current cluster status is unknown, have you run cluster.up yet?" ) - if not ready: + if not ready or not dashboard_ready: if timeout and time >= timeout: raise TimeoutError(f"wait() timed out after waiting {timeout}s") sleep(5) time += 5 - print("Requested cluster up and running!") + print("Requested cluster and dashboard are up and running!") @@ -1713,9 +1721,9 @@

create_app_wrapper
  • details
  • down
    • -
  • evaluate_config
  • evaluate_dispatch_priority
  • from_k8_cluster_object
    • +
  • is_dashboard_ready
  • job_logs
  • job_status
  • list_jobs
    • diff --git a/docs/utils/generate_yaml.html b/docs/utils/generate_yaml.html index fe0238991..ea1e73026 100644 --- a/docs/utils/generate_yaml.html +++ b/docs/utils/generate_yaml.html @@ -120,12 +120,21 @@

    Module codeflare_sdk.utils.generate_yaml

    metadata.pop("labels") -def update_priority(item, dispatch_priority): +def update_priority(yaml, item, dispatch_priority, priority_val): + spec = yaml.get("spec") if dispatch_priority is not None: + if priority_val: + spec["priority"] = priority_val + else: + raise ValueError( + "AW generation error: Priority value is None, while dispatch_priority is defined" + ) head = item.get("generictemplate").get("spec").get("headGroupSpec") worker = item.get("generictemplate").get("spec").get("workerGroupSpecs")[0] head["template"]["spec"]["priorityClassName"] = dispatch_priority worker["template"]["spec"]["priorityClassName"] = dispatch_priority + else: + spec.pop("priority") def update_custompodresources( @@ -214,11 +223,6 @@

    Module codeflare_sdk.utils.generate_yaml

    limits["nvidia.com/gpu"] = gpu -def update_scheduling_spec(yaml, workers): - spec = yaml.get("spec") - spec["schedulingSpec"]["minAvailable"] = workers + 1 - - def update_nodes( item, appwrapper_name, @@ -391,6 +395,7 @@

    Module codeflare_sdk.utils.generate_yaml

    local_interactive: bool, image_pull_secrets: list, dispatch_priority: str, + priority_val: int, ): user_yaml = read_template(template) appwrapper_name, cluster_name = gen_names(name) @@ -399,8 +404,7 @@

    Module codeflare_sdk.utils.generate_yaml

    route_item = resources["resources"].get("GenericItems")[1] update_names(user_yaml, item, appwrapper_name, cluster_name, namespace) update_labels(user_yaml, instascale, instance_types) - update_priority(item, dispatch_priority) - update_scheduling_spec(user_yaml, workers) + update_priority(user_yaml, item, dispatch_priority, priority_val) update_custompodresources( item, min_cpu, max_cpu, min_memory, max_memory, gpu, workers ) @@ -573,7 +577,7 @@

    Functions

    -def generate_appwrapper(name: str, namespace: str, min_cpu: int, max_cpu: int, min_memory: int, max_memory: int, gpu: int, workers: int, template: str, image: str, instascale: bool, instance_types: list, env, local_interactive: bool, image_pull_secrets: list, dispatch_priority: str) +def generate_appwrapper(name: str, namespace: str, min_cpu: int, max_cpu: int, min_memory: int, max_memory: int, gpu: int, workers: int, template: str, image: str, instascale: bool, instance_types: list, env, local_interactive: bool, image_pull_secrets: list, dispatch_priority: str, priority_val: int)
    @@ -598,6 +602,7 @@

    Functions

    local_interactive: bool, image_pull_secrets: list, dispatch_priority: str, + priority_val: int, ): user_yaml = read_template(template) appwrapper_name, cluster_name = gen_names(name) @@ -606,8 +611,7 @@

    Functions

    route_item = resources["resources"].get("GenericItems")[1] update_names(user_yaml, item, appwrapper_name, cluster_name, namespace) update_labels(user_yaml, instascale, instance_types) - update_priority(item, dispatch_priority) - update_scheduling_spec(user_yaml, workers) + update_priority(user_yaml, item, dispatch_priority, priority_val) update_custompodresources( item, min_cpu, max_cpu, min_memory, max_memory, gpu, workers ) @@ -898,7 +902,7 @@

    Functions

    -def update_priority(item, dispatch_priority) +def update_priority(yaml, item, dispatch_priority, priority_val)
    @@ -906,12 +910,21 @@

    Functions

    Expand source code -
    def update_priority(item, dispatch_priority):
+
    def update_priority(yaml, item, dispatch_priority, priority_val):
+    spec = yaml.get("spec")
     if dispatch_priority is not None:
+        if priority_val:
+            spec["priority"] = priority_val
+        else:
+            raise ValueError(
+                "AW generation error: Priority value is None, while dispatch_priority is defined"
+            )
         head = item.get("generictemplate").get("spec").get("headGroupSpec")
         worker = item.get("generictemplate").get("spec").get("workerGroupSpecs")[0]
         head["template"]["spec"]["priorityClassName"] = dispatch_priority
-        worker["template"]["spec"]["priorityClassName"] = dispatch_priority
    
+        worker["template"]["spec"]["priorityClassName"] = dispatch_priority
+    else:
+        spec.pop("priority")
    @@ -956,20 +969,6 @@

    Functions

    limits["nvidia.com/gpu"] = gpu     -
    -def update_scheduling_spec(yaml, workers) -
    -
    -
    -
    - -Expand source code - -
    def update_scheduling_spec(yaml, workers):
-    spec = yaml.get("spec")
-    spec["schedulingSpec"]["minAvailable"] = workers + 1
    -
    -
    def write_user_appwrapper(user_yaml, output_file_name)
    @@ -1021,7 +1020,6 @@

    Index

  • update_priority
  • update_rayclient_route
  • update_resources
    • -
  • update_scheduling_spec
  • write_user_appwrapper