Add RayCLuster Oauth Authentication test

Author: Srihari1192

docs/e2e.md

@@ -77,3 +77,7 @@ Pre-requisite for KinD clusters: please add in your local `/etc/hosts` file `127
   poetry install --with test,docs
   poetry run pytest -v -s ./tests/e2e/mnist_raycluster_sdk_test.py
   ```
+  - To run the multiple tests based on the cluster environment, we can run the e2e tests by marking -m with cluster environment (kind or openshift)
+  ```
+  poetry run pytest -v -s ./tests/e2e -m openshift
+  ```

pyproject.toml

@@ -49,3 +49,7 @@ filterwarnings = [
     "ignore::DeprecationWarning:pkg_resources",
     "ignore:pkg_resources is deprecated as an API:DeprecationWarning",
 ]
+markers = [
+    "kind",
+    "openshift"
+]

tests/e2e/mnist_raycluster_sdk_oauth_test.py

@@ -0,0 +1,158 @@
+import requests
+
+from time import sleep
+
+from torchx.specs.api import AppState, is_terminal
+
+from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration
+from codeflare_sdk.job.jobs import DDPJobDefinition
+
+import pytest
+
+from support import *
+
+# This test Creates a Ray cluster with openshift_oauth enable and covers the Ray Job submission with authentication and without authentication functionality
+
+
+@pytest.mark.openshift
+class TestRayClusterSDKOauth:
+    def setup_method(self):
+        initialize_kubernetes_client(self)
+
+    def teardown_method(self):
+        delete_namespace(self)
+
+    def test_mnist_ray_cluster_sdk_auth(self):
+        self.setup_method()
+        create_namespace(self)
+        self.run_mnist_raycluster_sdk_oauth()
+
+    def run_mnist_raycluster_sdk_oauth(self):
+        ray_image = get_ray_image()
+
+        cluster = Cluster(
+            ClusterConfiguration(
+                name="mnist",
+                namespace=self.namespace,
+                num_workers=1,
+                head_cpus="500m",
+                head_memory=2,
+                min_cpus="500m",
+                max_cpus=1,
+                min_memory=1,
+                max_memory=2,
+                num_gpus=0,
+                instascale=False,
+                image=ray_image,
+                openshift_oauth=True,
+            )
+        )
+
+        cluster.up()
+        self.assert_appwrapper_exists()
+
+        cluster.status()
+
+        cluster.wait_ready()
+        self.assert_raycluster_exists()
+
+        cluster.status()
+
+        cluster.details()
+
+        self.assert_jobsubmit_withoutLogin(cluster)
+
+        self.assert_jobsubmit_withlogin(cluster)
+
+    # Assertions
+
+    def assert_jobsubmit_withoutLogin(self, cluster):
+        dashboard_url = cluster.cluster_dashboard_uri()
+        jobdata = {
+            "entrypoint": "python mnist.py",
+            "runtime_env": {
+                "working_dir": "./tests/e2e/",
+                "pip": "mnist_pip_requirements.txt",
+            },
+        }
+        try:
+            response = requests.post(
+                dashboard_url + "/api/jobs/", verify=False, json=jobdata
+            )
+            if response.status_code == 403:
+                assert True
+            else:
+                response.raise_for_status()
+                assert False
+
+        except Exception as e:
+            print(f"An unexpected error occurred. Error: {e}")
+            assert False
+
+    def assert_jobsubmit_withlogin(self, cluster):
+        self.assert_appwrapper_exists()
+        jobdef = DDPJobDefinition(
+            name="mnist",
+            script="./tests/e2e/mnist.py",
+            scheduler_args={"requirements": "./tests/e2e/mnist_pip_requirements.txt"},
+        )
+        job = jobdef.submit(cluster)
+
+        done = False
+        time = 0
+        timeout = 900
+        while not done:
+            status = job.status()
+            if is_terminal(status.state):
+                break
+            if not done:
+                print(status)
+                if timeout and time >= timeout:
+                    raise TimeoutError(f"job has timed out after waiting {timeout}s")
+                sleep(5)
+                time += 5
+
+        print(job.status())
+        self.assert_job_completion(status)
+
+        print(job.logs())
+
+        cluster.down()
+
+    def assert_appwrapper_exists(self):
+        try:
+            self.custom_api.get_namespaced_custom_object(
+                "workload.codeflare.dev",
+                "v1beta1",
+                self.namespace,
+                "appwrappers",
+                "mnist",
+            )
+            print(
+                f"AppWrapper 'mnist' has been created in the namespace: '{self.namespace}'"
+            )
+            assert True
+        except Exception as e:
+            print(f"AppWrapper 'mnist' has not been created. Error: {e}")
+            assert False
+
+    def assert_raycluster_exists(self):
+        try:
+            self.custom_api.get_namespaced_custom_object(
+                "ray.io", "v1", self.namespace, "rayclusters", "mnist"
+            )
+            print(
+                f"RayCluster 'mnist' created successfully in the namespace: '{self.namespace}'"
+            )
+            assert True
+        except Exception as e:
+            print(f"RayCluster 'mnist' has not been created. Error: {e}")
+            assert False
+
+    def assert_job_completion(self, status):
+        if status.state == AppState.SUCCEEDED:
+            print(f"Job has completed: '{status.state}'")
+            assert True
+        else:
+            print(f"Job has completed: '{status.state}'")
+            assert False

tests/e2e/mnist_raycluster_sdk_test.py

@@ -14,30 +14,24 @@
 
 import pytest
 
-from support import random_choice, get_ray_image
+from support import *
 
 # Creates a Ray cluster, and trains the MNIST dataset using the CodeFlare SDK.
 # Asserts creation of AppWrapper, RayCluster, and successful completion of the training job.
 # Covers successfull installation of CodeFlare-SDK
 
 
+@pytest.mark.kind
+@pytest.mark.openshift
 class TestMNISTRayClusterSDK:
     def setup_method(self):
-        # Load the kube config from the environment or Kube config file.
-        config.load_kube_config()
-
-        # Initialize Kubernetes client
-        self.api_instance = client.CoreV1Api()
-        self.custom_api = kubernetes.client.CustomObjectsApi(
-            self.api_instance.api_client
-        )
+        initialize_kubernetes_client(self)
 
     def teardown_method(self):
-        if hasattr(self, "namespace"):
-            self.api_instance.delete_namespace(self.namespace)
+        delete_namespace(self)
 
     def test_mnist_ray_cluster_sdk(self):
-        self.create_test_namespace()
+        create_namespace(self)
         self.run_mnist_raycluster_sdk()
 
     def create_test_namespace(self):

tests/e2e/support.py

@@ -1,6 +1,8 @@
 import os
 import random
 import string
+from kubernetes import client, config
+import kubernetes.client
 
 
 def get_ray_image():
@@ -11,3 +13,23 @@ def get_ray_image():
 def random_choice():
     alphabet = string.ascii_lowercase + string.digits
     return "".join(random.choices(alphabet, k=5))
+
+
+def create_namespace(self):
+    self.namespace = f"test-ns-{random_choice()}"
+    namespace_body = client.V1Namespace(
+        metadata=client.V1ObjectMeta(name=self.namespace)
+    )
+    self.api_instance.create_namespace(namespace_body)
+
+
+def delete_namespace(self):
+    if hasattr(self, "namespace"):
+        self.api_instance.delete_namespace(self.namespace)
+
+
+def initialize_kubernetes_client(self):
+    config.load_kube_config()
+    # Initialize Kubernetes client
+    self.api_instance = client.CoreV1Api()
+    self.custom_api = kubernetes.client.CustomObjectsApi(self.api_instance.api_client)