Convert Go e2e test to Python Test to leverage SDK directly

.github/workflows/e2e_tests.yaml

@@ -93,6 +93,23 @@ jobs:
 
           cd ..
 
+      - name: Add user to KinD
+        uses: ./common/github-actions/kind-add-user
+        with:
+          user-name: sdk-user
+
+      - name: Configure RBAC for sdk user with limited permissions
+        run: |
+          kubectl create clusterrole list-ingresses --verb=get,list --resource=ingresses
+          kubectl create clusterrolebinding sdk-user-list-ingresses --clusterrole=list-ingresses --user=sdk-user
+          kubectl create clusterrole appwrapper-creator --verb=get,list,create,delete,patch --resource=appwrappers
+          kubectl create clusterrolebinding sdk-user-appwrapper-creator --clusterrole=appwrapper-creator --user=sdk-user
+          kubectl create clusterrole namespace-creator --verb=get,list,create,delete,patch --resource=namespaces
+          kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user
+          kubectl create clusterrole list-rayclusters --verb=get,list --resource=rayclusters
+          kubectl create clusterrolebinding sdk-user-list-rayclusters --clusterrole=list-rayclusters --user=sdk-user
+          kubectl config use-context sdk-user
+
       - name: Run e2e tests
         run: |
           export CODEFLARE_TEST_TIMEOUT_SHORT=1m
@@ -103,7 +120,10 @@ jobs:
           echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
 
           set -euo pipefail
-          go test -timeout 30m -v ./tests/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
+          pip install poetry
+          poetry install --with test,docs
+          echo "Running e2e tests..."
+          poetry run pytest -v -s ./tests/e2e/mnist_raycluster_sdk_test.py > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
 
       - name: Print CodeFlare operator logs
         if: always() && steps.deploy.outcome == 'success'

README.md

@@ -51,10 +51,15 @@ To build the codeflare-sdk pre-commit image run `podman build -f .github/build/C
 ### Testing
 
 - To install codeflare-sdk in editable mode, run `pip install -e .` from the repo root.
-- To run the unit tests, run `pytest -v tests/unit_test.py`
 - Any new test functions/scripts can be added into the `tests` folder
 - NOTE: Functional tests coming soon, will live in `tests/func_test.py`
 
+#### Unit Testing
+- To run the unit tests, run `pytest -v tests/unit_test.py`
+
+#### Local e2e Testing
+- Please follow the [e2e documentation](https://https://github.com/project-codeflare/codeflare-sdk/blob/main/docs/e2e.md)
+
 #### Code Coverage
 
 - Run tests with the following command: `coverage run -m --source=src pytest tests/unit_test.py`

docs/e2e.md

@@ -0,0 +1,79 @@
+# Running e2e tests locally
+#### Pre-requisites
+- We recommend using Python 3.9, along with Poetry.
+
+## On KinD clusters
+Pre-requisite for KinD clusters: please add in your local `/etc/hosts` file `127.0.0.1 kind`. This will map your localhost IP address to the KinD cluster's hostname. This is already performed on [GitHub Actions](https://github.com/project-codeflare/codeflare-common/blob/1edd775e2d4088a5a0bfddafb06ff3a773231c08/github-actions/kind/action.yml#L70-L72)
+
+- Setup Phase:
+  - Pull the [codeflare-operator repo](https://github.com/project-codeflare/codeflare-operator) and run the following make targets:
+  ```
+  make kind-e2e
+  export CLUSTER_HOSTNAME=kind
+  export CODEFLARE_TEST_TIMEOUT_LONG=20m
+  make deploy -e IMG=quay.io/project-codeflare/codeflare-operator:v1.1.0
+  make setup-e2e
+  ```
+
+  - **(Optional)** - Create and add `sdk-user` with limited permissions to the cluster to run through the e2e tests:
+  ```
+    # Get KinD certificates
+    docker cp kind-control-plane:/etc/kubernetes/pki/ca.crt .
+    docker cp kind-control-plane:/etc/kubernetes/pki/ca.key .
+
+    # Generate certificates for new user
+    openssl genrsa -out user.key 2048
+    openssl req -new -key user.key -out user.csr -subj '/CN=sdk-user/O=tenant'
+    openssl x509 -req -in user.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out user.crt -days 360
+
+    # Add generated certificated to KinD context
+    user_crt=$(base64 --wrap=0 user.crt)
+    user_key=$(base64 --wrap=0 user.key)
+    yq eval -i ".contexts += {\"context\": {\"cluster\": \"kind-kind\", \"user\": \"sdk-user\"}, \"name\": \"sdk-user\"}" $HOME/.kube/config
+    yq eval -i ".users += {\"name\": \"sdk-user\", \"user\": {\"client-certificate-data\": \"$user_crt\", \"client-key-data\": \"$user_key\"}}" $HOME/.kube/config
+    cat $HOME/.kube/config
+
+    # Cleanup
+    rm ca.crt
+    rm ca.srl
+    rm ca.key
+    rm user.crt
+    rm user.key
+    rm user.csr
+
+    # Add RBAC permissions to sdk-user
+    kubectl create clusterrole list-ingresses --verb=get,list --resource=ingresses
+    kubectl create clusterrolebinding sdk-user-list-ingresses --clusterrole=list-ingresses --user=sdk-user
+    kubectl create clusterrole appwrapper-creator --verb=get,list,create,delete,patch --resource=appwrappers
+    kubectl create clusterrolebinding sdk-user-appwrapper-creator --clusterrole=appwrapper-creator --user=sdk-user
+    kubectl create clusterrole namespace-creator --verb=get,list,create,delete,patch --resource=namespaces
+    kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user
+    kubectl create clusterrole list-rayclusters --verb=get,list --resource=rayclusters
+    kubectl create clusterrolebinding sdk-user-list-rayclusters --clusterrole=list-rayclusters --user=sdk-user
+    kubectl config use-context sdk-user
+
+  ```
+
+
+- Test Phase:
+   - Once we have the codeflare-operator and kuberay-operator running and ready, we can run the e2e test on the codeflare-sdk repository:
+  ```
+  poetry install --with test,docs
+  poetry run pytest -v -s ./tests/e2e/mnist_raycluster_sdk_test.py
+  ```
+
+
+
+## On OpenShift clusters
+- Setup Phase:
+  - Pull the [codeflare-operator repo](https://github.com/project-codeflare/codeflare-operator) and run the following make targets:
+  ```
+  make deploy -e IMG=quay.io/project-codeflare/codeflare-operator:v1.1.0
+  make setup-e2e
+  ```
+- Test Phase:
+   - Once we have the codeflare-operator and kuberay-operator running and ready, we can run the e2e test on the codeflare-sdk repository:
+  ```
+  poetry install --with test,docs
+  poetry run pytest -v -s ./tests/e2e/mnist_raycluster_sdk_test.py
+  ```