Merge pull request #870 from kpouget/codeflare

testing: codeflare: sdk_user/sample/sample: retry 'cluster.wait_ready' multiple times

Author: kpouget

testing/codeflare/images/Containerfile.codeflare_sdk

@@ -1,5 +1,5 @@
 # not used by the buildconfig should not be prefix with the image registry location
-FROM ci-artifacts:main 
+FROM ci-artifacts:main
 
-RUN pip install --quiet --no-cache-dir  \
+RUN pip install --no-cache-dir  \
            codeflare_sdk

testing/codeflare/sdk_user/sample/sample.py

@@ -35,7 +35,16 @@ def main():
 
     # Bring up the cluster
     cluster.up()
-    cluster.wait_ready()
+    while True:
+        try:
+            cluster.wait_ready()
+            break
+        except TypeError as e:
+            # there's a timeout in the calling script
+            logging.warning(f"cluster.wait_ready() failed with {e}")
+            import time
+            time.sleep(5)
+
     cluster.status()
     cluster.details()
 
@@ -56,7 +65,9 @@ def main():
         finished = (str(status.state) == "SUCCEEDED")
         failed = (str(status.state) == "FAILED")
 
-    print(job.logs())
+    with open(f"{name}-{job_name}.log", "w") as f:
+        print(job.logs(), file=f)
+
     print(status)
 
     cluster.down()