Add a way to easily test units

pyspark-notebook/test/test_spark.py

@@ -16,15 +16,3 @@ def test_spark_shell(container):
     logs = c.logs(stdout=True).decode('utf-8')
     LOGGER.debug(logs)
     assert 'res0: Int = 2' in logs, "spark-shell does not work"
-
-
-def test_pyspark(container):
-    """PySpark should be in the Python path"""
-    c = container.run(
-        tty=True,
-        command=['start.sh', 'python', '-c', 'import pyspark']
-    )
-    rv = c.wait(timeout=30)
-    logs = c.logs(stdout=True).decode('utf-8')
-    LOGGER.debug(logs)
-    assert rv == 0 or rv["StatusCode"] == 0, "pyspark not in PYTHONPATH"

pyspark-notebook/test/units/unit_spark.py

@@ -0,0 +1,4 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import pyspark  # noqa: F401

scipy-notebook/test/units/unit_pandas.py

@@ -0,0 +1,9 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import numpy as np
+import pandas as pd
+
+
+np.random.seed(0)
+print(pd.Series(np.random.randint(0, 7, size=10)).sum())

tensorflow-notebook/test/units/unit_tensorflow.py

@@ -0,0 +1,7 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import tensorflow as tf
+
+
+print(tf.constant('Hello, TensorFlow'))
+print(tf.reduce_sum(tf.random.normal([1000, 1000])))

test/README.md

@@ -0,0 +1,15 @@
+# Docker stacks testing
+
+We test our images using `pytest` module.
+
+`conftest.py` and `pytest.ini` in the root of our repository define the environment in which tests are run.
+More info on pytest can be found [here](https://docs.pytest.org/en/latest/contents.html).
+
+There are two kinds of tests we use:
+
+- General tests - these are located in [this](https://github.com/jupyter/docker-stacks/blob/master/test) folder
+- Image specific tests - for example, [base-notebook/test](https://github.com/jupyter/docker-stacks/blob/master/base-notebook/test) folder
+
+We also have a way to easily run arbitrary python files in a container.
+This is useful for running unit tests of packages we use, so we put these files in `{image}/test/units` folder.
+An example of such a test is [unit_pandas.py](https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/test/units/unit_pandas.py).

test/test_units.py

@@ -0,0 +1,35 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import logging
+import os
+
+LOGGER = logging.getLogger(__name__)
+THIS_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+def test_units(container):
+    """Various units tests
+    Add a py file in the {image}/test/units dir and it will be automatically tested
+    """
+    short_image_name = container.image_name[container.image_name.rfind('/') + 1:]
+    host_data_dir = os.path.join(THIS_DIR, f"../{short_image_name}/test/units")
+    LOGGER.info(f"Searching for units tests in {host_data_dir}")
+    cont_data_dir = "/home/jovyan/data"
+
+    if not os.path.exists(host_data_dir):
+        LOGGER.info(f"Not found unit tests for image: {container.image_name}")
+        return
+
+    for test_file in os.listdir(host_data_dir):
+        LOGGER.info(f"Running unit test: {test_file}")
+
+        c = container.run(
+            volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
+            tty=True,
+            command=['start.sh', 'python', f'{cont_data_dir}/{test_file}']
+        )
+        rv = c.wait(timeout=30)
+        logs = c.logs(stdout=True).decode('utf-8')
+        LOGGER.debug(logs)
+        assert rv == 0 or rv["StatusCode"] == 0