wip

Author: jakirkham

nanshe_ipython.ipynb

@@ -38,7 +38,11 @@
         "dataset = \"images\"\n",
         "\n",
         "num_workers = None\n",
-        "cluster_kwargs = {}\n",
+        "cluster_kwargs = {\n",
+        "    \"preexec_commands\": (\n",
+        "        \"singularity exec ~/nanshe_nanshe_workflow_latest.img \\\\\",\n",
+        "    ),\n",
+        "}\n",
         "client_kwargs = {}\n",
         "\n",
         "\n",
@@ -80,39 +84,7 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "import time\n",
-        "import distributed\n",
-        "import dask_drmaa\n",
-        "from nanshe_workflow.par import set_num_workers\n",
-        "\n",
-        "\n",
-        "def startup_distributed(nworkers):\n",
-        "    nworkers = int(nworkers)\n",
-        "\n",
-        "    if dask_drmaa:\n",
-        "        cluster = dask_drmaa.DRMAACluster(\n",
-        "            preexec_commands=(\n",
-        "                \"singularity exec ~/nanshe_nanshe_workflow_latest.img \\\\\",\n",
-        "            ),\n",
-        "            template={\"jobEnvironment\": os.environ}\n",
-        "        )\n",
-        "        cluster.start_workers(nworkers)\n",
-        "    else:\n",
-        "        # Either `dask_drmaa` is unavailable or DRMAA cannot start.\n",
-        "        # Fallback to a local Distributed client instead.\n",
-        "        cluster = distributed.LocalCluster(\n",
-        "            n_workers=nworkers, threads_per_worker=1\n",
-        "        )\n",
-        "\n",
-        "    client = distributed.Client(cluster)\n",
-        "    while (\n",
-        "              (client.status == \"running\") and\n",
-        "              (len(client.scheduler_info()[\"workers\"]) < nworkers)\n",
-        "          ):\n",
-        "        time.sleep(1)\n",
-        "\n",
-        "    return client\n",
-        "\n",
+        "from nanshe_workflow.par import set_num_workers, startup_distributed\n",
         "\n",
         "num_workers = set_num_workers(num_workers)\n",
         "\n",