Add deepforge create-env command for building conda env. Close #1466

bin/deepforge

@@ -1,4 +1,6 @@
 #!/usr/bin/env node
+const childProcess = require('child_process');
+const Conda = require('../utils/conda-utils');
 const os = require('os'),
     IS_WINDOWS = os.type() === 'WINDOWS_NT',
     SHELL = IS_WINDOWS ? true : '/bin/bash',
@@ -8,7 +10,6 @@ const os = require('os'),
 var Command = require('commander').Command,
     tcpPortUsed = require('tcp-port-used'),
     program = new Command(),
-    childProcess = require('child_process'),
     rawSpawn = childProcess.spawn,
     Q = require('q'),
     execSync = childProcess.execSync,
@@ -395,6 +396,63 @@ program
 program
     .command('extensions <command>', 'Manage deepforge extensions');
 
+program
+    .command('create-env')
+    .description('Create conda environment(s) with DeepForge python dependencies')
+    .option('-n, --name <name>', 'Name of environment to create')
+    .option('-s, --server', 'Create environment with server dependencies')
+    .option('-w, --worker', 'Create environment with worker dependencies')
+    .option('-f, --force', 'Overwrite any existing environments')
+    .action(async cmd => {
+        const createBoth = !cmd.server && !cmd.worker;
+        if (createBoth) {
+            cmd.server = cmd.worker = true;
+        }
+
+        const extender = require('../utils/extender');
+        const extensionData = extender.getExtensionsConfig();
+        const libraries = Object.values(extensionData.Library);
+        const dirs = libraries.map(lib => lib.project.root);
+        const name = typeof cmd.name === 'string' ? cmd.name : 'deepforge';
+
+        try {
+            if (cmd.server) {
+                const serverEnvName = createBoth ? `${name}-server` : name;
+                await createEnvFromDirs(serverEnvName, dirs, 'server', cmd.force);
+            }
+            if (cmd.worker) {
+                await createEnvFromDirs(name, dirs, 'worker', cmd.force);
+            }
+        } catch (errOrExitCode) {
+            const msg = '\n\nUnable to create environment.';
+            if (errOrExitCode instanceof Error) {
+                console.log(`${msg} An error occurred: ${errOrExitCode}`);
+            } else {
+                console.log(`${msg} Conda exited with exit code: ${errOrExitCode}`);
+            }
+        }
+    });
+
+async function createEnvFromDirs(name, dirs, type, force=false) {
+    const envFiles = getCondaEnvFiles(dirs, type);
+
+    const baseEnvFile = `environment.${type}.yml`;
+    const flags = `--name ${name} --file ${baseEnvFile}${force ? ' --force' : ''}`;
+    await Conda.spawn(`env create ${flags}`);
+    for (let i = 0; i < envFiles.length; i++) {
+        const envFile = envFiles[i];
+        await Conda.spawn(`env update -n ${name} --file ${envFile}`);
+    }
+}
+
+function getCondaEnvFiles(dirs, type) {
+    const validEnvFilenames = ['environment.yml', `environment.${type}.yml`];
+    const envFiles = dirs
+        .flatMap(dirname => validEnvFilenames.map(file => path.join(dirname, file)))
+        .filter(filepath => exists.sync(filepath));
+
+    return envFiles;
+}
 
 // user-management
 program.command('users', 'Manage deepforge users.');

package.json

@@ -9,7 +9,7 @@
   },
   "scripts": {
     "start": "node ./bin/deepforge start",
-    "postinstall": "node utils/reinstall-extensions.js && node utils/build-job-utils.js",
+    "postinstall": "node utils/reinstall-extensions.js && node utils/build-job-utils.js && ./bin/deepforge create-env",
     "start-dev": "NODE_ENV=dev node ./bin/deepforge start",
     "start-authenticated": "NODE_ENV=production ./bin/deepforge start",
     "server": "node ./bin/deepforge start --server",

test/unit/external-utils/conda-utils.spec.js

@@ -1,24 +1,29 @@
-describe('CondaUtils', function () {
-    const condaUtils = require('../../../utils/conda-utils'),
+describe.only('CondaUtils', function () {
+    const conda = require('../../../utils/conda-utils'),
         expect = require('chai').expect,
         path = require('path'),
-        ENV_FILE = path.join(__dirname, '..', '..', '..', 'environment.yml');
+        ENV_FILE = path.join(__dirname, '..', '..', '..', 'environment.server.yml');
 
     it('should find executable conda', () => {
-        expect(condaUtils.checkConda).to.not.throw();
+        expect(conda.check).to.not.throw();
     });
 
-    it('should throw an error when creating from a missing environment file', () => {
-        const badCreateFunc = () => {
-            condaUtils.createOrUpdateEnvironment('dummyfile');
-        };
-        expect(badCreateFunc).to.throw();
+    it('should throw an error when creating from a missing environment file', async () => {
+        const badCreateFunc = () => conda.createOrUpdateEnvironment('dummyfile');
+        await shouldThrow(badCreateFunc);
     });
 
-    it('should not throw an error from a proper environment file', () => {
-        const createFunc = () => {
-            condaUtils.createOrUpdateEnvironment(ENV_FILE);
-        };
-        expect(createFunc).to.not.throw();
+    it('should not throw an error from a proper environment file', async function() {
+        this.timeout(5000);
+        await conda.createOrUpdateEnvironment(ENV_FILE);
     });
-});
+
+    async function shouldThrow(fn) {
+        try {
+            await fn();
+        } catch (err) {
+            return err;
+        }
+        throw new Error('Function did not throw an exception.');
+    }
+});

utils/conda-utils.js

@@ -1,6 +1,8 @@
 /*eslint-env node*/
 /*eslint-disable no-console*/
 'use strict';
+const Conda = {};
+
 const {spawnSync, spawn} = require('child_process'),
     os = require('os'),
     path = require('path'),
@@ -33,39 +35,41 @@ const dumpYAML = function (environment, envFileName) {
     return envFileName;
 };
 
-const checkConda = function () {
+Conda.check = function () {
     const conda = spawnSyncCondaProcess(['-V']);
     if (conda.status !== 0) {
         throw new Error(`Please install conda before continuing. ${conda.stderr.toString()}`);
     }
 };
 
 
-const createOrUpdateEnvironment = function (envFile, envName) {
+Conda.createOrUpdateEnvironment = async function (envFile, envName) {
     const env = yaml.safeLoad(fs.readFileSync(envFile, 'utf8'));
     if (envName && envName !== env.name) {
         env.name = envName;
         envFile = dumpYAML(env, envFile);
     }
     const createOrUpdate = envExists(env.name) ? 'update' : 'create';
     console.log(`Environment ${env.name} will be ${createOrUpdate}d.`);
-    spawnCondaProcess(['env', createOrUpdate, '--file', envFile],
-        `Successfully ${createOrUpdate}d the environment ${env.name}`);
-
+    await Conda.spawn(`env ${createOrUpdate} --file ${envFile}`);
+    console.log(`Successfully ${createOrUpdate}d the environment ${env.name}`);
 };
 
-const spawnCondaProcess = function (args, onCompleteMessage, onErrorMessage) {
-    const condaProcess = spawn(CONDA_COMMAND, args, {
+Conda.spawn = function (command) {
+    const condaProcess = spawn(CONDA_COMMAND, command.split(' '), {
         shell: SHELL
     });
 
     condaProcess.stdout.pipe(process.stdout);
     condaProcess.stderr.pipe(process.stderr);
-    condaProcess.on('exit', (code) => {
-        if(code !== 0){
-            throw new Error(onErrorMessage || 'Spawned conda process failed.');
-        }
-        console.log(onCompleteMessage || 'Spawned conda process executed successfully');
+
+    return new Promise((resolve, reject) => {
+        condaProcess.on('exit', (code) => {
+            if(code !== 0){
+                return reject(code);
+            }
+            resolve();
+        });
     });
 };
 
@@ -75,6 +79,4 @@ const spawnSyncCondaProcess = function (args) {
     });
 };
 
-const CondaManager = {checkConda, createOrUpdateEnvironment};
-
-module.exports = CondaManager;
+module.exports = Conda;

utils/extender.js

@@ -153,7 +153,8 @@ let updateTemplateFile = (tplPath, type) => {
     fs.writeFileSync(dstPath, formatsIndex);
 };
 
-var makeInstallFor = function(typeCfg) {
+function makeInstallFor(typeCfg) {
+    allExtConfigs[typeCfg.type] = allExtConfigs[typeCfg.type] || {};
     var saveExtensions = () => {
         // regenerate the format.js file from the template
         var installedExts = values(allExtConfigs[typeCfg.type]),
@@ -209,7 +210,7 @@ var makeInstallFor = function(typeCfg) {
         // Re-generate template file
         saveExtensions();
     };
-};
+}
 
 //var PLUGIN_ROOT = path.join(__dirname, '..', 'src', 'plugins', 'Export');
 //makeInstallFor({