Add option to run compiler tests in parallel (#873)

Author: dcodeIO

package-lock.json

@@ -22,6 +22,7 @@
     "@types/node": "^12.7.5",
     "browser-process-hrtime": "^1.0.0",
     "diff": "^4.0.1",
+    "physical-cpu-count": "^2.0.0",
     "ts-loader": "^6.1.1",
     "ts-node": "^6.2.0",
     "tslint": "^5.20.0",

package.json

@@ -1,3 +1,4 @@
+const startTime = Date.now();
 const fs  = require("fs");
 const path = require("path");
 const os = require("os");
@@ -8,6 +9,8 @@ const optionsUtil = require("../cli/util/options");
 const diff = require("./util/diff");
 const asc = require("../cli/asc.js");
 const rtrace = require("../lib/rtrace");
+const cluster = require("cluster");
+const coreCount = require("physical-cpu-count");
 
 const config = {
   "create": {
@@ -34,13 +37,18 @@ const config = {
       "Enables verbose rtrace output."
     ]
   },
+  "parallel": {
+    "description": [
+      "Runs tests in parallel."
+    ]
+  },
   "help": {
     "description": "Prints this message and exits.",
     "type": "b",
     "alias": "h"
   }
 };
-const opts = optionsUtil.parse(process.argv.slice(2),config);
+const opts = optionsUtil.parse(process.argv.slice(2), config);
 const args = opts.options;
 const argv = opts.arguments;
 
@@ -65,23 +73,23 @@ var skippedMessages = new Map();
 
 const basedir = path.join(__dirname, "compiler");
 
-// Get a list of all tests
-var tests = glob.sync("**/!(_*).ts", { cwd: basedir });
-
-// Run specific tests only if arguments are provided
-if (argv.length) {
-  tests = tests.filter(filename => argv.indexOf(filename.replace(/\.ts$/, "")) >= 0);
-  if (!tests.length) {
-    console.error("No matching tests: " + argv.join(" "));
-    process.exit(1);
+// Gets a list of all relevant tests
+function getTests() {
+  var tests =  glob.sync("**/!(_*).ts", { cwd: basedir }).map(name => name.replace(/\.ts$/, ""));
+  if (argv.length) { // run matching tests only
+    tests = tests.filter(filename => argv.indexOf(filename.replace(/\.ts$/, "")) >= 0);
+    if (!tests.length) {
+      console.error("No matching tests: " + argv.join(" "));
+      process.exit(1);
+    }
   }
+  return tests;
 }
 
-// TODO: asc's callback is synchronous here. This might change.
-tests.forEach(filename => {
-  console.log(colorsUtil.white("Testing compiler/" + filename) + "\n");
+// Runs a single test
+function runTest(basename) {
+  console.log(colorsUtil.white("Testing compiler/" + basename) + "\n");
 
-  const basename = filename.replace(/\.ts$/, "");
   const configPath = path.join(basedir, basename + ".json");
   const config = fs.existsSync(configPath)
     ? require(configPath)
@@ -118,6 +126,7 @@ tests.forEach(filename => {
       console.log("- " + colorsUtil.yellow("feature SKIPPED") + " (" + missing_features.join(", ") + ")\n");
       skippedTests.add(basename);
       skippedMessages.set(basename, "feature not enabled");
+      if (cluster.isWorker) process.send({ cmd: "skipped", message: skippedMessages.get(basename) });
       return;
     }
   }
@@ -129,11 +138,9 @@ tests.forEach(filename => {
 
   var failed = false;
 
-  // TODO: also save stdout/stderr and diff it (-> expected failures)
-
   // Build unoptimized
   var cmd = [
-    filename,
+    basename + ".ts",
     "--baseDir", basedir,
     "--validate",
     "--measure",
@@ -142,10 +149,7 @@ tests.forEach(filename => {
   ];
   if (asc_flags)
     Array.prototype.push.apply(cmd, asc_flags);
-  if (args.createBinary)
-    cmd.push("--binaryFile", basename + ".untouched.wasm");
-  else
-    cmd.push("--binaryFile", "temp.wasm");
+  cmd.push("--binaryFile", basename + ".untouched.wasm");
   asc.main(cmd, {
     stdout: stdout,
     stderr: stderr
@@ -214,7 +218,7 @@ tests.forEach(filename => {
 
     // Build optimized
     var cmd = [
-      filename,
+      basename + ".ts",
       "--baseDir", basedir,
       "--validate",
       "--measure",
@@ -237,7 +241,7 @@ tests.forEach(filename => {
         failedTests.add(basename);
         return 1;
       }
-      let untouchedBuffer = fs.readFileSync(path.join(basedir, "temp.wasm"));
+      let untouchedBuffer = fs.readFileSync(path.join(basedir, basename + ".untouched.wasm"));
       let optimizedBuffer = stdout.toBuffer();
       const gluePath = path.join(basedir, basename + ".js");
       var glue = {};
@@ -258,29 +262,11 @@ tests.forEach(filename => {
     if (failed) return 1;
   });
   if (v8_no_flags) v8.setFlagsFromString(v8_no_flags);
-});
-
-if (skippedTests.size) {
-  console.log(colorsUtil.yellow("WARNING: ") + colorsUtil.white(skippedTests.size + " compiler tests have been skipped:\n"));
-  skippedTests.forEach(name => {
-    var message = skippedMessages.has(name) ? colorsUtil.gray("[" + skippedMessages.get(name) + "]") : "";
-    console.log("  " + name + " " + message);
-  });
-  console.log();
-}
-if (failedTests.size) {
-  process.exitCode = 1;
-  console.log(colorsUtil.red("ERROR: ") + colorsUtil.white(failedTests.size + " compiler tests had failures:\n"));
-  failedTests.forEach(name => {
-    var message = failedMessages.has(name) ? colorsUtil.gray("[" + failedMessages.get(name) + "]") : "";
-    console.log("  " + name + " " + message);
-  });
-  console.log();
-}
-if (!process.exitCode) {
-  console.log("[ " + colorsUtil.white("OK") + " ]");
+  if (!args.createBinary) fs.unlink(path.join(basedir, basename + ".untouched.wasm"), err => {});
+  if (cluster.isWorker) process.send({ cmd: "done", failed: failed, message: failedMessages.get(basename) });
 }
 
+// Tests if instantiation of a module succeeds
 function testInstantiate(basename, binaryBuffer, name, glue) {
   var failed = false;
   try {
@@ -370,3 +356,94 @@ function testInstantiate(basename, binaryBuffer, name, glue) {
   }
   return false;
 }
+
+// Evaluates the overall test result
+function evaluateResult() {
+  if (skippedTests.size) {
+    console.log(colorsUtil.yellow("WARNING: ") + colorsUtil.white(skippedTests.size + " compiler tests have been skipped:\n"));
+    skippedTests.forEach(name => {
+      var message = skippedMessages.has(name) ? colorsUtil.gray("[" + skippedMessages.get(name) + "]") : "";
+      console.log("  " + name + " " + message);
+    });
+    console.log();
+  }
+  if (failedTests.size) {
+    process.exitCode = 1;
+    console.log(colorsUtil.red("ERROR: ") + colorsUtil.white(failedTests.size + " compiler tests had failures:\n"));
+    failedTests.forEach(name => {
+      var message = failedMessages.has(name) ? colorsUtil.gray("[" + failedMessages.get(name) + "]") : "";
+      console.log("  " + name + " " + message);
+    });
+    console.log();
+  }
+  console.log("Time: " + (Date.now() - startTime) + " ms\n");
+  if (!process.exitCode) {
+    console.log("[ " + colorsUtil.white("OK") + " ]");
+  }
+}
+
+// Run tests in parallel if requested
+if (args.parallel && coreCount > 1) {
+  if (cluster.isWorker) {
+    colorsUtil.supported = true;
+    process.on("message", msg => {
+      if (msg.cmd != "run") throw Error("invalid command: " + msg.cmd);
+      try {
+        runTest(msg.test);
+      } catch (e) {
+        process.send({ cmd: "done", failed: true, message: e.message });
+      }
+    });
+    process.send({ cmd: "ready" });
+  } else {
+    const tests = getTests();
+    // const sizes = new Map();
+    // tests.forEach(name => sizes.set(name, fs.statSync(path.join(basedir, name + ".ts")).size));
+    // tests.sort((a, b) => sizes.get(b) - sizes.get(a));
+    const workers = [];
+    const current = [];
+    const outputs = [];
+    let numWorkers = Math.min(coreCount - 1, tests.length);
+    console.log("Spawning " + numWorkers + " workers ...");
+    cluster.settings.silent = true;
+    let index = 0;
+    for (let i = 0; i < numWorkers; ++i) {
+      let worker = cluster.fork();
+      workers[i] = worker;
+      current[i] = null;
+      outputs[i] = [];
+      worker.process.stdout.on("data", buf => outputs[i].push(buf));
+      worker.process.stderr.on("data", buf => outputs[i].push(buf));
+      worker.on("message", msg => {
+        if (msg.cmd == "done") {
+          process.stdout.write(Buffer.concat(outputs[i]).toString());
+          if (msg.failed) failedTests.add(current[i]);
+          if (msg.message) failedMessages.set(current[i], msg.message);
+        } else if (msg.cmd == "skipped") {
+          process.stdout.write(Buffer.concat(outputs[i]).toString());
+          skippedTests.add(current[i]);
+          if (msg.message) skippedMessages.set(current[i], msg.message);
+        } else if (msg.cmd != "ready") {
+          throw Error("invalid command: " + msg.cmd);
+        }
+        if (index >= tests.length) {
+          workers[i] = null;
+          worker.kill();
+          return;
+        }
+        current[i] = tests[index++];
+        outputs[i] = [];
+        worker.send({ cmd: "run", test: current[i] });
+      });
+      worker.on("disconnect", () => {
+        if (workers[i]) throw Error("worker#" + i + " died unexpectedly");
+        if (!--numWorkers) evaluateResult();
+      });
+    }
+  }
+
+// Otherwise run tests sequentially
+} else {
+  getTests().forEach(runTest);
+  evaluateResult();
+}