Defer aggregating results until after the instance returns to allow multi-instance load tests (#488)

* Expose aggregateResult

* Add skipAggregateResult flag. Init histograms in aggregateResult if they are not passed in. Docs

* Thin layer over aggregateResult for input validation

* More manageable docs

* Tests. Default opts

Author: milesnash

README.md

@@ -330,6 +330,7 @@ Start autocannon against the given target.
     * `excludeErrorStats`: A `Boolean` which allows you to disable tracking non-2xx code responses in latency and bytes per second calculations. _OPTIONAL_ default: `false`.
     * `expectBody`: A `String` representing the expected response body. Each request whose response body is not equal to `expectBody`is counted in `mismatches`. If enabled, mismatches count towards bailout. _OPTIONAL_
     * `tlsOptions`: An `Object` that is passed into `tls.connect` call ([Full list of options](https://nodejs.org/api/tls.html#tls_tls_connect_port_host_options_callback)). Note: this only applies if your URL is secure.
+    * `skipAggregateResult`: A `Boolean` which allows you to disable the aggregate result phase of an instance run. See [autocannon.aggregateResult](<#autocannon.aggregateResult(results[, opts])>)
 * `cb`: The callback which is called on completion of a benchmark. Takes the following params. _OPTIONAL_.
     * `err`: If there was an error encountered with the run.
     * `results`: The results of the run.
@@ -397,6 +398,22 @@ Print the result tables to the terminal, programmatically.
     * `renderResultsTable`: A truthy value to enable the rendering of the results table. default: `true`.
     * `renderLatencyTable`: A truthy value to enable the rendering of the latency table. default: `false`.
 
+### autocannon.aggregateResult(results[, opts])
+
+Aggregate the results of one or more autocannon instance runs, where the instances of autocannon have been run with the `skipAggregateResult` option.
+
+This is an advanced use case, where you might be running a load test using autocannon across multiple machines and therefore need to defer aggregating the results to a later time.
+
+* `results`: An array of autocannon instance results, where the instances have been run with the `skipAggregateResult` option set to true. _REQUIRED_.
+* `opts`: This is a subset of the options you would pass to the main autocannon API, so you could use the same options object as the one used to run the instances. See [autocannon](<#autocannon(opts[, cb])>) for full descriptions of the options. _REQUIRED_.
+    * `url`: _REQUIRED_
+    * `title`: _OPTIONAL_ default: `undefined`
+    * `socketPath`: _OPTIONAL_
+    * `connections`: _OPTIONAL_ default: `10`.
+    * `sampleInt`: _OPTIONAL_ default: `1`
+    * `pipelining`: _OPTIONAL_ default: `1`
+    * `workers`: _OPTIONAL_ default: `undefined`
+
 ### Autocannon events
 
 Because an autocannon instance is an `EventEmitter`, it emits several events. these are below:

autocannon.js

@@ -18,6 +18,8 @@ const track = require('./lib/progressTracker')
 const generateSubArgAliases = require('./lib/subargAliases')
 const { checkURL, ofURL } = require('./lib/url')
 const { parseHAR } = require('./lib/parseHAR')
+const _aggregateResult = require('./lib/aggregateResult')
+const validateOpts = require('./lib/validate')
 
 if (typeof URL !== 'function') {
   console.error('autocannon requires the WHATWG URL API, but it is not available. Please upgrade to Node 6.13+.')
@@ -30,6 +32,19 @@ module.exports.track = track
 module.exports.start = start
 module.exports.printResult = printResult
 module.exports.parseArguments = parseArguments
+module.exports.aggregateResult = function aggregateResult (results, opts = {}) {
+  if (!Array.isArray(results)) {
+    throw new Error('"results" must be an array of results')
+  }
+
+  opts = validateOpts(opts, false)
+
+  if (opts instanceof Error) {
+    throw opts
+  }
+
+  return _aggregateResult(results, opts)
+}
 const alias = {
   connections: 'c',
   pipelining: 'p',

lib/aggregateResult.js

@@ -1,9 +1,10 @@
 'use strict'
 
-const { decodeHist, histAsObj, addPercentiles } = require('./histUtil')
+const { decodeHist, getHistograms, histAsObj, addPercentiles } = require('./histUtil')
 
 function aggregateResult (results, opts, histograms) {
   results = Array.isArray(results) ? results : [results]
+  histograms = getHistograms(histograms)
 
   const aggregated = results.map(r => ({
     ...r,

lib/run.js

@@ -137,7 +137,7 @@ function run (opts, tracker, cb) {
 
       statusCodes.forEach((code, index) => { result[(index + 1) + 'xx'] = code })
 
-      const resultObj = isMainThread ? aggregateResult(result, opts, histograms) : result
+      const resultObj = isMainThread && !opts.skipAggregateResult ? aggregateResult(result, opts, histograms) : result
 
       if (opts.forever) {
         // we don't call callback when in forever mode, so this is the

test/aggregateResult.test.js

@@ -0,0 +1,33 @@
+const { test } = require('tap')
+const { startServer } = require('./helper')
+const autocannon = require('../autocannon')
+const aggregateResult = autocannon.aggregateResult
+const server = startServer()
+const url = 'http://localhost:' + server.address().port
+
+test('exec separate autocannon instances with skipAggregateResult, then aggregateResult afterwards', async (t) => {
+  t.plan(2)
+
+  const opts = {
+    url,
+    connections: 1,
+    maxOverallRequests: 10,
+    skipAggregateResult: true
+  }
+
+  const results = await Promise.all([
+    autocannon(opts),
+    autocannon(opts)
+  ])
+
+  const aggregateResults = aggregateResult(results, opts)
+
+  t.equal(aggregateResults['2xx'], 20)
+  t.equal(aggregateResults.requests.total, 20)
+})
+
+test('aggregateResult must be passed opts with at least a URL or socketPath property', async (t) => {
+  t.plan(2)
+  t.throws(() => aggregateResult([]), 'url or socketPath option required')
+  t.throws(() => aggregateResult([], {}), 'url or socketPath option required')
+})