5 files changed, 358 insertions, 232 deletions

diff --git a/benchmark/_cli.js b/benchmark/_cli.js
new file mode 100644
index 0000000000..be2f7ffff8
--- /dev/null
+++ b/benchmark/_cli.js
@@ -0,0 +1,99 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+// Create an object of all benchmark scripts
+const benchmarks = {};
+fs.readdirSync(__dirname)
+  .filter(function(name) {
+    return fs.statSync(path.resolve(__dirname, name)).isDirectory();
+  })
+  .forEach(function(category) {
+    benchmarks[category] = fs.readdirSync(path.resolve(__dirname, category))
+      .filter((filename) => filename[0] !== '.' && filename[0] !== '_');
+  });
+
+function CLI(usage, settings) {
+  if (!(this instanceof CLI)) return new CLI(usage, settings);
+
+  if (process.argv.length < 3) {
+    this.abort(usage); // abort will exit the process
+  }
+
+  this.usage = usage;
+  this.optional = {};
+  this.items = [];
+
+  for (const argName of settings.arrayArgs) {
+    this.optional[argName] = [];
+  }
+
+  let currentOptional = null;
+  let mode = 'both'; // possible states are: [both, option, item]
+
+  for (const arg of process.argv.slice(2)) {
+    if (arg === '--') {
+      // Only items can follow --
+      mode = 'item';
+    } else if (['both', 'option'].includes(mode) && arg[0] === '-') {
+      // Optional arguments declaration
+
+      if (arg[1] === '-') {
+        currentOptional = arg.slice(2);
+      } else {
+        currentOptional = arg.slice(1);
+      }
+
+      // Default the value to true
+      if (!settings.arrayArgs.includes(currentOptional)) {
+        this.optional[currentOptional] = true;
+      }
+
+      // expect the next value to be option related (either -- or the value)
+      mode = 'option';
+    } else if (mode === 'option') {
+      // Optional arguments value
+
+      if (settings.arrayArgs.includes(currentOptional)) {
+        this.optional[currentOptional].push(arg);
+      } else {
+        this.optional[currentOptional] = arg;
+      }
+
+      // the next value can be either an option or an item
+      mode = 'both';
+    } else if (['both', 'item'].includes(mode)) {
+      // item arguments
+      this.items.push(arg);
+
+      // the next value must be an item
+      mode = 'item';
+    } else {
+      // Bad case, abort
+      this.abort(usage);
+      return;
+    }
+  }
+}
+module.exports = CLI;
+
+CLI.prototype.abort = function(msg) {
+  console.error(msg);
+  process.exit(1);
+};
+
+CLI.prototype.benchmarks = function() {
+  const paths = [];
+  const filter = this.optional.filter || false;
+
+  for (const category of this.items) {
+    for (const scripts of benchmarks[category]) {
+      if (filter && scripts.lastIndexOf(filter) === -1) continue;
+
+      paths.push(path.join(category, scripts));
+    }
+  }
+
+  return paths;
+};
diff --git a/benchmark/common.js b/benchmark/common.js
index 3744a4420a..669a4c642b 100644
--- a/benchmark/common.js
+++ b/benchmark/common.js
@@ -1,201 +1,227 @@
 'use strict';
-var assert = require('assert');
-var fs = require('fs');
-var path = require('path');
-var child_process = require('child_process');
-
-var outputFormat = process.env.OUTPUT_FORMAT ||
-                   (+process.env.NODE_BENCH_SILENT ? 'silent' : false) ||
-                   'default';
-
-// verify outputFormat
-if (['default', 'csv', 'silent'].indexOf(outputFormat) == -1) {
-  throw new Error('OUTPUT_FORMAT set to invalid value');
-}
 
-exports.PORT = process.env.PORT || 12346;
+const child_process = require('child_process');
 
-function hasWrk() {
-  var result = child_process.spawnSync('wrk', ['-h']);
-  if (result.error && result.error.code === 'ENOENT') {
-    console.error('Couldn\'t locate `wrk` which is needed for running ' +
-      'benchmarks. Check benchmark/README.md for further instructions.');
-    process.exit(-1);
-  }
-}
+// The port used by servers and wrk
+exports.PORT = process.env.PORT || 12346;
 
 exports.createBenchmark = function(fn, options) {
   return new Benchmark(fn, options);
 };
 
 function Benchmark(fn, options) {
-  this.fn = fn;
-  this.options = options;
-  this.config = parseOpts(options);
-  this._name = require.main.filename.split(/benchmark[\/\\]/).pop();
-  this._start = [0, 0];
+  this.name = require.main.filename.slice(__dirname.length + 1);
+  this.options = this._parseArgs(process.argv.slice(2), options);
+  this.queue = this._queue(this.options);
+  this.config = this.queue[0];
+
+  this._time = [0, 0]; // holds process.hrtime value
   this._started = false;
 
-  var self = this;
+  // this._run will use fork() to create a new process for each configuration
+  // combination.
+  if (process.env.hasOwnProperty('NODE_RUN_BENCHMARK_FN')) {
+    process.nextTick(() => fn(this.config));
+  } else {
+    process.nextTick(() => this._run());
+  }
+}
 
-  process.nextTick(function() {
-    self._run();
-  });
+Benchmark.prototype._parseArgs = function(argv, options) {
+  const cliOptions = Object.assign({}, options);
+
+  // Parse configuarion arguments
+  for (const arg of argv) {
+    const match = arg.match(/^(.+?)=([\s\S]*)$/);
+    if (!match || !match[1]) {
+      console.error('bad argument: ' + arg);
+      process.exit(1);
+    }
+
+    // Infer the type from the options object and parse accordingly
+    const isNumber = typeof options[match[1]][0] === 'number';
+    const value = isNumber ? +match[2] : match[2];
+
+    cliOptions[match[1]] = [value];
+  }
+
+  return cliOptions;
+};
+
+Benchmark.prototype._queue = function(options) {
+  const queue = [];
+  const keys = Object.keys(options);
+
+  // Perform a depth-first walk though all options to genereate a
+  // configuration list that contains all combinations.
+  function recursive(keyIndex, prevConfig) {
+    const key = keys[keyIndex];
+    const values = options[key];
+    const type = typeof values[0];
+
+    for (const value of values) {
+      if (typeof value !== 'number' && typeof value !== 'string') {
+        throw new TypeError(`configuration "${key}" had type ${typeof value}`);
+      }
+      if (typeof value !== type) {
+        // This is a requirement for being able to consistently and predictably
+        // parse CLI provided configuration values.
+        throw new TypeError(`configuration "${key}" has mixed types`);
+      }
+
+      const currConfig = Object.assign({ [key]: value }, prevConfig);
+
+      if (keyIndex + 1 < keys.length) {
+        recursive(keyIndex + 1, currConfig);
+      } else {
+        queue.push(currConfig);
+      }
+    }
+  }
+
+  if (keys.length > 0) {
+    recursive(0, {});
+  } else {
+    queue.push({});
+  }
+
+  return queue;
+};
+
+function hasWrk() {
+  const result = child_process.spawnSync('wrk', ['-h']);
+  if (result.error && result.error.code === 'ENOENT') {
+    console.error('Couldn\'t locate `wrk` which is needed for running ' +
+      'benchmarks. Check benchmark/README.md for further instructions.');
+    process.exit(1);
+  }
 }
 
 // benchmark an http server.
-Benchmark.prototype.http = function(p, args, cb) {
+const WRK_REGEXP = /Requests\/sec:[ \t]+([0-9\.]+)/;
+Benchmark.prototype.http = function(urlPath, args, cb) {
   hasWrk();
-  var self = this;
-  var regexp = /Requests\/sec:[ \t]+([0-9\.]+)/;
-  var url = 'http://127.0.0.1:' + exports.PORT + p;
-
-  args = args.concat(url);
+  const self = this;
 
-  var out = '';
-  var child = child_process.spawn('wrk', args);
+  const urlFull = 'http://127.0.0.1:' + exports.PORT + urlPath;
+  args = args.concat(urlFull);
 
-  child.stdout.setEncoding('utf8');
+  const childStart = process.hrtime();
+  const child = child_process.spawn('wrk', args);
+  child.stderr.pipe(process.stderr);
 
-  child.stdout.on('data', function(chunk) {
-    out += chunk;
-  });
+  // Collect stdout
+  let stdout = '';
+  child.stdout.on('data', (chunk) => stdout += chunk.toString());
 
-  child.on('close', function(code) {
-    if (cb)
-      cb(code);
+  child.once('close', function(code) {
+    const elapsed = process.hrtime(childStart);
+    if (cb) cb(code);
 
     if (code) {
       console.error('wrk failed with ' + code);
       process.exit(code);
     }
-    var match = out.match(regexp);
-    var qps = match && +match[1];
-    if (!qps) {
-      console.error('%j', out);
-      console.error('wrk produced strange output');
+
+    // Extract requests pr second and check for odd results
+    const match = stdout.match(WRK_REGEXP);
+    if (!match || match.length <= 1) {
+      console.error('wrk produced strange output:');
+      console.error(stdout);
       process.exit(1);
     }
-    self.report(+qps);
+
+    // Report rate
+    self.report(+match[1], elapsed);
   });
 };
 
 Benchmark.prototype._run = function() {
-  if (this.config)
-    return this.fn(this.config);
-
-  // some options weren't set.
-  // run with all combinations
-  var main = require.main.filename;
-  var options = this.options;
-
-  var queue = Object.keys(options).reduce(function(set, key) {
-    var vals = options[key];
-    assert(Array.isArray(vals));
-
-    // match each item in the set with each item in the list
-    var newSet = new Array(set.length * vals.length);
-    var j = 0;
-    set.forEach(function(s) {
-      vals.forEach(function(val) {
-        if (typeof val !== 'number' && typeof val !== 'string') {
-          throw new TypeError(`configuration "${key}" had type ${typeof val}`);
-        }
-
-        newSet[j++] = s.concat(key + '=' + val);
-      });
-    });
-    return newSet;
-  }, [[main]]);
-
-  // output csv heading
-  if (outputFormat == 'csv')
-    console.log('filename,' + Object.keys(options).join(',') + ',result');
-
-  var node = process.execPath;
-  var i = 0;
-  function run() {
-    var argv = queue[i++];
-    if (!argv)
-      return;
-    argv = process.execArgv.concat(argv);
-    var child = child_process.spawn(node, argv, { stdio: 'inherit' });
-    child.on('close', function(code, signal) {
-      if (code)
-        console.error('child process exited with code ' + code);
-      else
-        run();
-    });
-  }
-  run();
-};
+  const self = this;
+
+  (function recursive(queueIndex) {
+    const config = self.queue[queueIndex];
 
-function parseOpts(options) {
-  // verify that there's an option provided for each of the options
-  // if they're not *all* specified, then we return null.
-  var keys = Object.keys(options);
-  var num = keys.length;
-  var conf = {};
-  for (var i = 2; i < process.argv.length; i++) {
-    var match = process.argv[i].match(/^(.+?)=([\s\S]*)$/);
-    if (!match || !match[1] || !options[match[1]]) {
-      return null;
-    } else {
-      conf[match[1]] = match[2];
-      num--;
+    // set NODE_RUN_BENCHMARK_FN to indicate that the child shouldn't construct
+    // a configuration queue, but just execute the benchmark function.
+    const childEnv = Object.assign({}, process.env);
+    childEnv.NODE_RUN_BENCHMARK_FN = '';
+
+    // Create configuration arguments
+    const childArgs = [];
+    for (const key of Object.keys(config)) {
+      childArgs.push(`${key}=${config[key]}`);
     }
-  }
-  // still go ahead and set whatever WAS set, if it was.
-  if (num !== 0) {
-    Object.keys(conf).forEach(function(k) {
-      options[k] = [conf[k]];
+
+    const child = child_process.fork(require.main.filename, childArgs, {
+      env: childEnv
     });
-  }
-  return num === 0 ? conf : null;
-}
+    child.on('message', sendResult);
+    child.on('close', function(code) {
+      if (code) {
+        process.exit(code);
+        return;
+      }
+
+      if (queueIndex + 1 < self.queue.length) {
+        recursive(queueIndex + 1);
+      }
+    });
+  })(0);
+};
 
 Benchmark.prototype.start = function() {
   if (this._started)
     throw new Error('Called start more than once in a single benchmark');
 
   this._started = true;
-  this._start = process.hrtime();
+  this._time = process.hrtime();
 };
 
 Benchmark.prototype.end = function(operations) {
-  var elapsed = process.hrtime(this._start);
+  // get elapsed time now and do error checking later for accuracy.
+  const elapsed = process.hrtime(this._time);
 
-  if (!this._started)
+  if (!this._started) {
     throw new Error('called end without start');
-  if (typeof operations !== 'number')
+  }
+  if (typeof operations !== 'number') {
     throw new Error('called end() without specifying operation count');
+  }
 
-  var time = elapsed[0] + elapsed[1] / 1e9;
-  var rate = operations / time;
-  this.report(rate);
+  const time = elapsed[0] + elapsed[1] / 1e9;
+  const rate = operations / time;
+  this.report(rate, elapsed);
 };
 
-Benchmark.prototype.report = function(value) {
-  var heading = this.getHeading();
+function formatResult(data) {
+  // Construct confiuration string, " A=a, B=b, ..."
+  let conf = '';
+  for (const key of Object.keys(data.conf)) {
+    conf += ' ' + key + '=' + JSON.stringify(data.conf[key]);
+  }
 
-  if (outputFormat == 'default')
-    console.log('%s: %s', heading, value.toFixed(5));
-  else if (outputFormat == 'csv')
-    console.log('%s,%s', heading, value.toFixed(5));
-};
+  return `${data.name}${conf}: ${data.rate}`;
+}
 
-Benchmark.prototype.getHeading = function() {
-  var conf = this.config;
-
-  if (outputFormat == 'default') {
-    return this._name + ' ' + Object.keys(conf).map(function(key) {
-      return key + '=' + JSON.stringify('' + conf[key]);
-    }).join(' ');
-  } else if (outputFormat == 'csv') {
-    return this._name + ',' + Object.keys(conf).map(function(key) {
-      return JSON.stringify('' + conf[key]);
-    }).join(',');
+function sendResult(data) {
+  if (process.send) {
+    // If forked, report by process send
+    process.send(data);
+  } else {
+    // Otherwise report by stdout
+    console.log(formatResult(data));
   }
+}
+exports.sendResult = sendResult;
+
+Benchmark.prototype.report = function(rate, elapsed) {
+  sendResult({
+    name: this.name,
+    conf: this.config,
+    rate: rate,
+    time: elapsed[0] + elapsed[1] / 1e9
+  });
 };
 
 exports.v8ForceOptimization = function(method, ...args) {
diff --git a/benchmark/http/http_server_for_chunky_client.js b/benchmark/http/http_server_for_chunky_client.js
index d85e15bcbe..fade895aa0 100644
--- a/benchmark/http/http_server_for_chunky_client.js
+++ b/benchmark/http/http_server_for_chunky_client.js
@@ -3,8 +3,8 @@
 var path = require('path');
 var http = require('http');
 var fs = require('fs');
-var spawn = require('child_process').spawn;
-require('../common.js');
+var fork = require('child_process').fork;
+var common = require('../common.js');
 var test = require('../../test/common.js');
 var pep = path.dirname(process.argv[1]) + '/_chunky_http_client.js';
 var PIPE = test.PIPE;
@@ -30,25 +30,10 @@ server.on('error', function(err) {
   throw new Error('server error: ' + err);
 });
 
-try {
-  var child;
-
-  server.listen(PIPE);
-
-  child = spawn(process.execPath, [pep], { });
-
-  child.on('error', function(err) {
-    throw new Error('spawn error: ' + err);
-  });
-
-  child.stdout.pipe(process.stdout);
-  child.stderr.pipe(process.stderr);
-
-  child.on('close', function(exitCode) {
-    server.close();
-  });
-
-} catch (e) {
-  throw new Error('error: ' + e);
-}
+server.listen(PIPE);
 
+var child = fork(pep, process.argv.slice(2));
+child.on('message', common.sendResult);
+child.on('close', function() {
+  server.close();
+});
diff --git a/benchmark/misc/v8-bench.js b/benchmark/misc/v8-bench.js
index 0b9a5139ba..9c0448a510 100644
--- a/benchmark/misc/v8-bench.js
+++ b/benchmark/misc/v8-bench.js
@@ -3,21 +3,49 @@
 var fs = require('fs');
 var path = require('path');
 var vm = require('vm');
+var common = require('../common.js');
 
 var dir = path.join(__dirname, '..', '..', 'deps', 'v8', 'benchmarks');
 
-global.print = function(s) {
-  if (s === '----') return;
-  console.log('misc/v8_bench.js %s', s);
-};
-
-global.load = function(filename) {
+function load(filename, inGlobal) {
   var source = fs.readFileSync(path.join(dir, filename), 'utf8');
-  // deps/v8/benchmarks/regexp.js breaks console.log() because it clobbers
-  // the RegExp global,  Restore the original when the script is done.
-  var $RegExp = global.RegExp;
-  vm.runInThisContext(source, { filename: filename });
-  global.RegExp = $RegExp;
-};
+  if (!inGlobal) source = '(function () {' + source + '\n})()';
+  vm.runInThisContext(source, { filename: 'v8/bechmark/' + filename });
+}
+
+load('base.js', true);
+load('richards.js');
+load('deltablue.js');
+load('crypto.js');
+load('raytrace.js');
+load('earley-boyer.js');
+load('regexp.js');
+load('splay.js');
+load('navier-stokes.js');
 
-global.load('run.js');
+const times = {};
+global.BenchmarkSuite.RunSuites({
+  NotifyStart: function(name) {
+    times[name] = process.hrtime();
+  },
+  NotifyResult: function(name, result) {
+    const elapsed = process.hrtime(times[name]);
+    common.sendResult({
+      name: name,
+      conf: {},
+      rate: result,
+      time: elapsed[0] + elapsed[1] / 1e9
+    });
+  },
+  NotifyError: function(name, error) {
+    console.error(name + ': ' + error);
+  },
+  NotifyScore: function(score) {
+    common.sendResult({
+      name: 'Score (version ' + global.BenchmarkSuite.version + ')',
+      conf: {},
+      rate: score,
+      time: 0
+    });
+  }
+});
diff --git a/benchmark/run.js b/benchmark/run.js
index ad590ea34a..756a7408bb 100644
--- a/benchmark/run.js
+++ b/benchmark/run.js
@@ -1,63 +1,51 @@
 'use strict';
 
-const fs = require('fs');
 const path = require('path');
-const child_process = require('child_process');
-
-var outputFormat = process.env.OUTPUT_FORMAT ||
-                   (+process.env.NODE_BENCH_SILENT ? 'silent' : false) ||
-                   'default';
-
-// If this is the main module, then run the benchmarks
-if (module === require.main) {
-  var type = process.argv[2];
-  var testFilter = process.argv[3];
-  if (!type) {
-    console.error('usage:\n ./node benchmark/run.js <type> [testFilter]');
-    process.exit(1);
-  }
-
-  var dir = path.join(__dirname, type);
-  var tests = fs.readdirSync(dir);
-
-  if (testFilter) {
-    var filteredTests = tests.filter(function(item) {
-      if (item.lastIndexOf(testFilter) >= 0) {
-        return item;
-      }
-    });
-
-    if (filteredTests.length === 0) {
-      console.error('%s is not found in \n %j', testFilter, tests);
-      return;
-    }
-    tests = filteredTests;
-  }
-
-  runBenchmarks();
+const fork = require('child_process').fork;
+const CLI = require('./_cli.js');
+
+const cli = CLI(`usage: ./node run.js [options] [--] <category> ...
+  Run each benchmark in the <category> directory a single time, more than one
+  <categoty> directory can be specified.
+
+  --filter pattern          string to filter benchmark scripts
+  --set    variable=value   set benchmark variable (can be repeated)
+`, {
+  arrayArgs: ['set']
+});
+const benchmarks = cli.benchmarks();
+
+if (benchmarks.length === 0) {
+  console.error('no benchmarks found');
+  process.exit(1);
 }
 
-function runBenchmarks() {
-  var test = tests.shift();
-  if (!test)
-    return;
+(function recursive(i) {
+  const filename = benchmarks[i];
+  const child = fork(path.resolve(__dirname, filename), cli.optional.set);
 
-  if (test.match(/^[\._]/))
-    return process.nextTick(runBenchmarks);
+  console.log();
+  console.log(filename);
 
-  if (outputFormat == 'default')
-    console.error(type + '/' + test);
+  child.on('message', function(data) {
+    // Construct configuration string, " A=a, B=b, ..."
+    let conf = '';
+    for (const key of Object.keys(data.conf)) {
+      conf += ' ' + key + '=' + JSON.stringify(data.conf[key]);
+    }
 
-  test = path.resolve(dir, test);
+    console.log(`${data.name}${conf}: ${data.rate}`);
+  });
 
-  var a = (process.execArgv || []).concat(test);
-  var child = child_process.spawn(process.execPath, a, { stdio: 'inherit' });
-  child.on('close', function(code) {
+  child.once('close', function(code) {
     if (code) {
       process.exit(code);
-    } else {
-      console.log('');
-      runBenchmarks();
+      return;
+    }
+
+    // If there are more benchmarks execute the next
+    if (i + 1 < benchmarks.length) {
+      recursive(i + 1);
     }
   });
-}
+})(0);