benchmark: ignore significance when using --runs 1

Because the standard deviation can't be calculated when there is only
one observation the R scripts raises an error. However it may still be
useful to run them for non-statistical purposes.

This changes the behaviour such when there is only one observation, the
values that depends on the standard deviation becomes Not Applicable
(NA).

Fixes: https://github.com/nodejs/node/issues/8288
PR-URL: https://github.com/nodejs/node/pull/8299
Reviewed-By: Anna Henningsen <anna@addaleax.net>

1 files changed, 25 insertions, 16 deletions

diff --git a/benchmark/compare.R b/benchmark/compare.R
index 01beb38046..1200340f32 100644
--- a/benchmark/compare.R
+++ b/benchmark/compare.R
@@ -33,30 +33,39 @@ if (!is.null(plot.filename)) {
 
 # Print a table with results
 statistics = ddply(dat, "name", function(subdat) {
-  # Perform a statistics test to see of there actually is a difference in
-  # performace.
-  w = t.test(rate ~ binary, data=subdat);
+  old.rate = subset(subdat, binary == "old")$rate;
+  new.rate = subset(subdat, binary == "new")$rate;
 
   # Calculate improvement for the "new" binary compared with the "old" binary
-  new_mu = mean(subset(subdat, binary == "new")$rate);
-  old_mu = mean(subset(subdat, binary == "old")$rate);
-  improvement = sprintf("%.2f %%", ((new_mu - old_mu) / old_mu * 100));
+  old.mu = mean(old.rate);
+  new.mu = mean(new.rate);
+  improvement = sprintf("%.2f %%", ((new.mu - old.mu) / old.mu * 100));
 
-  # Add user friendly stars to the table. There should be at least one star
-  # before you can say that there is an improvement.
-  significant = '';
-  if (w$p.value < 0.001) {
-    significant = '***';
-  } else if (w$p.value < 0.01) {
-    significant = '**';
-  } else if (w$p.value < 0.05) {
-    significant = '*';
+  p.value = NA;
+  significant = 'NA';
+  # Check if there is enough data to calulate the calculate the p-value
+  if (length(old.rate) > 1 && length(new.rate) > 1) {
+    # Perform a statistics test to see of there actually is a difference in
+    # performance.
+    w = t.test(rate ~ binary, data=subdat);
+    p.value = w$p.value;
+
+    # Add user friendly stars to the table. There should be at least one star
+    # before you can say that there is an improvement.
+    significant = '';
+    if (p.value < 0.001) {
+      significant = '***';
+    } else if (p.value < 0.01) {
+      significant = '**';
+    } else if (p.value < 0.05) {
+      significant = '*';
+    }
   }
 
   r = list(
     improvement = improvement,
     significant = significant,
-    p.value = w$p.value
+    p.value = p.value
   );
   return(data.frame(r));
 });