1 files changed, 34 insertions, 3 deletions
diff --git a/crosperf/benchmark.py b/crosperf/benchmark.py
index a2a34bca..bbb1cdfc 100644
--- a/crosperf/benchmark.py
+++ b/crosperf/benchmark.py
@@ -1,9 +1,40 @@
-
 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
-
 """Define a type that wraps a Benchmark instance."""
+from __future__ import print_function
+
+import math
+from scipy import stats
+
+# See crbug.com/673558 for how these are estimated.
+_estimated_stddev = {
+    'octane': 0.015,
+    'kraken': 0.019,
+    'speedometer': 0.007,
+    'dromaeo.domcoreattr': 0.023,
+    'dromaeo.domcoremodify': 0.011,
+    'smoothness.tough_webgl_cases': 0.025,
+    'graphics_WebGLAquarium': 0.008,
+    'page_cycler_v2.typical_25': 0.021,
+}
+
+
+# Get #samples needed to guarantee a given confidence interval, assuming the
+# samples follow normal distribution.
+def _samples(b):
+  # TODO: Make this an option
+  # CI = (0.9, 0.02), i.e., 90% chance that |sample mean - true mean| < 2%.
+  p = 0.9
+  e = 0.02
+  if b not in _estimated_stddev:
+    return 1
+  d = _estimated_stddev[b]
+  # Get at least 2 samples so as to calculate standard deviation, which is
+  # needed in T-test for p-value.
+  n = int(math.ceil((stats.norm.isf((1 - p) / 2) * d / e)**2))
+  return n if n > 1 else 2
+
 
 class Benchmark(object):
   """Class representing a benchmark to be run.
@@ -31,7 +62,7 @@ class Benchmark(object):
     self.test_name = test_name
     #For telemetry, this is the data.
     self.test_args = test_args
-    self.iterations = iterations
+    self.iterations = iterations if iterations > 0 else _samples(name)
     self.perf_args = perf_args
     self.rm_chroot_tmp = rm_chroot_tmp
     self.iteration_adjusted = False