crosperf: Feature to ignore min and max value in results

This patch provides an option for user to ignore min and max value in
the results when generating report. User can use this feature by
specifying `ignore_min_max: True` (which by default is False) in
experiment file.

When values count in a single test is smaller than 3, we automatically
ignore this option for it, and print out a warning.

BUG=chromium:938758
TEST=Tested with examples in general/cwp mode report generation. Passed
all unit tests.

Change-Id: I36a4c4d99836c201cdd2f2f9f2a4b1a4ffdaa47d
Reviewed-on: https://chromium-review.googlesource.com/1521054
Commit-Ready: Zhizhou Yang <zhizhouy@google.com>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Reviewed-by: Zhizhou Yang <zhizhouy@google.com>

2 files changed, 82 insertions, 43 deletions

diff --git a/cros_utils/tabulator.py b/cros_utils/tabulator.py
index 94c49424..ed93de7a 100644
--- a/cros_utils/tabulator.py
+++ b/cros_utils/tabulator.py
@@ -87,6 +87,17 @@ def _StripNone(results):
   return res
 
 
+def _RemoveMinMax(cell, values):
+  if len(values) < 3:
+    print('WARNING: Values count is less than 3, not ignoring min/max values')
+    print('WARNING: Cell name:', cell.name, 'Values:', values)
+    return values
+
+  values.remove(min(values))
+  values.remove(max(values))
+  return values
+
+
 class TableGenerator(object):
   """Creates a table from a list of list of dicts.
 
@@ -529,7 +540,13 @@ class StringMeanResult(Result):
 class AmeanResult(StringMeanResult):
   """Arithmetic mean."""
 
+  def __init__(self, ignore_min_max=False):
+    super(AmeanResult, self).__init__()
+    self.ignore_min_max = ignore_min_max
+
   def _ComputeFloat(self, cell, values, baseline_values):
+    if self.ignore_min_max:
+      values = _RemoveMinMax(cell, values)
     cell.value = numpy.mean(values)
 
 
@@ -579,14 +596,26 @@ class NumericalResult(Result):
 class StdResult(NumericalResult):
   """Standard deviation."""
 
+  def __init__(self, ignore_min_max=False):
+    super(StdResult, self).__init__()
+    self.ignore_min_max = ignore_min_max
+
   def _ComputeFloat(self, cell, values, baseline_values):
+    if self.ignore_min_max:
+      values = _RemoveMinMax(cell, values)
     cell.value = numpy.std(values)
 
 
 class CoeffVarResult(NumericalResult):
   """Standard deviation / Mean"""
 
+  def __init__(self, ignore_min_max=False):
+    super(CoeffVarResult, self).__init__()
+    self.ignore_min_max = ignore_min_max
+
   def _ComputeFloat(self, cell, values, baseline_values):
+    if self.ignore_min_max:
+      values = _RemoveMinMax(cell, values)
     if numpy.mean(values) != 0.0:
       noise = numpy.abs(numpy.std(values) / numpy.mean(values))
     else:
@@ -619,7 +648,14 @@ class ComparisonResult(Result):
 class PValueResult(ComparisonResult):
   """P-value."""
 
+  def __init__(self, ignore_min_max=False):
+    super(PValueResult, self).__init__()
+    self.ignore_min_max = ignore_min_max
+
   def _ComputeFloat(self, cell, values, baseline_values):
+    if self.ignore_min_max:
+      values = _RemoveMinMax(cell, values)
+      baseline_values = _RemoveMinMax(cell, baseline_values)
     if len(values) < 2 or len(baseline_values) < 2:
       cell.value = float('nan')
       return
@@ -674,7 +710,14 @@ class KeyAwareComparisonResult(ComparisonResult):
 class AmeanRatioResult(KeyAwareComparisonResult):
   """Ratio of arithmetic means of values vs. baseline values."""
 
+  def __init__(self, ignore_min_max=False):
+    super(AmeanRatioResult, self).__init__()
+    self.ignore_min_max = ignore_min_max
+
   def _ComputeFloat(self, cell, values, baseline_values):
+    if self.ignore_min_max:
+      values = _RemoveMinMax(cell, values)
+      baseline_values = _RemoveMinMax(cell, baseline_values)
     if numpy.mean(baseline_values) != 0:
       cell.value = numpy.mean(values) / numpy.mean(baseline_values)
     elif numpy.mean(values) != 0:
@@ -688,7 +731,14 @@ class AmeanRatioResult(KeyAwareComparisonResult):
 class GmeanRatioResult(KeyAwareComparisonResult):
   """Ratio of geometric means of values vs. baseline values."""
 
+  def __init__(self, ignore_min_max=False):
+    super(GmeanRatioResult, self).__init__()
+    self.ignore_min_max = ignore_min_max
+
   def _ComputeFloat(self, cell, values, baseline_values):
+    if self.ignore_min_max:
+      values = _RemoveMinMax(cell, values)
+      baseline_values = _RemoveMinMax(cell, baseline_values)
     if self._GetGmean(baseline_values) != 0:
       cell.value = self._GetGmean(values) / self._GetGmean(baseline_values)
     elif self._GetGmean(values) != 0:
diff --git a/cros_utils/tabulator_test.py b/cros_utils/tabulator_test.py
index 943d9349..33c8da25 100644
--- a/cros_utils/tabulator_test.py
+++ b/cros_utils/tabulator_test.py
@@ -73,20 +73,22 @@ class TabulatorTest(unittest.TestCase):
     b = tabulator.Result()._GetGmean(a)
     self.assertTrue(b >= 0.99e+308 and b <= 1.01e+308)
 
+  def testIgnoreMinMax(self):
+    amr = tabulator.AmeanResult(ignore_min_max=True)
+    cell = tabulator.Cell()
+    values = [1, 2]
+    amr.Compute(cell, values, None)
+    self.assertTrue(cell.value == 1.5)
+    values = [1, 2, 8]
+    amr.Compute(cell, values, None)
+    self.assertTrue(cell.value == 2)
+
   def testTableGenerator(self):
-    runs = [[{
-        'k1': '10',
-        'k2': '12'
-    }, {
-        'k1': '13',
-        'k2': '14',
-        'k3': '15'
-    }], [{
-        'k1': '50',
-        'k2': '51',
-        'k3': '52',
-        'k4': '53'
-    }]]
+    # yapf: disable
+    runs = [[{'k1': '10', 'k2': '12'},
+             {'k1': '13', 'k2': '14', 'k3': '15'}],
+            [{'k1': '50', 'k2': '51', 'k3': '52', 'k4': '53'}]]
+    # yapf: enable
     labels = ['vanilla', 'modified']
     tg = tabulator.TableGenerator(runs, labels)
     table = tg.GetTable()
@@ -113,24 +115,14 @@ class TabulatorTest(unittest.TestCase):
     self.assertTrue(table)
 
   def testSamplesTableGenerator(self):
+    # yapf: disable
     keyvals = {
-        'bench1': [[{
-            'samples': 1
-        }, {
-            'samples': 2
-        }], [{
-            'samples': 3
-        }, {
-            'samples': 4
-        }]],
-        'bench2': [[{
-            'samples': 5
-        }, {}], [{
-            'samples': 6
-        }, {
-            'samples': 7
-        }]]
+        'bench1': [[{'samples': 1}, {'samples': 2}],
+                   [{'samples': 3}, {'samples': 4}]],
+        'bench2': [[{'samples': 5}, {}],
+                   [{'samples': 6}, {'samples': 7}]]
     }
+    # yapf: enable
     weights = {'bench1': 0.2, 'bench2': 0.7}
     iter_counts = {'bench1': 2, 'bench2': 2}
     labels = ['vanilla', 'modified']
@@ -152,22 +144,19 @@ class TabulatorTest(unittest.TestCase):
     header = table.pop(0)
     self.assertTrue(header == ['Benchmarks', 'Weights', 'vanilla', 'modified'])
     row = table.pop(0)
-    self.assertTrue(row == [
-        'bench1', 0.2, ((2, 0), [1 * 0.2, 2 * 0.2]), ((2, 0),
-                                                      [3 * 0.2, 4 * 0.2])
-    ])
+    # yapf: disable
+    self.assertTrue(row == ['bench1', 0.2,
+                            ((2, 0), [1 * 0.2, 2 * 0.2]),
+                            ((2, 0), [3 * 0.2, 4 * 0.2])])
     row = table.pop(0)
-    self.assertTrue(row == [
-        'bench2', 0.7, ((1, 1), [5 * 0.7, None]), ((2, 0), [6 * 0.7, 7 * 0.7])
-    ])
+    self.assertTrue(row == ['bench2', 0.7,
+                            ((1, 1), [5 * 0.7, None]),
+                            ((2, 0), [6 * 0.7, 7 * 0.7])])
     row = table.pop(0)
-    self.assertTrue(row == [
-        'Composite Benchmark (samples)', 'N/A',
-        ((1, 1),
-         [1 * 0.2 +
-          5 * 0.7, None]), ((2, 0), [3 * 0.2 + 6 * 0.7, 4 * 0.2 + 7 * 0.7])
-    ])
-
+    self.assertTrue(row == ['Composite Benchmark (samples)', 'N/A',
+                            ((1, 1), [1 * 0.2 + 5 * 0.7, None]),
+                            ((2, 0), [3 * 0.2 + 6 * 0.7, 4 * 0.2 + 7 * 0.7])])
+    # yapf: enable
     self.assertTrue('Composite Benchmark' in new_keyvals.keys())
     self.assertTrue('Composite Benchmark' in new_iter_counts.keys())