crosperf: introduce cwp_dso and weight for CWP approximation

This patch does initial work to support CWP approximation in crosperf.
It introduceds a global option cwp_dso, which is used to specify the
type of DSO to use for approximation; an option weight in benchmark
setting, which decides the weight of each benchmark in test. This patch
also contains some value checks for these two options.

BUG=chromium:902785
TEST=Passed all unit tests and sample tests locally.

Change-Id: I565c4baf0630ce6c1b62ad0398d05a5f336aac6e
Reviewed-on: https://chromium-review.googlesource.com/1323473
Commit-Ready: Zhizhou Yang <zhizhouy@google.com>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Reviewed-by: Caroline Tice <cmtice@chromium.org>

5 files changed, 144 insertions, 7 deletions

diff --git a/crosperf/benchmark.py b/crosperf/benchmark.py
index 60ac778b..c1e44d5e 100644
--- a/crosperf/benchmark.py
+++ b/crosperf/benchmark.py
@@ -56,7 +56,8 @@ class Benchmark(object):
                suite='',
                show_all_results=False,
                retries=0,
-               run_local=False):
+               run_local=False,
+               weight=0):
     self.name = name
     #For telemetry, this is the benchmark name.
     self.test_name = test_name
@@ -74,3 +75,4 @@ class Benchmark(object):
     if run_local and self.suite != 'telemetry_Crosperf':
       raise RuntimeError('run_local is only supported by telemetry_Crosperf.')
     self.run_local = run_local
+    self.weight = weight
diff --git a/crosperf/experiment.py b/crosperf/experiment.py
index 987318a5..c4ed5967 100644
--- a/crosperf/experiment.py
+++ b/crosperf/experiment.py
@@ -27,7 +27,7 @@ class Experiment(object):
   def __init__(self, name, remote, working_directory, chromeos_root,
                cache_conditions, labels, benchmarks, experiment_file, email_to,
                acquire_timeout, log_dir, log_level, share_cache,
-               results_directory, locks_directory):
+               results_directory, locks_directory, cwp_dso):
     self.name = name
     self.working_directory = working_directory
     self.remote = remote
@@ -53,6 +53,7 @@ class Experiment(object):
     # locking mechanism.
     self.locks_dir = locks_directory
     self.locked_machines = []
+    self.cwp_dso = cwp_dso
 
     if not remote:
       raise RuntimeError('No remote hosts specified')
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index a6c163dc..c9c3e68a 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -82,6 +82,11 @@ crosbolt_perf_tests = [
 #    'cheets_LinpackTest',
 #]
 
+dso_list = [
+    'all',
+    'chrome',
+    'kallsyms',
+]
 
 class ExperimentFactory(object):
   """Factory class for building an Experiment, given an ExperimentFile as input.
@@ -137,6 +142,9 @@ class ExperimentFactory(object):
     log_level = global_settings.GetField('logging_level')
     if log_level not in ('quiet', 'average', 'verbose'):
       log_level = 'verbose'
+    cwp_dso = global_settings.GetField('cwp_dso')
+    if cwp_dso and not cwp_dso in dso_list:
+      raise RuntimeError('The DSO specified is not supported')
     # Default cache hit conditions. The image checksum in the cache and the
     # computed checksum of the image must match. Also a cache file must exist.
     cache_conditions = [
@@ -183,6 +191,18 @@ class ExperimentFactory(object):
       suite = benchmark_settings.GetField('suite')
       retries = benchmark_settings.GetField('retries')
       run_local = benchmark_settings.GetField('run_local')
+      weight = benchmark_settings.GetField('weight')
+      if weight:
+        if not cwp_dso:
+          raise RuntimeError('Weight can only be set when DSO specified')
+        if suite != 'telemetry_Crosperf':
+          raise RuntimeError('CWP approximation weight only works with '
+                             'telemetry_Crosperf suite')
+        if weight > 1 or weight < 0:
+          raise RuntimeError('Weight should be a float between 0 and 1')
+      elif cwp_dso:
+        raise RuntimeError('With DSO specified, each benchmark should have a '
+                           'weight')
 
       if suite == 'telemetry_Crosperf':
         if test_name == 'all_perfv2':
@@ -234,9 +254,9 @@ class ExperimentFactory(object):
                                   iterations, rm_chroot_tmp, perf_args, suite,
                                   show_all_results, retries, run_local)
         else:
-          benchmark = Benchmark(benchmark_name, test_name, test_args,
-                                iterations, rm_chroot_tmp, perf_args, suite,
-                                show_all_results, retries, run_local)
+          benchmark = Benchmark(benchmark_name, test_name, test_args, iterations,
+                                rm_chroot_tmp, perf_args, suite,
+                                show_all_results, retries, run_local, weight)
           benchmarks.append(benchmark)
       else:
         if test_name == 'all_graphics_perf':
@@ -330,7 +350,7 @@ class ExperimentFactory(object):
                             chromeos_root, cache_conditions, labels, benchmarks,
                             experiment_file.Canonicalize(), email,
                             acquire_timeout, log_dir, log_level, share_cache,
-                            results_dir, locks_dir)
+                            results_dir, locks_dir, cwp_dso)
 
     return experiment
 
@@ -352,4 +372,4 @@ class ExperimentFactory(object):
       raise RuntimeError('IOError while reading file {0}'
                          .format(default_remotes_file))
     else:
-      raise RuntimeError('There is not remote for {0}'.format(board))
+      raise RuntimeError('There is no remote for {0}'.format(board))
diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py
index 74521c7c..372dfbe6 100755
--- a/crosperf/experiment_factory_unittest.py
+++ b/crosperf/experiment_factory_unittest.py
@@ -43,6 +43,30 @@ EXPERIMENT_FILE_1 = """
   }
   """
 
+EXPERIMENT_FILE_2 = """
+  board: x86-alex
+  remote: chromeos-alex3
+
+  cwp_dso: kallsyms
+
+  benchmark: Octane {
+    iterations: 1
+    suite: telemetry_Crosperf
+    weight: 0.8
+  }
+
+  benchmark: Kraken {
+    iterations: 1
+    suite: telemetry_Crosperf
+    weight: 0.2
+  }
+
+  image1 {
+    chromeos_image: /usr/local/google/cros_image1.bin
+  }
+  """
+
+
 # pylint: disable=too-many-function-args
 
 
@@ -71,6 +95,15 @@ class ExperimentFactoryTest(unittest.TestCase):
                      '/usr/local/google/cros_image1.bin')
     self.assertEqual(exp.labels[0].board, 'x86-alex')
 
+  def testLoadExperimentFile2CWP(self):
+    experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_2))
+    exp = ExperimentFactory().GetExperiment(
+        experiment_file, working_directory='', log_dir='')
+    self.assertEqual(exp.cwp_dso, 'kallsyms')
+    self.assertEqual(len(exp.benchmarks), 2)
+    self.assertEqual(exp.benchmarks[0].weight, 0.8)
+    self.assertEqual(exp.benchmarks[1].weight, 0.2)
+
   def testDuplecateBenchmark(self):
     mock_experiment_file = ExperimentFile(StringIO.StringIO(''))
     mock_experiment_file.all_settings = []
@@ -83,6 +116,71 @@ class ExperimentFactoryTest(unittest.TestCase):
       ef = ExperimentFactory()
       ef.GetExperiment(mock_experiment_file, '', '')
 
+  def testCWPExceptions(self):
+    mock_experiment_file = ExperimentFile(StringIO.StringIO(''))
+    mock_experiment_file.all_settings = []
+    global_settings = settings_factory.GlobalSettings('test_name')
+
+    # Test 1: DSO type not supported
+    global_settings.SetField('cwp_dso', 'test')
+    self.assertEqual(global_settings.GetField('cwp_dso'), 'test')
+    mock_experiment_file.global_settings = global_settings
+    with self.assertRaises(RuntimeError) as msg:
+      ef = ExperimentFactory()
+      ef.GetExperiment(mock_experiment_file, '', '')
+    self.assertEqual('The DSO specified is not supported',
+                     str(msg.exception))
+
+    # Test 2: No weight after DSO specified
+    global_settings.SetField('cwp_dso', 'kallsyms')
+    mock_experiment_file.global_settings = global_settings
+    benchmark_settings = settings_factory.BenchmarkSettings('name')
+    mock_experiment_file.all_settings.append(benchmark_settings)
+    with self.assertRaises(RuntimeError) as msg:
+      ef = ExperimentFactory()
+      ef.GetExperiment(mock_experiment_file, '', '')
+    self.assertEqual('With DSO specified, each benchmark should have a weight',
+                     str(msg.exception))
+
+    # Test 3: Weight is set, but no dso specified
+    global_settings.SetField('cwp_dso', '')
+    mock_experiment_file.global_settings = global_settings
+    benchmark_settings = settings_factory.BenchmarkSettings('name')
+    benchmark_settings.SetField('weight', '0.8')
+    mock_experiment_file.all_settings = []
+    mock_experiment_file.all_settings.append(benchmark_settings)
+    with self.assertRaises(RuntimeError) as msg:
+      ef = ExperimentFactory()
+      ef.GetExperiment(mock_experiment_file, '', '')
+    self.assertEqual('Weight can only be set when DSO specified',
+                     str(msg.exception))
+
+    # Test 4: cwp_dso only works for telemetry_Crosperf benchmarks
+    global_settings.SetField('cwp_dso', 'kallsyms')
+    mock_experiment_file.global_settings = global_settings
+    benchmark_settings = settings_factory.BenchmarkSettings('name')
+    benchmark_settings.SetField('weight', '0.8')
+    mock_experiment_file.all_settings = []
+    mock_experiment_file.all_settings.append(benchmark_settings)
+    with self.assertRaises(RuntimeError) as msg:
+      ef = ExperimentFactory()
+      ef.GetExperiment(mock_experiment_file, '', '')
+    self.assertEqual('CWP approximation weight only works with '
+                     'telemetry_Crosperf suite',
+                     str(msg.exception))
+
+    # Test 5: weight should be float between 0 and 1
+    benchmark_settings = settings_factory.BenchmarkSettings('name')
+    benchmark_settings.SetField('weight', '1.2')
+    benchmark_settings.SetField('suite', 'telemetry_Crosperf')
+    mock_experiment_file.all_settings = []
+    mock_experiment_file.all_settings.append(benchmark_settings)
+    with self.assertRaises(RuntimeError) as msg:
+      ef = ExperimentFactory()
+      ef.GetExperiment(mock_experiment_file, '', '')
+    self.assertEqual('Weight should be a float between 0 and 1',
+                     str(msg.exception))
+
   def test_append_benchmark_set(self):
     ef = ExperimentFactory()
 
diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py
index efbb534f..edf269c1 100644
--- a/crosperf/settings_factory.py
+++ b/crosperf/settings_factory.py
@@ -6,6 +6,7 @@
 from __future__ import print_function
 
 from field import BooleanField
+from field import FloatField
 from field import IntegerField
 from field import ListField
 from field import TextField
@@ -51,6 +52,11 @@ class BenchmarkSettings(Settings):
             'telemetry_Crosperf.',
             required=False,
             default=True))
+    self.AddField(
+        FloatField(
+            'weight',
+            default=0.0,
+            description='Weight of the benchmark for CWP approximation'))
 
 
 class LabelSettings(Settings):
@@ -286,6 +292,16 @@ class GlobalSettings(Settings):
             default=0,
             description='Number of times to retry a '
             'benchmark run.'))
+    self.AddField(
+        TextField(
+            'cwp_dso',
+            description='The DSO type that we want to use for '
+            'CWP approximation. This is used to run telemetry '
+            'benchmarks. Valid DSO types can be found from dso_list '
+            'in experiment_factory.py. The default value is set to '
+            'be empty.',
+            required=False,
+            default=''))
 
 
 class SettingsFactory(object):