diff options
author | Zhizhou Yang <zhizhouy@google.com> | 2018-11-13 09:51:13 -0800 |
---|---|---|
committer | chrome-bot <chrome-bot@chromium.org> | 2018-11-15 02:16:59 -0800 |
commit | 69c558b42977054d6168ad13829ebdd80ad77a7c (patch) | |
tree | 3414dbbb7956fae95476fb6e761a3da10b3a0862 /crosperf | |
parent | 1a199b1827d125b89c8cf66c8d5680bf14a1b857 (diff) | |
download | toolchain-utils-69c558b42977054d6168ad13829ebdd80ad77a7c.tar.gz |
crosperf: do not allow more than one story tags in test_args
This patch added a check at stage of parsing benchmark, to ensure that
story tag only appear once in test_args. This is because more than one
story tags will cause an implicit failure when running benchmark on DUT
and crosperf cannot catch it.
BUG=chromium:902466
TEST=Passed unit tests and a simple test
Change-Id: Ic8e4aa19928911fb4ed03529ab89471856cec05c
Reviewed-on: https://chromium-review.googlesource.com/1330010
Commit-Ready: Zhizhou Yang <zhizhouy@google.com>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Reviewed-by: Caroline Tice <cmtice@chromium.org>
Diffstat (limited to 'crosperf')
-rw-r--r-- | crosperf/experiment_factory.py | 10 | ||||
-rwxr-xr-x | crosperf/experiment_factory_unittest.py | 14 |
2 files changed, 21 insertions, 3 deletions
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index c9c3e68a..60785fa3 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -174,12 +174,16 @@ class ExperimentFactory(object): test_args = benchmark_settings.GetField('test_args') # Rename benchmark name if 'story-filter' or 'story-tag-filter' specified - # in test_args. + # in test_args. Make sure these two tags only appear once. + story_count = 0 for arg in test_args.split(): - if '--story-filter' in arg or '--story-tag-filter' in arg: + if '--story-filter=' in arg or '--story-tag-filter=' in arg: + story_count += 1 + if story_count > 1: + raise RuntimeError("Only one story or story-tag filter allowed in " + "a single benchmark run") # Rename benchmark name with an extension of 'story'-option benchmark_name = '%s@@%s' % (benchmark_name, arg.split('=')[-1]) - break # Check for duplicated benchmark name after renaming if not benchmark_name in benchmark_names: diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py index 372dfbe6..d7805c72 100755 --- a/crosperf/experiment_factory_unittest.py +++ b/crosperf/experiment_factory_unittest.py @@ -181,6 +181,20 @@ class ExperimentFactoryTest(unittest.TestCase): self.assertEqual('Weight should be a float between 0 and 1', str(msg.exception)) + # Test 6: more than one story tag in test_args + benchmark_settings = settings_factory.BenchmarkSettings('name') + benchmark_settings.SetField('test_args', + '--story-filter=a --story-tag-filter=b') + benchmark_settings.SetField('weight', '1.2') + benchmark_settings.SetField('suite', 'telemetry_Crosperf') + mock_experiment_file.all_settings = [] + mock_experiment_file.all_settings.append(benchmark_settings) + with self.assertRaises(RuntimeError) as msg: + ef = ExperimentFactory() + ef.GetExperiment(mock_experiment_file, '', '') + self.assertEqual('Only one story or story-tag filter allowed in a single ' + 'benchmark run', str(msg.exception)) + def test_append_benchmark_set(self): ef = ExperimentFactory() |