aboutsummaryrefslogtreecommitdiff
path: root/crosperf
diff options
context:
space:
mode:
authorZhizhou Yang <zhizhouy@google.com>2018-11-13 09:51:13 -0800
committerchrome-bot <chrome-bot@chromium.org>2018-11-15 02:16:59 -0800
commit69c558b42977054d6168ad13829ebdd80ad77a7c (patch)
tree3414dbbb7956fae95476fb6e761a3da10b3a0862 /crosperf
parent1a199b1827d125b89c8cf66c8d5680bf14a1b857 (diff)
downloadtoolchain-utils-69c558b42977054d6168ad13829ebdd80ad77a7c.tar.gz
crosperf: do not allow more than one story tags in test_args
This patch added a check at stage of parsing benchmark, to ensure that story tag only appear once in test_args. This is because more than one story tags will cause an implicit failure when running benchmark on DUT and crosperf cannot catch it. BUG=chromium:902466 TEST=Passed unit tests and a simple test Change-Id: Ic8e4aa19928911fb4ed03529ab89471856cec05c Reviewed-on: https://chromium-review.googlesource.com/1330010 Commit-Ready: Zhizhou Yang <zhizhouy@google.com> Tested-by: Zhizhou Yang <zhizhouy@google.com> Reviewed-by: Caroline Tice <cmtice@chromium.org>
Diffstat (limited to 'crosperf')
-rw-r--r--crosperf/experiment_factory.py10
-rwxr-xr-xcrosperf/experiment_factory_unittest.py14
2 files changed, 21 insertions, 3 deletions
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index c9c3e68a..60785fa3 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -174,12 +174,16 @@ class ExperimentFactory(object):
test_args = benchmark_settings.GetField('test_args')
# Rename benchmark name if 'story-filter' or 'story-tag-filter' specified
- # in test_args.
+ # in test_args. Make sure these two tags only appear once.
+ story_count = 0
for arg in test_args.split():
- if '--story-filter' in arg or '--story-tag-filter' in arg:
+ if '--story-filter=' in arg or '--story-tag-filter=' in arg:
+ story_count += 1
+ if story_count > 1:
+ raise RuntimeError("Only one story or story-tag filter allowed in "
+ "a single benchmark run")
# Rename benchmark name with an extension of 'story'-option
benchmark_name = '%s@@%s' % (benchmark_name, arg.split('=')[-1])
- break
# Check for duplicated benchmark name after renaming
if not benchmark_name in benchmark_names:
diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py
index 372dfbe6..d7805c72 100755
--- a/crosperf/experiment_factory_unittest.py
+++ b/crosperf/experiment_factory_unittest.py
@@ -181,6 +181,20 @@ class ExperimentFactoryTest(unittest.TestCase):
self.assertEqual('Weight should be a float between 0 and 1',
str(msg.exception))
+ # Test 6: more than one story tag in test_args
+ benchmark_settings = settings_factory.BenchmarkSettings('name')
+ benchmark_settings.SetField('test_args',
+ '--story-filter=a --story-tag-filter=b')
+ benchmark_settings.SetField('weight', '1.2')
+ benchmark_settings.SetField('suite', 'telemetry_Crosperf')
+ mock_experiment_file.all_settings = []
+ mock_experiment_file.all_settings.append(benchmark_settings)
+ with self.assertRaises(RuntimeError) as msg:
+ ef = ExperimentFactory()
+ ef.GetExperiment(mock_experiment_file, '', '')
+ self.assertEqual('Only one story or story-tag filter allowed in a single '
+ 'benchmark run', str(msg.exception))
+
def test_append_benchmark_set(self):
ef = ExperimentFactory()