1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
|
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import optparse
from telemetry import decorators
from telemetry.internal import story_runner
from telemetry.internal.util import command_line
from telemetry.page import legacy_page_test
from telemetry.web_perf import timeline_based_measurement
Disabled = decorators.Disabled
Enabled = decorators.Enabled
Owner = decorators.Owner
class InvalidOptionsError(Exception):
"""Raised for invalid benchmark options."""
pass
class BenchmarkMetadata(object):
def __init__(self, name, description='', rerun_options=None):
self._name = name
self._description = description
self._rerun_options = rerun_options
@property
def name(self):
return self._name
@property
def description(self):
return self._description
@property
def rerun_options(self):
return self._rerun_options
def AsDict(self):
return {
'type': 'telemetry_benchmark',
'name': self._name,
'description': self._description,
'rerun_options': self._rerun_options,
}
class Benchmark(command_line.Command):
"""Base class for a Telemetry benchmark.
A benchmark packages a measurement and a PageSet together.
Benchmarks default to using TBM unless you override the value of
Benchmark.test, or override the CreatePageTest method.
New benchmarks should override CreateStorySet.
"""
options = {}
page_set = None
test = timeline_based_measurement.TimelineBasedMeasurement
def __init__(self, max_failures=None):
"""Creates a new Benchmark.
Args:
max_failures: The number of story run's failures before bailing
from executing subsequent page runs. If None, we never bail.
"""
self._max_failures = max_failures
self._has_original_tbm_options = (
self.CreateTimelineBasedMeasurementOptions.__func__ ==
Benchmark.CreateTimelineBasedMeasurementOptions.__func__)
has_original_create_page_test = (
self.CreatePageTest.__func__ == Benchmark.CreatePageTest.__func__)
assert self._has_original_tbm_options or has_original_create_page_test, (
'Cannot override both CreatePageTest and '
'CreateTimelineBasedMeasurementOptions.')
# pylint: disable=unused-argument
@classmethod
def ShouldDisable(cls, possible_browser):
"""Override this method to disable a benchmark under specific conditions.
Supports logic too complex for simple Enabled and Disabled decorators.
Decorators are still respected in cases where this function returns False.
"""
return False
def Run(self, finder_options):
"""Do not override this method."""
return story_runner.RunBenchmark(self, finder_options)
@property
def max_failures(self):
return self._max_failures
@classmethod
def Name(cls):
return '%s.%s' % (cls.__module__.split('.')[-1], cls.__name__)
@classmethod
def ShouldTearDownStateAfterEachStoryRun(cls):
"""Override to specify whether to tear down state after each story run.
Tearing down all states after each story run, e.g., clearing profiles,
stopping the browser, stopping local server, etc. So the browser will not be
reused among multiple stories. This is particularly useful to get the
startup part of launching the browser in each story.
This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
not by PageTest based benchmarks.
"""
return True
# NOTE: this is a temporary workaround for crbug.com/645329, do not rely on
# this as a stable public API as we may remove this without public notice.
@classmethod
def IsShouldTearDownStateAfterEachStoryRunOverriden(cls):
return (cls.ShouldTearDownStateAfterEachStoryRun.__func__ !=
Benchmark.ShouldTearDownStateAfterEachStoryRun.__func__)
@classmethod
def ShouldTearDownStateAfterEachStorySetRun(cls):
"""Override to specify whether to tear down state after each story set run.
Defaults to True in order to reset the state and make individual story set
repeats more independent of each other. The intended effect is to average
out noise in measurements between repeats.
Long running benchmarks willing to stess test the browser and have it run
for long periods of time may switch this value to False.
This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
not by PageTest based benchmarks.
"""
return True
@classmethod
def AddCommandLineArgs(cls, parser):
group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
cls.AddBenchmarkCommandLineArgs(group)
if cls.HasTraceRerunDebugOption():
group.add_option(
'--rerun-with-debug-trace',
action='store_true',
help='Rerun option that enables more extensive tracing.')
if group.option_list:
parser.add_option_group(group)
@classmethod
def AddBenchmarkCommandLineArgs(cls, group):
del group # unused
@classmethod
def HasTraceRerunDebugOption(cls):
return False
def GetTraceRerunCommands(self):
if self.HasTraceRerunDebugOption():
return [['Debug Trace', '--rerun-with-debug-trace']]
return []
def SetupTraceRerunOptions(self, browser_options, tbm_options):
if self.HasTraceRerunDebugOption():
if browser_options.rerun_with_debug_trace:
self.SetupBenchmarkDebugTraceRerunOptions(tbm_options)
else:
self.SetupBenchmarkDefaultTraceRerunOptions(tbm_options)
def SetupBenchmarkDefaultTraceRerunOptions(self, tbm_options):
"""Setup tracing categories associated with default trace option."""
def SetupBenchmarkDebugTraceRerunOptions(self, tbm_options):
"""Setup tracing categories associated with debug trace option."""
@classmethod
def SetArgumentDefaults(cls, parser):
default_values = parser.get_default_values()
invalid_options = [
o for o in cls.options if not hasattr(default_values, o)]
if invalid_options:
raise InvalidOptionsError('Invalid benchmark options: %s',
', '.join(invalid_options))
parser.set_defaults(**cls.options)
@classmethod
def ProcessCommandLineArgs(cls, parser, args):
pass
# pylint: disable=unused-argument
@classmethod
def ValueCanBeAddedPredicate(cls, value, is_first_result):
"""Returns whether |value| can be added to the test results.
Override this method to customize the logic of adding values to test
results.
Args:
value: a value.Value instance (except failure.FailureValue,
skip.SkipValue or trace.TraceValue which will always be added).
is_first_result: True if |value| is the first result for its
corresponding story.
Returns:
True if |value| should be added to the test results.
Otherwise, it returns False.
"""
return True
def CustomizeBrowserOptions(self, options):
"""Add browser options that are required by this benchmark."""
def GetMetadata(self):
return BenchmarkMetadata(
self.Name(), self.__doc__, self.GetTraceRerunCommands())
def CreateTimelineBasedMeasurementOptions(self):
"""Return the TimelineBasedMeasurementOptions for this Benchmark.
Override this method to configure a TimelineBasedMeasurement benchmark.
Otherwise, override CreatePageTest for PageTest tests. Do not override
both methods.
"""
return timeline_based_measurement.Options()
def CreatePageTest(self, options): # pylint: disable=unused-argument
"""Return the PageTest for this Benchmark.
Override this method for PageTest tests.
Override, override CreateTimelineBasedMeasurementOptions to configure
TimelineBasedMeasurement tests. Do not override both methods.
Args:
options: a browser_options.BrowserFinderOptions instance
Returns:
|test()| if |test| is a PageTest class.
Otherwise, a TimelineBasedMeasurement instance.
"""
is_page_test = issubclass(self.test, legacy_page_test.LegacyPageTest)
is_tbm = self.test == timeline_based_measurement.TimelineBasedMeasurement
if not is_page_test and not is_tbm:
raise TypeError('"%s" is not a PageTest or a TimelineBasedMeasurement.' %
self.test.__name__)
if is_page_test:
assert self._has_original_tbm_options, (
'Cannot override CreateTimelineBasedMeasurementOptions '
'with a PageTest.')
return self.test() # pylint: disable=no-value-for-parameter
opts = self.CreateTimelineBasedMeasurementOptions()
self.SetupTraceRerunOptions(options, opts)
return timeline_based_measurement.TimelineBasedMeasurement(opts)
def CreateStorySet(self, options):
"""Creates the instance of StorySet used to run the benchmark.
Can be overridden by subclasses.
"""
del options # unused
# TODO(aiolos, nednguyen, eakufner): replace class attribute page_set with
# story_set.
if not self.page_set:
raise NotImplementedError('This test has no "page_set" attribute.')
return self.page_set() # pylint: disable=not-callable
def AddCommandLineArgs(parser):
story_runner.AddCommandLineArgs(parser)
def ProcessCommandLineArgs(parser, args):
story_runner.ProcessCommandLineArgs(parser, args)
|