aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cros_utils/tabulator.py258
-rw-r--r--crosperf/results_cache.py2
-rw-r--r--crosperf/results_organizer.py12
-rw-r--r--crosperf/results_report.py96
4 files changed, 326 insertions, 42 deletions
diff --git a/cros_utils/tabulator.py b/cros_utils/tabulator.py
index 6936d35f..687ffeb8 100644
--- a/cros_utils/tabulator.py
+++ b/cros_utils/tabulator.py
@@ -207,6 +207,189 @@ class TableGenerator(object):
return table
+class CPUTableGenerator(TableGenerator):
+ """Creates a table with only cpu cycles from the results
+
+ The main public function is called GetTable().
+
+ Different than TableGenerator, self._runs is now a dict of {benchmark: runs}
+ We are expecting there is 'cpu_cycles' in `runs`.
+ """
+ def __init__(self, run_keyvals, labels, iter_counts, weights):
+ TableGenerator.__init__(self, run_keyvals, labels, key_name='Benchmarks')
+ self._iter_counts = iter_counts
+ self._weights = weights
+
+ def _GetKeys(self):
+ keys = self._runs.keys()
+ return self._SortKeys(keys)
+
+ def GetTable(self, number_of_rows=sys.maxint):
+ """Returns a tuple, which contains three args:
+ 1) a table from a list of list of dicts.
+ 2) updated benchmark_results run_keyvals with composite benchmark
+ 3) updated benchmark_results iter_count with composite benchmark
+
+ The dict of list of list of dicts is passed into the constructor of
+ CPUTableGenerator.
+ This method converts that into a canonical list of lists which represents a
+ table of values.
+
+ Args:
+ number_of_rows: Maximum number of rows to return from the table.
+
+ Returns:
+ A list of lists which is the table.
+
+ Example:
+ We have the following runs:
+ {bench1: [[{"cpu_cycles": "v1"}, {"cpu_cycles": "v2"}],
+ [{"cpu_cycles": "v3"}, {"cpu_cycles": "v4"}]]
+ bench2: [[{"cpu_cycles": "v21"}, None],
+ [{"cpu_cycles": "v22"}, {"cpu_cycles": "v23"}]]}
+ and weights of benchmarks:
+ {bench1: w1, bench2: w2}
+ and the following labels:
+ ["vanilla", "modified"]
+ it will return:
+ [["Benchmark", "Weights", "vanilla", "modified"]
+ ["bench1", w1,
+ ((2, 0), ["v1*w1", "v2*w1"]), ((2, 0), ["v3*w1", "v4*w1"])]
+ ["bench2", w2,
+ ((1, 1), ["v21*w2", None]), ((2, 0), ["v22*w2", "v23*w2"])]
+ ["Composite Benchmark", N/A,
+ ((1, 1), ["v1*w1+v21*w2", None]),
+ ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]]
+ The returned table can then be processed further by other classes in this
+ module.
+ """
+ keys = self._GetKeys()
+ header = [self._key_name, 'Weights'] + self._labels
+ table = [header]
+ rows = 0
+ iterations = 0
+
+ for k in keys:
+ runs = self._runs[k]
+ unit = None
+ all_runs_empty = all(not dict for label in runs for dict in label)
+ if all_runs_empty:
+ cell = Cell()
+ cell.string_value = 'Benchmark %s contains no result.' + \
+ ' Is the benchmark name valid?' % k
+ table.append([cell])
+ else:
+ row = [k]
+ row.append(self._weights[k])
+ for run_list in runs:
+ run_pass = 0
+ run_fail = 0
+ v = []
+ for run in run_list:
+ if 'cpu_cycles' in run:
+ if type(run['cpu_cycles']) is list:
+ val = run['cpu_cycles'][0] * self._weights[k]
+ unit = run['cpu_cycles'][1]
+ else:
+ val = run['cpu_cycles']
+ v.append(val)
+ run_pass += 1
+ else:
+ v.append(None)
+ run_fail += 1
+ t = ((run_pass, run_fail), v)
+ if iterations != 0 and iterations != run_pass + run_fail:
+ raise ValueError('Iterations of each benchmark run ' \
+ 'are not the same')
+ iterations = run_pass + run_fail
+ row.append(t)
+ if unit:
+ keyname = row[0] + ' (%s) ' % unit
+ row[0] = keyname
+ table.append(row)
+ rows += 1
+ if rows == number_of_rows:
+ break
+
+ k = 'Composite Benchmark'
+ if k in keys:
+ raise RuntimeError('Composite benchmark already exists in results')
+
+ # Create a new composite benchmark row at the bottom of the summary table
+ # The new row will be like the format in example:
+ # ["Composite Benchmark", N/A,
+ # ((1, 1), ["v1*w1+v21*w2", None]),
+ # ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]]
+ # First we will create a row of [key, weight, [[0] * iterations] * labels]
+ row = [None] * len(header)
+ row[0] = '%s (cycles)' % k
+ row[1] = 'N/A'
+ for label_index in xrange(2, len(row)):
+ row[label_index] = [0] * iterations
+
+ for cur_row in table[1:]:
+ # Iterate through each benchmark
+ if len(cur_row) > 1:
+ for label_index in xrange(2, len(cur_row)):
+ # Iterate through each run in a single benchmark
+ # each result should look like ((pass, fail), [values_list])
+ runs = cur_row[label_index][1]
+ for index in xrange(iterations):
+ # Accumulate each run result to composite benchmark run
+ # If any run fails, then we set this run for composite benchmark
+ # to None so that we know it fails.
+ if runs[index] and row[label_index][index] != None:
+ row[label_index][index] += runs[index]
+ else:
+ row[label_index][index] = None
+ else:
+ # One benchmark totally fails, no valid data will be in final result
+ for label_index in xrange(2, len(row)):
+ row[label_index] = [None] * iterations
+ break
+ # Calculate pass and fail count for composite benchmark
+ for label_index in xrange(2, len(row)):
+ run_pass = 0
+ run_fail = 0
+ for run in row[label_index]:
+ if run:
+ run_pass += 1
+ else:
+ run_fail += 1
+ row[label_index] = ((run_pass, run_fail), row[label_index])
+ table.append(row)
+
+ # Now that we have the table genearted, we want to store this new composite
+ # benchmark into the benchmark_result in ResultReport object.
+ # This will be used to generate a full table which contains our composite
+ # benchmark.
+ # We need to create composite benchmark result and add it to keyvals in
+ # benchmark_results.
+ v = []
+ for label in row[2:]:
+ # each label's result looks like ((pass, fail), [values])
+ runs = label[1]
+ # List of values of each label
+ single_run_list = []
+ for run in runs:
+ # Result of each run under the same label is a dict of keys.
+ # Here the only key we will add for composite benchmark is the
+ # weighted_cpu_cycles we added up.
+ one_dict = {}
+ if run:
+ one_dict[u'weighted_cpu_cycles'] = [run, u'cycles']
+ one_dict['retval'] = 0
+ else:
+ one_dict['retval'] = 1
+ single_run_list.append(one_dict)
+ v.append(single_run_list)
+
+ self._runs[k] = v
+ self._iter_counts[k] = iterations
+
+ return (table, self._runs, self._iter_counts)
+
+
class Result(object):
"""A class that respresents a single result.
@@ -345,11 +528,13 @@ class AmeanResult(StringMeanResult):
def _ComputeFloat(self, cell, values, baseline_values):
cell.value = numpy.mean(values)
-
class RawResult(Result):
"""Raw result."""
pass
+class IterationResult(Result):
+ """Iteration result."""
+ pass
class MinResult(Result):
"""Minimum."""
@@ -468,7 +653,7 @@ class KeyAwareComparisonResult(ComparisonResult):
'dropped_percent', '(ms)', '(seconds)', '--ms',
'--average_num_missing_tiles', '--experimental_jank',
'--experimental_mean_frame', '--experimental_median_frame_time',
- '--total_deferred_image_decode_count', '--seconds'
+ '--total_deferred_image_decode_count', '--seconds', 'cycles'
]
return any([l in key for l in lower_is_better_keys])
@@ -617,6 +802,13 @@ class PValueFormat(Format):
power=1)
+class WeightFormat(Format):
+ """Formatting for weight in cwp mode."""
+
+ def _ComputeFloat(self, cell):
+ cell.string_value = '%0.4f' % float(cell.value)
+
+
class StorageFormat(Format):
"""Format the cell as a storage number.
@@ -777,15 +969,18 @@ class TableFormatter(object):
formats to apply to the table and returns a table of cells.
"""
- def __init__(self, table, columns):
+ def __init__(self, table, columns, cpu_table=False):
"""The constructor takes in a table and a list of columns.
Args:
table: A list of lists of values.
columns: A list of column containing what to produce and how to format it.
+ cpu_table: A flag to check whether we are generating a table of cpu cycles
+ in CWP apporximation mode.
"""
self._table = table
self._columns = columns
+ self._cpu_table = cpu_table
self._table_columns = []
self._out_table = []
@@ -794,30 +989,48 @@ class TableFormatter(object):
all_failed = False
for row in self._table[1:]:
+ # If we are generating cpu_table, the second value will be weight rather
+ # than values.
+ start_col = 2 if self._cpu_table else 1
# It does not make sense to put retval in the summary table.
if str(row[0]) == 'retval' and table_type == 'summary':
# Check to see if any runs passed, and update all_failed.
all_failed = True
- for values in row[1:]:
+ for values in row[start_col:]:
if 0 in values:
all_failed = False
continue
key = Cell()
key.string_value = str(row[0])
out_row = [key]
+ if self._cpu_table:
+ # Add one column for weight if in cpu_table mode
+ weight = Cell()
+ weight.value = row[1]
+ f = WeightFormat()
+ f.Compute(weight)
+ out_row.append(weight)
baseline = None
- for values in row[1:]:
- for column in self._columns:
+ for results in row[start_col:]:
+ column_start = 0
+ values = None
+ # If generating cpu table, we will split a tuple of iterations info
+ # from the results
+ if isinstance(results, tuple):
+ it, values = results
+ column_start = 1
+ cell = Cell()
+ cell.string_value = '[%d: %d]' % (it[0], it[1])
+ out_row.append(cell)
+ if not row_index:
+ self._table_columns.append(self._columns[0])
+ else:
+ values = results
+ # Parse each column
+ for column in self._columns[column_start:]:
cell = Cell()
cell.name = key.string_value
- if column.result.NeedsBaseline():
- if baseline is not None:
- column.result.Compute(cell, values, baseline)
- column.fmt.Compute(cell)
- out_row.append(cell)
- if not row_index:
- self._table_columns.append(column)
- else:
+ if not column.result.NeedsBaseline() or baseline is not None:
column.result.Compute(cell, values, baseline)
column.fmt.Compute(cell)
out_row.append(cell)
@@ -853,8 +1066,13 @@ class TableFormatter(object):
"""Generate Column name at the top of table."""
key = Cell()
key.header = True
- key.string_value = 'Keys'
+ key.string_value = 'Keys'if not self._cpu_table else 'Benchmarks'
header = [key]
+ if self._cpu_table:
+ weight = Cell()
+ weight.header = True
+ weight.string_value = 'Weights'
+ header.append(weight)
for column in self._table_columns:
cell = Cell()
cell.header = True
@@ -891,7 +1109,7 @@ class TableFormatter(object):
fails = fails + 1
return passes, fails
- def AddLabelName(self):
+ def AddLabelName(self, table_type):
"""Put label on the top of the table."""
top_header = []
base_colspan = len(
@@ -924,9 +1142,11 @@ class TableFormatter(object):
else:
cell.string_value = str(label)
if top_header:
- cell.colspan = base_colspan
+ if not self._cpu_table or (self._cpu_table and len(top_header) == 2):
+ cell.colspan = base_colspan
if len(top_header) > 1:
- cell.colspan = compare_colspan
+ if not self._cpu_table or (self._cpu_table and len(top_header) > 2):
+ cell.colspan = compare_colspan
top_header.append(cell)
column_position = column_position + 1
self._out_table = [top_header] + self._out_table
@@ -958,7 +1178,7 @@ class TableFormatter(object):
self.GenerateCellTable(table_type)
if headers:
self.AddColumnName()
- self.AddLabelName()
+ self.AddLabelName(table_type)
return self._out_table
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index 67c3b93b..3ff58dd3 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -385,7 +385,7 @@ class Result(object):
# If we are in CWP approximation mode, we want to collect DSO CPU cycles
# for each perf.data file
if self.cwp_dso:
- self.keyvals['cpu_cycles'] = [self.GetCPUCycles(), u'count']
+ self.keyvals['cpu_cycles'] = [self.GetCPUCycles(), u'cycles']
self.keyvals['retval'] = self.retval
# Generate report from all perf.data files.
# Now parse all perf report files and include them in keyvals.
diff --git a/crosperf/results_organizer.py b/crosperf/results_organizer.py
index c39119bb..5410d6d8 100644
--- a/crosperf/results_organizer.py
+++ b/crosperf/results_organizer.py
@@ -188,9 +188,15 @@ def OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False):
else:
# Did not find test_name in json file; show everything.
show_all_results = True
- for test_key in benchmark_run.result.keyvals:
- if show_all_results or test_key in summary_list:
- cur_dict[test_key] = benchmark_run.result.keyvals[test_key]
+ if benchmark_run.result.cwp_dso:
+ # If we are in cwp approximation mode, we only care about cpu_cycles
+ if 'cpu_cycles' in benchmark_run.result.keyvals:
+ cur_dict['cpu_cycles'] = benchmark_run.result.keyvals['cpu_cycles']
+ cur_dict['retval'] = benchmark_run.result.keyvals['retval']
+ else:
+ for test_key in benchmark_run.result.keyvals:
+ if show_all_results or test_key in summary_list:
+ cur_dict[test_key] = benchmark_run.result.keyvals[test_key]
# Occasionally Telemetry tests will not fail but they will not return a
# result, either. Look for those cases, and force them to be a fail.
# (This can happen if, for example, the test has been disabled.)
diff --git a/crosperf/results_report.py b/crosperf/results_report.py
index fac044fb..c766f9b7 100644
--- a/crosperf/results_report.py
+++ b/crosperf/results_report.py
@@ -16,7 +16,9 @@ from cros_utils.tabulator import Cell
from cros_utils.tabulator import CoeffVarFormat
from cros_utils.tabulator import CoeffVarResult
from cros_utils.tabulator import Column
+from cros_utils.tabulator import CPUTableGenerator
from cros_utils.tabulator import Format
+from cros_utils.tabulator import IterationResult
from cros_utils.tabulator import GmeanRatioResult
from cros_utils.tabulator import LiteralResult
from cros_utils.tabulator import MaxResult
@@ -157,6 +159,12 @@ def _GetResultsTableHeader(ben_name, iterations):
cell.header = True
return [[cell]]
+def _GetDSOHeader(cwp_dso):
+ info = 'CWP_DSO: %s' % cwp_dso
+ cell = Cell()
+ cell.string_value = info
+ cell.header = False
+ return [[cell]]
def _ParseColumn(columns, iteration):
new_column = []
@@ -222,6 +230,20 @@ def _GetPerfTables(benchmark_results, columns, table_type):
tables.append(table)
return tables
+def _GetCPUTables(benchmark_results, columns, table_type):
+ tables = []
+ dso_header_table = _GetDSOHeader(benchmark_results.cwp_dso)
+ tables.append(dso_header_table)
+ (table, new_keyvals, iter_counts) = CPUTableGenerator(
+ benchmark_results.run_keyvals,
+ benchmark_results.label_names,
+ benchmark_results.iter_counts,
+ benchmark_results.weights).GetTable()
+ parsed_columns = _ParseColumn(columns, 1)
+ tf = TableFormatter(table, parsed_columns, cpu_table=True)
+ cell_table = tf.GetCellTable(table_type)
+ tables.append(cell_table)
+ return (tables, new_keyvals, iter_counts)
class ResultsReport(object):
"""Class to handle the report format."""
@@ -231,9 +253,22 @@ class ResultsReport(object):
def __init__(self, results):
self.benchmark_results = results
- def _GetTablesWithColumns(self, columns, table_type, perf):
- get_tables = _GetPerfTables if perf else _GetTables
- return get_tables(self.benchmark_results, columns, table_type)
+ def _GetTablesWithColumns(self, columns, table_type, summary_type):
+ if summary_type == 'perf':
+ get_tables = _GetPerfTables
+ elif summary_type == 'cpu':
+ get_tables = _GetCPUTables
+ else:
+ get_tables = _GetTables
+ ret = get_tables(self.benchmark_results, columns, table_type)
+ # If we are generating a CPU summary table, the return value of get_tables
+ # will be a tuple, and we will update the benchmark_results for composite
+ # benchmark so that full table can use it.
+ if isinstance(ret, tuple):
+ self.benchmark_results.run_keyvals = ret[1]
+ self.benchmark_results.iter_counts = ret[2]
+ ret = ret[0]
+ return ret
def GetFullTables(self, perf=False):
columns = [
@@ -246,15 +281,21 @@ class ResultsReport(object):
]
return self._GetTablesWithColumns(columns, 'full', perf)
- def GetSummaryTables(self, perf=False):
- columns = [
- Column(AmeanResult(), Format()), Column(StdResult(), Format(),
- 'StdDev'),
- Column(CoeffVarResult(), CoeffVarFormat(), 'StdDev/Mean'), Column(
- GmeanRatioResult(), RatioFormat(), 'GmeanSpeedup'), Column(
- PValueResult(), PValueFormat(), 'p-value')
- ]
- return self._GetTablesWithColumns(columns, 'summary', perf)
+ def GetSummaryTables(self, summary_type=''):
+ if summary_type == 'cpu':
+ columns = [Column(IterationResult(), Format(), 'Iterations [Pass:Fail]'),
+ Column(AmeanResult(), Format(), 'Weighted CPU-cycles Amean'),
+ Column(StdResult(), Format(), 'StdDev'),
+ Column(CoeffVarResult(), CoeffVarFormat(), 'StdDev/Mean'),
+ Column(GmeanRatioResult(), RatioFormat(), 'GmeanSpeedup'),
+ Column(PValueResult(), PValueFormat(), 'p-value')]
+ else:
+ columns = [Column(AmeanResult(), Format()),
+ Column(StdResult(), Format(), 'StdDev'),
+ Column(CoeffVarResult(), CoeffVarFormat(), 'StdDev/Mean'),
+ Column(GmeanRatioResult(), RatioFormat(), 'GmeanSpeedup'),
+ Column(PValueResult(), PValueFormat(), 'p-value')]
+ return self._GetTablesWithColumns(columns, 'summary', summary_type)
def _PrintTable(tables, out_to):
@@ -336,15 +377,20 @@ class TextResultsReport(ResultsReport):
title_contents = 'Results report'
sections.append(self._MakeTitle(title_contents))
- summary_table = _PrintTable(self.GetSummaryTables(perf=False), output_type)
+ if not self.experiment.cwp_dso:
+ summary_table = _PrintTable(self.GetSummaryTables(), output_type)
+ else:
+ summary_table = _PrintTable(self.GetSummaryTables(summary_type='cpu'),
+ output_type)
sections.append(self._MakeSection('Summary', summary_table))
if experiment is not None:
table = _PrintTable(self.GetStatusTable(), output_type)
sections.append(self._MakeSection('Benchmark Run Status', table))
- perf_table = _PrintTable(self.GetSummaryTables(perf=True), output_type)
- if perf_table:
+ perf_table = _PrintTable(self.GetSummaryTables(summary_type='perf'),
+ output_type)
+ if perf_table and not self.experiment.cwp_dso:
sections.append(self._MakeSection('Perf Data', perf_table))
if experiment is not None:
@@ -414,9 +460,14 @@ class HTMLResultsReport(ResultsReport):
chart_javascript = ''.join(chart.GetJavascript() for chart in charts)
chart_divs = ''.join(chart.GetDiv() for chart in charts)
- summary_table = self.GetSummaryTables()
+ if not self.experiment.cwp_dso:
+ summary_table = self.GetSummaryTables()
+ perf_table = self.GetSummaryTables(summary_type='perf')
+ else:
+ summary_table = self.GetSummaryTables(summary_type='cpu')
+ perf_table = None
full_table = self.GetFullTables()
- perf_table = self.GetSummaryTables(perf=True)
+
experiment_file = ''
if self.experiment is not None:
experiment_file = self.experiment.experiment_file
@@ -540,7 +591,9 @@ class BenchmarkResults(object):
label_names,
benchmark_names_and_iterations,
run_keyvals,
- read_perf_report=None):
+ read_perf_report=None,
+ cwp_dso=None,
+ weights=None):
if read_perf_report is None:
def _NoPerfReport(*_args, **_kwargs):
@@ -553,6 +606,8 @@ class BenchmarkResults(object):
self.iter_counts = dict(benchmark_names_and_iterations)
self.run_keyvals = run_keyvals
self.read_perf_report = read_perf_report
+ self.cwp_dso = cwp_dso
+ self.weights = dict(weights)
@staticmethod
def FromExperiment(experiment, for_json_report=False):
@@ -562,8 +617,11 @@ class BenchmarkResults(object):
run_keyvals = _ExperimentToKeyvals(experiment, for_json_report)
read_perf_report = functools.partial(_ReadExperimentPerfReport,
experiment.results_directory)
+ cwp_dso = experiment.cwp_dso
+ weights = [(benchmark.name, benchmark.weight)
+ for benchmark in experiment.benchmarks]
return BenchmarkResults(label_names, benchmark_names_and_iterations,
- run_keyvals, read_perf_report)
+ run_keyvals, read_perf_report, cwp_dso, weights)
def _GetElemByName(name, from_list):
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-05 06:20:43 +0000