diff options
-rw-r--r-- | cros_utils/tabulator.py | 258 | ||||
-rw-r--r-- | crosperf/results_cache.py | 2 | ||||
-rw-r--r-- | crosperf/results_organizer.py | 12 | ||||
-rw-r--r-- | crosperf/results_report.py | 96 |
4 files changed, 326 insertions, 42 deletions
diff --git a/cros_utils/tabulator.py b/cros_utils/tabulator.py index 6936d35f..687ffeb8 100644 --- a/cros_utils/tabulator.py +++ b/cros_utils/tabulator.py @@ -207,6 +207,189 @@ class TableGenerator(object): return table +class CPUTableGenerator(TableGenerator): + """Creates a table with only cpu cycles from the results + + The main public function is called GetTable(). + + Different than TableGenerator, self._runs is now a dict of {benchmark: runs} + We are expecting there is 'cpu_cycles' in `runs`. + """ + def __init__(self, run_keyvals, labels, iter_counts, weights): + TableGenerator.__init__(self, run_keyvals, labels, key_name='Benchmarks') + self._iter_counts = iter_counts + self._weights = weights + + def _GetKeys(self): + keys = self._runs.keys() + return self._SortKeys(keys) + + def GetTable(self, number_of_rows=sys.maxint): + """Returns a tuple, which contains three args: + 1) a table from a list of list of dicts. + 2) updated benchmark_results run_keyvals with composite benchmark + 3) updated benchmark_results iter_count with composite benchmark + + The dict of list of list of dicts is passed into the constructor of + CPUTableGenerator. + This method converts that into a canonical list of lists which represents a + table of values. + + Args: + number_of_rows: Maximum number of rows to return from the table. + + Returns: + A list of lists which is the table. + + Example: + We have the following runs: + {bench1: [[{"cpu_cycles": "v1"}, {"cpu_cycles": "v2"}], + [{"cpu_cycles": "v3"}, {"cpu_cycles": "v4"}]] + bench2: [[{"cpu_cycles": "v21"}, None], + [{"cpu_cycles": "v22"}, {"cpu_cycles": "v23"}]]} + and weights of benchmarks: + {bench1: w1, bench2: w2} + and the following labels: + ["vanilla", "modified"] + it will return: + [["Benchmark", "Weights", "vanilla", "modified"] + ["bench1", w1, + ((2, 0), ["v1*w1", "v2*w1"]), ((2, 0), ["v3*w1", "v4*w1"])] + ["bench2", w2, + ((1, 1), ["v21*w2", None]), ((2, 0), ["v22*w2", "v23*w2"])] + ["Composite Benchmark", N/A, + ((1, 1), ["v1*w1+v21*w2", None]), + ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]] + The returned table can then be processed further by other classes in this + module. + """ + keys = self._GetKeys() + header = [self._key_name, 'Weights'] + self._labels + table = [header] + rows = 0 + iterations = 0 + + for k in keys: + runs = self._runs[k] + unit = None + all_runs_empty = all(not dict for label in runs for dict in label) + if all_runs_empty: + cell = Cell() + cell.string_value = 'Benchmark %s contains no result.' + \ + ' Is the benchmark name valid?' % k + table.append([cell]) + else: + row = [k] + row.append(self._weights[k]) + for run_list in runs: + run_pass = 0 + run_fail = 0 + v = [] + for run in run_list: + if 'cpu_cycles' in run: + if type(run['cpu_cycles']) is list: + val = run['cpu_cycles'][0] * self._weights[k] + unit = run['cpu_cycles'][1] + else: + val = run['cpu_cycles'] + v.append(val) + run_pass += 1 + else: + v.append(None) + run_fail += 1 + t = ((run_pass, run_fail), v) + if iterations != 0 and iterations != run_pass + run_fail: + raise ValueError('Iterations of each benchmark run ' \ + 'are not the same') + iterations = run_pass + run_fail + row.append(t) + if unit: + keyname = row[0] + ' (%s) ' % unit + row[0] = keyname + table.append(row) + rows += 1 + if rows == number_of_rows: + break + + k = 'Composite Benchmark' + if k in keys: + raise RuntimeError('Composite benchmark already exists in results') + + # Create a new composite benchmark row at the bottom of the summary table + # The new row will be like the format in example: + # ["Composite Benchmark", N/A, + # ((1, 1), ["v1*w1+v21*w2", None]), + # ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]] + # First we will create a row of [key, weight, [[0] * iterations] * labels] + row = [None] * len(header) + row[0] = '%s (cycles)' % k + row[1] = 'N/A' + for label_index in xrange(2, len(row)): + row[label_index] = [0] * iterations + + for cur_row in table[1:]: + # Iterate through each benchmark + if len(cur_row) > 1: + for label_index in xrange(2, len(cur_row)): + # Iterate through each run in a single benchmark + # each result should look like ((pass, fail), [values_list]) + runs = cur_row[label_index][1] + for index in xrange(iterations): + # Accumulate each run result to composite benchmark run + # If any run fails, then we set this run for composite benchmark + # to None so that we know it fails. + if runs[index] and row[label_index][index] != None: + row[label_index][index] += runs[index] + else: + row[label_index][index] = None + else: + # One benchmark totally fails, no valid data will be in final result + for label_index in xrange(2, len(row)): + row[label_index] = [None] * iterations + break + # Calculate pass and fail count for composite benchmark + for label_index in xrange(2, len(row)): + run_pass = 0 + run_fail = 0 + for run in row[label_index]: + if run: + run_pass += 1 + else: + run_fail += 1 + row[label_index] = ((run_pass, run_fail), row[label_index]) + table.append(row) + + # Now that we have the table genearted, we want to store this new composite + # benchmark into the benchmark_result in ResultReport object. + # This will be used to generate a full table which contains our composite + # benchmark. + # We need to create composite benchmark result and add it to keyvals in + # benchmark_results. + v = [] + for label in row[2:]: + # each label's result looks like ((pass, fail), [values]) + runs = label[1] + # List of values of each label + single_run_list = [] + for run in runs: + # Result of each run under the same label is a dict of keys. + # Here the only key we will add for composite benchmark is the + # weighted_cpu_cycles we added up. + one_dict = {} + if run: + one_dict[u'weighted_cpu_cycles'] = [run, u'cycles'] + one_dict['retval'] = 0 + else: + one_dict['retval'] = 1 + single_run_list.append(one_dict) + v.append(single_run_list) + + self._runs[k] = v + self._iter_counts[k] = iterations + + return (table, self._runs, self._iter_counts) + + class Result(object): """A class that respresents a single result. @@ -345,11 +528,13 @@ class AmeanResult(StringMeanResult): def _ComputeFloat(self, cell, values, baseline_values): cell.value = numpy.mean(values) - class RawResult(Result): """Raw result.""" pass +class IterationResult(Result): + """Iteration result.""" + pass class MinResult(Result): """Minimum.""" @@ -468,7 +653,7 @@ class KeyAwareComparisonResult(ComparisonResult): 'dropped_percent', '(ms)', '(seconds)', '--ms', '--average_num_missing_tiles', '--experimental_jank', '--experimental_mean_frame', '--experimental_median_frame_time', - '--total_deferred_image_decode_count', '--seconds' + '--total_deferred_image_decode_count', '--seconds', 'cycles' ] return any([l in key for l in lower_is_better_keys]) @@ -617,6 +802,13 @@ class PValueFormat(Format): power=1) +class WeightFormat(Format): + """Formatting for weight in cwp mode.""" + + def _ComputeFloat(self, cell): + cell.string_value = '%0.4f' % float(cell.value) + + class StorageFormat(Format): """Format the cell as a storage number. @@ -777,15 +969,18 @@ class TableFormatter(object): formats to apply to the table and returns a table of cells. """ - def __init__(self, table, columns): + def __init__(self, table, columns, cpu_table=False): """The constructor takes in a table and a list of columns. Args: table: A list of lists of values. columns: A list of column containing what to produce and how to format it. + cpu_table: A flag to check whether we are generating a table of cpu cycles + in CWP apporximation mode. """ self._table = table self._columns = columns + self._cpu_table = cpu_table self._table_columns = [] self._out_table = [] @@ -794,30 +989,48 @@ class TableFormatter(object): all_failed = False for row in self._table[1:]: + # If we are generating cpu_table, the second value will be weight rather + # than values. + start_col = 2 if self._cpu_table else 1 # It does not make sense to put retval in the summary table. if str(row[0]) == 'retval' and table_type == 'summary': # Check to see if any runs passed, and update all_failed. all_failed = True - for values in row[1:]: + for values in row[start_col:]: if 0 in values: all_failed = False continue key = Cell() key.string_value = str(row[0]) out_row = [key] + if self._cpu_table: + # Add one column for weight if in cpu_table mode + weight = Cell() + weight.value = row[1] + f = WeightFormat() + f.Compute(weight) + out_row.append(weight) baseline = None - for values in row[1:]: - for column in self._columns: + for results in row[start_col:]: + column_start = 0 + values = None + # If generating cpu table, we will split a tuple of iterations info + # from the results + if isinstance(results, tuple): + it, values = results + column_start = 1 + cell = Cell() + cell.string_value = '[%d: %d]' % (it[0], it[1]) + out_row.append(cell) + if not row_index: + self._table_columns.append(self._columns[0]) + else: + values = results + # Parse each column + for column in self._columns[column_start:]: cell = Cell() cell.name = key.string_value - if column.result.NeedsBaseline(): - if baseline is not None: - column.result.Compute(cell, values, baseline) - column.fmt.Compute(cell) - out_row.append(cell) - if not row_index: - self._table_columns.append(column) - else: + if not column.result.NeedsBaseline() or baseline is not None: column.result.Compute(cell, values, baseline) column.fmt.Compute(cell) out_row.append(cell) @@ -853,8 +1066,13 @@ class TableFormatter(object): """Generate Column name at the top of table.""" key = Cell() key.header = True - key.string_value = 'Keys' + key.string_value = 'Keys'if not self._cpu_table else 'Benchmarks' header = [key] + if self._cpu_table: + weight = Cell() + weight.header = True + weight.string_value = 'Weights' + header.append(weight) for column in self._table_columns: cell = Cell() cell.header = True @@ -891,7 +1109,7 @@ class TableFormatter(object): fails = fails + 1 return passes, fails - def AddLabelName(self): + def AddLabelName(self, table_type): """Put label on the top of the table.""" top_header = [] base_colspan = len( @@ -924,9 +1142,11 @@ class TableFormatter(object): else: cell.string_value = str(label) if top_header: - cell.colspan = base_colspan + if not self._cpu_table or (self._cpu_table and len(top_header) == 2): + cell.colspan = base_colspan if len(top_header) > 1: - cell.colspan = compare_colspan + if not self._cpu_table or (self._cpu_table and len(top_header) > 2): + cell.colspan = compare_colspan top_header.append(cell) column_position = column_position + 1 self._out_table = [top_header] + self._out_table @@ -958,7 +1178,7 @@ class TableFormatter(object): self.GenerateCellTable(table_type) if headers: self.AddColumnName() - self.AddLabelName() + self.AddLabelName(table_type) return self._out_table diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index 67c3b93b..3ff58dd3 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -385,7 +385,7 @@ class Result(object): # If we are in CWP approximation mode, we want to collect DSO CPU cycles # for each perf.data file if self.cwp_dso: - self.keyvals['cpu_cycles'] = [self.GetCPUCycles(), u'count'] + self.keyvals['cpu_cycles'] = [self.GetCPUCycles(), u'cycles'] self.keyvals['retval'] = self.retval # Generate report from all perf.data files. # Now parse all perf report files and include them in keyvals. diff --git a/crosperf/results_organizer.py b/crosperf/results_organizer.py index c39119bb..5410d6d8 100644 --- a/crosperf/results_organizer.py +++ b/crosperf/results_organizer.py @@ -188,9 +188,15 @@ def OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False): else: # Did not find test_name in json file; show everything. show_all_results = True - for test_key in benchmark_run.result.keyvals: - if show_all_results or test_key in summary_list: - cur_dict[test_key] = benchmark_run.result.keyvals[test_key] + if benchmark_run.result.cwp_dso: + # If we are in cwp approximation mode, we only care about cpu_cycles + if 'cpu_cycles' in benchmark_run.result.keyvals: + cur_dict['cpu_cycles'] = benchmark_run.result.keyvals['cpu_cycles'] + cur_dict['retval'] = benchmark_run.result.keyvals['retval'] + else: + for test_key in benchmark_run.result.keyvals: + if show_all_results or test_key in summary_list: + cur_dict[test_key] = benchmark_run.result.keyvals[test_key] # Occasionally Telemetry tests will not fail but they will not return a # result, either. Look for those cases, and force them to be a fail. # (This can happen if, for example, the test has been disabled.) diff --git a/crosperf/results_report.py b/crosperf/results_report.py index fac044fb..c766f9b7 100644 --- a/crosperf/results_report.py +++ b/crosperf/results_report.py @@ -16,7 +16,9 @@ from cros_utils.tabulator import Cell from cros_utils.tabulator import CoeffVarFormat from cros_utils.tabulator import CoeffVarResult from cros_utils.tabulator import Column +from cros_utils.tabulator import CPUTableGenerator from cros_utils.tabulator import Format +from cros_utils.tabulator import IterationResult from cros_utils.tabulator import GmeanRatioResult from cros_utils.tabulator import LiteralResult from cros_utils.tabulator import MaxResult @@ -157,6 +159,12 @@ def _GetResultsTableHeader(ben_name, iterations): cell.header = True return [[cell]] +def _GetDSOHeader(cwp_dso): + info = 'CWP_DSO: %s' % cwp_dso + cell = Cell() + cell.string_value = info + cell.header = False + return [[cell]] def _ParseColumn(columns, iteration): new_column = [] @@ -222,6 +230,20 @@ def _GetPerfTables(benchmark_results, columns, table_type): tables.append(table) return tables +def _GetCPUTables(benchmark_results, columns, table_type): + tables = [] + dso_header_table = _GetDSOHeader(benchmark_results.cwp_dso) + tables.append(dso_header_table) + (table, new_keyvals, iter_counts) = CPUTableGenerator( + benchmark_results.run_keyvals, + benchmark_results.label_names, + benchmark_results.iter_counts, + benchmark_results.weights).GetTable() + parsed_columns = _ParseColumn(columns, 1) + tf = TableFormatter(table, parsed_columns, cpu_table=True) + cell_table = tf.GetCellTable(table_type) + tables.append(cell_table) + return (tables, new_keyvals, iter_counts) class ResultsReport(object): """Class to handle the report format.""" @@ -231,9 +253,22 @@ class ResultsReport(object): def __init__(self, results): self.benchmark_results = results - def _GetTablesWithColumns(self, columns, table_type, perf): - get_tables = _GetPerfTables if perf else _GetTables - return get_tables(self.benchmark_results, columns, table_type) + def _GetTablesWithColumns(self, columns, table_type, summary_type): + if summary_type == 'perf': + get_tables = _GetPerfTables + elif summary_type == 'cpu': + get_tables = _GetCPUTables + else: + get_tables = _GetTables + ret = get_tables(self.benchmark_results, columns, table_type) + # If we are generating a CPU summary table, the return value of get_tables + # will be a tuple, and we will update the benchmark_results for composite + # benchmark so that full table can use it. + if isinstance(ret, tuple): + self.benchmark_results.run_keyvals = ret[1] + self.benchmark_results.iter_counts = ret[2] + ret = ret[0] + return ret def GetFullTables(self, perf=False): columns = [ @@ -246,15 +281,21 @@ class ResultsReport(object): ] return self._GetTablesWithColumns(columns, 'full', perf) - def GetSummaryTables(self, perf=False): - columns = [ - Column(AmeanResult(), Format()), Column(StdResult(), Format(), - 'StdDev'), - Column(CoeffVarResult(), CoeffVarFormat(), 'StdDev/Mean'), Column( - GmeanRatioResult(), RatioFormat(), 'GmeanSpeedup'), Column( - PValueResult(), PValueFormat(), 'p-value') - ] - return self._GetTablesWithColumns(columns, 'summary', perf) + def GetSummaryTables(self, summary_type=''): + if summary_type == 'cpu': + columns = [Column(IterationResult(), Format(), 'Iterations [Pass:Fail]'), + Column(AmeanResult(), Format(), 'Weighted CPU-cycles Amean'), + Column(StdResult(), Format(), 'StdDev'), + Column(CoeffVarResult(), CoeffVarFormat(), 'StdDev/Mean'), + Column(GmeanRatioResult(), RatioFormat(), 'GmeanSpeedup'), + Column(PValueResult(), PValueFormat(), 'p-value')] + else: + columns = [Column(AmeanResult(), Format()), + Column(StdResult(), Format(), 'StdDev'), + Column(CoeffVarResult(), CoeffVarFormat(), 'StdDev/Mean'), + Column(GmeanRatioResult(), RatioFormat(), 'GmeanSpeedup'), + Column(PValueResult(), PValueFormat(), 'p-value')] + return self._GetTablesWithColumns(columns, 'summary', summary_type) def _PrintTable(tables, out_to): @@ -336,15 +377,20 @@ class TextResultsReport(ResultsReport): title_contents = 'Results report' sections.append(self._MakeTitle(title_contents)) - summary_table = _PrintTable(self.GetSummaryTables(perf=False), output_type) + if not self.experiment.cwp_dso: + summary_table = _PrintTable(self.GetSummaryTables(), output_type) + else: + summary_table = _PrintTable(self.GetSummaryTables(summary_type='cpu'), + output_type) sections.append(self._MakeSection('Summary', summary_table)) if experiment is not None: table = _PrintTable(self.GetStatusTable(), output_type) sections.append(self._MakeSection('Benchmark Run Status', table)) - perf_table = _PrintTable(self.GetSummaryTables(perf=True), output_type) - if perf_table: + perf_table = _PrintTable(self.GetSummaryTables(summary_type='perf'), + output_type) + if perf_table and not self.experiment.cwp_dso: sections.append(self._MakeSection('Perf Data', perf_table)) if experiment is not None: @@ -414,9 +460,14 @@ class HTMLResultsReport(ResultsReport): chart_javascript = ''.join(chart.GetJavascript() for chart in charts) chart_divs = ''.join(chart.GetDiv() for chart in charts) - summary_table = self.GetSummaryTables() + if not self.experiment.cwp_dso: + summary_table = self.GetSummaryTables() + perf_table = self.GetSummaryTables(summary_type='perf') + else: + summary_table = self.GetSummaryTables(summary_type='cpu') + perf_table = None full_table = self.GetFullTables() - perf_table = self.GetSummaryTables(perf=True) + experiment_file = '' if self.experiment is not None: experiment_file = self.experiment.experiment_file @@ -540,7 +591,9 @@ class BenchmarkResults(object): label_names, benchmark_names_and_iterations, run_keyvals, - read_perf_report=None): + read_perf_report=None, + cwp_dso=None, + weights=None): if read_perf_report is None: def _NoPerfReport(*_args, **_kwargs): @@ -553,6 +606,8 @@ class BenchmarkResults(object): self.iter_counts = dict(benchmark_names_and_iterations) self.run_keyvals = run_keyvals self.read_perf_report = read_perf_report + self.cwp_dso = cwp_dso + self.weights = dict(weights) @staticmethod def FromExperiment(experiment, for_json_report=False): @@ -562,8 +617,11 @@ class BenchmarkResults(object): run_keyvals = _ExperimentToKeyvals(experiment, for_json_report) read_perf_report = functools.partial(_ReadExperimentPerfReport, experiment.results_directory) + cwp_dso = experiment.cwp_dso + weights = [(benchmark.name, benchmark.weight) + for benchmark in experiment.benchmarks] return BenchmarkResults(label_names, benchmark_names_and_iterations, - run_keyvals, read_perf_report) + run_keyvals, read_perf_report, cwp_dso, weights) def _GetElemByName(name, from_list): |