# -*- coding: utf-8 -*- # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Table generating, analyzing and printing functions. This defines several classes that are used to generate, analyze and print tables. Example usage: from cros_utils import tabulator data = [["benchmark1", "33", "44"],["benchmark2", "44", "33"]] tabulator.GetSimpleTable(data) You could also use it to generate more complex tables with analysis such as p-values, custom colors, etc. Tables are generated by TableGenerator and analyzed/formatted by TableFormatter. TableFormatter can take in a list of columns with custom result computation and coloring, and will compare values in each row according to taht scheme. Here is a complex example on printing a table: from cros_utils import tabulator runs = [[{"k1": "10", "k2": "12", "k5": "40", "k6": "40", "ms_1": "20", "k7": "FAIL", "k8": "PASS", "k9": "PASS", "k10": "0"}, {"k1": "13", "k2": "14", "k3": "15", "ms_1": "10", "k8": "PASS", "k9": "FAIL", "k10": "0"}], [{"k1": "50", "k2": "51", "k3": "52", "k4": "53", "k5": "35", "k6": "45", "ms_1": "200", "ms_2": "20", "k7": "FAIL", "k8": "PASS", "k9": "PASS"}]] labels = ["vanilla", "modified"] tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) table = tg.GetTable() columns = [Column(LiteralResult(), Format(), "Literal"), Column(AmeanResult(), Format()), Column(StdResult(), Format()), Column(CoeffVarResult(), CoeffVarFormat()), Column(NonEmptyCountResult(), Format()), Column(AmeanRatioResult(), PercentFormat()), Column(AmeanRatioResult(), RatioFormat()), Column(GmeanRatioResult(), RatioFormat()), Column(PValueResult(), PValueFormat()), ] tf = TableFormatter(table, columns) cell_table = tf.GetCellTable() tp = TablePrinter(cell_table, out_to) print tp.Print() """ from __future__ import division from __future__ import print_function import collections import getpass import math import statistics import sys # TODO(crbug.com/980719): Drop scipy in the future. # pylint: disable=import-error import scipy from cros_utils.email_sender import EmailSender from cros_utils import misc def _AllFloat(values): return all([misc.IsFloat(v) for v in values]) def _GetFloats(values): return [float(v) for v in values] def _StripNone(results): res = [] for result in results: if result is not None: res.append(result) return res def _RemoveMinMax(cell, values): if len(values) < 3: print('WARNING: Values count is less than 3, not ignoring min/max values') print('WARNING: Cell name:', cell.name, 'Values:', values) return values values.remove(min(values)) values.remove(max(values)) return values class TableGenerator(object): """Creates a table from a list of list of dicts. The main public function is called GetTable(). """ SORT_BY_KEYS = 0 SORT_BY_KEYS_DESC = 1 SORT_BY_VALUES = 2 SORT_BY_VALUES_DESC = 3 NO_SORT = 4 MISSING_VALUE = 'x' def __init__(self, d, l, sort=NO_SORT, key_name='keys'): self._runs = d self._labels = l self._sort = sort self._key_name = key_name def _AggregateKeys(self): keys = collections.OrderedDict() for run_list in self._runs: for run in run_list: keys.update(dict.fromkeys(run.keys())) return list(keys.keys()) def _GetHighestValue(self, key): values = [] for run_list in self._runs: for run in run_list: if key in run: values.append(run[key]) values = _StripNone(values) if _AllFloat(values): values = _GetFloats(values) return max(values) def _GetLowestValue(self, key): values = [] for run_list in self._runs: for run in run_list: if key in run: values.append(run[key]) values = _StripNone(values) if _AllFloat(values): values = _GetFloats(values) return min(values) def _SortKeys(self, keys): if self._sort == self.SORT_BY_KEYS: return sorted(keys) elif self._sort == self.SORT_BY_VALUES: # pylint: disable=unnecessary-lambda return sorted(keys, key=lambda x: self._GetLowestValue(x)) elif self._sort == self.SORT_BY_VALUES_DESC: # pylint: disable=unnecessary-lambda return sorted(keys, key=lambda x: self._GetHighestValue(x), reverse=True) elif self._sort == self.NO_SORT: return keys else: assert 0, 'Unimplemented sort %s' % self._sort def _GetKeys(self): keys = self._AggregateKeys() return self._SortKeys(keys) def GetTable(self, number_of_rows=sys.maxsize): """Returns a table from a list of list of dicts. Examples: We have the following runs: [[{"k1": "v1", "k2": "v2"}, {"k1": "v3"}], [{"k1": "v4", "k4": "v5"}]] and the following labels: ["vanilla", "modified"] it will return: [["Key", "vanilla", "modified"] ["k1", ["v1", "v3"], ["v4"]] ["k2", ["v2"], []] ["k4", [], ["v5"]]] The returned table can then be processed further by other classes in this module. The list of list of dicts is passed into the constructor of TableGenerator. This method converts that into a canonical list of lists which represents a table of values. Args: number_of_rows: Maximum number of rows to return from the table. Returns: A list of lists which is the table. """ keys = self._GetKeys() header = [self._key_name] + self._labels table = [header] rows = 0 for k in keys: row = [k] unit = None for run_list in self._runs: v = [] for run in run_list: if k in run: if isinstance(run[k], list): val = run[k][0] unit = run[k][1] else: val = run[k] v.append(val) else: v.append(None) row.append(v) # If we got a 'unit' value, append the units name to the key name. if unit: keyname = row[0] + ' (%s) ' % unit row[0] = keyname table.append(row) rows += 1 if rows == number_of_rows: break return table class SamplesTableGenerator(TableGenerator): """Creates a table with only samples from the results The main public function is called GetTable(). Different than TableGenerator, self._runs is now a dict of {benchmark: runs} We are expecting there is 'samples' in `runs`. """ def __init__(self, run_keyvals, label_list, iter_counts, weights): TableGenerator.__init__( self, run_keyvals, label_list, key_name='Benchmarks') self._iter_counts = iter_counts self._weights = weights def _GetKeys(self): keys = self._runs.keys() return self._SortKeys(keys) def GetTable(self, number_of_rows=sys.maxsize): """Returns a tuple, which contains three args: 1) a table from a list of list of dicts. 2) updated benchmark_results run_keyvals with composite benchmark 3) updated benchmark_results iter_count with composite benchmark The dict of list of list of dicts is passed into the constructor of SamplesTableGenerator. This method converts that into a canonical list of lists which represents a table of values. Examples: We have the following runs: {bench1: [[{"samples": "v1"}, {"samples": "v2"}], [{"samples": "v3"}, {"samples": "v4"}]] bench2: [[{"samples": "v21"}, None], [{"samples": "v22"}, {"samples": "v23"}]]} and weights of benchmarks: {bench1: w1, bench2: w2} and the following labels: ["vanilla", "modified"] it will return: [["Benchmark", "Weights", "vanilla", "modified"] ["bench1", w1, ((2, 0), ["v1*w1", "v2*w1"]), ((2, 0), ["v3*w1", "v4*w1"])] ["bench2", w2, ((1, 1), ["v21*w2", None]), ((2, 0), ["v22*w2", "v23*w2"])] ["Composite Benchmark", N/A, ((1, 1), ["v1*w1+v21*w2", None]), ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]] The returned table can then be processed further by other classes in this module. Args: number_of_rows: Maximum number of rows to return from the table. Returns: A list of lists which is the table. """ keys = self._GetKeys() header = [self._key_name, 'Weights'] + self._labels table = [header] rows = 0 iterations = 0 for k in keys: bench_runs = self._runs[k] unit = None all_runs_empty = all(not dict for label in bench_runs for dict in label) if all_runs_empty: cell = Cell() cell.string_value = ('Benchmark %s contains no result.' ' Is the benchmark name valid?' % k) table.append([cell]) else: row = [k] row.append(self._weights[k]) for run_list in bench_runs: run_pass = 0 run_fail = 0 v = [] for run in run_list: if 'samples' in run: if isinstance(run['samples'], list): val = run['samples'][0] * self._weights[k] unit = run['samples'][1] else: val = run['samples'] * self._weights[k] v.append(val) run_pass += 1 else: v.append(None) run_fail += 1 one_tuple = ((run_pass, run_fail), v) if iterations not in (0, run_pass + run_fail): raise ValueError('Iterations of each benchmark run ' \ 'are not the same') iterations = run_pass + run_fail row.append(one_tuple) if unit: keyname = row[0] + ' (%s) ' % unit row[0] = keyname table.append(row) rows += 1 if rows == number_of_rows: break k = 'Composite Benchmark' if k in keys: raise RuntimeError('Composite benchmark already exists in results') # Create a new composite benchmark row at the bottom of the summary table # The new row will be like the format in example: # ["Composite Benchmark", N/A, # ((1, 1), ["v1*w1+v21*w2", None]), # ((2, 0), ["v3*w1+v22*w2", "v4*w1+ v23*w2"])]] # First we will create a row of [key, weight, [[0] * iterations] * labels] row = [None] * len(header) row[0] = '%s (samples)' % k row[1] = 'N/A' for label_index in range(2, len(row)): row[label_index] = [0] * iterations for cur_row in table[1:]: # Iterate through each benchmark if len(cur_row) > 1: for label_index in range(2, len(cur_row)): # Iterate through each run in a single benchmark # each result should look like ((pass, fail), [values_list]) bench_runs = cur_row[label_index][1] for index in range(iterations): # Accumulate each run result to composite benchmark run # If any run fails, then we set this run for composite benchmark # to None so that we know it fails. if bench_runs[index] and row[label_index][index] is not None: row[label_index][index] += bench_runs[index] else: row[label_index][index] = None else: # One benchmark totally fails, no valid data will be in final result for label_index in range(2, len(row)): row[label_index] = [None] * iterations break # Calculate pass and fail count for composite benchmark for label_index in range(2, len(row)): run_pass = 0 run_fail = 0 for run in row[label_index]: if run: run_pass += 1 else: run_fail += 1 row[label_index] = ((run_pass, run_fail), row[label_index]) table.append(row) # Now that we have the table genearted, we want to store this new composite # benchmark into the benchmark_result in ResultReport object. # This will be used to generate a full table which contains our composite # benchmark. # We need to create composite benchmark result and add it to keyvals in # benchmark_results. v = [] for label in row[2:]: # each label's result looks like ((pass, fail), [values]) benchmark_runs = label[1] # List of values of each label single_run_list = [] for run in benchmark_runs: # Result of each run under the same label is a dict of keys. # Here the only key we will add for composite benchmark is the # weighted_samples we added up. one_dict = {} if run: one_dict[u'weighted_samples'] = [run, u'samples'] one_dict['retval'] = 0 else: one_dict['retval'] = 1 single_run_list.append(one_dict) v.append(single_run_list) self._runs[k] = v self._iter_counts[k] = iterations return (table, self._runs, self._iter_counts) class Result(object): """A class that respresents a single result. This single result is obtained by condensing the information from a list of runs and a list of baseline runs. """ def __init__(self): pass def _AllStringsSame(self, values): values_set = set(values) return len(values_set) == 1 def NeedsBaseline(self): return False # pylint: disable=unused-argument def _Literal(self, cell, values, baseline_values): cell.value = ' '.join([str(v) for v in values]) def _ComputeFloat(self, cell, values, baseline_values): self._Literal(cell, values, baseline_values) def _ComputeString(self, cell, values, baseline_values): self._Literal(cell, values, baseline_values) def _InvertIfLowerIsBetter(self, cell): pass def _GetGmean(self, values): if not values: return float('nan') if any([v < 0 for v in values]): return float('nan') if any([v == 0 for v in values]): return 0.0 log_list = [math.log(v) for v in values] gmean_log = sum(log_list) / len(log_list) return math.exp(gmean_log) def Compute(self, cell, values, baseline_values): """Compute the result given a list of values and baseline values. Args: cell: A cell data structure to populate. values: List of values. baseline_values: List of baseline values. Can be none if this is the baseline itself. """ all_floats = True values = _StripNone(values) if not values: cell.value = '' return if _AllFloat(values): float_values = _GetFloats(values) else: all_floats = False if baseline_values: baseline_values = _StripNone(baseline_values) if baseline_values: if _AllFloat(baseline_values): float_baseline_values = _GetFloats(baseline_values) else: all_floats = False else: if self.NeedsBaseline(): cell.value = '' return float_baseline_values = None if all_floats: self._ComputeFloat(cell, float_values, float_baseline_values) self._InvertIfLowerIsBetter(cell) else: self._ComputeString(cell, values, baseline_values) class LiteralResult(Result): """A literal result.""" def __init__(self, iteration=0): super(LiteralResult, self).__init__() self.iteration = iteration def Compute(self, cell, values, baseline_values): try: cell.value = values[self.iteration] except IndexError: cell.value = '-' class NonEmptyCountResult(Result): """A class that counts the number of non-empty results. The number of non-empty values will be stored in the cell. """ def Compute(self, cell, values, baseline_values): """Put the number of non-empty values in the cell result. Args: cell: Put the result in cell.value. values: A list of values for the row. baseline_values: A list of baseline values for the row. """ cell.value = len(_StripNone(values)) if not baseline_values: return base_value = len(_StripNone(baseline_values)) if cell.value == base_value: return f = ColorBoxFormat() len_values = len(values) len_baseline_values = len(baseline_values) tmp_cell = Cell() tmp_cell.value = 1.0 + ( float(cell.value - base_value) / (max(len_values, len_baseline_values))) f.Compute(tmp_cell) cell.bgcolor = tmp_cell.bgcolor class StringMeanResult(Result): """Mean of string values.""" def _ComputeString(self, cell, values, baseline_values): if self._AllStringsSame(values): cell.value = str(values[0]) else: cell.value = '?' class AmeanResult(StringMeanResult): """Arithmetic mean.""" def __init__(self, ignore_min_max=False): super(AmeanResult, self).__init__() self.ignore_min_max = ignore_min_max def _ComputeFloat(self, cell, values, baseline_values): if self.ignore_min_max: values = _RemoveMinMax(cell, values) cell.value = statistics.mean(values) class RawResult(Result): """Raw result.""" class IterationResult(Result): """Iteration result.""" class MinResult(Result): """Minimum.""" def _ComputeFloat(self, cell, values, baseline_values): cell.value = min(values) def _ComputeString(self, cell, values, baseline_values): if values: cell.value = min(values) else: cell.value = '' class MaxResult(Result): """Maximum.""" def _ComputeFloat(self, cell, values, baseline_values): cell.value = max(values) def _ComputeString(self, cell, values, baseline_values): if values: cell.value = max(values) else: cell.value = '' class NumericalResult(Result): """Numerical result.""" def _ComputeString(self, cell, values, baseline_values): cell.value = '?' class StdResult(NumericalResult): """Standard deviation.""" def __init__(self, ignore_min_max=False): super(StdResult, self).__init__() self.ignore_min_max = ignore_min_max def _ComputeFloat(self, cell, values, baseline_values): if self.ignore_min_max: values = _RemoveMinMax(cell, values) cell.value = statistics.pstdev(values) class CoeffVarResult(NumericalResult): """Standard deviation / Mean""" def __init__(self, ignore_min_max=False): super(CoeffVarResult, self).__init__() self.ignore_min_max = ignore_min_max def _ComputeFloat(self, cell, values, baseline_values): if self.ignore_min_max: values = _RemoveMinMax(cell, values) if statistics.mean(values) != 0.0: noise = abs(statistics.pstdev(values) / statistics.mean(values)) else: noise = 0.0 cell.value = noise class ComparisonResult(Result): """Same or Different.""" def NeedsBaseline(self): return True def _ComputeString(self, cell, values, baseline_values): value = None baseline_value = None if self._AllStringsSame(values): value = values[0] if self._AllStringsSame(baseline_values): baseline_value = baseline_values[0] if value is not None and baseline_value is not None: if value == baseline_value: cell.value = 'SAME' else: cell.value = 'DIFFERENT' else: cell.value = '?' class PValueResult(ComparisonResult): """P-value.""" def __init__(self, ignore_min_max=False): super(PValueResult, self).__init__() self.ignore_min_max = ignore_min_max def _ComputeFloat(self, cell, values, baseline_values): if self.ignore_min_max: values = _RemoveMinMax(cell, values) baseline_values = _RemoveMinMax(cell, baseline_values) if len(values) < 2 or len(baseline_values) < 2: cell.value = float('nan') return _, cell.value = scipy.stats.ttest_ind(values, baseline_values) def _ComputeString(self, cell, values, baseline_values): return float('nan') class KeyAwareComparisonResult(ComparisonResult): """Automatic key aware comparison.""" def _IsLowerBetter(self, key): # Units in histograms should include directions if 'smallerIsBetter' in key: return True if 'biggerIsBetter' in key: return False # For units in chartjson: # TODO(llozano): Trying to guess direction by looking at the name of the # test does not seem like a good idea. Test frameworks should provide this # info explicitly. I believe Telemetry has this info. Need to find it out. # # Below are some test names for which we are not sure what the # direction is. # # For these we dont know what the direction is. But, since we dont # specify anything, crosperf will assume higher is better: # --percent_impl_scrolled--percent_impl_scrolled--percent # --solid_color_tiles_analyzed--solid_color_tiles_analyzed--count # --total_image_cache_hit_count--total_image_cache_hit_count--count # --total_texture_upload_time_by_url # # About these we are doubtful but we made a guess: # --average_num_missing_tiles_by_url--*--units (low is good) # --experimental_mean_frame_time_by_url--*--units (low is good) # --experimental_median_frame_time_by_url--*--units (low is good) # --texture_upload_count--texture_upload_count--count (high is good) # --total_deferred_image_decode_count--count (low is good) # --total_tiles_analyzed--total_tiles_analyzed--count (high is good) lower_is_better_keys = [ 'milliseconds', 'ms_', 'seconds_', 'KB', 'rdbytes', 'wrbytes', 'dropped_percent', '(ms)', '(seconds)', '--ms', '--average_num_missing_tiles', '--experimental_jank', '--experimental_mean_frame', '--experimental_median_frame_time', '--total_deferred_image_decode_count', '--seconds', 'samples', 'bytes' ] return any([l in key for l in lower_is_better_keys]) def _InvertIfLowerIsBetter(self, cell): if self._IsLowerBetter(cell.name): if cell.value: cell.value = 1.0 / cell.value class AmeanRatioResult(KeyAwareComparisonResult): """Ratio of arithmetic means of values vs. baseline values.""" def __init__(self, ignore_min_max=False): super(AmeanRatioResult, self).__init__() self.ignore_min_max = ignore_min_max def _ComputeFloat(self, cell, values, baseline_values): if self.ignore_min_max: values = _RemoveMinMax(cell, values) baseline_values = _RemoveMinMax(cell, baseline_values) baseline_mean = statistics.mean(baseline_values) values_mean = statistics.mean(values) if baseline_mean != 0: cell.value = values_mean / baseline_mean elif values_mean != 0: cell.value = 0.00 # cell.value = 0 means the values and baseline_values have big difference else: cell.value = 1.00 # no difference if both values and baseline_values are 0 class GmeanRatioResult(KeyAwareComparisonResult): """Ratio of geometric means of values vs. baseline values.""" def __init__(self, ignore_min_max=False): super(GmeanRatioResult, self).__init__() self.ignore_min_max = ignore_min_max def _ComputeFloat(self, cell, values, baseline_values): if self.ignore_min_max: values = _RemoveMinMax(cell, values) baseline_values = _RemoveMinMax(cell, baseline_values) if self._GetGmean(baseline_values) != 0: cell.value = self._GetGmean(values) / self._GetGmean(baseline_values) elif self._GetGmean(values) != 0: cell.value = 0.00 else: cell.value = 1.00 class Color(object): """Class that represents color in RGBA format.""" def __init__(self, r=0, g=0, b=0, a=0): self.r = r self.g = g self.b = b self.a = a def __str__(self): return 'r: %s g: %s: b: %s: a: %s' % (self.r, self.g, self.b, self.a) def Round(self): """Round RGBA values to the nearest integer.""" self.r = int(self.r) self.g = int(self.g) self.b = int(self.b) self.a = int(self.a) def GetRGB(self): """Get a hex representation of the color.""" return '%02x%02x%02x' % (self.r, self.g, self.b) @classmethod def Lerp(cls, ratio, a, b): """Perform linear interpolation between two colors. Args: ratio: The ratio to use for linear polation. a: The first color object (used when ratio is 0). b: The second color object (used when ratio is 1). Returns: Linearly interpolated color. """ ret = cls() ret.r = (b.r - a.r) * ratio + a.r ret.g = (b.g - a.g) * ratio + a.g ret.b = (b.b - a.b) * ratio + a.b ret.a = (b.a - a.a) * ratio + a.a return ret class Format(object): """A class that represents the format of a column.""" def __init__(self): pass def Compute(self, cell): """Computes the attributes of a cell based on its value. Attributes typically are color, width, etc. Args: cell: The cell whose attributes are to be populated. """ if cell.value is None: cell.string_value = '' if isinstance(cell.value, float): self._ComputeFloat(cell) else: self._ComputeString(cell) def _ComputeFloat(self, cell): cell.string_value = '{0:.2f}'.format(cell.value) def _ComputeString(self, cell): cell.string_value = str(cell.value) def _GetColor(self, value, low, mid, high, power=6, mid_value=1.0): min_value = 0.0 max_value = 2.0 if math.isnan(value): return mid if value > mid_value: value = max_value - mid_value / value return self._GetColorBetweenRange(value, min_value, mid_value, max_value, low, mid, high, power) def _GetColorBetweenRange(self, value, min_value, mid_value, max_value, low_color, mid_color, high_color, power): assert value <= max_value assert value >= min_value if value > mid_value: value = (max_value - value) / (max_value - mid_value) value **= power ret = Color.Lerp(value, high_color, mid_color) else: value = (value - min_value) / (mid_value - min_value) value **= power ret = Color.Lerp(value, low_color, mid_color) ret.Round() return ret class PValueFormat(Format): """Formatting for p-value.""" def _ComputeFloat(self, cell): cell.string_value = '%0.2f' % float(cell.value) if float(cell.value) < 0.05: cell.bgcolor = self._GetColor( cell.value, Color(255, 255, 0, 0), Color(255, 255, 255, 0), Color(255, 255, 255, 0), mid_value=0.05, power=1) class WeightFormat(Format): """Formatting for weight in cwp mode.""" def _ComputeFloat(self, cell): cell.string_value = '%0.4f' % float(cell.value) class StorageFormat(Format): """Format the cell as a storage number. Examples: If the cell contains a value of 1024, the string_value will be 1.0K. """ def _ComputeFloat(self, cell): base = 1024 suffices = ['K', 'M', 'G'] v = float(cell.value) current = 0 while v >= base**(current + 1) and current < len(suffices): current += 1 if current: divisor = base**current cell.string_value = '%1.1f%s' % ((v / divisor), suffices[current - 1]) else: cell.string_value = str(cell.value) class CoeffVarFormat(Format): """Format the cell as a percent. Examples: If the cell contains a value of 1.5, the string_value will be +150%. """ def _ComputeFloat(self, cell): cell.string_value = '%1.1f%%' % (float(cell.value) * 100) cell.color = self._GetColor( cell.value, Color(0, 255, 0, 0), Color(0, 0, 0, 0), Color(255, 0, 0, 0), mid_value=0.02, power=1) class PercentFormat(Format): """Format the cell as a percent. Examples: If the cell contains a value of 1.5, the string_value will be +50%. """ def _ComputeFloat(self, cell): cell.string_value = '%+1.1f%%' % ((float(cell.value) - 1) * 100) cell.color = self._GetColor(cell.value, Color(255, 0, 0, 0), Color(0, 0, 0, 0), Color(0, 255, 0, 0)) class RatioFormat(Format): """Format the cell as a ratio. Examples: If the cell contains a value of 1.5642, the string_value will be 1.56. """ def _ComputeFloat(self, cell): cell.string_value = '%+1.1f%%' % ((cell.value - 1) * 100) cell.color = self._GetColor(cell.value, Color(255, 0, 0, 0), Color(0, 0, 0, 0), Color(0, 255, 0, 0)) class ColorBoxFormat(Format): """Format the cell as a color box. Examples: If the cell contains a value of 1.5, it will get a green color. If the cell contains a value of 0.5, it will get a red color. The intensity of the green/red will be determined by how much above or below 1.0 the value is. """ def _ComputeFloat(self, cell): cell.string_value = '--' bgcolor = self._GetColor(cell.value, Color(255, 0, 0, 0), Color(255, 255, 255, 0), Color(0, 255, 0, 0)) cell.bgcolor = bgcolor cell.color = bgcolor class Cell(object): """A class to represent a cell in a table. Attributes: value: The raw value of the cell. color: The color of the cell. bgcolor: The background color of the cell. string_value: The string value of the cell. suffix: A string suffix to be attached to the value when displaying. prefix: A string prefix to be attached to the value when displaying. color_row: Indicates whether the whole row is to inherit this cell's color. bgcolor_row: Indicates whether the whole row is to inherit this cell's bgcolor. width: Optional specifier to make a column narrower than the usual width. The usual width of a column is the max of all its cells widths. colspan: Set the colspan of the cell in the HTML table, this is used for table headers. Default value is 1. name: the test name of the cell. header: Whether this is a header in html. """ def __init__(self): self.value = None self.color = None self.bgcolor = None self.string_value = None self.suffix = None self.prefix = None # Entire row inherits this color. self.color_row = False self.bgcolor_row = False self.width = 0 self.colspan = 1 self.name = None self.header = False def __str__(self): l = [] l.append('value: %s' % self.value) l.append('string_value: %s' % self.string_value) return ' '.join(l) class Column(object): """Class representing a column in a table. Attributes: result: an object of the Result class. fmt: an object of the Format class. """ def __init__(self, result, fmt, name=''): self.result = result self.fmt = fmt self.name = name # Takes in: # ["Key", "Label1", "Label2"] # ["k", ["v", "v2"], [v3]] # etc. # Also takes in a format string. # Returns a table like: # ["Key", "Label1", "Label2"] # ["k", avg("v", "v2"), stddev("v", "v2"), etc.]] # according to format string class TableFormatter(object): """Class to convert a plain table into a cell-table. This class takes in a table generated by TableGenerator and a list of column formats to apply to the table and returns a table of cells. """ def __init__(self, table, columns, samples_table=False): """The constructor takes in a table and a list of columns. Args: table: A list of lists of values. columns: A list of column containing what to produce and how to format it. samples_table: A flag to check whether we are generating a table of samples in CWP apporximation mode. """ self._table = table self._columns = columns self._samples_table = samples_table self._table_columns = [] self._out_table = [] def GenerateCellTable(self, table_type): row_index = 0 all_failed = False for row in self._table[1:]: # If we are generating samples_table, the second value will be weight # rather than values. start_col = 2 if self._samples_table else 1 # It does not make sense to put retval in the summary table. if str(row[0]) == 'retval' and table_type == 'summary': # Check to see if any runs passed, and update all_failed. all_failed = True for values in row[start_col:]: if 0 in values: all_failed = False continue key = Cell() key.string_value = str(row[0]) out_row = [key] if self._samples_table: # Add one column for weight if in samples_table mode weight = Cell() weight.value = row[1] f = WeightFormat() f.Compute(weight) out_row.append(weight) baseline = None for results in row[start_col:]: column_start = 0 values = None # If generating sample table, we will split a tuple of iterations info # from the results if isinstance(results, tuple): it, values = results column_start = 1 cell = Cell() cell.string_value = '[%d: %d]' % (it[0], it[1]) out_row.append(cell) if not row_index: self._table_columns.append(self._columns[0]) else: values = results # Parse each column for column in self._columns[column_start:]: cell = Cell() cell.name = key.string_value if not column.result.NeedsBaseline() or baseline is not None: column.result.Compute(cell, values, baseline) column.fmt.Compute(cell) out_row.append(cell) if not row_index: self._table_columns.append(column) if baseline is None: baseline = values self._out_table.append(out_row) row_index += 1 # If this is a summary table, and the only row in it is 'retval', and # all the test runs failed, we need to a 'Results' row to the output # table. if table_type == 'summary' and all_failed and len(self._table) == 2: labels_row = self._table[0] key = Cell() key.string_value = 'Results' out_row = [key] baseline = None for _ in labels_row[1:]: for column in self._columns: cell = Cell() cell.name = key.string_value column.result.Compute(cell, ['Fail'], baseline) column.fmt.Compute(cell) out_row.append(cell) if not row_index: self._table_columns.append(column) self._out_table.append(out_row) def AddColumnName(self): """Generate Column name at the top of table.""" key = Cell() key.header = True key.string_value = 'Keys' if not self._samples_table else 'Benchmarks' header = [key] if self._samples_table: weight = Cell() weight.header = True weight.string_value = 'Weights' header.append(weight) for column in self._table_columns: cell = Cell() cell.header = True if column.name: cell.string_value = column.name else: result_name = column.result.__class__.__name__ format_name = column.fmt.__class__.__name__ cell.string_value = '%s %s' % ( result_name.replace('Result', ''), format_name.replace('Format', ''), ) header.append(cell) self._out_table = [header] + self._out_table def AddHeader(self, s): """Put additional string on the top of the table.""" cell = Cell() cell.header = True cell.string_value = str(s) header = [cell] colspan = max(1, max(len(row) for row in self._table)) cell.colspan = colspan self._out_table = [header] + self._out_table def GetPassesAndFails(self, values): passes = 0 fails = 0 for val in values: if val == 0: passes = passes + 1 else: fails = fails + 1 return passes, fails def AddLabelName(self): """Put label on the top of the table.""" top_header = [] base_colspan = len( [c for c in self._columns if not c.result.NeedsBaseline()]) compare_colspan = len(self._columns) # Find the row with the key 'retval', if it exists. This # will be used to calculate the number of iterations that passed and # failed for each image label. retval_row = None for row in self._table: if row[0] == 'retval': retval_row = row # The label is organized as follows # "keys" label_base, label_comparison1, label_comparison2 # The first cell has colspan 1, the second is base_colspan # The others are compare_colspan column_position = 0 for label in self._table[0]: cell = Cell() cell.header = True # Put the number of pass/fail iterations in the image label header. if column_position > 0 and retval_row: retval_values = retval_row[column_position] if isinstance(retval_values, list): passes, fails = self.GetPassesAndFails(retval_values) cell.string_value = str(label) + ' (pass:%d fail:%d)' % (passes, fails) else: cell.string_value = str(label) else: cell.string_value = str(label) if top_header: if not self._samples_table or (self._samples_table and len(top_header) == 2): cell.colspan = base_colspan if len(top_header) > 1: if not self._samples_table or (self._samples_table and len(top_header) > 2): cell.colspan = compare_colspan top_header.append(cell) column_position = column_position + 1 self._out_table = [top_header] + self._out_table def _PrintOutTable(self): o = '' for row in self._out_table: for cell in row: o += str(cell) + ' ' o += '\n' print(o) def GetCellTable(self, table_type='full', headers=True): """Function to return a table of cells. The table (list of lists) is converted into a table of cells by this function. Args: table_type: Can be 'full' or 'summary' headers: A boolean saying whether we want default headers Returns: A table of cells with each cell having the properties and string values as requiested by the columns passed in the constructor. """ # Generate the cell table, creating a list of dynamic columns on the fly. if not self._out_table: self.GenerateCellTable(table_type) if headers: self.AddColumnName() self.AddLabelName() return self._out_table class TablePrinter(object): """Class to print a cell table to the console, file or html.""" PLAIN = 0 CONSOLE = 1 HTML = 2 TSV = 3 EMAIL = 4 def __init__(self, table, output_type): """Constructor that stores the cell table and output type.""" self._table = table self._output_type = output_type self._row_styles = [] self._column_styles = [] # Compute whole-table properties like max-size, etc. def _ComputeStyle(self): self._row_styles = [] for row in self._table: row_style = Cell() for cell in row: if cell.color_row: assert cell.color, 'Cell color not set but color_row set!' assert not row_style.color, 'Multiple row_style.colors found!' row_style.color = cell.color if cell.bgcolor_row: assert cell.bgcolor, 'Cell bgcolor not set but bgcolor_row set!' assert not row_style.bgcolor, 'Multiple row_style.bgcolors found!' row_style.bgcolor = cell.bgcolor self._row_styles.append(row_style) self._column_styles = [] if len(self._table) < 2: return for i in range(max(len(row) for row in self._table)): column_style = Cell() for row in self._table: if not any([cell.colspan != 1 for cell in row]): column_style.width = max(column_style.width, len(row[i].string_value)) self._column_styles.append(column_style) def _GetBGColorFix(self, color): if self._output_type == self.CONSOLE: prefix = misc.rgb2short(color.r, color.g, color.b) # pylint: disable=anomalous-backslash-in-string prefix = '\033[48;5;%sm' % prefix suffix = '\033[0m' elif self._output_type in [self.EMAIL, self.HTML]: rgb = color.GetRGB() prefix = (''.format(rgb)) suffix = '' elif self._output_type in [self.PLAIN, self.TSV]: prefix = '' suffix = '' return prefix, suffix def _GetColorFix(self, color): if self._output_type == self.CONSOLE: prefix = misc.rgb2short(color.r, color.g, color.b) # pylint: disable=anomalous-backslash-in-string prefix = '\033[38;5;%sm' % prefix suffix = '\033[0m' elif self._output_type in [self.EMAIL, self.HTML]: rgb = color.GetRGB() prefix = ''.format(rgb) suffix = '' elif self._output_type in [self.PLAIN, self.TSV]: prefix = '' suffix = '' return prefix, suffix def Print(self): """Print the table to a console, html, etc. Returns: A string that contains the desired representation of the table. """ self._ComputeStyle() return self._GetStringValue() def _GetCellValue(self, i, j): cell = self._table[i][j] out = cell.string_value raw_width = len(out) if cell.color: p, s = self._GetColorFix(cell.color) out = '%s%s%s' % (p, out, s) if cell.bgcolor: p, s = self._GetBGColorFix(cell.bgcolor) out = '%s%s%s' % (p, out, s) if self._output_type in [self.PLAIN, self.CONSOLE, self.EMAIL]: if cell.width: width = cell.width else: if self._column_styles: width = self._column_styles[j].width else: width = len(cell.string_value) if cell.colspan > 1: width = 0 start = 0 for k in range(j): start += self._table[i][k].colspan for k in range(cell.colspan): width += self._column_styles[start + k].width if width > raw_width: padding = ('%' + str(width - raw_width) + 's') % '' out = padding + out if self._output_type == self.HTML: if cell.header: tag = 'th' else: tag = 'td' out = '<{0} colspan = "{2}"> {1} '.format(tag, out, cell.colspan) return out def _GetHorizontalSeparator(self): if self._output_type in [self.CONSOLE, self.PLAIN, self.EMAIL]: return ' ' if self._output_type == self.HTML: return '' if self._output_type == self.TSV: return '\t' def _GetVerticalSeparator(self): if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]: return '\n' if self._output_type == self.HTML: return '\n' def _GetPrefix(self): if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]: return '' if self._output_type == self.HTML: return '

\n' def _GetSuffix(self): if self._output_type in [self.PLAIN, self.CONSOLE, self.TSV, self.EMAIL]: return '' if self._output_type == self.HTML: return '\n
' def _GetStringValue(self): o = '' o += self._GetPrefix() for i in range(len(self._table)): row = self._table[i] # Apply row color and bgcolor. p = s = bgp = bgs = '' if self._row_styles[i].bgcolor: bgp, bgs = self._GetBGColorFix(self._row_styles[i].bgcolor) if self._row_styles[i].color: p, s = self._GetColorFix(self._row_styles[i].color) o += p + bgp for j in range(len(row)): out = self._GetCellValue(i, j) o += out + self._GetHorizontalSeparator() o += s + bgs o += self._GetVerticalSeparator() o += self._GetSuffix() return o # Some common drivers def GetSimpleTable(table, out_to=TablePrinter.CONSOLE): """Prints a simple table. This is used by code that has a very simple list-of-lists and wants to produce a table with ameans, a percentage ratio of ameans and a colorbox. Examples: GetSimpleConsoleTable([["binary", "b1", "b2"],["size", "300", "400"]]) will produce a colored table that can be printed to the console. Args: table: a list of lists. out_to: specify the fomat of output. Currently it supports HTML and CONSOLE. Returns: A string version of the table that can be printed to the console. """ columns = [ Column(AmeanResult(), Format()), Column(AmeanRatioResult(), PercentFormat()), Column(AmeanRatioResult(), ColorBoxFormat()), ] our_table = [table[0]] for row in table[1:]: our_row = [row[0]] for v in row[1:]: our_row.append([v]) our_table.append(our_row) tf = TableFormatter(our_table, columns) cell_table = tf.GetCellTable() tp = TablePrinter(cell_table, out_to) return tp.Print() # pylint: disable=redefined-outer-name def GetComplexTable(runs, labels, out_to=TablePrinter.CONSOLE): """Prints a complex table. This can be used to generate a table with arithmetic mean, standard deviation, coefficient of variation, p-values, etc. Args: runs: A list of lists with data to tabulate. labels: A list of labels that correspond to the runs. out_to: specifies the format of the table (example CONSOLE or HTML). Returns: A string table that can be printed to the console or put in an HTML file. """ tg = TableGenerator(runs, labels, TableGenerator.SORT_BY_VALUES_DESC) table = tg.GetTable() columns = [ Column(LiteralResult(), Format(), 'Literal'), Column(AmeanResult(), Format()), Column(StdResult(), Format()), Column(CoeffVarResult(), CoeffVarFormat()), Column(NonEmptyCountResult(), Format()), Column(AmeanRatioResult(), PercentFormat()), Column(AmeanRatioResult(), RatioFormat()), Column(GmeanRatioResult(), RatioFormat()), Column(PValueResult(), PValueFormat()) ] tf = TableFormatter(table, columns) cell_table = tf.GetCellTable() tp = TablePrinter(cell_table, out_to) return tp.Print() if __name__ == '__main__': # Run a few small tests here. run1 = { 'k1': '10', 'k2': '12', 'k5': '40', 'k6': '40', 'ms_1': '20', 'k7': 'FAIL', 'k8': 'PASS', 'k9': 'PASS', 'k10': '0' } run2 = { 'k1': '13', 'k2': '14', 'k3': '15', 'ms_1': '10', 'k8': 'PASS', 'k9': 'FAIL', 'k10': '0' } run3 = { 'k1': '50', 'k2': '51', 'k3': '52', 'k4': '53', 'k5': '35', 'k6': '45', 'ms_1': '200', 'ms_2': '20', 'k7': 'FAIL', 'k8': 'PASS', 'k9': 'PASS' } runs = [[run1, run2], [run3]] labels = ['vanilla', 'modified'] t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) print(t) email = GetComplexTable(runs, labels, TablePrinter.EMAIL) runs = [[{ 'k1': '1' }, { 'k1': '1.1' }, { 'k1': '1.2' }], [{ 'k1': '5' }, { 'k1': '5.1' }, { 'k1': '5.2' }]] t = GetComplexTable(runs, labels, TablePrinter.CONSOLE) print(t) simple_table = [ ['binary', 'b1', 'b2', 'b3'], ['size', 100, 105, 108], ['rodata', 100, 80, 70], ['data', 100, 100, 100], ['debug', 100, 140, 60], ] t = GetSimpleTable(simple_table) print(t) email += GetSimpleTable(simple_table, TablePrinter.HTML) email_to = [getpass.getuser()] email = "
%s
" % email EmailSender().SendEmail(email_to, 'SimpleTableTest', email, msg_type='html')