diff options
Diffstat (limited to 'crosperf/results_report.py')
| -rw-r--r-- | crosperf/results_report.py | 1474 |
1 files changed, 784 insertions, 690 deletions
diff --git a/crosperf/results_report.py b/crosperf/results_report.py index dc80b53b..045e623b 100644 --- a/crosperf/results_report.py +++ b/crosperf/results_report.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Copyright 2013 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """A module to handle the report format.""" -from __future__ import print_function import datetime import functools @@ -14,15 +13,15 @@ import os import re import time +from column_chart import ColumnChart from cros_utils.tabulator import AmeanResult from cros_utils.tabulator import Cell from cros_utils.tabulator import CoeffVarFormat from cros_utils.tabulator import CoeffVarResult from cros_utils.tabulator import Column -from cros_utils.tabulator import SamplesTableGenerator from cros_utils.tabulator import Format -from cros_utils.tabulator import IterationResult from cros_utils.tabulator import GmeanRatioResult +from cros_utils.tabulator import IterationResult from cros_utils.tabulator import LiteralResult from cros_utils.tabulator import MaxResult from cros_utils.tabulator import MinResult @@ -30,20 +29,18 @@ from cros_utils.tabulator import PValueFormat from cros_utils.tabulator import PValueResult from cros_utils.tabulator import RatioFormat from cros_utils.tabulator import RawResult +from cros_utils.tabulator import SamplesTableGenerator from cros_utils.tabulator import StdResult from cros_utils.tabulator import TableFormatter from cros_utils.tabulator import TableGenerator from cros_utils.tabulator import TablePrinter -from update_telemetry_defaults import TelemetryDefaults - -from column_chart import ColumnChart from results_organizer import OrganizeResults - import results_report_templates as templates +from update_telemetry_defaults import TelemetryDefaults def ParseChromeosImage(chromeos_image): - """Parse the chromeos_image string for the image and version. + """Parse the chromeos_image string for the image and version. The chromeos_image string will probably be in one of two formats: 1: <path-to-chroot>/src/build/images/<board>/<ChromeOS-version>.<datetime>/ \ @@ -64,760 +61,857 @@ def ParseChromeosImage(chromeos_image): version, image: The results of parsing the input string, as explained above. """ - # Find the Chromeos Version, e.g. R45-2345.0.0..... - # chromeos_image should have been something like: - # <path>/<board-trybot-release>/<chromeos-version>/chromiumos_test_image.bin" - if chromeos_image.endswith('/chromiumos_test_image.bin'): - full_version = chromeos_image.split('/')[-2] - # Strip the date and time off of local builds (which have the format - # "R43-2345.0.0.date-and-time"). - version, _ = os.path.splitext(full_version) - else: - version = '' - - # Find the chromeos image. If it's somewhere in .../chroot/tmp/..., then - # it's an official image that got downloaded, so chop off the download path - # to make the official image name more clear. - official_image_path = '/chroot/tmp' - if official_image_path in chromeos_image: - image = chromeos_image.split(official_image_path, 1)[1] - else: - image = chromeos_image - return version, image + # Find the Chromeos Version, e.g. R45-2345.0.0..... + # chromeos_image should have been something like: + # <path>/<board-trybot-release>/<chromeos-version>/chromiumos_test_image.bin" + if chromeos_image.endswith("/chromiumos_test_image.bin"): + full_version = chromeos_image.split("/")[-2] + # Strip the date and time off of local builds (which have the format + # "R43-2345.0.0.date-and-time"). + version, _ = os.path.splitext(full_version) + else: + version = "" + + # Find the chromeos image. If it's somewhere in .../chroot/tmp/..., then + # it's an official image that got downloaded, so chop off the download path + # to make the official image name more clear. + official_image_path = "/chroot/tmp" + if official_image_path in chromeos_image: + image = chromeos_image.split(official_image_path, 1)[1] + else: + image = chromeos_image + return version, image def _AppendUntilLengthIs(gen, the_list, target_len): - """Appends to `list` until `list` is `target_len` elements long. + """Appends to `list` until `list` is `target_len` elements long. - Uses `gen` to generate elements. - """ - the_list.extend(gen() for _ in range(target_len - len(the_list))) - return the_list + Uses `gen` to generate elements. + """ + the_list.extend(gen() for _ in range(target_len - len(the_list))) + return the_list def _FilterPerfReport(event_threshold, report): - """Filters out entries with `< event_threshold` percent in a perf report.""" + """Filters out entries with `< event_threshold` percent in a perf report.""" - def filter_dict(m): - return { - fn_name: pct for fn_name, pct in m.items() if pct >= event_threshold - } + def filter_dict(m): + return { + fn_name: pct for fn_name, pct in m.items() if pct >= event_threshold + } - return {event: filter_dict(m) for event, m in report.items()} + return {event: filter_dict(m) for event, m in report.items()} class _PerfTable(object): - """Generates dicts from a perf table. - - Dicts look like: - {'benchmark_name': {'perf_event_name': [LabelData]}} - where LabelData is a list of perf dicts, each perf dict coming from the same - label. - Each perf dict looks like {'function_name': 0.10, ...} (where 0.10 is the - percentage of time spent in function_name). - """ - - def __init__(self, - benchmark_names_and_iterations, - label_names, - read_perf_report, - event_threshold=None): - """Constructor. - - read_perf_report is a function that takes a label name, benchmark name, and - benchmark iteration, and returns a dictionary describing the perf output for - that given run. + """Generates dicts from a perf table. + + Dicts look like: + {'benchmark_name': {'perf_event_name': [LabelData]}} + where LabelData is a list of perf dicts, each perf dict coming from the same + label. + Each perf dict looks like {'function_name': 0.10, ...} (where 0.10 is the + percentage of time spent in function_name). """ - self.event_threshold = event_threshold - self._label_indices = {name: i for i, name in enumerate(label_names)} - self.perf_data = {} - for label in label_names: - for bench_name, bench_iterations in benchmark_names_and_iterations: - for i in range(bench_iterations): - report = read_perf_report(label, bench_name, i) - self._ProcessPerfReport(report, label, bench_name, i) - - def _ProcessPerfReport(self, perf_report, label, benchmark_name, iteration): - """Add the data from one run to the dict.""" - perf_of_run = perf_report - if self.event_threshold is not None: - perf_of_run = _FilterPerfReport(self.event_threshold, perf_report) - if benchmark_name not in self.perf_data: - self.perf_data[benchmark_name] = {event: [] for event in perf_of_run} - ben_data = self.perf_data[benchmark_name] - label_index = self._label_indices[label] - for event in ben_data: - _AppendUntilLengthIs(list, ben_data[event], label_index + 1) - data_for_label = ben_data[event][label_index] - _AppendUntilLengthIs(dict, data_for_label, iteration + 1) - data_for_label[iteration] = perf_of_run[event] if perf_of_run else {} + + def __init__( + self, + benchmark_names_and_iterations, + label_names, + read_perf_report, + event_threshold=None, + ): + """Constructor. + + read_perf_report is a function that takes a label name, benchmark name, and + benchmark iteration, and returns a dictionary describing the perf output for + that given run. + """ + self.event_threshold = event_threshold + self._label_indices = {name: i for i, name in enumerate(label_names)} + self.perf_data = {} + for label in label_names: + for bench_name, bench_iterations in benchmark_names_and_iterations: + for i in range(bench_iterations): + report = read_perf_report(label, bench_name, i) + self._ProcessPerfReport(report, label, bench_name, i) + + def _ProcessPerfReport(self, perf_report, label, benchmark_name, iteration): + """Add the data from one run to the dict.""" + perf_of_run = perf_report + if self.event_threshold is not None: + perf_of_run = _FilterPerfReport(self.event_threshold, perf_report) + if benchmark_name not in self.perf_data: + self.perf_data[benchmark_name] = { + event: [] for event in perf_of_run + } + ben_data = self.perf_data[benchmark_name] + label_index = self._label_indices[label] + for event in ben_data: + _AppendUntilLengthIs(list, ben_data[event], label_index + 1) + data_for_label = ben_data[event][label_index] + _AppendUntilLengthIs(dict, data_for_label, iteration + 1) + data_for_label[iteration] = ( + perf_of_run[event] if perf_of_run else {} + ) def _GetResultsTableHeader(ben_name, iterations): - benchmark_info = ('Benchmark: {0}; Iterations: {1}'.format( - ben_name, iterations)) - cell = Cell() - cell.string_value = benchmark_info - cell.header = True - return [[cell]] + benchmark_info = "Benchmark: {0}; Iterations: {1}".format( + ben_name, iterations + ) + cell = Cell() + cell.string_value = benchmark_info + cell.header = True + return [[cell]] def _GetDSOHeader(cwp_dso): - info = 'CWP_DSO: %s' % cwp_dso - cell = Cell() - cell.string_value = info - cell.header = False - return [[cell]] + info = "CWP_DSO: %s" % cwp_dso + cell = Cell() + cell.string_value = info + cell.header = False + return [[cell]] def _ParseColumn(columns, iteration): - new_column = [] - for column in columns: - if column.result.__class__.__name__ != 'RawResult': - new_column.append(column) - else: - new_column.extend( - Column(LiteralResult(i), Format(), str(i + 1)) - for i in range(iteration)) - return new_column + new_column = [] + for column in columns: + if column.result.__class__.__name__ != "RawResult": + new_column.append(column) + else: + new_column.extend( + Column(LiteralResult(i), Format(), str(i + 1)) + for i in range(iteration) + ) + return new_column def _GetTables(benchmark_results, columns, table_type): - iter_counts = benchmark_results.iter_counts - result = benchmark_results.run_keyvals - tables = [] - for bench_name, runs in result.items(): - iterations = iter_counts[bench_name] - ben_table = _GetResultsTableHeader(bench_name, iterations) - - all_runs_empty = all(not dict for label in runs for dict in label) - if all_runs_empty: - cell = Cell() - cell.string_value = ('This benchmark contains no result.' - ' Is the benchmark name valid?') - cell_table = [[cell]] - else: - table = TableGenerator(runs, benchmark_results.label_names).GetTable() - parsed_columns = _ParseColumn(columns, iterations) - tf = TableFormatter(table, parsed_columns) - cell_table = tf.GetCellTable(table_type) - tables.append(ben_table) - tables.append(cell_table) - return tables + iter_counts = benchmark_results.iter_counts + result = benchmark_results.run_keyvals + tables = [] + for bench_name, runs in result.items(): + iterations = iter_counts[bench_name] + ben_table = _GetResultsTableHeader(bench_name, iterations) + + all_runs_empty = all(not dict for label in runs for dict in label) + if all_runs_empty: + cell = Cell() + cell.string_value = ( + "This benchmark contains no result." + " Is the benchmark name valid?" + ) + cell_table = [[cell]] + else: + table = TableGenerator( + runs, benchmark_results.label_names + ).GetTable() + parsed_columns = _ParseColumn(columns, iterations) + tf = TableFormatter(table, parsed_columns) + cell_table = tf.GetCellTable(table_type) + tables.append(ben_table) + tables.append(cell_table) + return tables def _GetPerfTables(benchmark_results, columns, table_type): - p_table = _PerfTable(benchmark_results.benchmark_names_and_iterations, - benchmark_results.label_names, - benchmark_results.read_perf_report) - - tables = [] - for benchmark in p_table.perf_data: - iterations = benchmark_results.iter_counts[benchmark] - ben_table = _GetResultsTableHeader(benchmark, iterations) - tables.append(ben_table) - benchmark_data = p_table.perf_data[benchmark] - table = [] - for event in benchmark_data: - tg = TableGenerator( - benchmark_data[event], - benchmark_results.label_names, - sort=TableGenerator.SORT_BY_VALUES_DESC) - table = tg.GetTable(ResultsReport.PERF_ROWS) - parsed_columns = _ParseColumn(columns, iterations) - tf = TableFormatter(table, parsed_columns) - tf.GenerateCellTable(table_type) - tf.AddColumnName() - tf.AddLabelName() - tf.AddHeader(str(event)) - table = tf.GetCellTable(table_type, headers=False) - tables.append(table) - return tables + p_table = _PerfTable( + benchmark_results.benchmark_names_and_iterations, + benchmark_results.label_names, + benchmark_results.read_perf_report, + ) + + tables = [] + for benchmark in p_table.perf_data: + iterations = benchmark_results.iter_counts[benchmark] + ben_table = _GetResultsTableHeader(benchmark, iterations) + tables.append(ben_table) + benchmark_data = p_table.perf_data[benchmark] + table = [] + for event in benchmark_data: + tg = TableGenerator( + benchmark_data[event], + benchmark_results.label_names, + sort=TableGenerator.SORT_BY_VALUES_DESC, + ) + table = tg.GetTable(ResultsReport.PERF_ROWS) + parsed_columns = _ParseColumn(columns, iterations) + tf = TableFormatter(table, parsed_columns) + tf.GenerateCellTable(table_type) + tf.AddColumnName() + tf.AddLabelName() + tf.AddHeader(str(event)) + table = tf.GetCellTable(table_type, headers=False) + tables.append(table) + return tables def _GetSamplesTables(benchmark_results, columns, table_type): - tables = [] - dso_header_table = _GetDSOHeader(benchmark_results.cwp_dso) - tables.append(dso_header_table) - (table, new_keyvals, iter_counts) = SamplesTableGenerator( - benchmark_results.run_keyvals, benchmark_results.label_names, - benchmark_results.iter_counts, benchmark_results.weights).GetTable() - parsed_columns = _ParseColumn(columns, 1) - tf = TableFormatter(table, parsed_columns, samples_table=True) - cell_table = tf.GetCellTable(table_type) - tables.append(cell_table) - return (tables, new_keyvals, iter_counts) + tables = [] + dso_header_table = _GetDSOHeader(benchmark_results.cwp_dso) + tables.append(dso_header_table) + (table, new_keyvals, iter_counts) = SamplesTableGenerator( + benchmark_results.run_keyvals, + benchmark_results.label_names, + benchmark_results.iter_counts, + benchmark_results.weights, + ).GetTable() + parsed_columns = _ParseColumn(columns, 1) + tf = TableFormatter(table, parsed_columns, samples_table=True) + cell_table = tf.GetCellTable(table_type) + tables.append(cell_table) + return (tables, new_keyvals, iter_counts) class ResultsReport(object): - """Class to handle the report format.""" - MAX_COLOR_CODE = 255 - PERF_ROWS = 5 - - def __init__(self, results): - self.benchmark_results = results - - def _GetTablesWithColumns(self, columns, table_type, summary_type): - if summary_type == 'perf': - get_tables = _GetPerfTables - elif summary_type == 'samples': - get_tables = _GetSamplesTables - else: - get_tables = _GetTables - ret = get_tables(self.benchmark_results, columns, table_type) - # If we are generating a samples summary table, the return value of - # get_tables will be a tuple, and we will update the benchmark_results for - # composite benchmark so that full table can use it. - if isinstance(ret, tuple): - self.benchmark_results.run_keyvals = ret[1] - self.benchmark_results.iter_counts = ret[2] - ret = ret[0] - return ret - - def GetFullTables(self, perf=False): - ignore_min_max = self.benchmark_results.ignore_min_max - columns = [ - Column(RawResult(), Format()), - Column(MinResult(), Format()), - Column(MaxResult(), Format()), - Column(AmeanResult(ignore_min_max), Format()), - Column(StdResult(ignore_min_max), Format(), 'StdDev'), - Column(CoeffVarResult(ignore_min_max), CoeffVarFormat(), 'StdDev/Mean'), - Column(GmeanRatioResult(ignore_min_max), RatioFormat(), 'GmeanSpeedup'), - Column(PValueResult(ignore_min_max), PValueFormat(), 'p-value') - ] - return self._GetTablesWithColumns(columns, 'full', perf) - - def GetSummaryTables(self, summary_type=''): - ignore_min_max = self.benchmark_results.ignore_min_max - columns = [] - if summary_type == 'samples': - columns += [Column(IterationResult(), Format(), 'Iterations [Pass:Fail]')] - columns += [ - Column( - AmeanResult(ignore_min_max), Format(), - 'Weighted Samples Amean' if summary_type == 'samples' else ''), - Column(StdResult(ignore_min_max), Format(), 'StdDev'), - Column(CoeffVarResult(ignore_min_max), CoeffVarFormat(), 'StdDev/Mean'), - Column(GmeanRatioResult(ignore_min_max), RatioFormat(), 'GmeanSpeedup'), - Column(PValueResult(ignore_min_max), PValueFormat(), 'p-value') - ] - return self._GetTablesWithColumns(columns, 'summary', summary_type) + """Class to handle the report format.""" + + MAX_COLOR_CODE = 255 + PERF_ROWS = 5 + + def __init__(self, results): + self.benchmark_results = results + + def _GetTablesWithColumns(self, columns, table_type, summary_type): + if summary_type == "perf": + get_tables = _GetPerfTables + elif summary_type == "samples": + get_tables = _GetSamplesTables + else: + get_tables = _GetTables + ret = get_tables(self.benchmark_results, columns, table_type) + # If we are generating a samples summary table, the return value of + # get_tables will be a tuple, and we will update the benchmark_results for + # composite benchmark so that full table can use it. + if isinstance(ret, tuple): + self.benchmark_results.run_keyvals = ret[1] + self.benchmark_results.iter_counts = ret[2] + ret = ret[0] + return ret + + def GetFullTables(self, perf=False): + ignore_min_max = self.benchmark_results.ignore_min_max + columns = [ + Column(RawResult(), Format()), + Column(MinResult(), Format()), + Column(MaxResult(), Format()), + Column(AmeanResult(ignore_min_max), Format()), + Column(StdResult(ignore_min_max), Format(), "StdDev"), + Column( + CoeffVarResult(ignore_min_max), CoeffVarFormat(), "StdDev/Mean" + ), + Column( + GmeanRatioResult(ignore_min_max), RatioFormat(), "GmeanSpeedup" + ), + Column(PValueResult(ignore_min_max), PValueFormat(), "p-value"), + ] + return self._GetTablesWithColumns(columns, "full", perf) + + def GetSummaryTables(self, summary_type=""): + ignore_min_max = self.benchmark_results.ignore_min_max + columns = [] + if summary_type == "samples": + columns += [ + Column(IterationResult(), Format(), "Iterations [Pass:Fail]") + ] + columns += [ + Column( + AmeanResult(ignore_min_max), + Format(), + "Weighted Samples Amean" if summary_type == "samples" else "", + ), + Column(StdResult(ignore_min_max), Format(), "StdDev"), + Column( + CoeffVarResult(ignore_min_max), CoeffVarFormat(), "StdDev/Mean" + ), + Column( + GmeanRatioResult(ignore_min_max), RatioFormat(), "GmeanSpeedup" + ), + Column(PValueResult(ignore_min_max), PValueFormat(), "p-value"), + ] + return self._GetTablesWithColumns(columns, "summary", summary_type) def _PrintTable(tables, out_to): - # tables may be None. - if not tables: - return '' - - if out_to == 'HTML': - out_type = TablePrinter.HTML - elif out_to == 'PLAIN': - out_type = TablePrinter.PLAIN - elif out_to == 'CONSOLE': - out_type = TablePrinter.CONSOLE - elif out_to == 'TSV': - out_type = TablePrinter.TSV - elif out_to == 'EMAIL': - out_type = TablePrinter.EMAIL - else: - raise ValueError('Invalid out_to value: %s' % (out_to,)) - - printers = (TablePrinter(table, out_type) for table in tables) - return ''.join(printer.Print() for printer in printers) + # tables may be None. + if not tables: + return "" + + if out_to == "HTML": + out_type = TablePrinter.HTML + elif out_to == "PLAIN": + out_type = TablePrinter.PLAIN + elif out_to == "CONSOLE": + out_type = TablePrinter.CONSOLE + elif out_to == "TSV": + out_type = TablePrinter.TSV + elif out_to == "EMAIL": + out_type = TablePrinter.EMAIL + else: + raise ValueError("Invalid out_to value: %s" % (out_to,)) + printers = (TablePrinter(table, out_type) for table in tables) + return "".join(printer.Print() for printer in printers) -class TextResultsReport(ResultsReport): - """Class to generate text result report.""" - - H1_STR = '===========================================' - H2_STR = '-------------------------------------------' - - def __init__(self, results, email=False, experiment=None): - super(TextResultsReport, self).__init__(results) - self.email = email - self.experiment = experiment - - @staticmethod - def _MakeTitle(title): - header_line = TextResultsReport.H1_STR - # '' at the end gives one newline. - return '\n'.join([header_line, title, header_line, '']) - - @staticmethod - def _MakeSection(title, body): - header_line = TextResultsReport.H2_STR - # '\n' at the end gives us two newlines. - return '\n'.join([header_line, title, header_line, body, '\n']) - - @staticmethod - def FromExperiment(experiment, email=False): - results = BenchmarkResults.FromExperiment(experiment) - return TextResultsReport(results, email, experiment) - - def GetStatusTable(self): - """Generate the status table by the tabulator.""" - table = [['', '']] - columns = [ - Column(LiteralResult(iteration=0), Format(), 'Status'), - Column(LiteralResult(iteration=1), Format(), 'Failing Reason') - ] - - for benchmark_run in self.experiment.benchmark_runs: - status = [ - benchmark_run.name, - [benchmark_run.timeline.GetLastEvent(), benchmark_run.failure_reason] - ] - table.append(status) - cell_table = TableFormatter(table, columns).GetCellTable('status') - return [cell_table] - - def GetTotalWaitCooldownTime(self): - """Get cooldown wait time in seconds from experiment benchmark runs. - - Returns: - Dictionary {'dut': int(wait_time_in_seconds)} - """ - waittime_dict = {} - for dut in self.experiment.machine_manager.GetMachines(): - waittime_dict[dut.name] = dut.GetCooldownWaitTime() - return waittime_dict - - def GetReport(self): - """Generate the report for email and console.""" - output_type = 'EMAIL' if self.email else 'CONSOLE' - experiment = self.experiment - - sections = [] - if experiment is not None: - title_contents = "Results report for '%s'" % (experiment.name,) - else: - title_contents = 'Results report' - sections.append(self._MakeTitle(title_contents)) - if not self.benchmark_results.cwp_dso: - summary_table = _PrintTable(self.GetSummaryTables(), output_type) - else: - summary_table = _PrintTable( - self.GetSummaryTables(summary_type='samples'), output_type) - sections.append(self._MakeSection('Summary', summary_table)) - - if experiment is not None: - table = _PrintTable(self.GetStatusTable(), output_type) - sections.append(self._MakeSection('Benchmark Run Status', table)) - - if not self.benchmark_results.cwp_dso: - perf_table = _PrintTable( - self.GetSummaryTables(summary_type='perf'), output_type) - sections.append(self._MakeSection('Perf Data', perf_table)) - - if experiment is not None: - experiment_file = experiment.experiment_file - sections.append(self._MakeSection('Experiment File', experiment_file)) - - cpu_info = experiment.machine_manager.GetAllCPUInfo(experiment.labels) - sections.append(self._MakeSection('CPUInfo', cpu_info)) - - totaltime = (time.time() - - experiment.start_time) if experiment.start_time else 0 - totaltime_str = 'Total experiment time:\n%d min' % (totaltime // 60) - cooldown_waittime_list = ['Cooldown wait time:'] - # When running experiment on multiple DUTs cooldown wait time may vary - # on different devices. In addition its combined time may exceed total - # experiment time which will look weird but it is reasonable. - # For this matter print cooldown time per DUT. - for dut, waittime in sorted(self.GetTotalWaitCooldownTime().items()): - cooldown_waittime_list.append('DUT %s: %d min' % (dut, waittime // 60)) - cooldown_waittime_str = '\n'.join(cooldown_waittime_list) - sections.append( - self._MakeSection('Duration', - '\n\n'.join([totaltime_str, - cooldown_waittime_str]))) - - return '\n'.join(sections) +class TextResultsReport(ResultsReport): + """Class to generate text result report.""" + + H1_STR = "===========================================" + H2_STR = "-------------------------------------------" + + def __init__(self, results, email=False, experiment=None): + super(TextResultsReport, self).__init__(results) + self.email = email + self.experiment = experiment + + @staticmethod + def _MakeTitle(title): + header_line = TextResultsReport.H1_STR + # '' at the end gives one newline. + return "\n".join([header_line, title, header_line, ""]) + + @staticmethod + def _MakeSection(title, body): + header_line = TextResultsReport.H2_STR + # '\n' at the end gives us two newlines. + return "\n".join([header_line, title, header_line, body, "\n"]) + + @staticmethod + def FromExperiment(experiment, email=False): + results = BenchmarkResults.FromExperiment(experiment) + return TextResultsReport(results, email, experiment) + + def GetStatusTable(self): + """Generate the status table by the tabulator.""" + table = [["", ""]] + columns = [ + Column(LiteralResult(iteration=0), Format(), "Status"), + Column(LiteralResult(iteration=1), Format(), "Failing Reason"), + ] + + for benchmark_run in self.experiment.benchmark_runs: + status = [ + benchmark_run.name, + [ + benchmark_run.timeline.GetLastEvent(), + benchmark_run.failure_reason, + ], + ] + table.append(status) + cell_table = TableFormatter(table, columns).GetCellTable("status") + return [cell_table] + + def GetTotalWaitCooldownTime(self): + """Get cooldown wait time in seconds from experiment benchmark runs. + + Returns: + Dictionary {'dut': int(wait_time_in_seconds)} + """ + waittime_dict = {} + for dut in self.experiment.machine_manager.GetMachines(): + waittime_dict[dut.name] = dut.GetCooldownWaitTime() + return waittime_dict + + def GetReport(self): + """Generate the report for email and console.""" + output_type = "EMAIL" if self.email else "CONSOLE" + experiment = self.experiment + + sections = [] + if experiment is not None: + title_contents = "Results report for '%s'" % (experiment.name,) + else: + title_contents = "Results report" + sections.append(self._MakeTitle(title_contents)) + + if not self.benchmark_results.cwp_dso: + summary_table = _PrintTable(self.GetSummaryTables(), output_type) + else: + summary_table = _PrintTable( + self.GetSummaryTables(summary_type="samples"), output_type + ) + sections.append(self._MakeSection("Summary", summary_table)) + + if experiment is not None: + table = _PrintTable(self.GetStatusTable(), output_type) + sections.append(self._MakeSection("Benchmark Run Status", table)) + + if not self.benchmark_results.cwp_dso: + perf_table = _PrintTable( + self.GetSummaryTables(summary_type="perf"), output_type + ) + sections.append(self._MakeSection("Perf Data", perf_table)) + + if experiment is not None: + experiment_file = experiment.experiment_file + sections.append( + self._MakeSection("Experiment File", experiment_file) + ) + + cpu_info = experiment.machine_manager.GetAllCPUInfo( + experiment.labels + ) + sections.append(self._MakeSection("CPUInfo", cpu_info)) + + totaltime = ( + (time.time() - experiment.start_time) + if experiment.start_time + else 0 + ) + totaltime_str = "Total experiment time:\n%d min" % (totaltime // 60) + cooldown_waittime_list = ["Cooldown wait time:"] + # When running experiment on multiple DUTs cooldown wait time may vary + # on different devices. In addition its combined time may exceed total + # experiment time which will look weird but it is reasonable. + # For this matter print cooldown time per DUT. + for dut, waittime in sorted( + self.GetTotalWaitCooldownTime().items() + ): + cooldown_waittime_list.append( + "DUT %s: %d min" % (dut, waittime // 60) + ) + cooldown_waittime_str = "\n".join(cooldown_waittime_list) + sections.append( + self._MakeSection( + "Duration", + "\n\n".join([totaltime_str, cooldown_waittime_str]), + ) + ) + + return "\n".join(sections) def _GetHTMLCharts(label_names, test_results): - charts = [] - for item, runs in test_results.items(): - # Fun fact: label_names is actually *entirely* useless as a param, since we - # never add headers. We still need to pass it anyway. - table = TableGenerator(runs, label_names).GetTable() - columns = [ - Column(AmeanResult(), Format()), - Column(MinResult(), Format()), - Column(MaxResult(), Format()) - ] - tf = TableFormatter(table, columns) - data_table = tf.GetCellTable('full', headers=False) - - for cur_row_data in data_table: - test_key = cur_row_data[0].string_value - title = '{0}: {1}'.format(item, test_key.replace('/', '')) - chart = ColumnChart(title, 300, 200) - chart.AddColumn('Label', 'string') - chart.AddColumn('Average', 'number') - chart.AddColumn('Min', 'number') - chart.AddColumn('Max', 'number') - chart.AddSeries('Min', 'line', 'black') - chart.AddSeries('Max', 'line', 'black') - cur_index = 1 - for label in label_names: - chart.AddRow([ - label, cur_row_data[cur_index].value, - cur_row_data[cur_index + 1].value, cur_row_data[cur_index + 2].value - ]) - if isinstance(cur_row_data[cur_index].value, str): - chart = None - break - cur_index += 3 - if chart: - charts.append(chart) - return charts + charts = [] + for item, runs in test_results.items(): + # Fun fact: label_names is actually *entirely* useless as a param, since we + # never add headers. We still need to pass it anyway. + table = TableGenerator(runs, label_names).GetTable() + columns = [ + Column(AmeanResult(), Format()), + Column(MinResult(), Format()), + Column(MaxResult(), Format()), + ] + tf = TableFormatter(table, columns) + data_table = tf.GetCellTable("full", headers=False) + + for cur_row_data in data_table: + test_key = cur_row_data[0].string_value + title = "{0}: {1}".format(item, test_key.replace("/", "")) + chart = ColumnChart(title, 300, 200) + chart.AddColumn("Label", "string") + chart.AddColumn("Average", "number") + chart.AddColumn("Min", "number") + chart.AddColumn("Max", "number") + chart.AddSeries("Min", "line", "black") + chart.AddSeries("Max", "line", "black") + cur_index = 1 + for label in label_names: + chart.AddRow( + [ + label, + cur_row_data[cur_index].value, + cur_row_data[cur_index + 1].value, + cur_row_data[cur_index + 2].value, + ] + ) + if isinstance(cur_row_data[cur_index].value, str): + chart = None + break + cur_index += 3 + if chart: + charts.append(chart) + return charts class HTMLResultsReport(ResultsReport): - """Class to generate html result report.""" - - def __init__(self, benchmark_results, experiment=None): - super(HTMLResultsReport, self).__init__(benchmark_results) - self.experiment = experiment - - @staticmethod - def FromExperiment(experiment): - return HTMLResultsReport( - BenchmarkResults.FromExperiment(experiment), experiment=experiment) - - def GetReport(self): - label_names = self.benchmark_results.label_names - test_results = self.benchmark_results.run_keyvals - charts = _GetHTMLCharts(label_names, test_results) - chart_javascript = ''.join(chart.GetJavascript() for chart in charts) - chart_divs = ''.join(chart.GetDiv() for chart in charts) - - if not self.benchmark_results.cwp_dso: - summary_table = self.GetSummaryTables() - perf_table = self.GetSummaryTables(summary_type='perf') - else: - summary_table = self.GetSummaryTables(summary_type='samples') - perf_table = None - full_table = self.GetFullTables() - - experiment_file = '' - if self.experiment is not None: - experiment_file = self.experiment.experiment_file - # Use kwargs for code readability, and so that testing is a bit easier. - return templates.GenerateHTMLPage( - perf_table=perf_table, - chart_js=chart_javascript, - summary_table=summary_table, - print_table=_PrintTable, - chart_divs=chart_divs, - full_table=full_table, - experiment_file=experiment_file) + """Class to generate html result report.""" + + def __init__(self, benchmark_results, experiment=None): + super(HTMLResultsReport, self).__init__(benchmark_results) + self.experiment = experiment + + @staticmethod + def FromExperiment(experiment): + return HTMLResultsReport( + BenchmarkResults.FromExperiment(experiment), experiment=experiment + ) + + def GetReport(self): + label_names = self.benchmark_results.label_names + test_results = self.benchmark_results.run_keyvals + charts = _GetHTMLCharts(label_names, test_results) + chart_javascript = "".join(chart.GetJavascript() for chart in charts) + chart_divs = "".join(chart.GetDiv() for chart in charts) + + if not self.benchmark_results.cwp_dso: + summary_table = self.GetSummaryTables() + perf_table = self.GetSummaryTables(summary_type="perf") + else: + summary_table = self.GetSummaryTables(summary_type="samples") + perf_table = None + full_table = self.GetFullTables() + + experiment_file = "" + if self.experiment is not None: + experiment_file = self.experiment.experiment_file + # Use kwargs for code readability, and so that testing is a bit easier. + return templates.GenerateHTMLPage( + perf_table=perf_table, + chart_js=chart_javascript, + summary_table=summary_table, + print_table=_PrintTable, + chart_divs=chart_divs, + full_table=full_table, + experiment_file=experiment_file, + ) def ParseStandardPerfReport(report_data): - """Parses the output of `perf report`. + """Parses the output of `perf report`. - It'll parse the following: - {{garbage}} - # Samples: 1234M of event 'foo' + It'll parse the following: + {{garbage}} + # Samples: 1234M of event 'foo' - 1.23% command shared_object location function::name + 1.23% command shared_object location function::name - 1.22% command shared_object location function2::name + 1.22% command shared_object location function2::name - # Samples: 999K of event 'bar' + # Samples: 999K of event 'bar' - 0.23% command shared_object location function3::name - {{etc.}} + 0.23% command shared_object location function3::name + {{etc.}} - Into: - {'foo': {'function::name': 1.23, 'function2::name': 1.22}, - 'bar': {'function3::name': 0.23, etc.}} - """ - # This function fails silently on its if it's handed a string (as opposed to a - # list of lines). So, auto-split if we do happen to get a string. - if isinstance(report_data, str): - report_data = report_data.splitlines() - # When switching to python3 catch the case when bytes are passed. - elif isinstance(report_data, bytes): - raise TypeError() - - # Samples: N{K,M,G} of event 'event-name' - samples_regex = re.compile(r"#\s+Samples: \d+\S? of event '([^']+)'") - - # We expect lines like: - # N.NN% command samples shared_object [location] symbol - # - # Note that we're looking at stripped lines, so there is no space at the - # start. - perf_regex = re.compile(r'^(\d+(?:.\d*)?)%' # N.NN% - r'\s*\d+' # samples count (ignored) - r'\s*\S+' # command (ignored) - r'\s*\S+' # shared_object (ignored) - r'\s*\[.\]' # location (ignored) - r'\s*(\S.+)' # function - ) - - stripped_lines = (l.strip() for l in report_data) - nonempty_lines = (l for l in stripped_lines if l) - # Ignore all lines before we see samples_regex - interesting_lines = itertools.dropwhile(lambda x: not samples_regex.match(x), - nonempty_lines) - - first_sample_line = next(interesting_lines, None) - # Went through the entire file without finding a 'samples' header. Quit. - if first_sample_line is None: - return {} - - sample_name = samples_regex.match(first_sample_line).group(1) - current_result = {} - results = {sample_name: current_result} - for line in interesting_lines: - samples_match = samples_regex.match(line) - if samples_match: - sample_name = samples_match.group(1) - current_result = {} - results[sample_name] = current_result - continue - - match = perf_regex.match(line) - if not match: - continue - percentage_str, func_name = match.groups() - try: - percentage = float(percentage_str) - except ValueError: - # Couldn't parse it; try to be "resilient". - continue - current_result[func_name] = percentage - return results + Into: + {'foo': {'function::name': 1.23, 'function2::name': 1.22}, + 'bar': {'function3::name': 0.23, etc.}} + """ + # This function fails silently on its if it's handed a string (as opposed to a + # list of lines). So, auto-split if we do happen to get a string. + if isinstance(report_data, str): + report_data = report_data.splitlines() + # When switching to python3 catch the case when bytes are passed. + elif isinstance(report_data, bytes): + raise TypeError() + + # Samples: N{K,M,G} of event 'event-name' + samples_regex = re.compile(r"#\s+Samples: \d+\S? of event '([^']+)'") + + # We expect lines like: + # N.NN% command samples shared_object [location] symbol + # + # Note that we're looking at stripped lines, so there is no space at the + # start. + perf_regex = re.compile( + r"^(\d+(?:.\d*)?)%" # N.NN% + r"\s*\d+" # samples count (ignored) + r"\s*\S+" # command (ignored) + r"\s*\S+" # shared_object (ignored) + r"\s*\[.\]" # location (ignored) + r"\s*(\S.+)" # function + ) + + stripped_lines = (l.strip() for l in report_data) + nonempty_lines = (l for l in stripped_lines if l) + # Ignore all lines before we see samples_regex + interesting_lines = itertools.dropwhile( + lambda x: not samples_regex.match(x), nonempty_lines + ) + + first_sample_line = next(interesting_lines, None) + # Went through the entire file without finding a 'samples' header. Quit. + if first_sample_line is None: + return {} + sample_name = samples_regex.match(first_sample_line).group(1) + current_result = {} + results = {sample_name: current_result} + for line in interesting_lines: + samples_match = samples_regex.match(line) + if samples_match: + sample_name = samples_match.group(1) + current_result = {} + results[sample_name] = current_result + continue + + match = perf_regex.match(line) + if not match: + continue + percentage_str, func_name = match.groups() + try: + percentage = float(percentage_str) + except ValueError: + # Couldn't parse it; try to be "resilient". + continue + current_result[func_name] = percentage + return results -def _ReadExperimentPerfReport(results_directory, label_name, benchmark_name, - benchmark_iteration): - """Reads a perf report for the given benchmark. Returns {} on failure. - The result should be a map of maps; it should look like: - {perf_event_name: {function_name: pct_time_spent}}, e.g. - {'cpu_cycles': {'_malloc': 10.0, '_free': 0.3, ...}} - """ - raw_dir_name = label_name + benchmark_name + str(benchmark_iteration + 1) - dir_name = ''.join(c for c in raw_dir_name if c.isalnum()) - file_name = os.path.join(results_directory, dir_name, 'perf.data.report.0') - try: - with open(file_name) as in_file: - return ParseStandardPerfReport(in_file) - except IOError: - # Yes, we swallow any IO-related errors. - return {} +def _ReadExperimentPerfReport( + results_directory, label_name, benchmark_name, benchmark_iteration +): + """Reads a perf report for the given benchmark. Returns {} on failure. + + The result should be a map of maps; it should look like: + {perf_event_name: {function_name: pct_time_spent}}, e.g. + {'cpu_cycles': {'_malloc': 10.0, '_free': 0.3, ...}} + """ + raw_dir_name = label_name + benchmark_name + str(benchmark_iteration + 1) + dir_name = "".join(c for c in raw_dir_name if c.isalnum()) + file_name = os.path.join(results_directory, dir_name, "perf.data.report.0") + try: + with open(file_name) as in_file: + return ParseStandardPerfReport(in_file) + except IOError: + # Yes, we swallow any IO-related errors. + return {} # Split out so that testing (specifically: mocking) is easier def _ExperimentToKeyvals(experiment, for_json_report): - """Converts an experiment to keyvals.""" - return OrganizeResults( - experiment.benchmark_runs, experiment.labels, json_report=for_json_report) + """Converts an experiment to keyvals.""" + return OrganizeResults( + experiment.benchmark_runs, + experiment.labels, + json_report=for_json_report, + ) class BenchmarkResults(object): - """The minimum set of fields that any ResultsReport will take.""" - - def __init__(self, - label_names, - benchmark_names_and_iterations, - run_keyvals, - ignore_min_max=False, - read_perf_report=None, - cwp_dso=None, - weights=None): - if read_perf_report is None: - - def _NoPerfReport(*_args, **_kwargs): - return {} - - read_perf_report = _NoPerfReport - - self.label_names = label_names - self.benchmark_names_and_iterations = benchmark_names_and_iterations - self.iter_counts = dict(benchmark_names_and_iterations) - self.run_keyvals = run_keyvals - self.ignore_min_max = ignore_min_max - self.read_perf_report = read_perf_report - self.cwp_dso = cwp_dso - self.weights = dict(weights) if weights else None - - @staticmethod - def FromExperiment(experiment, for_json_report=False): - label_names = [label.name for label in experiment.labels] - benchmark_names_and_iterations = [(benchmark.name, benchmark.iterations) - for benchmark in experiment.benchmarks] - run_keyvals = _ExperimentToKeyvals(experiment, for_json_report) - ignore_min_max = experiment.ignore_min_max - read_perf_report = functools.partial(_ReadExperimentPerfReport, - experiment.results_directory) - cwp_dso = experiment.cwp_dso - weights = [(benchmark.name, benchmark.weight) - for benchmark in experiment.benchmarks] - return BenchmarkResults(label_names, benchmark_names_and_iterations, - run_keyvals, ignore_min_max, read_perf_report, - cwp_dso, weights) + """The minimum set of fields that any ResultsReport will take.""" + + def __init__( + self, + label_names, + benchmark_names_and_iterations, + run_keyvals, + ignore_min_max=False, + read_perf_report=None, + cwp_dso=None, + weights=None, + ): + if read_perf_report is None: + + def _NoPerfReport(*_args, **_kwargs): + return {} + + read_perf_report = _NoPerfReport + + self.label_names = label_names + self.benchmark_names_and_iterations = benchmark_names_and_iterations + self.iter_counts = dict(benchmark_names_and_iterations) + self.run_keyvals = run_keyvals + self.ignore_min_max = ignore_min_max + self.read_perf_report = read_perf_report + self.cwp_dso = cwp_dso + self.weights = dict(weights) if weights else None + + @staticmethod + def FromExperiment(experiment, for_json_report=False): + label_names = [label.name for label in experiment.labels] + benchmark_names_and_iterations = [ + (benchmark.name, benchmark.iterations) + for benchmark in experiment.benchmarks + ] + run_keyvals = _ExperimentToKeyvals(experiment, for_json_report) + ignore_min_max = experiment.ignore_min_max + read_perf_report = functools.partial( + _ReadExperimentPerfReport, experiment.results_directory + ) + cwp_dso = experiment.cwp_dso + weights = [ + (benchmark.name, benchmark.weight) + for benchmark in experiment.benchmarks + ] + return BenchmarkResults( + label_names, + benchmark_names_and_iterations, + run_keyvals, + ignore_min_max, + read_perf_report, + cwp_dso, + weights, + ) def _GetElemByName(name, from_list): - """Gets an element from the given list by its name field. + """Gets an element from the given list by its name field. - Raises an error if it doesn't find exactly one match. - """ - elems = [e for e in from_list if e.name == name] - if len(elems) != 1: - raise ValueError('Expected 1 item named %s, found %d' % (name, len(elems))) - return elems[0] + Raises an error if it doesn't find exactly one match. + """ + elems = [e for e in from_list if e.name == name] + if len(elems) != 1: + raise ValueError( + "Expected 1 item named %s, found %d" % (name, len(elems)) + ) + return elems[0] def _Unlist(l): - """If l is a list, extracts the first element of l. Otherwise, returns l.""" - return l[0] if isinstance(l, list) else l + """If l is a list, extracts the first element of l. Otherwise, returns l.""" + return l[0] if isinstance(l, list) else l class JSONResultsReport(ResultsReport): - """Class that generates JSON reports for experiments.""" - - def __init__(self, - benchmark_results, - benchmark_date=None, - benchmark_time=None, - experiment=None, - json_args=None): - """Construct a JSONResultsReport. - - json_args is the dict of arguments we pass to json.dumps in GetReport(). - """ - super(JSONResultsReport, self).__init__(benchmark_results) - - defaults = TelemetryDefaults() - defaults.ReadDefaultsFile() - summary_field_defaults = defaults.GetDefault() - if summary_field_defaults is None: - summary_field_defaults = {} - self.summary_field_defaults = summary_field_defaults - - if json_args is None: - json_args = {} - self.json_args = json_args - - self.experiment = experiment - if not benchmark_date: - timestamp = datetime.datetime.strftime(datetime.datetime.now(), - '%Y-%m-%d %H:%M:%S') - benchmark_date, benchmark_time = timestamp.split(' ') - self.date = benchmark_date - self.time = benchmark_time - - @staticmethod - def FromExperiment(experiment, - benchmark_date=None, - benchmark_time=None, - json_args=None): - benchmark_results = BenchmarkResults.FromExperiment( - experiment, for_json_report=True) - return JSONResultsReport(benchmark_results, benchmark_date, benchmark_time, - experiment, json_args) - - def GetReportObjectIgnoringExperiment(self): - """Gets the JSON report object specifically for the output data. - - Ignores any experiment-specific fields (e.g. board, machine checksum, ...). - """ - benchmark_results = self.benchmark_results - label_names = benchmark_results.label_names - summary_field_defaults = self.summary_field_defaults - final_results = [] - for test, test_results in benchmark_results.run_keyvals.items(): - for label_name, label_results in zip(label_names, test_results): - for iter_results in label_results: - passed = iter_results.get('retval') == 0 - json_results = { - 'date': self.date, - 'time': self.time, - 'label': label_name, - 'test_name': test, - 'pass': passed, - } - final_results.append(json_results) - - if not passed: - continue - - # Get overall results. - summary_fields = summary_field_defaults.get(test) - if summary_fields is not None: - value = [] - json_results['overall_result'] = value - for f in summary_fields: - v = iter_results.get(f) - if v is None: + """Class that generates JSON reports for experiments.""" + + def __init__( + self, + benchmark_results, + benchmark_date=None, + benchmark_time=None, + experiment=None, + json_args=None, + ): + """Construct a JSONResultsReport. + + json_args is the dict of arguments we pass to json.dumps in GetReport(). + """ + super(JSONResultsReport, self).__init__(benchmark_results) + + defaults = TelemetryDefaults() + defaults.ReadDefaultsFile() + summary_field_defaults = defaults.GetDefault() + if summary_field_defaults is None: + summary_field_defaults = {} + self.summary_field_defaults = summary_field_defaults + + if json_args is None: + json_args = {} + self.json_args = json_args + + self.experiment = experiment + if not benchmark_date: + timestamp = datetime.datetime.strftime( + datetime.datetime.now(), "%Y-%m-%d %H:%M:%S" + ) + benchmark_date, benchmark_time = timestamp.split(" ") + self.date = benchmark_date + self.time = benchmark_time + + @staticmethod + def FromExperiment( + experiment, benchmark_date=None, benchmark_time=None, json_args=None + ): + benchmark_results = BenchmarkResults.FromExperiment( + experiment, for_json_report=True + ) + return JSONResultsReport( + benchmark_results, + benchmark_date, + benchmark_time, + experiment, + json_args, + ) + + def GetReportObjectIgnoringExperiment(self): + """Gets the JSON report object specifically for the output data. + + Ignores any experiment-specific fields (e.g. board, machine checksum, ...). + """ + benchmark_results = self.benchmark_results + label_names = benchmark_results.label_names + summary_field_defaults = self.summary_field_defaults + final_results = [] + for test, test_results in benchmark_results.run_keyvals.items(): + for label_name, label_results in zip(label_names, test_results): + for iter_results in label_results: + passed = iter_results.get("retval") == 0 + json_results = { + "date": self.date, + "time": self.time, + "label": label_name, + "test_name": test, + "pass": passed, + } + final_results.append(json_results) + + if not passed: + continue + + # Get overall results. + summary_fields = summary_field_defaults.get(test) + if summary_fields is not None: + value = [] + json_results["overall_result"] = value + for f in summary_fields: + v = iter_results.get(f) + if v is None: + continue + # New telemetry results format: sometimes we get a list of lists + # now. + v = _Unlist(_Unlist(v)) + value.append((f, float(v))) + + # Get detailed results. + detail_results = {} + json_results["detailed_results"] = detail_results + for k, v in iter_results.items(): + if ( + k == "retval" + or k == "PASS" + or k == ["PASS"] + or v == "PASS" + ): + continue + + v = _Unlist(v) + if "machine" in k: + json_results[k] = v + elif v is not None: + if isinstance(v, list): + detail_results[k] = [float(d) for d in v] + else: + detail_results[k] = float(v) + return final_results + + def GetReportObject(self): + """Generate the JSON report, returning it as a python object.""" + report_list = self.GetReportObjectIgnoringExperiment() + if self.experiment is not None: + self._AddExperimentSpecificFields(report_list) + return report_list + + def _AddExperimentSpecificFields(self, report_list): + """Add experiment-specific data to the JSON report.""" + board = self.experiment.labels[0].board + manager = self.experiment.machine_manager + for report in report_list: + label_name = report["label"] + label = _GetElemByName(label_name, self.experiment.labels) + + img_path = os.path.realpath( + os.path.expanduser(label.chromeos_image) + ) + ver, img = ParseChromeosImage(img_path) + + report.update( + { + "board": board, + "chromeos_image": img, + "chromeos_version": ver, + "chrome_version": label.chrome_version, + "compiler": label.compiler, + } + ) + + if not report["pass"]: continue - # New telemetry results format: sometimes we get a list of lists - # now. - v = _Unlist(_Unlist(v)) - value.append((f, float(v))) - - # Get detailed results. - detail_results = {} - json_results['detailed_results'] = detail_results - for k, v in iter_results.items(): - if k == 'retval' or k == 'PASS' or k == ['PASS'] or v == 'PASS': - continue - - v = _Unlist(v) - if 'machine' in k: - json_results[k] = v - elif v is not None: - if isinstance(v, list): - detail_results[k] = [float(d) for d in v] - else: - detail_results[k] = float(v) - return final_results - - def GetReportObject(self): - """Generate the JSON report, returning it as a python object.""" - report_list = self.GetReportObjectIgnoringExperiment() - if self.experiment is not None: - self._AddExperimentSpecificFields(report_list) - return report_list - - def _AddExperimentSpecificFields(self, report_list): - """Add experiment-specific data to the JSON report.""" - board = self.experiment.labels[0].board - manager = self.experiment.machine_manager - for report in report_list: - label_name = report['label'] - label = _GetElemByName(label_name, self.experiment.labels) - - img_path = os.path.realpath(os.path.expanduser(label.chromeos_image)) - ver, img = ParseChromeosImage(img_path) - - report.update({ - 'board': board, - 'chromeos_image': img, - 'chromeos_version': ver, - 'chrome_version': label.chrome_version, - 'compiler': label.compiler - }) - - if not report['pass']: - continue - if 'machine_checksum' not in report: - report['machine_checksum'] = manager.machine_checksum[label_name] - if 'machine_string' not in report: - report['machine_string'] = manager.machine_checksum_string[label_name] - - def GetReport(self): - """Dump the results of self.GetReportObject() to a string as JSON.""" - # This exists for consistency with the other GetReport methods. - # Specifically, they all return strings, so it's a bit awkward if the JSON - # results reporter returns an object. - return json.dumps(self.GetReportObject(), **self.json_args) + if "machine_checksum" not in report: + report["machine_checksum"] = manager.machine_checksum[ + label_name + ] + if "machine_string" not in report: + report["machine_string"] = manager.machine_checksum_string[ + label_name + ] + + def GetReport(self): + """Dump the results of self.GetReportObject() to a string as JSON.""" + # This exists for consistency with the other GetReport methods. + # Specifically, they all return strings, so it's a bit awkward if the JSON + # results reporter returns an object. + return json.dumps(self.GetReportObject(), **self.json_args) |