diff options
Diffstat (limited to 'crosperf/generate_report.py')
| -rwxr-xr-x | crosperf/generate_report.py | 277 |
1 files changed, 277 insertions, 0 deletions
diff --git a/crosperf/generate_report.py b/crosperf/generate_report.py new file mode 100755 index 00000000..e0add994 --- /dev/null +++ b/crosperf/generate_report.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python2 +# +# Copyright 2016 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Given a specially-formatted JSON object, generates results report(s). + +The JSON object should look like: +{"data": BenchmarkData, "platforms": BenchmarkPlatforms} + +BenchmarkPlatforms is a [str], each of which names a platform the benchmark + was run on (e.g. peppy, shamu, ...). Note that the order of this list is + related with the order of items in BenchmarkData. + +BenchmarkData is a {str: [PlatformData]}. The str is the name of the benchmark, +and a PlatformData is a set of data for a given platform. There must be one +PlatformData for each benchmark, for each element in BenchmarkPlatforms. + +A PlatformData is a [{str: float}], where each str names a metric we recorded, +and the float is the value for that metric. Each element is considered to be +the metrics collected from an independent run of this benchmark. NOTE: Each +PlatformData is expected to have a "retval" key, with the return value of +the benchmark. If the benchmark is successful, said return value should be 0. +Otherwise, this will break some of our JSON functionality. + +Putting it all together, a JSON object will end up looking like: + { "platforms": ["peppy", "peppy-new-crosstool"], + "data": { + "bench_draw_line": [ + [{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0}, + {"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}], + [{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0}, + {"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}] + ] + } + } + +Which says that we ran a benchmark on platforms named peppy, and + peppy-new-crosstool. +We ran one benchmark, named bench_draw_line. +It was run twice on each platform. +Peppy's runs took 1.321ms and 1.920ms, while peppy-new-crosstool's took 1.221ms + and 1.423ms. None of the runs failed to complete. +""" + +from __future__ import division +from __future__ import print_function + +import argparse +import functools +import json +import os +import sys +import traceback + +from results_report import BenchmarkResults +from results_report import HTMLResultsReport +from results_report import JSONResultsReport +from results_report import TextResultsReport + + +def CountBenchmarks(benchmark_runs): + """Counts the number of iterations for each benchmark in benchmark_runs.""" + # Example input for benchmark_runs: + # {"bench": [[run1, run2, run3], [run1, run2, run3, run4]]} + def _MaxLen(results): + return 0 if not results else max(len(r) for r in results) + return [(name, _MaxLen(results)) + for name, results in benchmark_runs.iteritems()] + + +def CutResultsInPlace(results, max_keys=50, complain_on_update=True): + """Limits the given benchmark results to max_keys keys in-place. + + This takes the `data` field from the benchmark input, and mutates each + benchmark run to contain `max_keys` elements (ignoring special elements, like + "retval"). At the moment, it just selects the first `max_keys` keyvals, + alphabetically. + + If complain_on_update is true, this will print a message noting that a + truncation occurred. + + This returns the `results` object that was passed in, for convenience. + + e.g. + >>> benchmark_data = { + ... "bench_draw_line": [ + ... [{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0}, + ... {"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}], + ... [{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0}, + ... {"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}] + ... ] + ... } + >>> CutResultsInPlace(benchmark_data, max_keys=1, complain_on_update=False) + { + 'bench_draw_line': [ + [{'memory (mb)': 128.1, 'retval': 0}, + {'memory (mb)': 128.4, 'retval': 0}], + [{'memory (mb)': 124.3, 'retval': 0}, + {'memory (mb)': 123.9, 'retval': 0}] + ] + } + """ + actually_updated = False + for bench_results in results.itervalues(): + for platform_results in bench_results: + for i, result in enumerate(platform_results): + # Keep the keys that come earliest when sorted alphabetically. + # Forcing alphabetical order is arbitrary, but necessary; otherwise, + # the keyvals we'd emit would depend on our iteration order through a + # map. + removable_keys = sorted(k for k in result if k != 'retval') + retained_keys = removable_keys[:max_keys] + platform_results[i] = {k: result[k] for k in retained_keys} + # retval needs to be passed through all of the time. + retval = result.get('retval') + if retval is not None: + platform_results[i]['retval'] = retval + actually_updated = actually_updated or \ + len(retained_keys) != len(removable_keys) + + if actually_updated and complain_on_update: + print("Warning: Some benchmark keyvals have been truncated.", + file=sys.stderr) + return results + + +def _ConvertToASCII(obj): + """Convert an object loaded from JSON to ASCII; JSON gives us unicode.""" + + # Using something like `object_hook` is insufficient, since it only fires on + # actual JSON objects. `encoding` fails, too, since the default decoder always + # uses unicode() to decode strings. + if isinstance(obj, unicode): + return str(obj) + if isinstance(obj, dict): + return {_ConvertToASCII(k): _ConvertToASCII(v) for k, v in obj.iteritems()} + if isinstance(obj, list): + return [_ConvertToASCII(v) for v in obj] + return obj + + +def _PositiveInt(s): + i = int(s) + if i < 0: + raise argparse.ArgumentTypeError('%d is not a positive integer.' % (i, )) + return i + + +def _AccumulateActions(args): + """Given program arguments, determines what actions we want to run. + + Returns [(ResultsReportCtor, str)], where ResultsReportCtor can construct a + ResultsReport, and the str is the file extension for the given report. + """ + results = [] + # The order of these is arbitrary. + if args.json: + results.append((JSONResultsReport, 'json')) + if args.text: + results.append((TextResultsReport, 'txt')) + if args.email: + email_ctor = functools.partial(TextResultsReport, email=True) + results.append((email_ctor, 'email')) + # We emit HTML if nothing else was specified. + if args.html or not results: + results.append((HTMLResultsReport, 'html')) + return results + + +# Note: get_contents is a function, because it may be expensive (generating some +# HTML reports takes O(seconds) on my machine, depending on the size of the +# input data). +def WriteFile(output_prefix, extension, get_contents, overwrite, verbose): + """Writes `contents` to a file named "${output_prefix}.${extension}". + + get_contents should be a zero-args function that returns a string (of the + contents to write). + If output_prefix == '-', this writes to stdout. + If overwrite is False, this will not overwrite files. + """ + if output_prefix == '-': + if verbose: + print('Writing %s report to stdout' % (extension, ), file=sys.stderr) + sys.stdout.write(get_contents()) + return + + file_name = '%s.%s' % (output_prefix, extension) + if not overwrite and os.path.exists(file_name): + raise IOError('Refusing to write %s -- it already exists' % (file_name, )) + + with open(file_name, 'w') as out_file: + if verbose: + print('Writing %s report to %s' % (extension, file_name), file=sys.stderr) + out_file.write(get_contents()) + + +def RunActions(actions, benchmark_results, output_prefix, overwrite, verbose): + """Runs `actions`, returning True if all succeeded.""" + failed = False + + report_ctor = None # Make the linter happy + for report_ctor, extension in actions: + try: + get_contents = lambda: report_ctor(benchmark_results).GetReport() + WriteFile(output_prefix, extension, get_contents, overwrite, verbose) + except Exception: + # Complain and move along; we may have more actions that might complete + # successfully. + failed = True + traceback.print_exc() + return not failed + + +def PickInputFile(input_name): + """Given program arguments, returns file to read for benchmark input.""" + return sys.stdin if input_name == '-' else open(input_name) + + +def _NoPerfReport(_label_name, _benchmark_name, _benchmark_iteration): + return {} + + +def _ParseArgs(argv): + parser = argparse.ArgumentParser(description='Turns JSON into results ' + 'report(s).') + parser.add_argument('-v', '--verbose', action='store_true', + help='Be a tiny bit more verbose.') + parser.add_argument('-f', '--force', action='store_true', + help='Overwrite existing results files.') + parser.add_argument('-o', '--output', default='report', type=str, + help='Prefix of the output filename (default: report). ' + '- means stdout.') + parser.add_argument('-i', '--input', required=True, type=str, + help='Where to read the JSON from. - means stdin.') + parser.add_argument('-l', '--statistic-limit', default=0, type=_PositiveInt, + help='The maximum number of benchmark statistics to ' + 'display from a single run. 0 implies unlimited.') + parser.add_argument('--json', action='store_true', + help='Output a JSON report.') + parser.add_argument('--text', action='store_true', + help='Output a text report.') + parser.add_argument('--email', action='store_true', + help='Output a text report suitable for email.') + parser.add_argument('--html', action='store_true', + help='Output an HTML report (this is the default if no ' + 'other output format is specified).') + return parser.parse_args(argv) + + +def Main(argv): + args = _ParseArgs(argv) + # JSON likes to load UTF-8; our results reporter *really* doesn't like + # UTF-8. + with PickInputFile(args.input) as in_file: + raw_results = _ConvertToASCII(json.load(in_file)) + + platform_names = raw_results['platforms'] + results = raw_results['data'] + if args.statistic_limit: + results = CutResultsInPlace(results, max_keys=args.statistic_limit) + benches = CountBenchmarks(results) + # In crosperf, a label is essentially a platform+configuration. So, a name of + # a label and a name of a platform are equivalent for our purposes. + bench_results = BenchmarkResults(label_names=platform_names, + benchmark_names_and_iterations=benches, + run_keyvals=results, + read_perf_report=_NoPerfReport) + actions = _AccumulateActions(args) + ok = RunActions(actions, bench_results, args.output, args.force, + args.verbose) + return 0 if ok else 1 + + +if __name__ == '__main__': + sys.exit(Main(sys.argv[1:])) |