diff options
Diffstat (limited to 'afdo_tools/generate_afdo_from_tryjob.py')
| -rwxr-xr-x | afdo_tools/generate_afdo_from_tryjob.py | 266 |
1 files changed, 139 insertions, 127 deletions
diff --git a/afdo_tools/generate_afdo_from_tryjob.py b/afdo_tools/generate_afdo_from_tryjob.py index 3ed578ea..e398f8a1 100755 --- a/afdo_tools/generate_afdo_from_tryjob.py +++ b/afdo_tools/generate_afdo_from_tryjob.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2019 The Chromium OS Authors. All rights reserved. +# Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Given a tryjob and perf profile, generates an AFDO profile.""" -from __future__ import print_function import argparse import distutils.spawn @@ -17,149 +16,162 @@ import subprocess import sys import tempfile -_CREATE_LLVM_PROF = 'create_llvm_prof' -_GS_PREFIX = 'gs://' + +_CREATE_LLVM_PROF = "create_llvm_prof" +_GS_PREFIX = "gs://" def _fetch_gs_artifact(remote_name, local_name): - assert remote_name.startswith(_GS_PREFIX) - subprocess.check_call(['gsutil', 'cp', remote_name, local_name]) + assert remote_name.startswith(_GS_PREFIX) + subprocess.check_call(["gsutil", "cp", remote_name, local_name]) def _fetch_and_maybe_unpack(remote_name, local_name): - unpackers = [ - ('.tar.bz2', ['tar', 'xaf']), - ('.bz2', ['bunzip2']), - ('.tar.xz', ['tar', 'xaf']), - ('.xz', ['xz', '-d']), - ] - - unpack_ext = None - unpack_cmd = None - for ext, unpack in unpackers: - if remote_name.endswith(ext): - unpack_ext, unpack_cmd = ext, unpack - break - - download_to = local_name + unpack_ext if unpack_ext else local_name - _fetch_gs_artifact(remote_name, download_to) - if unpack_cmd is not None: - print('Unpacking', download_to) - subprocess.check_output(unpack_cmd + [download_to]) - assert os.path.exists(local_name) + unpackers = [ + (".tar.bz2", ["tar", "xaf"]), + (".bz2", ["bunzip2"]), + (".tar.xz", ["tar", "xaf"]), + (".xz", ["xz", "-d"]), + ] + + unpack_ext = None + unpack_cmd = None + for ext, unpack in unpackers: + if remote_name.endswith(ext): + unpack_ext, unpack_cmd = ext, unpack + break + + download_to = local_name + unpack_ext if unpack_ext else local_name + _fetch_gs_artifact(remote_name, download_to) + if unpack_cmd is not None: + print("Unpacking", download_to) + subprocess.check_output(unpack_cmd + [download_to]) + assert os.path.exists(local_name) def _generate_afdo(perf_profile_loc, tryjob_loc, output_name): - if perf_profile_loc.startswith(_GS_PREFIX): - local_loc = 'perf.data' - _fetch_and_maybe_unpack(perf_profile_loc, local_loc) - perf_profile_loc = local_loc - - chrome_in_debug_loc = 'debug/opt/google/chrome/chrome.debug' - debug_out = 'debug.tgz' - _fetch_gs_artifact(os.path.join(tryjob_loc, 'debug.tgz'), debug_out) - - print('Extracting chrome.debug.') - # This has tons of artifacts, and we only want Chrome; don't waste time - # extracting the rest in _fetch_and_maybe_unpack. - subprocess.check_call(['tar', 'xaf', 'debug.tgz', chrome_in_debug_loc]) - - # Note that the AFDO tool *requires* a binary named `chrome` to be present if - # we're generating a profile for chrome. It's OK for this to be split debug - # information. - os.rename(chrome_in_debug_loc, 'chrome') - - print('Generating AFDO profile.') - subprocess.check_call([ - _CREATE_LLVM_PROF, '--out=' + output_name, '--binary=chrome', - '--profile=' + perf_profile_loc - ]) + if perf_profile_loc.startswith(_GS_PREFIX): + local_loc = "perf.data" + _fetch_and_maybe_unpack(perf_profile_loc, local_loc) + perf_profile_loc = local_loc + + chrome_in_debug_loc = "debug/opt/google/chrome/chrome.debug" + debug_out = "debug.tgz" + _fetch_gs_artifact(os.path.join(tryjob_loc, "debug.tgz"), debug_out) + + print("Extracting chrome.debug.") + # This has tons of artifacts, and we only want Chrome; don't waste time + # extracting the rest in _fetch_and_maybe_unpack. + subprocess.check_call(["tar", "xaf", "debug.tgz", chrome_in_debug_loc]) + + # Note that the AFDO tool *requires* a binary named `chrome` to be present if + # we're generating a profile for chrome. It's OK for this to be split debug + # information. + os.rename(chrome_in_debug_loc, "chrome") + + print("Generating AFDO profile.") + subprocess.check_call( + [ + _CREATE_LLVM_PROF, + "--out=" + output_name, + "--binary=chrome", + "--profile=" + perf_profile_loc, + ] + ) def _abspath_or_gs_link(path): - if path.startswith(_GS_PREFIX): - return path - return os.path.abspath(path) + if path.startswith(_GS_PREFIX): + return path + return os.path.abspath(path) def _tryjob_arg(tryjob_arg): - # Forward gs args through - if tryjob_arg.startswith(_GS_PREFIX): - return tryjob_arg + # Forward gs args through + if tryjob_arg.startswith(_GS_PREFIX): + return tryjob_arg - # Clicking on the 'Artifacts' link gives us a pantheon link that's basically - # a preamble and gs path. - pantheon = 'https://pantheon.corp.google.com/storage/browser/' - if tryjob_arg.startswith(pantheon): - return _GS_PREFIX + tryjob_arg[len(pantheon):] + # Clicking on the 'Artifacts' link gives us a pantheon link that's basically + # a preamble and gs path. + pantheon = "https://pantheon.corp.google.com/storage/browser/" + if tryjob_arg.startswith(pantheon): + return _GS_PREFIX + tryjob_arg[len(pantheon) :] - # Otherwise, only do things with a tryjob ID (e.g. R75-11965.0.0-b3648595) - if not tryjob_arg.startswith('R'): - raise ValueError('Unparseable tryjob arg; give a tryjob ID, pantheon ' - 'link, or gs:// link. Please see source for more.') + # Otherwise, only do things with a tryjob ID (e.g. R75-11965.0.0-b3648595) + if not tryjob_arg.startswith("R"): + raise ValueError( + "Unparseable tryjob arg; give a tryjob ID, pantheon " + "link, or gs:// link. Please see source for more." + ) - chell_path = 'chromeos-image-archive/chell-chrome-pfq-tryjob/' - # ...And assume it's from chell, since that's the only thing we generate - # profiles with today. - return _GS_PREFIX + chell_path + tryjob_arg + chell_path = "chromeos-image-archive/chell-chrome-pfq-tryjob/" + # ...And assume it's from chell, since that's the only thing we generate + # profiles with today. + return _GS_PREFIX + chell_path + tryjob_arg def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - '--perf_profile', - required=True, - help='Path to our perf profile. Accepts either a gs:// path or local ' - 'filepath.') - parser.add_argument( - '--tryjob', - required=True, - type=_tryjob_arg, - help="Path to our tryjob's artifacts. Accepts a gs:// path, pantheon " - 'link, or tryjob ID, e.g. R75-11965.0.0-b3648595. In the last case, ' - 'the assumption is that you ran a chell-chrome-pfq-tryjob.') - parser.add_argument( - '-o', - '--output', - default='afdo.prof', - help='Where to put the AFDO profile. Default is afdo.prof.') - parser.add_argument( - '-k', - '--keep_artifacts_on_failure', - action='store_true', - help="Don't remove the tempdir on failure") - args = parser.parse_args() - - if not distutils.spawn.find_executable(_CREATE_LLVM_PROF): - sys.exit(_CREATE_LLVM_PROF + ' not found; are you in the chroot?') - - profile = _abspath_or_gs_link(args.perf_profile) - afdo_output = os.path.abspath(args.output) - - initial_dir = os.getcwd() - temp_dir = tempfile.mkdtemp(prefix='generate_afdo') - success = True - try: - os.chdir(temp_dir) - _generate_afdo(profile, args.tryjob, afdo_output) - - # The AFDO tooling is happy to generate essentially empty profiles for us. - # Chrome's profiles are often 8+ MB; if we only see a small fraction of - # that, something's off. 512KB was arbitrarily selected. - if os.path.getsize(afdo_output) < 512 * 1024: - raise ValueError('The AFDO profile is suspiciously small for Chrome. ' - 'Something might have gone wrong.') - except: - success = False - raise - finally: - os.chdir(initial_dir) - - if success or not args.keep_artifacts_on_failure: - shutil.rmtree(temp_dir, ignore_errors=True) - else: - print('Artifacts are available at', temp_dir) - - -if __name__ == '__main__': - sys.exit(main()) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--perf_profile", + required=True, + help="Path to our perf profile. Accepts either a gs:// path or local " + "filepath.", + ) + parser.add_argument( + "--tryjob", + required=True, + type=_tryjob_arg, + help="Path to our tryjob's artifacts. Accepts a gs:// path, pantheon " + "link, or tryjob ID, e.g. R75-11965.0.0-b3648595. In the last case, " + "the assumption is that you ran a chell-chrome-pfq-tryjob.", + ) + parser.add_argument( + "-o", + "--output", + default="afdo.prof", + help="Where to put the AFDO profile. Default is afdo.prof.", + ) + parser.add_argument( + "-k", + "--keep_artifacts_on_failure", + action="store_true", + help="Don't remove the tempdir on failure", + ) + args = parser.parse_args() + + if not distutils.spawn.find_executable(_CREATE_LLVM_PROF): + sys.exit(_CREATE_LLVM_PROF + " not found; are you in the chroot?") + + profile = _abspath_or_gs_link(args.perf_profile) + afdo_output = os.path.abspath(args.output) + + initial_dir = os.getcwd() + temp_dir = tempfile.mkdtemp(prefix="generate_afdo") + success = True + try: + os.chdir(temp_dir) + _generate_afdo(profile, args.tryjob, afdo_output) + + # The AFDO tooling is happy to generate essentially empty profiles for us. + # Chrome's profiles are often 8+ MB; if we only see a small fraction of + # that, something's off. 512KB was arbitrarily selected. + if os.path.getsize(afdo_output) < 512 * 1024: + raise ValueError( + "The AFDO profile is suspiciously small for Chrome. " + "Something might have gone wrong." + ) + except: + success = False + raise + finally: + os.chdir(initial_dir) + + if success or not args.keep_artifacts_on_failure: + shutil.rmtree(temp_dir, ignore_errors=True) + else: + print("Artifacts are available at", temp_dir) + + +if __name__ == "__main__": + sys.exit(main()) |