Add AFDO generation scripts

People want to generate AFDO profiles, which is a somewhat error-prone
and complicated process for a few reasons. So, have two scripts.

The first of these allows you to easily generate an AFDO perf profile on
a trybot. The second gives you a convenient way to turn that into an
AFDO profile. Both have accompanying documentation, so the hope is that
they're usable.

BUG=None
TEST=None

Change-Id: I4afb665ffc8550c718b3867ca3d5b98aa4cd0901
Reviewed-on: https://chromium-review.googlesource.com/1526721
Commit-Ready: ChromeOS CL Exonerator Bot <chromiumos-cl-exonerator@appspot.gserviceaccount.com>
Tested-by: George Burgess <gbiv@chromium.org>
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>

2 files changed, 336 insertions, 0 deletions

diff --git a/afdo_tools/generate_afdo_from_tryjob.py b/afdo_tools/generate_afdo_from_tryjob.py
new file mode 100755
index 00000000..b8a2d669
--- /dev/null
+++ b/afdo_tools/generate_afdo_from_tryjob.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Given a tryjob and perf profile, generates an AFDO profile."""
+
+from __future__ import print_function
+
+import argparse
+import distutils.spawn
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+import tempfile
+
+_CREATE_LLVM_PROF = 'create_llvm_prof'
+_GS_PREFIX = 'gs://'
+
+
+def _fetch_gs_artifact(remote_name, local_name):
+  assert remote_name.startswith(_GS_PREFIX)
+  subprocess.check_call(['gsutil', 'cp', remote_name, local_name])
+
+
+def _fetch_and_maybe_unpack(remote_name, local_name):
+  unpackers = [
+      ('.tar.bz2', ['tar', 'xaf']),
+      ('.bz2', ['bunzip2']),
+      ('.tar.xz', ['tar', 'xaf']),
+      ('.xz', ['xz', '-d']),
+  ]
+
+  unpack_ext = None
+  unpack_cmd = None
+  for ext, unpack in unpackers:
+    if remote_name.endswith(ext):
+      unpack_ext, unpack_cmd = ext, unpack
+      break
+
+  download_to = local_name + unpack_ext if unpack_ext else local_name
+  _fetch_gs_artifact(remote_name, download_to)
+  if unpack_cmd is not None:
+    print('Unpacking', download_to)
+    subprocess.check_output(unpack_cmd + [download_to])
+    assert os.path.exists(local_name)
+
+
+def _generate_afdo(perf_profile_loc, tryjob_loc, output_name):
+  if perf_profile_loc.startswith(_GS_PREFIX):
+    local_loc = 'perf.data'
+    _fetch_and_maybe_unpack(perf_profile_loc, local_loc)
+    perf_profile_loc = local_loc
+
+  chrome_in_debug_loc = 'debug/opt/google/chrome/chrome.debug'
+  debug_out = 'debug.tgz'
+  _fetch_gs_artifact(os.path.join(tryjob_loc, 'debug.tgz'), debug_out)
+
+  print('Extracting chrome.debug.')
+  # This has tons of artifacts, and we only want Chrome; don't waste time
+  # extracting the rest in _fetch_and_maybe_unpack.
+  subprocess.check_call(['tar', 'xaf', 'debug.tgz', chrome_in_debug_loc])
+
+  # Note that the AFDO tool *requires* a binary named `chrome` to be present if
+  # we're generating a profile for chrome. It's OK for this to be split debug
+  # information.
+  os.rename(chrome_in_debug_loc, 'chrome')
+
+  print('Generating AFDO profile.')
+  subprocess.check_call([
+      _CREATE_LLVM_PROF, '--out=' + output_name, '--binary=chrome',
+      '--profile=' + perf_profile_loc
+  ])
+
+
+def _abspath_or_gs_link(path):
+  if path.startswith(_GS_PREFIX):
+    return path
+  return os.path.abspath(path)
+
+
+def _tryjob_arg(tryjob_arg):
+  # Forward gs args through
+  if tryjob_arg.startswith(_GS_PREFIX):
+    return tryjob_arg
+
+  # Clicking on the 'Artifacts' link gives us a pantheon link that's basically
+  # a preamble and gs path.
+  pantheon = 'https://pantheon.corp.google.com/storage/browser/'
+  if tryjob_arg.startswith(pantheon):
+    return _GS_PREFIX + tryjob_arg[len(pantheon):]
+
+  # Otherwise, only do things with a tryjob ID (e.g. R75-11965.0.0-b3648595)
+  if not tryjob_arg.startswith('R'):
+    raise ValueError('Unparseable tryjob arg; give a tryjob ID, pantheon '
+                     'link, or gs:// link. Please see source for more.')
+
+  chell_path = 'chromeos-image-archive/chell-chrome-pfq-tryjob/'
+  # ...And assume it's from chell, since that's the only thing we generate
+  # profiles with today.
+  return _GS_PREFIX + chell_path + tryjob_arg
+
+
+def main():
+  parser = argparse.ArgumentParser(description=__doc__)
+  parser.add_argument(
+      '--perf_profile',
+      required=True,
+      help='Path to our perf profile. Accepts either a gs:// path or local '
+      'filepath.')
+  parser.add_argument(
+      '--tryjob',
+      required=True,
+      type=_tryjob_arg,
+      help='Path to our tryjob\'s artifacts. Accepts a gs:// path, pantheon '
+      'link, or tryjob ID, e.g. R75-11965.0.0-b3648595. In the last case, '
+      'the assumption is that you ran a chell-chrome-pfq-tryjob.')
+  parser.add_argument(
+      '-o',
+      '--output',
+      default='afdo.prof',
+      help='Where to put the AFDO profile. Default is afdo.prof.')
+  parser.add_argument(
+      '-k',
+      '--keep_artifacts_on_failure',
+      action='store_true',
+      help='Don\'t remove the tempdir on failure')
+  args = parser.parse_args()
+
+  if not distutils.spawn.find_executable(_CREATE_LLVM_PROF):
+    sys.exit(_CREATE_LLVM_PROF + ' not found; are you in the chroot?')
+
+  profile = _abspath_or_gs_link(args.perf_profile)
+  afdo_output = os.path.abspath(args.output)
+
+  initial_dir = os.getcwd()
+  temp_dir = tempfile.mkdtemp(prefix='generate_afdo')
+  success = True
+  try:
+    os.chdir(temp_dir)
+    _generate_afdo(profile, args.tryjob, afdo_output)
+
+    # The AFDO tooling is happy to generate essentially empty profiles for us.
+    # Chrome's profiles are often 8+ MB; if we only see a small fraction of
+    # that, something's off. 512KB was arbitrarily selected.
+    if os.path.getsize(afdo_output) < 512 * 1024:
+      raise ValueError('The AFDO profile is suspiciously small for Chrome. '
+                       'Something might have gone wrong.')
+  except:
+    success = False
+    raise
+  finally:
+    os.chdir(initial_dir)
+
+    if success or not args.keep_artifacts_on_failure:
+      shutil.rmtree(temp_dir, ignore_errors=True)
+    else:
+      print('Artifacts are available at', temp_dir)
+
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/afdo_tools/run_afdo_tryjob.py b/afdo_tools/run_afdo_tryjob.py
new file mode 100755
index 00000000..de45af0b
--- /dev/null
+++ b/afdo_tools/run_afdo_tryjob.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Spawns off an AFDO tryjob.
+
+This tryjob will cause perf profiles to be collected as though we were running
+our benchmark AFDO pipeline. Depending on the set of flags that you use,
+different things will happen. Any artifacts will land in
+gs://chromeos-localmirror/distfiles/afdo/experimental/approximation
+
+This tryjob will generate *either* a full (AFDO profile, perf.data,
+chrome.debug) combo, or just a perf.data, depending on the arguments you feed
+it.
+
+The thing to be careful of is that our localmirror bucket is shared between
+everyone, so it's super easy for two AFDO profile runs to 'collide'. Hence, if
+you provide the --tag_profiles_with_current_time flag, the script will generate
+*only* a perf.data, but that perf.data will have a timestamp (with second
+resolution) on it. This makes collisions super unlikely.
+
+If you'd like to know which perf profile was yours:
+  - Go to the tryjob output page
+  - Look for 'HWTest [AFDO_Record]'
+  - Click on its stdout
+  - Find "Links to test logs:" in the stdout
+  - Follow the link by telemetry_AFDOGenerate
+  - Find and click the link to debug/autoserv.DEBUG
+  - Look for a gs:// link ending in `.perf.data` with a compression suffix
+    (currently `.bz2`; maybe `.xz` eventually). That's the gs:// path to your
+    perf profile.
+
+The downside to this option is that there's no (reliable + trivial to
+implement) way for the bot that converts AFDO profiles into perf profiles to
+know the profile to choose. So, you're stuck generating a profile on your own.
+We have a tool for just that. Please see `generate_afdo_from_tryjob.py`.
+
+If you don't like that tool, generating your own profile isn't super difficult.
+Just grab the perf profile that your logs note from gs://, grab a copy of
+chrome.debug from your tryjob, and use `create_llvm_prof` to create a profile.
+
+On the other hand, if you're 100% sure that your profile won't collide, you can
+make your life easier by providing --use_afdo_generation_stage.
+
+If you provide neither --use_afdo_generation_stage nor
+--tag_profiles_with_current_time, --tag_profiles_with_current_time is implied,
+since it's safer.
+"""
+
+from __future__ import print_function
+
+import argparse
+import collections
+import pipes
+import subprocess
+import sys
+import time
+
+
+def main():
+  parser = argparse.ArgumentParser(
+      description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+  parser.add_argument(
+      '--force_no_patches',
+      action='store_true',
+      help='Run even if no patches are provided')
+  parser.add_argument(
+      '--tag_profiles_with_current_time',
+      action='store_true',
+      help='Perf profile names will have the current time added to them.')
+  parser.add_argument(
+      '--use_afdo_generation_stage',
+      action='store_true',
+      help='Perf profiles will be automatically converted to AFDO profiles.')
+  parser.add_argument(
+      '-g',
+      '--patch',
+      action='append',
+      default=[],
+      help='A patch to add to the AFDO run')
+  parser.add_argument(
+      '-n',
+      '--dry_run',
+      action='store_true',
+      help='Just print the command that would be run')
+  args = parser.parse_args()
+
+  dry_run = args.dry_run
+  force_no_patches = args.force_no_patches
+  tag_profiles_with_current_time = args.tag_profiles_with_current_time
+  use_afdo_generation_stage = args.use_afdo_generation_stage
+  user_patches = args.patch
+
+  if tag_profiles_with_current_time and use_afdo_generation_stage:
+    raise ValueError('You can\'t tag profiles with the time + have '
+                     'afdo-generate')
+
+  if not tag_profiles_with_current_time and not use_afdo_generation_stage:
+    print('Neither current_time nor afdo_generate asked for. Assuming you '
+          'prefer current time tagging.')
+    print('You have 5 seconds to cancel and try again.')
+    print()
+    if not dry_run:
+      time.sleep(5)
+    tag_profiles_with_current_time = True
+
+  patches = [
+      # Send profiles to localmirror instead of chromeos-prebuilt. This should
+      # always be done, since sending profiles into production is bad. :)
+      # https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/1436158
+      1436158,
+      # Force profile generation. Otherwise, we'll decide to not spawn off the
+      # perf hwtests.
+      # https://chromium-review.googlesource.com/c/chromiumos/chromite/+/1313291
+      1313291,
+  ]
+
+  if tag_profiles_with_current_time:
+    # Tags the profiles with the current time of day. As detailed in the
+    # docstring, this is desirable unless you're sure that this is the only
+    # experimental profile that will be generated today.
+    # https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/1436157
+    patches.append(1436157)
+
+  if use_afdo_generation_stage:
+    # Make the profile generation stage look in localmirror, instead of having
+    # it look in chromeos-prebuilt. Without this, we'll never upload
+    # chrome.debug or try to generate an AFDO profile.
+    # https://chromium-review.googlesource.com/c/chromiumos/chromite/+/1436583
+    patches.append(1436583)
+
+  if not user_patches and not force_no_patches:
+    raise ValueError('No patches given; pass --force_no_patches to force a '
+                     'tryjob')
+
+  for patch in user_patches:
+    # We accept two formats. Either a URL that ends with a number, or a number.
+    if patch.startswith('http'):
+      patch = patch.split('/')[-1]
+    patches.append(int(patch))
+
+  count = collections.Counter(patches)
+  too_many = [k for k, v in count.items() if v > 1]
+  if too_many:
+    too_many.sort()
+    raise ValueError(
+        'Patch(es) asked for application more than once: %s' % too_many)
+
+  args = [
+      'cros',
+      'tryjob',
+  ]
+
+  for patch in patches:
+    args += ['-g', str(patch)]
+
+  args += [
+      '--nochromesdk',
+      '--hwtest',
+      'chell-chrome-pfq-tryjob',
+  ]
+
+  print(' '.join(pipes.quote(a) for a in args))
+  if not dry_run:
+    sys.exit(subprocess.call(args))
+
+
+if __name__ == '__main__':
+  main()