1 files changed, 385 insertions, 0 deletions
diff --git a/cros_utils/buildbot_utils.py b/cros_utils/buildbot_utils.py
new file mode 100644
index 00000000..d1403557
--- /dev/null
+++ b/cros_utils/buildbot_utils.py
@@ -0,0 +1,385 @@
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Utilities for launching and accessing ChromeOS buildbots."""
+
+from __future__ import print_function
+
+import base64
+import json
+import os
+import time
+import urllib2
+
+# pylint: disable=no-name-in-module
+from oauth2client.service_account import ServiceAccountCredentials
+
+from cros_utils import command_executer
+from cros_utils import logger
+from cros_utils import buildbot_json
+
+INITIAL_SLEEP_TIME = 7200  # 2 hours; wait time before polling buildbot.
+SLEEP_TIME = 600  # 10 minutes; time between polling of buildbot.
+TIME_OUT = 28800  # Decide the build is dead or will never finish
+# after this time (8 hours).
+OK_STATUS = [  # List of result status values that are 'ok'.
+    # This was obtained from:
+    #   https://chromium.googlesource.com/chromium/tools/build/+/
+    #       master/third_party/buildbot_8_4p1/buildbot/status/results.py
+    0,  # "success"
+    1,  # "warnings"
+    6,  # "retry"
+]
+
+
+class BuildbotTimeout(Exception):
+  """Exception to throw when a buildbot operation timesout."""
+  pass
+
+
+def ParseReportLog(url, build):
+  """Scrape the trybot image name off the Reports log page.
+
+  This takes the URL for a trybot Reports Stage web page,
+  and a trybot build type, such as 'daisy-release'.  It
+  opens the web page and parses it looking for the trybot
+  artifact name (e.g. something like
+  'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the
+  artifact name, if found.
+  """
+  trybot_image = ''
+  url += '/text'
+  newurl = url.replace('uberchromegw', 'chromegw')
+  webpage = urllib2.urlopen(newurl)
+  data = webpage.read()
+  lines = data.split('\n')
+  for l in lines:
+    if l.find('Artifacts') > 0 and l.find('trybot') > 0:
+      trybot_name = 'trybot-%s' % build
+      start_pos = l.find(trybot_name)
+      end_pos = l.find('@https://storage')
+      trybot_image = l[start_pos:end_pos]
+
+  return trybot_image
+
+
+def GetBuildData(buildbot_queue, build_id):
+  """Find the Reports stage web page for a trybot build.
+
+  This takes the name of a buildbot_queue, such as 'daisy-release'
+  and a build id (the build number), and uses the json buildbot api to
+  find the Reports stage web page for that build, if it exists.
+  """
+  builder = buildbot_json.Buildbot(
+      'http://chromegw/p/tryserver.chromiumos/').builders[buildbot_queue]
+  build_data = builder.builds[build_id].data
+  logs = build_data['logs']
+  for l in logs:
+    fname = l[1]
+    if 'steps/Report/' in fname:
+      return fname
+
+  return ''
+
+
+def FindBuildRecordFromLog(description, build_info):
+  """Find the right build record in the build logs.
+
+  Get the first build record from build log with a reason field
+  that matches 'description'. ('description' is a special tag we
+  created when we launched the buildbot, so we could find it at this
+  point.)
+  """
+  for build_log in build_info:
+    if description in build_log['reason']:
+      return build_log
+  return {}
+
+
+def GetBuildInfo(file_dir, waterfall_builder):
+  """Get all the build records for the trybot builds."""
+
+  builder = ''
+  if waterfall_builder.endswith('-release'):
+    builder = 'release'
+  elif waterfall_builder.endswith('-gcc-toolchain'):
+    builder = 'gcc_toolchain'
+  elif waterfall_builder.endswith('-llvm-toolchain'):
+    builder = 'llvm_toolchain'
+  elif waterfall_builder.endswith('-llvm-next-toolchain'):
+    builder = 'llvm_next_toolchain'
+
+  sa_file = os.path.expanduser(
+      os.path.join(file_dir, 'cros_utils',
+                   'chromeos-toolchain-credentials.json'))
+  scopes = ['https://www.googleapis.com/auth/userinfo.email']
+
+  credentials = ServiceAccountCredentials.from_json_keyfile_name(
+      sa_file, scopes=scopes)
+  url = (
+      'https://luci-milo.appspot.com/prpc/milo.Buildbot/GetBuildbotBuildsJSON')
+
+  # NOTE: If we want to get build logs for the main waterfall builders, the
+  # 'master' field below should be 'chromeos' instead of 'chromiumos.tryserver'.
+  # Builder would be 'amd64-gcc-toolchain' or 'arm-llvm-toolchain', etc.
+
+  body = json.dumps({
+      'master': 'chromiumos.tryserver',
+      'builder': builder,
+      'include_current': True,
+      'limit': 100
+  })
+  access_token = credentials.get_access_token()
+  headers = {
+      'Accept': 'application/json',
+      'Content-Type': 'application/json',
+      'Authorization': 'Bearer %s' % access_token.access_token
+  }
+  r = urllib2.Request(url, body, headers)
+  u = urllib2.urlopen(r, timeout=60)
+  u.read(4)
+  o = json.load(u)
+  data = [base64.b64decode(item['data']) for item in o['builds']]
+  result = []
+  for d in data:
+    tmp = json.loads(d)
+    result.append(tmp)
+  return result
+
+
+def FindArchiveImage(chromeos_root, build, build_id):
+  """Returns name of the trybot artifact for board/build_id."""
+  ce = command_executer.GetCommandExecuter()
+  command = ('gsutil ls gs://chromeos-image-archive/trybot-%s/*b%s'
+             '/chromiumos_test_image.tar.xz' % (build, build_id))
+  _, out, _ = ce.ChrootRunCommandWOutput(
+      chromeos_root, command, print_to_console=False)
+  #
+  # If build_id is not unique, there may be multiple archive images
+  # to choose from; sort them & pick the first (newest).
+  #
+  # If there are multiple archive images found, out will look something
+  # like this:
+  #
+  # 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz
+  #  gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz'
+  #
+  out = out.rstrip('\n')
+  tmp_list = out.split('\n')
+  # After stripping the final '\n' and splitting on any other '\n', we get
+  # something like this:
+  #  tmp_list = [ 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz' ,
+  #               'gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz' ]
+  #
+  #  If we sort this in descending order, we should end up with the most
+  #  recent test image first, so that's what we do here.
+  #
+  if len(tmp_list) > 1:
+    tmp_list = sorted(tmp_list, reverse=True)
+  out = tmp_list[0]
+
+  trybot_image = ''
+  trybot_name = 'trybot-%s' % build
+  if out and out.find(trybot_name) > 0:
+    start_pos = out.find(trybot_name)
+    end_pos = out.find('/chromiumos_test_image')
+    trybot_image = out[start_pos:end_pos]
+
+  return trybot_image
+
+
+def GetTrybotImage(chromeos_root,
+                   buildbot_name,
+                   patch_list,
+                   build_tag,
+                   other_flags=None,
+                   build_toolchain=False,
+                   async=False):
+  """Launch buildbot and get resulting trybot artifact name.
+
+  This function launches a buildbot with the appropriate flags to
+  build the test ChromeOS image, with the current ToT mobile compiler.  It
+  checks every 10 minutes to see if the trybot has finished.  When the trybot
+  has finished, it parses the resulting report logs to find the trybot
+  artifact (if one was created), and returns that artifact name.
+
+  chromeos_root is the path to the ChromeOS root, needed for finding chromite
+  and launching the buildbot.
+
+  buildbot_name is the name of the buildbot queue, such as lumpy-release or
+  daisy-paladin.
+
+  patch_list a python list of the patches, if any, for the buildbot to use.
+
+  build_tag is a (unique) string to be used to look up the buildbot results
+  from among all the build records.
+  """
+  ce = command_executer.GetCommandExecuter()
+  cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot')
+  base_dir = os.getcwd()
+  patch_arg = ''
+  if patch_list:
+    for p in patch_list:
+      patch_arg = patch_arg + ' -g ' + repr(p)
+  toolchain_flags = ''
+  if build_toolchain:
+    toolchain_flags += '--latest-toolchain'
+  os.chdir(cbuildbot_path)
+  if other_flags:
+    optional_flags = ' '.join(other_flags)
+  else:
+    optional_flags = ''
+
+  # Launch buildbot with appropriate flags.
+  build = buildbot_name
+  description = build_tag
+  command_prefix = ''
+  if not patch_arg:
+    command_prefix = 'yes | '
+  command = ('%s ./cbuildbot --remote --nochromesdk %s'
+             ' --remote-description=%s %s %s %s' % (command_prefix,
+                                                    optional_flags, description,
+                                                    toolchain_flags, patch_arg,
+                                                    build))
+  _, out, _ = ce.RunCommandWOutput(command)
+  if 'Tryjob submitted!' not in out:
+    logger.GetLogger().LogFatal('Error occurred while launching trybot job: '
+                                '%s' % command)
+
+  os.chdir(base_dir)
+
+  build_id = 0
+  build_status = None
+  # Wait for  buildbot to finish running (check every 10 minutes).  Wait
+  # 10 minutes before the first check to give the buildbot time to launch
+  # (so we don't start looking for build data before it's out there).
+  time.sleep(SLEEP_TIME)
+  done = False
+  pending = True
+  # pending_time is the time between when we submit the job and when the
+  # buildbot actually launches the build.  running_time is the time between
+  # when the buildbot job launches and when it finishes.  The job is
+  # considered 'pending' until we can find an entry for it in the buildbot
+  # logs.
+  pending_time = SLEEP_TIME
+  running_time = 0
+  long_slept = False
+  while not done:
+    done = True
+    build_info = GetBuildInfo(base_dir, build)
+    if not build_info:
+      if pending_time > TIME_OUT:
+        logger.GetLogger().LogFatal('Unable to get build logs for target %s.' %
+                                    build)
+      else:
+        pending_message = 'Unable to find build log; job may be pending.'
+        done = False
+
+    if done:
+      data_dict = FindBuildRecordFromLog(description, build_info)
+      if not data_dict:
+        # Trybot job may be pending (not actually launched yet).
+        if pending_time > TIME_OUT:
+          logger.GetLogger().LogFatal('Unable to find build record for trybot'
+                                      ' %s.' % description)
+        else:
+          pending_message = 'Unable to find build record; job may be pending.'
+          done = False
+
+      else:
+        # Now that we have actually found the entry for the build
+        # job in the build log, we know the job is actually
+        # runnning, not pending, so we flip the 'pending' flag.  We
+        # still have to wait for the buildbot job to finish running
+        # however.
+        pending = False
+        build_id = data_dict['number']
+
+        if async:
+          # Do not wait for trybot job to finish; return immediately
+          return build_id, ' '
+
+        if not long_slept:
+          # The trybot generally takes more than 2 hours to finish.
+          # Wait two hours before polling the status.
+          long_slept = True
+          time.sleep(INITIAL_SLEEP_TIME)
+          pending_time += INITIAL_SLEEP_TIME
+        if True == data_dict['finished']:
+          build_status = data_dict['results']
+        else:
+          done = False
+
+    if not done:
+      if pending:
+        logger.GetLogger().LogOutput(pending_message)
+        logger.GetLogger().LogOutput('Current pending time: %d minutes.' %
+                                     (pending_time / 60))
+        pending_time += SLEEP_TIME
+      else:
+        logger.GetLogger().LogOutput('{0} minutes passed.'.format(running_time /
+                                                                  60))
+        logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
+        running_time += SLEEP_TIME
+
+      time.sleep(SLEEP_TIME)
+      if running_time > TIME_OUT:
+        done = True
+
+  trybot_image = ''
+
+  if build.endswith('-toolchain'):
+    # For rotating testers, we don't care about their build_status
+    # result, because if any HWTest failed it will be non-zero.
+    trybot_image = FindArchiveImage(chromeos_root, build, build_id)
+  else:
+    # The nightly performance tests do not run HWTests, so if
+    # their build_status is non-zero, we do care.  In this case
+    # non-zero means the image itself probably did not build.
+    if build_status in OK_STATUS:
+      trybot_image = FindArchiveImage(chromeos_root, build, build_id)
+  if not trybot_image:
+    logger.GetLogger().LogError('Trybot job %s failed with status %d;'
+                                ' no trybot image generated.' %
+                                (description, build_status))
+
+  logger.GetLogger().LogOutput("trybot_image is '%s'" % trybot_image)
+  logger.GetLogger().LogOutput('build_status is %d' % build_status)
+  return build_id, trybot_image
+
+
+def GetGSContent(chromeos_root, path):
+  """gsutil cat path"""
+
+  ce = command_executer.GetCommandExecuter()
+  command = ('gsutil cat gs://chromeos-image-archive/%s' % path)
+  _, out, _ = ce.ChrootRunCommandWOutput(
+      chromeos_root, command, print_to_console=False)
+  return out
+
+
+def DoesImageExist(chromeos_root, build):
+  """Check if the image for the given build exists."""
+
+  ce = command_executer.GetCommandExecuter()
+  command = ('gsutil ls gs://chromeos-image-archive/%s'
+             '/chromiumos_test_image.tar.xz' % (build))
+  ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False)
+  return not ret
+
+
+def WaitForImage(chromeos_root, build):
+  """Wait for an image to be ready."""
+
+  elapsed_time = 0
+  while elapsed_time < TIME_OUT:
+    if DoesImageExist(chromeos_root, build):
+      return
+    logger.GetLogger().LogOutput('Image %s not ready, waiting for 10 minutes' %
+                                 build)
+    time.sleep(SLEEP_TIME)
+    elapsed_time += SLEEP_TIME
+
+  logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' %
+                               (build, (TIME_OUT / 3600)))
+  raise BuildbotTimeout('Timeout while waiting for image %s' % build)