[toolchain-utils] Use new system for accessing build logs.

The old system for accessing build logs is being deprecated and
turned off.  This CL switches over to using the new recommended
system for accessing build log records.

BUG=chromium:664537
TEST=Tested in my account & role account.

Change-Id: Id964c43ce09df9c52a6337ea57fb469423c3cf51
Reviewed-on: https://chrome-internal-review.googlesource.com/304258
Commit-Ready: Caroline Tice <cmtice@google.com>
Tested-by: Caroline Tice <cmtice@google.com>
Reviewed-by: Yunlian Jiang <yunlian@google.com>

2 files changed, 100 insertions, 81 deletions

diff --git a/cros_utils/buildbot_utils.py b/cros_utils/buildbot_utils.py
index 3b0b916b..4cca08ed 100644
--- a/cros_utils/buildbot_utils.py
+++ b/cros_utils/buildbot_utils.py
@@ -5,10 +5,15 @@
 
 from __future__ import print_function
 
+import base64
+import json
 import os
 import time
 import urllib2
 
+# pylint: disable=no-name-in-module
+from oauth2client.service_account import ServiceAccountCredentials
+
 from cros_utils import command_executer
 from cros_utils import logger
 from cros_utils import buildbot_json
@@ -34,13 +39,13 @@ class BuildbotTimeout(Exception):
 def ParseReportLog(url, build):
   """Scrape the trybot image name off the Reports log page.
 
-    This takes the URL for a trybot Reports Stage web page,
-    and a trybot build type, such as 'daisy-release'.  It
-    opens the web page and parses it looking for the trybot
-    artifact name (e.g. something like
-    'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the
-    artifact name, if found.
-    """
+  This takes the URL for a trybot Reports Stage web page,
+  and a trybot build type, such as 'daisy-release'.  It
+  opens the web page and parses it looking for the trybot
+  artifact name (e.g. something like
+  'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the
+  artifact name, if found.
+  """
   trybot_image = ''
   url += '/text'
   newurl = url.replace('uberchromegw', 'chromegw')
@@ -60,10 +65,10 @@ def ParseReportLog(url, build):
 def GetBuildData(buildbot_queue, build_id):
   """Find the Reports stage web page for a trybot build.
 
-    This takes the name of a buildbot_queue, such as 'daisy-release'
-    and a build id (the build number), and uses the json buildbot api to
-    find the Reports stage web page for that build, if it exists.
-    """
+  This takes the name of a buildbot_queue, such as 'daisy-release'
+  and a build id (the build number), and uses the json buildbot api to
+  find the Reports stage web page for that build, if it exists.
+  """
   builder = buildbot_json.Buildbot(
       'http://chromegw/p/tryserver.chromiumos/').builders[buildbot_queue]
   build_data = builder.builds[build_id].data
@@ -76,65 +81,67 @@ def GetBuildData(buildbot_queue, build_id):
   return ''
 
 
-def FindBuildRecordFromLog(description, log_info):
+def FindBuildRecordFromLog(description, build_info):
   """Find the right build record in the build logs.
 
-    Get the first build record from build log with a reason field
-    that matches 'description'. ('description' is a special tag we
-    created when we launched the buildbot, so we could find it at this
-    point.)
-    """
-
-  current_line = 1
-  while current_line < len(log_info):
-    my_dict = {}
-    # Read all the lines from one "Build" to the next into my_dict
-    while True:
-      key = log_info[current_line].split(':')[0].strip()
-      value = log_info[current_line].split(':', 1)[1].strip()
-      my_dict[key] = value
-      current_line += 1
-      if 'Build' in key or current_line == len(log_info):
-        break
-    try:
-      # Check to see of the build record is the right one.
-      if str(description) in my_dict['reason']:
-        # We found a match; we're done.
-        return my_dict
-    except KeyError:
-      print("reason is not in dictionary: '%s'" % repr(my_dict))
-    else:
-      # Keep going.
-      continue
-
-  # We hit the bottom of the log without a match.
+  Get the first build record from build log with a reason field
+  that matches 'description'. ('description' is a special tag we
+  created when we launched the buildbot, so we could find it at this
+  point.)
+  """
+  for build_log in build_info:
+    if description in build_log['reason']:
+      return build_log
   return {}
 
 
-def GetBuildInfo(file_dir, builder):
-  """Get all the build records for the trybot builds.
-
-    file_dir is the toolchain_utils directory.
-    """
-  ce = command_executer.GetCommandExecuter()
-  commands = ('{0}/cros_utils/buildbot_json.py builds '
-              'http://chromegw/i/tryserver.chromiumos/'.format(file_dir))
-
-  if builder:
-    # For release builds, get logs from the 'release' builder.
-    if builder.endswith('-release'):
-      commands += ' -b release'
-    elif builder.endswith('-gcc-toolchain'):
-      commands += ' -b gcc_toolchain'
-    elif builder.endswith('-llvm-toolchain'):
-      commands += ' -b llvm_toolchain'
-    elif builder.endswith('-toolchain'):
-      commands += ' -b etc'
-    else:
-      commands += ' -b %s' % builder
-  _, buildinfo, _ = ce.RunCommandWOutput(commands, print_to_console=False)
-  build_log = buildinfo.splitlines()
-  return build_log
+def GetBuildInfo(file_dir, waterfall_builder):
+  """Get all the build records for the trybot builds."""
+
+  builder = ''
+  if waterfall_builder.endswith('-release'):
+    builder = 'release'
+  elif waterfall_builder.endswith('-gcc-toolchain'):
+    builder = 'gcc_toolchain'
+  elif waterfall_builder.endswith('-llvm-toolchain'):
+    builder = 'llvm_toolchain'
+
+  sa_file = os.path.expanduser(
+      os.path.join(file_dir, 'cros_utils',
+                   'chromeos-toolchain-credentials.json'))
+  scopes = ['https://www.googleapis.com/auth/userinfo.email']
+
+  credentials = ServiceAccountCredentials.from_json_keyfile_name(
+      sa_file, scopes=scopes)
+  url = (
+      'https://luci-milo.appspot.com/prpc/milo.Buildbot/GetBuildbotBuildsJSON')
+
+  # NOTE: If we want to get build logs for the main waterfall builders, the
+  # 'master' field below should be 'chromeos' instead of 'chromiumos.tryserver'.
+  # Builder would be 'amd64-gcc-toolchain' or 'arm-llvm-toolchain', etc.
+
+  body = json.dumps({
+      'master': 'chromiumos.tryserver',
+      'builder': builder,
+      'include_current': True,
+      'limit': 100
+  })
+  access_token = credentials.get_access_token()
+  headers = {
+      'Accept': 'application/json',
+      'Content-Type': 'application/json',
+      'Authorization': 'Bearer %s' % access_token.access_token
+  }
+  r = urllib2.Request(url, body, headers)
+  u = urllib2.urlopen(r, timeout=60)
+  u.read(4)
+  o = json.load(u)
+  data = [base64.b64decode(item['data']) for item in o['builds']]
+  result = []
+  for d in data:
+    tmp = json.loads(d)
+    result.append(tmp)
+  return result
 
 
 def FindArchiveImage(chromeos_root, build, build_id):
@@ -182,28 +189,28 @@ def GetTrybotImage(chromeos_root,
                    buildbot_name,
                    patch_list,
                    build_tag,
-                   other_flags=[],
+                   other_flags=None,
                    build_toolchain=False,
                    async=False):
   """Launch buildbot and get resulting trybot artifact name.
 
-    This function launches a buildbot with the appropriate flags to
-    build the test ChromeOS image, with the current ToT mobile compiler.  It
-    checks every 10 minutes to see if the trybot has finished.  When the trybot
-    has finished, it parses the resulting report logs to find the trybot
-    artifact (if one was created), and returns that artifact name.
+  This function launches a buildbot with the appropriate flags to
+  build the test ChromeOS image, with the current ToT mobile compiler.  It
+  checks every 10 minutes to see if the trybot has finished.  When the trybot
+  has finished, it parses the resulting report logs to find the trybot
+  artifact (if one was created), and returns that artifact name.
 
-    chromeos_root is the path to the ChromeOS root, needed for finding chromite
-    and launching the buildbot.
+  chromeos_root is the path to the ChromeOS root, needed for finding chromite
+  and launching the buildbot.
 
-    buildbot_name is the name of the buildbot queue, such as lumpy-release or
-    daisy-paladin.
+  buildbot_name is the name of the buildbot queue, such as lumpy-release or
+  daisy-paladin.
 
-    patch_list a python list of the patches, if any, for the buildbot to use.
+  patch_list a python list of the patches, if any, for the buildbot to use.
 
-    build_tag is a (unique) string to be used to look up the buildbot results
-    from among all the build records.
-    """
+  build_tag is a (unique) string to be used to look up the buildbot results
+  from among all the build records.
+  """
   ce = command_executer.GetCommandExecuter()
   cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot')
   base_dir = os.getcwd()
@@ -286,9 +293,9 @@ def GetTrybotImage(chromeos_root,
         # still have to wait for the buildbot job to finish running
         # however.
         pending = False
-        if 'True' in data_dict['completed']:
+        if True == data_dict['finished']:
           build_id = data_dict['number']
-          build_status = int(data_dict['result'])
+          build_status = data_dict['results']
         else:
           done = False
 
diff --git a/cros_utils/chromeos-toolchain-credentials.json b/cros_utils/chromeos-toolchain-credentials.json
new file mode 100644
index 00000000..aed9a527
--- /dev/null
+++ b/cros_utils/chromeos-toolchain-credentials.json
@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "chromeos-toolchain-u",
+  "private_key_id": "d0efe593ad39aad4c685273ee80e4c24bb3f2e92",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC5Rm6aqSjMNrRp\ntYNc++ec79L3QZ2MxLMYKyhlgGhppVt6p/wiSvLdI19nS5TAkKMjKv71rb9DbQlG\nfQVckiY+MlADJKi29lJrwqqNDCcRgB7CL6hgDfmhWMKonZn2MwvBVROD0gi7sY+A\nipIe92jVeqG8Gvp5kOgsBxCRV5YQok8j1FxE5fIsS2sg93VS1YAzH8uPYadWb/Z5\n9uwc8U7SL0mEPjXjsLEm8Y70zovGVjv7kOLqYBMUmROLvSouG/HrZWy9uTgxFOb5\njOhxKhDcDMPVM3g8lfc0EwPUB1NxXztoST9qBJVqdzQmHpPjeDxgru0A+zaQMkWA\ne8Sn5EQrAgMBAAECggEAUnhbe1SY6G3hWLyHQLiQnHbUPWNjselMnxF25deGoqAM\nXEiyHsGl4XGBYgVurVi0hU76NnmkjWrXmhzEajRT+ZODsiJ7RxXWEkmQiUBDk7Kn\n/mAgXsFZwMw1ucCNa93E+cXY7fBsGsAq1FjaOhZ+/6eanpSTsdEix5ZNdaS7E6Op\n9zIba9EjLIvSl435+eWq0C3aU9nd1RbbRwD6vGpgG8L/r957s+AAALTqdSZGWxJX\nEC9OKT07e76qvwAsq2BoBx5vW0xmeQdZgKrA10LLDWa7UjFbwSDJIBESYtd4rYMj\nAqg5eND0bC1RrgzI+RD/10l6Vj8bBFo/403s0P5LYQKBgQDiVGVFkrw5LSy82CGC\nvSraxPriivEweMfpkp6buMbD2Je0RMR4glc1vW5m0QUJmy+ymiIHVMCmE9xNBwbS\nRyCBnrs2+3FtdnruNdcaGh6sbTlY+qJI0rEZUdbb5OhlHZF47KW66hI6sWJ1YF8O\niLQTokW8ejybprCtl1HvEHhEbwKBgQDRkD/acZrvmcnqqmorqW6mgJEkrRF/i5Th\npDo3WegXA4irX0tNqh5w+wms8r41vUZSCZYvyi0Of9LMObVdB/gA/qVzETE0p5he\ns3Skp/VK8nF53pAUd+4dKlnCvD3TOEkIq+kxuEOs2iHJcvSjmKtMgqfMK/UtieB4\n7+MaOcbyBQKBgHOUndMVyEF6rGoUBaj6abQm++hNBDa4t360fYMQrZC+P1qz85GH\nHno3LvYar/Pj6EvRIqeTxH4LjmlXuUgRQqxvHzRI2/gGlWio3hxaUmfDr5GdDNsb\nnY1MmejZ0UQyAWQ7lbcKahzHEXzXpjOJ5ExShkJmOiVSzs8Xg6QOSRzJAoGAemYs\nRWQxQFysqJlcZaASdsGFBMzo+XwHOzt2nTmv6zEvNBj2nKgOG6MkZZVqL20bk3Lx\n+3u0kVFrR8k0+t9apQoWjHywJrb0setS55EKHfo4+RtbP/lEZFiGEM1ttt6bGat/\nCoE7VcwaC9VOufbDpm5xnzjVfQGY0EocdQbmAhkCgYB/isdqeDyafawr+38fcU1r\nX2+cK5JCrEzHIwg2QN3Z56cRrqrb+kK1H3w/F7ZfrlPSmS8XMwZV73QwieoGNIYL\nie9UZqRoZSG73FzIw5mXhWWA1adFz8HpGG5wRNshnPI2liOPwhnblfadJUfXb2br\n021vPgpsxamLjHSDSmSf6Q==\n-----END PRIVATE KEY-----\n",
+  "client_email": "mobiletc-prebuild-2@chromeos-toolchain-u.iam.gserviceaccount.com",
+  "client_id": "114495086044042319417",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://accounts.google.com/o/oauth2/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/mobiletc-prebuild-2%40chromeos-toolchain-u.iam.gserviceaccount.com"
+}