diff options
| -rw-r--r-- | cros_utils/buildbot_utils.py | 169 | ||||
| -rw-r--r-- | cros_utils/chromeos-toolchain-credentials.json | 12 |
2 files changed, 100 insertions, 81 deletions
diff --git a/cros_utils/buildbot_utils.py b/cros_utils/buildbot_utils.py index 3b0b916b..4cca08ed 100644 --- a/cros_utils/buildbot_utils.py +++ b/cros_utils/buildbot_utils.py @@ -5,10 +5,15 @@ from __future__ import print_function +import base64 +import json import os import time import urllib2 +# pylint: disable=no-name-in-module +from oauth2client.service_account import ServiceAccountCredentials + from cros_utils import command_executer from cros_utils import logger from cros_utils import buildbot_json @@ -34,13 +39,13 @@ class BuildbotTimeout(Exception): def ParseReportLog(url, build): """Scrape the trybot image name off the Reports log page. - This takes the URL for a trybot Reports Stage web page, - and a trybot build type, such as 'daisy-release'. It - opens the web page and parses it looking for the trybot - artifact name (e.g. something like - 'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the - artifact name, if found. - """ + This takes the URL for a trybot Reports Stage web page, + and a trybot build type, such as 'daisy-release'. It + opens the web page and parses it looking for the trybot + artifact name (e.g. something like + 'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the + artifact name, if found. + """ trybot_image = '' url += '/text' newurl = url.replace('uberchromegw', 'chromegw') @@ -60,10 +65,10 @@ def ParseReportLog(url, build): def GetBuildData(buildbot_queue, build_id): """Find the Reports stage web page for a trybot build. - This takes the name of a buildbot_queue, such as 'daisy-release' - and a build id (the build number), and uses the json buildbot api to - find the Reports stage web page for that build, if it exists. - """ + This takes the name of a buildbot_queue, such as 'daisy-release' + and a build id (the build number), and uses the json buildbot api to + find the Reports stage web page for that build, if it exists. + """ builder = buildbot_json.Buildbot( 'http://chromegw/p/tryserver.chromiumos/').builders[buildbot_queue] build_data = builder.builds[build_id].data @@ -76,65 +81,67 @@ def GetBuildData(buildbot_queue, build_id): return '' -def FindBuildRecordFromLog(description, log_info): +def FindBuildRecordFromLog(description, build_info): """Find the right build record in the build logs. - Get the first build record from build log with a reason field - that matches 'description'. ('description' is a special tag we - created when we launched the buildbot, so we could find it at this - point.) - """ - - current_line = 1 - while current_line < len(log_info): - my_dict = {} - # Read all the lines from one "Build" to the next into my_dict - while True: - key = log_info[current_line].split(':')[0].strip() - value = log_info[current_line].split(':', 1)[1].strip() - my_dict[key] = value - current_line += 1 - if 'Build' in key or current_line == len(log_info): - break - try: - # Check to see of the build record is the right one. - if str(description) in my_dict['reason']: - # We found a match; we're done. - return my_dict - except KeyError: - print("reason is not in dictionary: '%s'" % repr(my_dict)) - else: - # Keep going. - continue - - # We hit the bottom of the log without a match. + Get the first build record from build log with a reason field + that matches 'description'. ('description' is a special tag we + created when we launched the buildbot, so we could find it at this + point.) + """ + for build_log in build_info: + if description in build_log['reason']: + return build_log return {} -def GetBuildInfo(file_dir, builder): - """Get all the build records for the trybot builds. - - file_dir is the toolchain_utils directory. - """ - ce = command_executer.GetCommandExecuter() - commands = ('{0}/cros_utils/buildbot_json.py builds ' - 'http://chromegw/i/tryserver.chromiumos/'.format(file_dir)) - - if builder: - # For release builds, get logs from the 'release' builder. - if builder.endswith('-release'): - commands += ' -b release' - elif builder.endswith('-gcc-toolchain'): - commands += ' -b gcc_toolchain' - elif builder.endswith('-llvm-toolchain'): - commands += ' -b llvm_toolchain' - elif builder.endswith('-toolchain'): - commands += ' -b etc' - else: - commands += ' -b %s' % builder - _, buildinfo, _ = ce.RunCommandWOutput(commands, print_to_console=False) - build_log = buildinfo.splitlines() - return build_log +def GetBuildInfo(file_dir, waterfall_builder): + """Get all the build records for the trybot builds.""" + + builder = '' + if waterfall_builder.endswith('-release'): + builder = 'release' + elif waterfall_builder.endswith('-gcc-toolchain'): + builder = 'gcc_toolchain' + elif waterfall_builder.endswith('-llvm-toolchain'): + builder = 'llvm_toolchain' + + sa_file = os.path.expanduser( + os.path.join(file_dir, 'cros_utils', + 'chromeos-toolchain-credentials.json')) + scopes = ['https://www.googleapis.com/auth/userinfo.email'] + + credentials = ServiceAccountCredentials.from_json_keyfile_name( + sa_file, scopes=scopes) + url = ( + 'https://luci-milo.appspot.com/prpc/milo.Buildbot/GetBuildbotBuildsJSON') + + # NOTE: If we want to get build logs for the main waterfall builders, the + # 'master' field below should be 'chromeos' instead of 'chromiumos.tryserver'. + # Builder would be 'amd64-gcc-toolchain' or 'arm-llvm-toolchain', etc. + + body = json.dumps({ + 'master': 'chromiumos.tryserver', + 'builder': builder, + 'include_current': True, + 'limit': 100 + }) + access_token = credentials.get_access_token() + headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + 'Authorization': 'Bearer %s' % access_token.access_token + } + r = urllib2.Request(url, body, headers) + u = urllib2.urlopen(r, timeout=60) + u.read(4) + o = json.load(u) + data = [base64.b64decode(item['data']) for item in o['builds']] + result = [] + for d in data: + tmp = json.loads(d) + result.append(tmp) + return result def FindArchiveImage(chromeos_root, build, build_id): @@ -182,28 +189,28 @@ def GetTrybotImage(chromeos_root, buildbot_name, patch_list, build_tag, - other_flags=[], + other_flags=None, build_toolchain=False, async=False): """Launch buildbot and get resulting trybot artifact name. - This function launches a buildbot with the appropriate flags to - build the test ChromeOS image, with the current ToT mobile compiler. It - checks every 10 minutes to see if the trybot has finished. When the trybot - has finished, it parses the resulting report logs to find the trybot - artifact (if one was created), and returns that artifact name. + This function launches a buildbot with the appropriate flags to + build the test ChromeOS image, with the current ToT mobile compiler. It + checks every 10 minutes to see if the trybot has finished. When the trybot + has finished, it parses the resulting report logs to find the trybot + artifact (if one was created), and returns that artifact name. - chromeos_root is the path to the ChromeOS root, needed for finding chromite - and launching the buildbot. + chromeos_root is the path to the ChromeOS root, needed for finding chromite + and launching the buildbot. - buildbot_name is the name of the buildbot queue, such as lumpy-release or - daisy-paladin. + buildbot_name is the name of the buildbot queue, such as lumpy-release or + daisy-paladin. - patch_list a python list of the patches, if any, for the buildbot to use. + patch_list a python list of the patches, if any, for the buildbot to use. - build_tag is a (unique) string to be used to look up the buildbot results - from among all the build records. - """ + build_tag is a (unique) string to be used to look up the buildbot results + from among all the build records. + """ ce = command_executer.GetCommandExecuter() cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot') base_dir = os.getcwd() @@ -286,9 +293,9 @@ def GetTrybotImage(chromeos_root, # still have to wait for the buildbot job to finish running # however. pending = False - if 'True' in data_dict['completed']: + if True == data_dict['finished']: build_id = data_dict['number'] - build_status = int(data_dict['result']) + build_status = data_dict['results'] else: done = False diff --git a/cros_utils/chromeos-toolchain-credentials.json b/cros_utils/chromeos-toolchain-credentials.json new file mode 100644 index 00000000..aed9a527 --- /dev/null +++ b/cros_utils/chromeos-toolchain-credentials.json @@ -0,0 +1,12 @@ +{ + "type": "service_account", + "project_id": "chromeos-toolchain-u", + "private_key_id": "d0efe593ad39aad4c685273ee80e4c24bb3f2e92", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC5Rm6aqSjMNrRp\ntYNc++ec79L3QZ2MxLMYKyhlgGhppVt6p/wiSvLdI19nS5TAkKMjKv71rb9DbQlG\nfQVckiY+MlADJKi29lJrwqqNDCcRgB7CL6hgDfmhWMKonZn2MwvBVROD0gi7sY+A\nipIe92jVeqG8Gvp5kOgsBxCRV5YQok8j1FxE5fIsS2sg93VS1YAzH8uPYadWb/Z5\n9uwc8U7SL0mEPjXjsLEm8Y70zovGVjv7kOLqYBMUmROLvSouG/HrZWy9uTgxFOb5\njOhxKhDcDMPVM3g8lfc0EwPUB1NxXztoST9qBJVqdzQmHpPjeDxgru0A+zaQMkWA\ne8Sn5EQrAgMBAAECggEAUnhbe1SY6G3hWLyHQLiQnHbUPWNjselMnxF25deGoqAM\nXEiyHsGl4XGBYgVurVi0hU76NnmkjWrXmhzEajRT+ZODsiJ7RxXWEkmQiUBDk7Kn\n/mAgXsFZwMw1ucCNa93E+cXY7fBsGsAq1FjaOhZ+/6eanpSTsdEix5ZNdaS7E6Op\n9zIba9EjLIvSl435+eWq0C3aU9nd1RbbRwD6vGpgG8L/r957s+AAALTqdSZGWxJX\nEC9OKT07e76qvwAsq2BoBx5vW0xmeQdZgKrA10LLDWa7UjFbwSDJIBESYtd4rYMj\nAqg5eND0bC1RrgzI+RD/10l6Vj8bBFo/403s0P5LYQKBgQDiVGVFkrw5LSy82CGC\nvSraxPriivEweMfpkp6buMbD2Je0RMR4glc1vW5m0QUJmy+ymiIHVMCmE9xNBwbS\nRyCBnrs2+3FtdnruNdcaGh6sbTlY+qJI0rEZUdbb5OhlHZF47KW66hI6sWJ1YF8O\niLQTokW8ejybprCtl1HvEHhEbwKBgQDRkD/acZrvmcnqqmorqW6mgJEkrRF/i5Th\npDo3WegXA4irX0tNqh5w+wms8r41vUZSCZYvyi0Of9LMObVdB/gA/qVzETE0p5he\ns3Skp/VK8nF53pAUd+4dKlnCvD3TOEkIq+kxuEOs2iHJcvSjmKtMgqfMK/UtieB4\n7+MaOcbyBQKBgHOUndMVyEF6rGoUBaj6abQm++hNBDa4t360fYMQrZC+P1qz85GH\nHno3LvYar/Pj6EvRIqeTxH4LjmlXuUgRQqxvHzRI2/gGlWio3hxaUmfDr5GdDNsb\nnY1MmejZ0UQyAWQ7lbcKahzHEXzXpjOJ5ExShkJmOiVSzs8Xg6QOSRzJAoGAemYs\nRWQxQFysqJlcZaASdsGFBMzo+XwHOzt2nTmv6zEvNBj2nKgOG6MkZZVqL20bk3Lx\n+3u0kVFrR8k0+t9apQoWjHywJrb0setS55EKHfo4+RtbP/lEZFiGEM1ttt6bGat/\nCoE7VcwaC9VOufbDpm5xnzjVfQGY0EocdQbmAhkCgYB/isdqeDyafawr+38fcU1r\nX2+cK5JCrEzHIwg2QN3Z56cRrqrb+kK1H3w/F7ZfrlPSmS8XMwZV73QwieoGNIYL\nie9UZqRoZSG73FzIw5mXhWWA1adFz8HpGG5wRNshnPI2liOPwhnblfadJUfXb2br\n021vPgpsxamLjHSDSmSf6Q==\n-----END PRIVATE KEY-----\n", + "client_email": "mobiletc-prebuild-2@chromeos-toolchain-u.iam.gserviceaccount.com", + "client_id": "114495086044042319417", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/mobiletc-prebuild-2%40chromeos-toolchain-u.iam.gserviceaccount.com" +} |