diff options
| -rwxr-xr-x | remote_gcc_build.py | 166 | ||||
| -rwxr-xr-x | setup_chromeos.py | 33 | ||||
| -rw-r--r-- | utils/manifest_versions.py | 100 |
3 files changed, 199 insertions, 100 deletions
diff --git a/remote_gcc_build.py b/remote_gcc_build.py index ad6be7b1..1b367a74 100755 --- a/remote_gcc_build.py +++ b/remote_gcc_build.py @@ -18,6 +18,7 @@ import time from utils import command_executer from utils import logger +from utils import manifest_versions from utils import misc BRANCH = "the_actual_branch_used_in_this_script" @@ -60,13 +61,13 @@ def FindVersionForToolchain(branch, chromeos_root): return version -def FindResultIndex(reason): +def FindBuildId(description): """Find the build id of the build at trybot server.""" running_time = 0 while True: - num = GetBuildNumber(reason) - if num >= 0: - return num + (result, number) = FindBuildIdFromLog(description) + if result >= 0: + return (result, number) logger.GetLogger().LogOutput("{0} minutes passed." .format(running_time / 60)) logger.GetLogger().LogOutput("Sleeping {0} seconds.".format(SLEEP_TIME)) @@ -74,11 +75,14 @@ def FindResultIndex(reason): running_time += SLEEP_TIME -def GetBuildNumber(reason): - """Get the build num from build log.""" - # returns 0 if only failed build found. - # returns -1 if no finished build found or one build is running. - # returns build number is successful build found. +def FindBuildIdFromLog(description): + """Get the build id from build log.""" + # returns tuple (result, buildid) + # result == 0, buildid > 0, the build was successful and we have a build id + # result > 0, buildid > 0, the whole build failed for some reason but we + # do have a build id. + # result == -1, buildid == -1, we have not found a finished build for this + # description yet file_dir = os.path.dirname(os.path.realpath(__file__)) commands = ("{0}/utils/buildbot_json.py builds " @@ -91,10 +95,11 @@ def GetBuildNumber(reason): my_info = buildinfo.splitlines() current_line = 1 running_job = False - number = -1 - # number > 0, we have a successful build. - # number = 0, we have a failed build. - # number = -1, we do not have a finished build for this description. + result = -1 + + # result == 0, we have a successful build + # result > 0, we have a failed build but build id may be valid + # result == -1, we have not found a finished build for this description while current_line < len(my_info): my_dict = {} while True: @@ -105,32 +110,35 @@ def GetBuildNumber(reason): if "Build" in key or current_line == len(my_info): break if ("True" not in my_dict["completed"] and - str(reason) in my_dict["reason"]): + str(description) in my_dict["reason"]): running_job = True if ("True" not in my_dict["completed"] or - str(reason) not in my_dict["reason"]): + str(description) not in my_dict["reason"]): continue - if my_dict["result"] == "0": - number = int(my_dict["number"]) - return number - else: - # Record found a finished failed build. - # Keep searching to find a successful one. - number = 0 - if number == 0 and not running_job: - return 0 - return -1 + result = int(my_dict["result"]) + build_id = int(my_dict["number"]) + if result == 0: + return (result, build_id) + else + # Found a finished failed build. + # Keep searching to find a successful one + pass + + if result > 0 and not running_job: + return (result, build_id) + return (-1, -1) def DownloadImage(target, index, dest, version): """Download artifacts from cloud.""" - match = re.search(r"(.*-\d+\.\d+\.\d+)", version) - version = match.group(0) if not os.path.exists(dest): os.makedirs(dest) - ls_cmd = ("gsutil ls gs://chromeos-image-archive/trybot-{0}/{1}-b{2}" - .format(target, version, index)) + rversion = manifest_versions.RFormatCrosVersion(version) +# ls_cmd = ("gsutil ls gs://chromeos-image-archive/trybot-{0}/{1}-b{2}" +# .format(target, rversion, index)) + ls_cmd = ("gsutil ls gs://chromeos-image-archive/trybot-{0}/*-b{2}" + .format(target, rversion, index)) download_cmd = ("$(which gsutil) cp {0} {1}".format("{0}", dest)) ce = command_executer.GetCommandExecuter() @@ -163,30 +171,6 @@ def UnpackImage(dest): return ce.RunCommand(commands) -def GetManifest(version, to_file): - """Get the manifest file from a given chromeos-internal version.""" - temp_dir = tempfile.mkdtemp() - version = version.split("-", 1)[1] - os.chdir(temp_dir) - command = ("git clone " - "ssh://gerrit-int.chromium.org:29419/" - "chromeos/manifest-versions.git") - ce = command_executer.GetCommandExecuter() - ce.RunCommand(command) - files = [os.path.join(r, f) - for r, _, fs in os.walk(".") - for f in fs if version in f] - if files: - command = "cp {0} {1} && rm -rf {2}".format(files[0], to_file, temp_dir) - ret = ce.RunCommand(command) - if ret: - raise Exception("Cannot copy manifest to {0}".format(to_file)) - else: - command = "rm -rf {0}".format(temp_dir) - ce.RunCommand(command) - raise Exception("Version {0} is not available.".format(version)) - - def RemoveOldBranch(): """Remove the branch with name BRANCH.""" ce = command_executer.GetCommandExecuter() @@ -226,11 +210,11 @@ def UploadManifest(manifest, chromeos_root, branch="master"): return UploadPatch(manifest) -def GetManifestPatch(version, chromeos_root, branch="master"): +def GetManifestPatch(manifests, version, chromeos_root, branch="master"): """Return a gerrit patch number given a version of manifest file.""" temp_dir = tempfile.mkdtemp() to_file = os.path.join(temp_dir, "default.xml") - GetManifest(version, to_file) + manifests.GetManifest(version, to_file) return UploadManifest(to_file, chromeos_root, branch) @@ -342,9 +326,11 @@ def RunRemote(chromeos_root, branch, patches, is_local, description = "{0}_{1}_{2}".format(branch, GetPatchString(patches), target) command = ("yes | ./cbuildbot {0} {1} {2} {3} {4} {5}" " --remote-description={6}" + " --chrome_rev=tot" .format(patch, branch_flag, chrome_version, local_flag, chrome_version_flag, target, description)) ce.RunCommand(command) + return description @@ -368,8 +354,11 @@ def Main(argv): default="", help="The chrome version to use. " "Default it will use the latest one.") parser.add_argument("--chromeos_version", dest="chromeos_version", - default="", help="The chromeos version to use.") - + default="", + help=("The chromeos version to use." + "(1) A release version in the format: " + "'\d+\.\d+\.\d+\.\d+.*'" + "(2) 'latest_lkgm' for the latest lkgm version")) parser.add_argument("-r", "--replace_sysroot", action="store_true", help=("Whether or not to replace the build/$board dir" "under the chroot of chromeos_root and copy " @@ -390,45 +379,72 @@ def Main(argv): else: patch = [] chromeos_root = misc.CanonicalizePath(args.chromeos_root) - branch = args.branch - if not branch: - branch = chromeos_version - # descritption is the keyword of the build in build log. - # Here we use [{branch)_{patchnumber}_{target}] - description = "{0}_{1}_{2}".format(branch, GetPatchString(patch), target) if args.chromeos_version and args.branch: raise Exception("You can not set chromeos_version and branch at the " "same time.") - chromeos_version = args.chromeos_version + + manifests = None if args.branch: - chromeos_version = 0 + chromeos_version = "" + branch = args.branch else: chromeos_version = args.chromeos_version + manifests = manifest_versions.ManifestVersions() + if chromeos_version == "latest_lkgm": + chromeos_version = manifests.TimeToVersion(time.mktime(time.gmtime())) + logger.GetLogger().LogOutput("found version %s for latest LKGM" % ( + chromeos_version)) + # TODO: this script currently does not handle the case where the version + # is not in the "master" branch + branch = "master" + if chromeos_version: - manifest_patch = GetManifestPatch(chromeos_version, + manifest_patch = GetManifestPatch(manifests, chromeos_version, chromeos_root) patch.append(manifest_patch) if args.gcc_dir: + # TODO: everytime we invoke this script we are getting a different + # patch for GCC even if GCC has not changed. The description should + # be based on the MD5 of the GCC patch contents. patch.append(UploadGccPatch(chromeos_root, args.gcc_dir, branch)) - index = 0 description = RunRemote(chromeos_root, branch, patch, args.local, target, args.chrome_version, args.dest_dir) if args.local or not args.dest_dir: + # TODO: We are not checktng the result of cbuild_bot in here! return 0 + + # return value: + # 0 => build bot was successful and image was put where requested + # 1 => Build bot FAILED but image was put where requested + # 2 => Build bot failed or BUild bot was successful but and image was + # not generated or could not be put where expected + os.chdir(script_dir) dest_dir = misc.CanonicalizePath(args.dest_dir) - if index <= 0: - index = FindResultIndex(description) - if not index: - logger.GetLogger().LogFatal("Remote trybot failed.") + (bot_result, build_id) = FindBuildId(description) + if bot_result > 0 and build_id > 0: + logger.GetLogger().LogError("Remote trybot failed but image was generated") + bot_result = 1 + elif bot_result > 0: + logger.GetLogger().LogError("Remote trybot failed. No image was generated") + return 2 if "toolchain" in branch: chromeos_version = FindVersionForToolchain(branch, chromeos_root) - DownloadImage(target, index, dest_dir, chromeos_version) + assert not manifest_versions.IsRFormatCrosVersion(chromeos_version) + DownloadImage(target, build_id, dest_dir, chromeos_version) ret = UnpackImage(dest_dir) + if ret != 0: + return 2 + # todo: return a more inteligent return value if not args.replace_sysroot: - return ret - else: - return ReplaceSysroot(chromeos_root, args.dest_dir, target) + return bot_result + + ret = ReplaceSysroot(chromeos_root, args.dest_dir, target) + if ret != 0: + return 2 + + # got an image and we were successful in placing it where requested + return bot_result if __name__ == "__main__": retval = Main(sys.argv) diff --git a/setup_chromeos.py b/setup_chromeos.py index 7d3c22da..f753d968 100755 --- a/setup_chromeos.py +++ b/setup_chromeos.py @@ -12,6 +12,7 @@ __author__ = "raymes@google.com (Raymes Khoury)" from datetime import datetime import getpass + import optparse import os import pickle @@ -20,6 +21,7 @@ import tempfile import time from utils import command_executer from utils import logger +from utils import manifest_versions GCLIENT_FILE = """solutions = [ { "name" : "CHROME_DEPS", @@ -43,29 +45,6 @@ def Usage(parser): parser.print_help() sys.exit(0) - -def TimeToVersion(my_time, versions_git): - """Convert timestamp to version number.""" - cur_time = time.mktime(time.gmtime()) - des_time = float(my_time) - if cur_time - des_time > 7000000: - logger.GetLogger().LogFatal("The time you specify is too early.") - temp = tempfile.mkdtemp() - commands = ["cd {0}".format(temp), "git clone {0}".format(versions_git), - "cd manifest-versions", "git checkout -f $(git rev-list" + - " --max-count=1 --before={0} origin/master)".format(my_time)] - cmd_executer = command_executer.GetCommandExecuter() - ret = cmd_executer.RunCommands(commands) - if ret: - logger.GetLogger().LogFatal("Failed to checkout manifest-versions.") - path = os.path.realpath("{0}/manifest-versions/LKGM/lkgm.xml".format(temp)) - pp = path.split("/") - small = os.path.basename(path).split(".xml")[0] - version = pp[-2] + "." + small - commands = ["rm -rf {0}".format(temp)] - cmd_executer.RunCommands(commands) - return version - # Get version spec file, either from "paladin" or "buildspec" directory. def GetVersionSpecFile(version, versions_git): temp = tempfile.mkdtemp() @@ -176,22 +155,26 @@ Use in combination with --version=latest or --version=common. Use if version == "top": init = "repo init -u %s" % manifest_repo elif version == "latest_lkgm": - version = TimeToVersion(time.mktime(time.gmtime()), versions_repo) + manifests = manifest_versions.ManifestVersions() + version = manifests.TimeToVersion(time.mktime(time.gmtime())) version, manifest = version.split(".", 1) logger.GetLogger().LogOutput("found version %s.%s for latest LKGM" % ( version, manifest)) init = ("repo init -u %s -m paladin/buildspecs/%s/%s.xml" % ( versions_repo, version, manifest)) + del manifests elif version == "lkgm": if not timestamp: parser.print_help() logger.GetLogger().LogFatal("No timestamp specified for version=lkgm") - version = TimeToVersion(timestamp, versions_repo) + manifests = manifest_versions.ManifestVersions() + version = manifests.TimeToVersion(timestamp) version, manifest = version.split(".", 1) logger.GetLogger().LogOutput("found version %s.%s for LKGM at timestamp %s" % (version, manifest, timestamp)) init = ("repo init -u %s -m paladin/buildspecs/%s/%s.xml" % ( versions_repo, version, manifest)) + del manifests elif version == "latest_common": version = TimeToCommonVersion(time.mktime(time.gmtime())) version, manifest = version.split(".", 1) diff --git a/utils/manifest_versions.py b/utils/manifest_versions.py new file mode 100644 index 00000000..7b16f8a2 --- /dev/null +++ b/utils/manifest_versions.py @@ -0,0 +1,100 @@ +#!/usr/bin/python + +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tools for searching/manipulating the manifests repository.""" + +__author__ = "llozano@google.com (Luis Lozano)" + +import os +import re +import shutil +import tempfile +import time + +import command_executer +import logger + + +def IsCrosVersion(version): + match = re.search(r"(\d+\.\d+\.\d+\.\d+)", version) + return match is not None + + +def IsRFormatCrosVersion(version): + match = re.search(r"(R\d+-\d+\.\d+\.\d+)", version) + return match is not None + + +def RFormatCrosVersion(version): + assert IsCrosVersion(version) + tmp_major, tmp_minor = version.split(".", 1) + rformat = "R" + tmp_major + "-" + tmp_minor + assert IsRFormatCrosVersion(rformat) + return rformat + + +class ManifestVersions(object): + """This class handles interactions with the manifests repo.""" + + def __init__(self, internal=True): + self.internal = internal + self.clone_location = tempfile.mkdtemp() + self.ce = command_executer.GetCommandExecuter() + if internal: + versions_git = ( + "ssh://gerrit-int.chromium.org:29419/chromeos/manifest-versions.git") + else: + versions_git = ( + "http://git.chromium.org/chromiumos/manifest-versions.git") + commands = ["cd {0}".format(self.clone_location), + "git clone {0}".format(versions_git)] + ret = self.ce.RunCommands(commands) + if ret: + logger.GetLogger().LogFatal("Failed to clone manifest-versions.") + + def __del__(self): + if self.clone_location: + shutil.rmtree(self.clone_location) + + def TimeToVersion(self, my_time): + """Convert timestamp to version number.""" + cur_time = time.mktime(time.gmtime()) + des_time = float(my_time) + if cur_time - des_time > 7000000: + logger.GetLogger().LogFatal("The time you specify is too early.") + commands = ["cd {0}".format(self.clone_location), + "cd manifest-versions", + "git checkout -f $(git rev-list" + + " --max-count=1 --before={0} origin/master)".format(my_time)] + ret = self.ce.RunCommands(commands) + if ret: + logger.GetLogger().LogFatal("Failed to checkout manifest at " + "specified time") + path = os.path.realpath("{0}/manifest-versions/LKGM/lkgm.xml". + format(self.clone_location)) + pp = path.split("/") + small = os.path.basename(path).split(".xml")[0] + version = pp[-2] + "." + small + commands = ["cd {0}".format(self.clone_location), + "cd manifest-versions", "git checkout master"] + self.ce.RunCommands(commands) + return version + + def GetManifest(self, version, to_file): + """Get the manifest file from a given chromeos-internal version.""" + assert not IsRFormatCrosVersion(version) + version = version.split(".", 1)[1] + os.chdir(self.clone_location) + files = [os.path.join(r, f) + for r, _, fs in os.walk(".") + for f in fs if version in f] + if files: + command = "cp {0} {1}".format(files[0], to_file) + ret = self.ce.RunCommand(command) + if ret: + raise Exception("Cannot copy manifest to {0}".format(to_file)) + else: + raise Exception("Version {0} is not available.".format(version)) |