#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright 2019 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Performs bisection on LLVM based off a .JSON file.""" import enum import json import os import subprocess import sys import time import traceback import chroot import llvm_bisection from llvm_bisection import BisectionExitStatus import update_tryjob_status # Used to re-try for 'llvm_bisection.py' to attempt to launch more tryjobs. BISECTION_RETRY_TIME_SECS = 10 * 60 # Wait time to then poll each tryjob whose 'status' value is 'pending'. POLL_RETRY_TIME_SECS = 30 * 60 # The number of attempts for 'llvm_bisection.py' to launch more tryjobs. # # It is reset (break out of the `for` loop/ exit the program) if successfully # launched more tryjobs or bisection is finished (no more revisions between # start and end of the bisection). BISECTION_ATTEMPTS = 3 # The limit for updating all tryjobs whose 'status' is 'pending'. # # If the time that has passed for polling exceeds this value, then the program # will exit with the appropriate exit code. POLLING_LIMIT_SECS = 18 * 60 * 60 class BuilderStatus(enum.Enum): """Actual values given via 'cros buildresult'.""" PASS = "pass" FAIL = "fail" RUNNING = "running" builder_status_mapping = { BuilderStatus.PASS.value: update_tryjob_status.TryjobStatus.GOOD.value, BuilderStatus.FAIL.value: update_tryjob_status.TryjobStatus.BAD.value, BuilderStatus.RUNNING.value: update_tryjob_status.TryjobStatus.PENDING.value, } def GetBuildResult(chroot_path, buildbucket_id): """Returns the conversion of the result of 'cros buildresult'.""" # Calls 'cros buildresult' to get the status of the tryjob. try: tryjob_json = subprocess.check_output( [ "cros_sdk", "--", "cros", "buildresult", "--buildbucket-id", str(buildbucket_id), "--report", "json", ], cwd=chroot_path, stderr=subprocess.STDOUT, encoding="UTF-8", ) except subprocess.CalledProcessError as err: if "No build found. Perhaps not started" not in err.output: raise return None tryjob_content = json.loads(tryjob_json) build_result = str(tryjob_content["%d" % buildbucket_id]["status"]) # The string returned by 'cros buildresult' might not be in the mapping. if build_result not in builder_status_mapping: raise ValueError( '"cros buildresult" return value is invalid: %s' % build_result ) return builder_status_mapping[build_result] def main(): """Bisects LLVM using the result of `cros buildresult` of each tryjob. Raises: AssertionError: The script was run inside the chroot. """ chroot.VerifyOutsideChroot() args_output = llvm_bisection.GetCommandLineArgs() if os.path.isfile(args_output.last_tested): print("Resuming bisection for %s" % args_output.last_tested) else: print("Starting a new bisection for %s" % args_output.last_tested) while True: # Update the status of existing tryjobs if os.path.isfile(args_output.last_tested): update_start_time = time.time() with open(args_output.last_tested) as json_file: json_dict = json.load(json_file) while True: print( '\nAttempting to update all tryjobs whose "status" is ' '"pending":' ) print("-" * 40) completed = True for tryjob in json_dict["jobs"]: if ( tryjob["status"] == update_tryjob_status.TryjobStatus.PENDING.value ): status = GetBuildResult( args_output.chroot_path, tryjob["buildbucket_id"] ) if status: tryjob["status"] = status else: completed = False print("-" * 40) # Proceed to the next step if all the existing tryjobs have completed. if completed: break delta_time = time.time() - update_start_time if delta_time > POLLING_LIMIT_SECS: # Something is wrong with updating the tryjobs's 'status' via # `cros buildresult` (e.g. network issue, etc.). sys.exit("Failed to update pending tryjobs.") print("-" * 40) print("Sleeping for %d minutes." % (POLL_RETRY_TIME_SECS // 60)) time.sleep(POLL_RETRY_TIME_SECS) # There should always be update from the tryjobs launched in the # last iteration. temp_filename = "%s.new" % args_output.last_tested with open(temp_filename, "w") as temp_file: json.dump( json_dict, temp_file, indent=4, separators=(",", ": ") ) os.rename(temp_filename, args_output.last_tested) # Launch more tryjobs. for cur_try in range(1, BISECTION_ATTEMPTS + 1): try: print("\nAttempting to launch more tryjobs if possible:") print("-" * 40) bisection_ret = llvm_bisection.main(args_output) print("-" * 40) # Stop if the bisection has completed. if ( bisection_ret == BisectionExitStatus.BISECTION_COMPLETE.value ): sys.exit(0) # Successfully launched more tryjobs. break except Exception: traceback.print_exc() print("-" * 40) # Exceeded the number of times to launch more tryjobs. if cur_try == BISECTION_ATTEMPTS: sys.exit("Unable to continue bisection.") num_retries_left = BISECTION_ATTEMPTS - cur_try print( "Retries left to continue bisection %d." % num_retries_left ) print( "Sleeping for %d minutes." % (BISECTION_RETRY_TIME_SECS // 60) ) time.sleep(BISECTION_RETRY_TIME_SECS) if __name__ == "__main__": main()