LLVM tools: Added auto bisection of LLVM

Similar to 'llvm_bisection.py' script but uses `cros buildresult` to
update each tryjob's 'status' field. The script sleeps for X minutes and
continues where bisection left off (similar behavior if script
terminates). The script is an infinite loop but terminates when there
are no more revisions between the new 'start' and new 'end' or if an
exception happens.

This script is using the scripts such as bisection script and updating a
tryjob's status script in a loop.

BUG=None
TEST=Ran 'auto_llvm_bisection.py' script with a file that was not
created and start revision of 369410 and end revision of 369420.
Successfully created tryjobs between the start and end and the script
went to sleep for X minutes and woke up to resume where bisection
left off.

Change-Id: I711988b164c41f56ecc2c9478527bdcfe8f5bb88
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1776330
Reviewed-by: George Burgess <gbiv@chromium.org>
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Tested-by: Salud Lemus <saludlemus@google.com>

5 files changed, 396 insertions, 79 deletions

diff --git a/llvm_tools/auto_llvm_bisection.py b/llvm_tools/auto_llvm_bisection.py
new file mode 100755
index 00000000..125b504b
--- /dev/null
+++ b/llvm_tools/auto_llvm_bisection.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Performs bisection on LLVM based off a .JSON file."""
+
+from __future__ import division
+from __future__ import print_function
+
+import os
+import subprocess
+import sys
+import time
+import traceback
+
+from assert_not_in_chroot import VerifyOutsideChroot
+from update_all_tryjobs_with_auto import GetPathToUpdateAllTryjobsWithAutoScript
+from llvm_bisection import BisectionExitStatus
+import llvm_bisection
+
+
+def main():
+  """Bisects LLVM using the result of `cros buildresult` of each tryjob.
+
+  Raises:
+    AssertionError: The script was run inside the chroot.
+  """
+
+  VerifyOutsideChroot()
+
+  args_output = llvm_bisection.GetCommandLineArgs()
+
+  exec_update_tryjobs = [
+      GetPathToUpdateAllTryjobsWithAutoScript(), '--chroot_path',
+      args_output.chroot_path, '--last_tested', args_output.last_tested
+  ]
+
+  # Used to re-try for 'llvm_bisection.py' to attempt to launch more tryjobs.
+  BISECTION_RETRY_TIME_SECS = 10 * 60
+
+  # Wait time to then poll each tryjob whose 'status' value is 'pending'.
+  POLL_RETRY_TIME_SECS = 30 * 60
+
+  # The number of attempts for 'llvm_bisection.py' to launch more tryjobs.
+  #
+  # It is reset (break out of the `for` loop/ exit the program) if successfully
+  # launched more tryjobs or bisection is finished (no more revisions between
+  # start and end of the bisection).
+  BISECTION_ATTEMPTS = 3
+
+  # The limit for updating all tryjobs whose 'status' is 'pending'.
+  #
+  # If the time that has passed for polling exceeds this value, then the program
+  # will exit with the appropriate exit code.
+  POLLING_LIMIT_SECS = 18 * 60 * 60
+
+  if os.path.isfile(args_output.last_tested):
+    print('Resuming bisection for %s' % args_output.last_tested)
+  else:
+    print('Starting a new bisection for %s' % args_output.last_tested)
+
+  while True:
+    if os.path.isfile(args_output.last_tested):
+      update_start_time = time.time()
+
+      # Update all tryjobs whose status is 'pending' to the result of `cros
+      # buildresult`.
+      while True:
+        print('\nAttempting to update all tryjobs whose \'status\' is '
+              '\'pending\':')
+        print('-' * 40)
+
+        update_ret = subprocess.call(exec_update_tryjobs)
+
+        print('-' * 40)
+
+        # Successfully updated all tryjobs whose 'status' was 'pending'/ no
+        # updates were needed (all tryjobs already have been updated).
+        if update_ret == 0:
+          break
+
+        delta_time = time.time() - update_start_time
+
+        if delta_time > POLLING_LIMIT_SECS:
+          print('Unable to update tryjobs whose status is \'pending\' to '
+                'the result of `cros buildresult`.')
+
+          # Something is wrong with updating the tryjobs's 'status' via
+          # `cros buildresult` (e.g. network issue, etc.).
+          sys.exit(1)
+
+        print('Sleeping for %d minutes.' % (POLL_RETRY_TIME_SECS // 60))
+        time.sleep(POLL_RETRY_TIME_SECS)
+
+    # Launch more tryjobs if possible to narrow down the bad commit/revision or
+    # terminate the bisection because the bad commit/revision was found.
+    for cur_try in range(1, BISECTION_ATTEMPTS + 1):
+      try:
+        print('\nAttempting to launch more tryjobs if possible:')
+        print('-' * 40)
+
+        bisection_ret = llvm_bisection.main(args_output)
+
+        print('-' * 40)
+
+        # Exit code 126 means that there are no more revisions to test between
+        # 'start' and 'end', so bisection is complete.
+        if bisection_ret == BisectionExitStatus.BISECTION_COMPLETE.value:
+          sys.exit(0)
+
+        # Successfully launched more tryjobs.
+        break
+      except Exception:
+        traceback.print_exc()
+
+        print('-' * 40)
+
+        # Exceeded the number of times to launch more tryjobs.
+        if cur_try == BISECTION_ATTEMPTS:
+          print('Unable to continue bisection.')
+
+          sys.exit(1)
+
+        num_retries_left = BISECTION_ATTEMPTS - cur_try
+
+        print('Retries left to continue bisection %d.' % num_retries_left)
+
+        print('Sleeping for %d minutes.' % (BISECTION_RETRY_TIME_SECS // 60))
+        time.sleep(BISECTION_RETRY_TIME_SECS)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py
index da26a540..d9eecce6 100755
--- a/llvm_tools/llvm_bisection.py
+++ b/llvm_tools/llvm_bisection.py
@@ -10,9 +10,11 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
+import enum
 import errno
 import json
 import os
+import sys
 
 from assert_not_in_chroot import VerifyOutsideChroot
 from get_llvm_hash import CreateTempLLVMRepo
@@ -23,6 +25,13 @@ from update_tryjob_status import FindTryjobIndex
 from update_tryjob_status import TryjobStatus
 
 
+class BisectionExitStatus(enum.Enum):
+  """Exit code when performing bisection."""
+
+  # Means that there are no more revisions available to bisect.
+  BISECTION_COMPLETE = 126
+
+
 def is_file_and_json(json_file):
   """Validates that the file exists and is a JSON file."""
   return os.path.isfile(json_file) and json_file.endswith('.json')
@@ -302,7 +311,69 @@ def GetRevisionsListAndHashList(start, end, parallel, src_path,
   return revisions, git_hashes
 
 
-def main():
+def DieWithNoRevisionsError(start, end, skip_revisions, pending_revisions):
+  """Raises a ValueError exception with useful information."""
+
+  no_revisions_message = ('No revisions between start %d and end '
+                          '%d to create tryjobs' % (start, end))
+
+  if pending_revisions:
+    no_revisions_message += '\nThe following tryjobs are pending:\n' \
+        + '\n'.join(str(rev) for rev in pending_revisions)
+
+  if skip_revisions:
+    no_revisions_message += '\nThe following tryjobs were skipped:\n' \
+        + '\n'.join(str(rev) for rev in skip_revisions)
+
+  raise ValueError(no_revisions_message)
+
+
+def CheckForExistingTryjobsInRevisionsToLaunch(revisions, jobs):
+  """Checks if a revision in 'revisions' exists in 'jobs' list."""
+
+  for rev in revisions:
+    if FindTryjobIndex(rev, jobs) is not None:
+      raise ValueError('Revision %d exists already in \'jobs\'' % rev)
+
+
+def UpdateBisection(revisions, git_hashes, bisect_contents, last_tested,
+                    update_packages, chroot_path, patch_metadata_file,
+                    extra_change_lists, options, builder, log_level):
+  """Adds tryjobs and updates the status file with the new tryjobs."""
+
+  try:
+    for svn_revision, git_hash in zip(revisions, git_hashes):
+      tryjob_dict = AddTryjob(update_packages, git_hash, svn_revision,
+                              chroot_path, patch_metadata_file,
+                              extra_change_lists, options, builder, log_level,
+                              svn_revision)
+
+      bisect_contents['jobs'].append(tryjob_dict)
+  finally:
+    # Do not want to lose progress if there is an exception.
+    if last_tested:
+      new_file = '%s.new' % last_tested
+      with open(new_file, 'w') as json_file:
+        json.dump(bisect_contents, json_file, indent=4, separators=(',', ': '))
+
+      os.rename(new_file, last_tested)
+
+
+def _NoteCompletedBisection(last_tested, src_path, end):
+  """Prints that bisection is complete."""
+
+  print('Finished bisecting for %s' % last_tested)
+
+  if src_path:
+    bad_llvm_hash = LLVMHash().GetGitHashForVersion(src_path, end)
+  else:
+    bad_llvm_hash = LLVMHash().GetLLVMHash(end)
+
+  print(
+      'The bad revision is %d and its commit hash is %s' % (end, bad_llvm_hash))
+
+
+def main(args_output):
   """Bisects LLVM based off of a .JSON file.
 
   Raises:
@@ -311,8 +382,6 @@ def main():
 
   VerifyOutsideChroot()
 
-  args_output = GetCommandLineArgs()
-
   update_packages = [
       'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx',
       'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind'
@@ -333,8 +402,7 @@ def main():
 
   _ValidateStartAndEndAgainstJSONStartAndEnd(
       start, end, bisect_contents['start'], bisect_contents['end'])
-  print(start, end)
-  print(bisect_contents['jobs'])
+
   # Pending and skipped revisions are between 'start_revision' and
   # 'end_revision'.
   start_revision, end_revision, pending_revisions, skip_revisions = \
@@ -344,41 +412,47 @@ def main():
       start_revision, end_revision, args_output.parallel, args_output.src_path,
       pending_revisions, skip_revisions)
 
+  # No more revisions between 'start_revision' and 'end_revision', so
+  # bisection is complete.
+  #
+  # This is determined by finding all valid revisions between 'start_revision'
+  # and 'end_revision' and that are NOT in the 'pending' and 'skipped' set.
   if not revisions:
-    no_revisions_message = (
-        'No revisions between start %d and end '
-        '%d to create tryjobs' % (start_revision, end_revision))
+    # Successfully completed bisection where there are 2 cases:
+    # 1) 'start_revision' and 'end_revision' are back-to-back (example:
+    # 'start_revision' is 369410 and 'end_revision' is 369411).
+    #
+    # 2) 'start_revision' and 'end_revision' are NOT back-to-back, so there must
+    # be tryjobs in between which are labeled as 'skip' for their 'status'
+    # value.
+    #
+    # In either case, there are no 'pending' jobs.
+    if not pending_revisions:
+      _NoteCompletedBisection(args_output.last_tested, args_output.src_path,
+                              end_revision)
 
-    if skip_revisions:
-      no_revisions_message += '\nThe following tryjobs were skipped:\n' \
-          + '\n'.join(str(rev) for rev in skip_revisions)
+      if skip_revisions:
+        skip_revisions_message = ('\nThe following revisions were skipped:\n' +
+                                  '\n'.join(str(rev) for rev in skip_revisions))
 
-    raise ValueError(no_revisions_message)
+        print(skip_revisions_message)
 
-  # Check if any revisions that are going to be added as a tryjob exist already
-  # in the 'jobs' list.
-  for rev in revisions:
-    if FindTryjobIndex(rev, bisect_contents['jobs']) is not None:
-      raise ValueError('Revision %d exists already in \'jobs\'' % rev)
+      return BisectionExitStatus.BISECTION_COMPLETE.value
 
-  try:
-    for svn_revision, git_hash in zip(revisions, git_hashes):
-      tryjob_dict = AddTryjob(update_packages, git_hash, svn_revision,
-                              args_output.chroot_path, patch_metadata_file,
-                              args_output.extra_change_lists,
-                              args_output.options, args_output.builder,
-                              args_output.log_level, svn_revision)
+    # Some tryjobs are not finished which may change the actual bad
+    # commit/revision when those tryjobs are finished.
+    DieWithNoRevisionsError(start_revision, end_revision, skip_revisions,
+                            pending_revisions)
 
-      bisect_contents['jobs'].append(tryjob_dict)
-  finally:
-    # Do not want to lose progress if there is an exception.
-    if args_output.last_tested:
-      new_file = '%s.new' % args_output.last_tested
-      with open(new_file, 'w') as json_file:
-        json.dump(bisect_contents, json_file, indent=4, separators=(',', ': '))
+  CheckForExistingTryjobsInRevisionsToLaunch(revisions, bisect_contents['jobs'])
 
-        os.rename(new_file, args_output.last_tested)
+  UpdateBisection(revisions, git_hashes, bisect_contents,
+                  args_output.last_tested, update_packages,
+                  args_output.chroot_path, patch_metadata_file,
+                  args_output.extra_change_lists, args_output.options,
+                  args_output.builder, args_output.log_level)
 
 
 if __name__ == '__main__':
-  main()
+  args_output = GetCommandLineArgs()
+  sys.exit(main(args_output))
diff --git a/llvm_tools/update_all_tryjobs_with_auto.py b/llvm_tools/update_all_tryjobs_with_auto.py
new file mode 100755
index 00000000..4a670c34
--- /dev/null
+++ b/llvm_tools/update_all_tryjobs_with_auto.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Updates the status of all tryjobs to the result of `cros buildresult`."""
+
+from __future__ import print_function
+
+import argparse
+import json
+import os
+
+from assert_not_in_chroot import VerifyOutsideChroot
+from patch_manager import _ConvertToASCII
+from update_tryjob_status import GetAutoResult
+from update_tryjob_status import TryjobStatus
+
+
+def GetPathToUpdateAllTryjobsWithAutoScript():
+  """Returns the absolute path to this script."""
+
+  return os.path.abspath(__file__)
+
+
+def GetCommandLineArgs():
+  """Parses the command line for the command line arguments."""
+
+  # Default absoute path to the chroot if not specified.
+  cros_root = os.path.expanduser('~')
+  cros_root = os.path.join(cros_root, 'chromiumos')
+
+  # Create parser and add optional command-line arguments.
+  parser = argparse.ArgumentParser(description=__doc__)
+
+  # Add argument for the JSON file to use for the update of a tryjob.
+  parser.add_argument(
+      '--last_tested',
+      required=True,
+      help='The absolute path to the JSON file that contains the tryjobs used '
+      'for bisecting LLVM.')
+
+  # Add argument for a specific chroot path.
+  parser.add_argument(
+      '--chroot_path',
+      default=cros_root,
+      help='the path to the chroot (default: %(default)s)')
+
+  args_output = parser.parse_args()
+
+  if not os.path.isfile(args_output.last_tested) or \
+      not args_output.last_tested.endswith('.json'):
+    raise ValueError('File does not exist or does not ending in \'.json\' '
+                     ': %s' % args_output.last_tested)
+
+  return args_output
+
+
+def main():
+  """Updates the status of a tryjob."""
+
+  VerifyOutsideChroot()
+
+  args_output = GetCommandLineArgs()
+
+  with open(args_output.last_tested) as tryjobs:
+    bisect_contents = _ConvertToASCII(json.load(tryjobs))
+
+  for tryjob in bisect_contents['jobs']:
+    if tryjob['status'] == TryjobStatus.PENDING.value:
+      tryjob['status'] = GetAutoResult(args_output.chroot_path,
+                                       tryjob['buildbucket_id'])
+
+  new_file = '%s.new' % args_output.last_tested
+  with open(new_file, 'w') as update_tryjobs:
+    json.dump(bisect_contents, update_tryjobs, indent=4, separators=(',', ': '))
+  os.rename(new_file, args_output.last_tested)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/llvm_tools/update_packages_and_run_tryjobs.py b/llvm_tools/update_packages_and_run_tryjobs.py
index c9b4bfe1..487b7f6a 100755
--- a/llvm_tools/update_packages_and_run_tryjobs.py
+++ b/llvm_tools/update_packages_and_run_tryjobs.py
@@ -10,6 +10,7 @@ from __future__ import print_function
 
 from pipes import quote
 import argparse
+import datetime
 import json
 import os
 import sys
@@ -217,11 +218,14 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path,
     # stderr can be noisy e.g. warnings when entering chroot, so ignore it.
     # e.g. cros_sdk:enter_chroot: Gclient cache dir "/tmp/git-cache" is not...
 
+    tryjob_launch_time = datetime.datetime.utcnow()
+
     buildbucket_id = int(ParseTryjobBuildbucketId(out.rstrip()))
 
     tryjob_contents = json.loads(out.rstrip())
 
     new_tryjob = {
+        'launch_time': str(tryjob_launch_time),
         'link': str(tryjob_contents[0]['url']),
         'buildbucket_id': buildbucket_id,
         'extra_cls': extra_change_lists,
diff --git a/llvm_tools/update_tryjob_status.py b/llvm_tools/update_tryjob_status.py
index 8425f2c6..4648db12 100755
--- a/llvm_tools/update_tryjob_status.py
+++ b/llvm_tools/update_tryjob_status.py
@@ -188,6 +188,71 @@ def GetStatusFromCrosBuildResult(chroot_path, buildbucket_id):
   return str(tryjob_contents['%d' % buildbucket_id]['status'])
 
 
+def GetAutoResult(chroot_path, buildbucket_id):
+  """Returns the conversion of the result of 'cros buildresult'."""
+
+  # Calls 'cros buildresult' to get the status of the tryjob.
+  build_result = GetStatusFromCrosBuildResult(chroot_path, buildbucket_id)
+
+  # The string returned by 'cros buildresult' might not be in the mapping.
+  if build_result not in builder_status_mapping:
+    raise ValueError(
+        '\'cros buildresult\' return value is invalid: %s' % build_result)
+
+  return builder_status_mapping[build_result]
+
+
+def GetCustomScriptResult(custom_script, status_file, tryjob_contents):
+  """Returns the conversion of the exit code of the custom script.
+
+  Args:
+    custom_script: Absolute path to the script to be executed.
+    status_file: Absolute path to the file that contains information about the
+    bisection of LLVM.
+    tryjob_contents: A dictionary of the contents of the tryjob (e.g. 'status',
+    'url', 'link', 'buildbucket_id', etc.).
+
+  Returns:
+    The exit code conversion to either return 'good', 'bad', or 'skip'.
+
+  Raises:
+    ValueError: The custom script failed to provide the correct exit code.
+  """
+
+  # Create a temporary file to write the contents of the tryjob at index
+  # 'tryjob_index' (the temporary file path will be passed into the custom
+  # script as a command line argument).
+  with CreateTemporaryJsonFile() as temp_json_file:
+    with open(temp_json_file, 'w') as tryjob_file:
+      json.dump(tryjob_contents, tryjob_file, indent=4, separators=(',', ': '))
+
+    exec_script_cmd = [custom_script, temp_json_file]
+
+    # Execute the custom script to get the exit code.
+    exec_script_cmd_obj = subprocess.Popen(
+        exec_script_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    _, stderr = exec_script_cmd_obj.communicate()
+
+    # Invalid exit code by the custom script.
+    if exec_script_cmd_obj.returncode not in custom_script_exit_value_mapping:
+      # Save the .JSON file to the directory of 'status_file'.
+      name_of_json_file = os.path.join(
+          os.path.dirname(status_file), os.path.basename(temp_json_file))
+
+      os.rename(temp_json_file, name_of_json_file)
+
+      raise ValueError(
+          'Custom script %s exit code %d did not match '
+          'any of the expected exit codes: %d for \'good\', %d '
+          'for \'bad\', or %d for \'skip\'.\nPlease check %s for information '
+          'about the tryjob: %s' %
+          (custom_script, exec_script_cmd_obj.returncode,
+           CustomScriptStatus.GOOD.value, CustomScriptStatus.BAD.value,
+           CustomScriptStatus.SKIP.value, name_of_json_file, stderr))
+
+  return custom_script_exit_value_mapping[exec_script_cmd_obj.returncode]
+
+
 def UpdateTryjobStatus(revision, set_status, status_file, chroot_path,
                        custom_script):
   """Updates a tryjob's 'status' field based off of 'set_status'.
@@ -236,56 +301,13 @@ def UpdateTryjobStatus(revision, set_status, status_file, chroot_path,
   elif set_status == TryjobStatus.PENDING:
     bisect_contents['jobs'][tryjob_index]['status'] = TryjobStatus.PENDING.value
   elif set_status == TryjobStatus.AUTO:
-    # Calls 'cros buildresult' and uses the mapping to assign the value to
-    # 'status'.
-    build_result = GetStatusFromCrosBuildResult(
+    bisect_contents['jobs'][tryjob_index]['status'] = GetAutoResult(
         chroot_path, bisect_contents['jobs'][tryjob_index]['buildbucket_id'])
-
-    # The string returned by 'cros buildresult' might not be in the mapping.
-    if build_result not in builder_status_mapping:
-      raise ValueError(
-          '\'cros buildresult\' return value is invalid: %s' % build_result)
-
-    bisect_contents['jobs'][tryjob_index]['status'] = builder_status_mapping[
-        build_result]
   elif set_status == TryjobStatus.SKIP:
     bisect_contents['jobs'][tryjob_index]['status'] = TryjobStatus.SKIP.value
   elif set_status == TryjobStatus.CUSTOM_SCRIPT:
-    # Create a temporary file to write the contents of the tryjob at index
-    # 'tryjob_index' (the temporary file path will be passed into the custom
-    # script as a command line argument).
-    with CreateTemporaryJsonFile() as temp_json_file:
-      with open(temp_json_file, 'w') as tryjob_file:
-        json.dump(
-            bisect_contents['jobs'][tryjob_index],
-            tryjob_file,
-            indent=4,
-            separators=(',', ': '))
-
-      exec_script_cmd = [custom_script, temp_json_file]
-
-      # Execute the custom script to get the exit code.
-      exec_script_cmd_obj = subprocess.Popen(
-          exec_script_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-      _, stderr = exec_script_cmd_obj.communicate()
-
-      # Invalid exit code by the custom script.
-      if exec_script_cmd_obj.returncode not in custom_script_exit_value_mapping:
-        name_of_json_file = os.path.join(
-            os.path.dirname(status_file), os.path.basename(temp_json_file))
-        os.rename(temp_json_file, name_of_json_file)
-        raise ValueError(
-            'Custom script %s exit code %d did not match '
-            'any of the expected exit codes: %d for \'good\', %d '
-            'for \'bad\', or %d for \'skip\'\nPlease check %s for information '
-            'about the tryjob: %s' %
-            (custom_script, exec_script_cmd_obj.returncode,
-             CustomScriptStatus.GOOD.value, CustomScriptStatus.BAD.value,
-             CustomScriptStatus.SKIP.value, name_of_json_file, stderr))
-
-    bisect_contents['jobs'][tryjob_index][
-        'status'] = custom_script_exit_value_mapping[exec_script_cmd_obj
-                                                     .returncode]
+    bisect_contents['jobs'][tryjob_index]['status'] = GetCustomScriptResult(
+        custom_script, status_file, bisect_contents['jobs'][tryjob_index])
   else:
     raise ValueError('Invalid \'set_status\' option provided: %s' % set_status)