llvm_tot: add an option for recipe builders.

Add an option to launch LLVM TOT builders with recipe builders to
update_packages_and_run_tests.py. The builders launched will not run
hwtests for now.

BUG=chromium:1072063

TEST=local tests.

Change-Id: I1c34090f2d2ea6d9b7efab0a0dccf795396c96d1
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2157624
Tested-by: Jian Cai <jiancai@google.com>
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>

3 files changed, 220 insertions, 119 deletions

diff --git a/llvm_tools/README.md b/llvm_tools/README.md
index ce094e98..b0fbea5a 100644
--- a/llvm_tools/README.md
+++ b/llvm_tools/README.md
@@ -21,6 +21,8 @@ password)**
 This script is used for updating a package's LLVM hash (sys-devel/llvm,
 sys-libs/compiler-rt, sys-libs/libcxx, sys-libs/libcxxabi, and
 sys-libs/llvm-libunwind) and then run tests after updating the git hash.
+There are three ways to test the change, including starting tryjobs,
+recipe builders or using cq+1.
 
 An example when this script should be run is when certain boards would like
 to be tested with the updated `LLVM_NEXT_HASH`.
@@ -56,15 +58,15 @@ $ ./update_packages_and_run_tests.py \
 ```
 
 Similarly as the previous example, but for updating `LLVM_NEXT_HASH` to
-the git hash of revision 367622 and test with tryobs:
+the git hash of revision 367622 and test with recipe builders:
 
 ```
 $ ./update_packages_and_run_tests.py \
   --is_llvm_next \
   --llvm_version 367622 \
-  tryjobs \
-  --options nochromesdk latest-toolchain \
-  --builders kevin-release-tryjob nocturne-release-tryjob
+  recipe \
+  --options -nocanary \
+  --builders chromeos/toolchain/kevin-llvm chromeos/toolchain/nocturne-llvm
 ```
 
 ## `update_chromeos_llvm_hash.py`
diff --git a/llvm_tools/update_packages_and_run_tests.py b/llvm_tools/update_packages_and_run_tests.py
index 9fe4fe02..f47b5cda 100755
--- a/llvm_tools/update_packages_and_run_tests.py
+++ b/llvm_tools/update_packages_and_run_tests.py
@@ -12,11 +12,11 @@ import argparse
 import datetime
 import json
 import os
+import subprocess
 
 import chroot
 import failure_modes
 import get_llvm_hash
-import subprocess_helpers
 import update_chromeos_llvm_hash
 
 
@@ -93,8 +93,10 @@ def GetCommandLineArgs():
       '(default: %(default)s)')
 
   subparsers = parser.add_subparsers(dest='subparser_name')
+  subparser_names = []
   # Testing with the tryjobs.
   tryjob_subparser = subparsers.add_parser('tryjobs')
+  subparser_names.append('tryjobs')
   tryjob_subparser.add_argument(
       '--builders',
       required=True,
@@ -110,12 +112,29 @@ def GetCommandLineArgs():
       default=[],
       help='options to use for the tryjob testing')
 
+  # Testing with the recipe builders
+  recipe_subparser = subparsers.add_parser('recipe')
+  subparser_names.append('recipe')
+  recipe_subparser.add_argument(
+      '--options',
+      required=False,
+      nargs='+',
+      default=[],
+      help='options passed to the recipe builders')
+
+  recipe_subparser.add_argument(
+      '--builders',
+      required=True,
+      nargs='+',
+      default=[],
+      help='recipe builders to launch')
+
   # Testing with CQ.
   subparsers.add_parser('cq')
+  subparser_names.append('cq')
 
   args_output = parser.parse_args()
 
-  subparser_names = ['tryjobs', 'cq']
   if args_output.subparser_name not in subparser_names:
     parser.error('one of %s must be specified' % subparser_names)
 
@@ -157,11 +176,11 @@ def AddReviewers(cl, reviewers, chroot_path):
   for reviewer in reviewers:
     cmd = [gerrit_abs_path, 'reviewers', str(cl), reviewer]
 
-    subprocess_helpers.ExecCommandAndCaptureOutput(cmd)
+    subprocess.check_output(cmd)
 
 
-def AddTryjobLinkToCL(tryjobs, cl, chroot_path):
-  """Adds the tryjob link(s) to the CL as a comment."""
+def AddLinksToCL(tests, cl, chroot_path):
+  """Adds the test link(s) to the CL as a comment."""
 
   # NOTE: Invoking `cros_sdk` does not make each tryjob link appear on its own
   # line, so invoking the `gerrit` command directly instead of using `cros_sdk`
@@ -171,15 +190,12 @@ def AddTryjobLinkToCL(tryjobs, cl, chroot_path):
   # newline.
   gerrit_abs_path = os.path.join(chroot_path, 'chromite/bin/gerrit')
 
-  tryjob_links = ['Started the following tryjobs:']
-  tryjob_links.extend(tryjob['link'] for tryjob in tryjobs)
+  links = ['Started the following tests:']
+  links.extend(test['link'] for test in tests)
 
-  add_message_cmd = [
-      gerrit_abs_path, 'message',
-      str(cl), '\n'.join(tryjob_links)
-  ]
+  add_message_cmd = [gerrit_abs_path, 'message', str(cl), '\n'.join(links)]
 
-  subprocess_helpers.ExecCommandAndCaptureOutput(add_message_cmd)
+  subprocess.check_output(add_message_cmd)
 
 
 # Testing with tryjobs
@@ -209,16 +225,15 @@ def GetTryJobCommand(change_list, extra_change_lists, options, builder):
     for extra_cl in extra_change_lists:
       tryjob_cmd.extend(['-g', '%d' % extra_cl])
 
-  tryjob_cmd.append(builder)
-
   if options:
     tryjob_cmd.extend('--%s' % option for option in options)
 
+  tryjob_cmd.append(builder)
+
   return tryjob_cmd
 
 
-def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path,
-               verbose):
+def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path):
   """Runs a tryjob/tryjobs.
 
   Args:
@@ -228,7 +243,6 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path,
     options: Any options to be passed into the 'tryjob' command.
     builders: All the builders to run the 'tryjob' with.
     chroot_path: The absolute path to the chroot.
-    verbose: Print command contents to `stdout`.
 
   Returns:
     A list that contains stdout contents of each tryjob, where stdout is
@@ -239,42 +253,91 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path,
     ValueError: Failed to submit a tryjob.
   """
 
-  # Contains the results of each tryjob. The results are retrieved from 'out'
-  # which is stdout of the command executer.
-  tryjob_results = []
+  # Contains the results of each builder.
+  tests = []
 
-  # For each builder passed into the command line:
-  #
-  # Run a tryjob with the change list number obtained from updating the
+  # Run tryjobs with the change list number obtained from updating the
   # packages and append additional changes lists and options obtained from the
   # command line.
-  for cur_builder in builders:
-    tryjob_cmd = GetTryJobCommand(cl_number, extra_change_lists, options,
-                                  cur_builder)
+  for builder in builders:
+    cmd = GetTryJobCommand(cl_number, extra_change_lists, options, builder)
+
+    out = subprocess.check_output(cmd, cwd=chroot_path, encoding='utf-8')
+
+    test_output = json.loads(out)
+
+    tests.append({
+        'launch_time': str(GetCurrentTimeInUTC()),
+        'link': str(test_output[0]['url']),
+        'buildbucket_id': int(test_output[0]['buildbucket_id']),
+        'extra_cls': extra_change_lists,
+        'options': options,
+        'builder': [builder]
+    })
 
-    out = subprocess_helpers.ChrootRunCommand(
-        chroot_path, tryjob_cmd, verbose=verbose)
+  AddLinksToCL(tests, cl_number, chroot_path)
 
-    tryjob_launch_time = GetCurrentTimeInUTC()
+  return tests
 
-    tryjob_contents = json.loads(out)
 
-    buildbucket_id = int(tryjob_contents[0]['buildbucket_id'])
+def StartRecipeBuilders(cl_number, extra_change_lists, options, builders,
+                        chroot_path):
+  """Launch recipe builders.
 
-    new_tryjob = {
-        'launch_time': str(tryjob_launch_time),
-        'link': str(tryjob_contents[0]['url']),
-        'buildbucket_id': buildbucket_id,
+  Args:
+    cl_number: The CL created by updating the packages.
+    extra_change_lists: Any extra change lists that would run alongside the CL
+    that was created by updating the packages ('cl_number').
+    options: Any options to be passed into the 'tryjob' command.
+    builders: All the builders to run the 'tryjob' with.
+    chroot_path: The absolute path to the chroot.
+
+  Returns:
+    A list that contains stdout contents of each builder, where stdout is
+    information (a hashmap) about the tryjob. The hashmap also contains stderr
+    if there was an error when running a tryjob.
+
+  Raises:
+    ValueError: Failed to start a builder.
+  """
+
+  # Contains the results of each builder.
+  tests = []
+
+  # Launch a builders with the change list number obtained from updating the
+  # packages and append additional changes lists and options obtained from the
+  # command line.
+  for builder in builders:
+    cmd = ['bb', 'add', '-json']
+
+    if cl_number:
+      cmd.extend(['-cl', 'crrev.com/c/%d' % cl_number])
+
+    if extra_change_lists:
+      for cl in extra_change_lists:
+        cmd.extend(['-cl', 'crrev.com/c/%d' % cl])
+
+    if options:
+      cmd.extend(options)
+
+    cmd.append(builder)
+
+    out = subprocess.check_output(cmd, cwd=chroot_path, encoding='utf-8')
+
+    test_output = json.loads(out)
+
+    tests.append({
+        'launch_time': test_output['createTime'],
+        'link': 'http://ci.chromium.org/b/%s' % test_output['id'],
+        'buildbucket_id': test_output['id'],
         'extra_cls': extra_change_lists,
         'options': options,
-        'builder': [cur_builder]
-    }
-
-    tryjob_results.append(new_tryjob)
+        'builder': [builder]
+    })
 
-  AddTryjobLinkToCL(tryjob_results, cl_number, chroot_path)
+  AddLinksToCL(tests, cl_number, chroot_path)
 
-  return tryjob_results
+  return tests
 
 
 # Testing with CQ
@@ -299,7 +362,7 @@ def StartCQDryRun(cl, dependent_cls, chroot_path):
   for changes in cl_list:
     cq_dry_run_cmd = [gerrit_abs_path, 'label-cq', str(changes), '1']
 
-    subprocess_helpers.ExecCommandAndCaptureOutput(cq_dry_run_cmd)
+    subprocess.check_output(cq_dry_run_cmd)
 
 
 def main():
@@ -338,7 +401,7 @@ def main():
         'ebuilds': chroot_file_paths,
         'extra_cls': args_output.extra_change_lists,
     }
-    if args_output.subparser_name == 'tryjobs':
+    if args_output.subparser_name in ('tryjobs', 'recipe'):
       arg_dict['builders'] = args_output.builders
       arg_dict['tryjob_options'] = args_output.options
     if UnchangedSinceLastRun(args_output.last_tested, arg_dict):
@@ -373,13 +436,20 @@ def main():
   print('Change list number: %d' % change_list.cl_number)
 
   if args_output.subparser_name == 'tryjobs':
-    tryjob_results = RunTryJobs(change_list.cl_number,
-                                args_output.extra_change_lists,
-                                args_output.options, args_output.builders,
-                                args_output.chroot_path, args_output.verbose)
-    print('Tryjobs:')
-    for tryjob in tryjob_results:
-      print(tryjob)
+    tests = RunTryJobs(change_list.cl_number, args_output.extra_change_lists,
+                       args_output.options, args_output.builders,
+                       args_output.chroot_path)
+    print('Tests:')
+    for test in tests:
+      print(test)
+  elif args_output.subparser_name == 'recipe':
+    tests = StartRecipeBuilders(
+        change_list.cl_number, args_output.extra_change_lists,
+        args_output.options, args_output.builders, args_output.chroot_path)
+    print('Tests:')
+    for test in tests:
+      print(test)
+
   else:
     StartCQDryRun(change_list.cl_number, args_output.extra_change_lists,
                   args_output.chroot_path)
diff --git a/llvm_tools/update_packages_and_run_tests_unittest.py b/llvm_tools/update_packages_and_run_tests_unittest.py
index 70ecfffe..25a8f83c 100755
--- a/llvm_tools/update_packages_and_run_tests_unittest.py
+++ b/llvm_tools/update_packages_and_run_tests_unittest.py
@@ -9,13 +9,13 @@
 from __future__ import print_function
 
 import json
+import subprocess
 import unittest
 import unittest.mock as mock
 
 import chroot
 import get_llvm_hash
 import git
-import subprocess_helpers
 import test_helpers
 import update_chromeos_llvm_hash
 import update_packages_and_run_tests
@@ -68,112 +68,146 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase):
               last_tested_file, arg_dict), True)
 
   def testGetTryJobCommandWithNoExtraInformation(self):
-    test_change_list = 1234
+    change_list = 1234
 
-    test_builder = 'nocturne'
+    builder = 'nocturne'
 
-    expected_tryjob_cmd_list = [
+    expected_cmd = [
         'cros', 'tryjob', '--yes', '--json', '-g',
-        '%d' % test_change_list, test_builder
+        '%d' % change_list, builder
     ]
 
     self.assertEqual(
-        update_packages_and_run_tests.GetTryJobCommand(test_change_list, None,
-                                                       None, test_builder),
-        expected_tryjob_cmd_list)
+        update_packages_and_run_tests.GetTryJobCommand(change_list, None, None,
+                                                       builder), expected_cmd)
 
   def testGetTryJobCommandWithExtraInformation(self):
-    test_change_list = 4321
-    test_extra_cls = [1000, 10]
-    test_options = ['report_error', 'delete_tryjob']
-    test_builder = 'kevin'
+    change_list = 4321
+    extra_cls = [1000, 10]
+    options = ['option1', 'option2']
+    builder = 'kevin'
 
-    expected_tryjob_cmd_list = [
+    expected_cmd = [
         'cros',
         'tryjob',
         '--yes',
         '--json',
         '-g',
-        '%d' % test_change_list,
+        '%d' % change_list,
         '-g',
-        '%d' % test_extra_cls[0],
+        '%d' % extra_cls[0],
         '-g',
-        '%d' % test_extra_cls[1],
-        test_builder,
-        '--%s' % test_options[0],
-        '--%s' % test_options[1],
+        '%d' % extra_cls[1],
+        '--%s' % options[0],
+        '--%s' % options[1],
+        builder,
     ]
 
     self.assertEqual(
         update_packages_and_run_tests.GetTryJobCommand(
-            test_change_list, test_extra_cls, test_options, test_builder),
-        expected_tryjob_cmd_list)
+            change_list, extra_cls, options, builder), expected_cmd)
 
   @mock.patch.object(
       update_packages_and_run_tests,
       'GetCurrentTimeInUTC',
       return_value='2019-09-09')
-  @mock.patch.object(update_packages_and_run_tests, 'AddTryjobLinkToCL')
-  @mock.patch.object(subprocess_helpers, 'ChrootRunCommand')
-  def testSuccessfullySubmittedTryJob(
-      self, mock_chroot_cmd, mock_add_tryjob_link_to_cl, mock_launch_time):
+  @mock.patch.object(update_packages_and_run_tests, 'AddLinksToCL')
+  @mock.patch.object(subprocess, 'check_output')
+  def testSuccessfullySubmittedTryJob(self, mock_cmd, mock_add_links_to_cl,
+                                      mock_launch_time):
 
-    expected_tryjob_cmd_list = [
+    expected_cmd = [
         'cros', 'tryjob', '--yes', '--json', '-g',
         '%d' % 900, '-g',
-        '%d' % 1200, 'builder1', '--some_option'
+        '%d' % 1200, '--some_option', 'builder1'
     ]
 
-    buildbucket_id = '1234'
+    bb_id = '1234'
     url = 'https://some_tryjob_url.com'
 
-    tryjob_launch_contents = [{'buildbucket_id': buildbucket_id, 'url': url}]
+    mock_cmd.return_value = json.dumps([{'buildbucket_id': bb_id, 'url': url}])
 
-    mock_chroot_cmd.return_value = json.dumps(tryjob_launch_contents)
-
-    extra_cls = [1200]
-    tryjob_options = ['some_option']
-    builder_list = ['builder1']
     chroot_path = '/some/path/to/chroot'
-    cl_to_launch_tryjob = 900
-    verbose = False
+    cl = 900
+    extra_cls = [1200]
+    options = ['some_option']
+    builders = ['builder1']
 
-    tryjob_results_list = update_packages_and_run_tests.RunTryJobs(
-        cl_to_launch_tryjob, extra_cls, tryjob_options, builder_list,
-        chroot_path, verbose)
+    tests = update_packages_and_run_tests.RunTryJobs(cl, extra_cls, options,
+                                                     builders, chroot_path)
 
-    expected_tryjob_dict = {
+    expected_tests = [{
         'launch_time': mock_launch_time.return_value,
         'link': url,
-        'buildbucket_id': int(buildbucket_id),
+        'buildbucket_id': int(bb_id),
         'extra_cls': extra_cls,
-        'options': tryjob_options,
-        'builder': builder_list
-    }
+        'options': options,
+        'builder': builders
+    }]
 
-    self.assertEqual(tryjob_results_list, [expected_tryjob_dict])
+    self.assertEqual(tests, expected_tests)
 
-    mock_chroot_cmd.assert_called_once_with(
-        chroot_path, expected_tryjob_cmd_list, verbose=False)
+    mock_cmd.assert_called_once_with(
+        expected_cmd, cwd=chroot_path, encoding='utf-8')
 
-    mock_add_tryjob_link_to_cl.assert_called_once()
+    mock_add_links_to_cl.assert_called_once()
 
-  @mock.patch.object(
-      subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
-  def testSuccessfullyAddedTryjobLinkToCL(self, mock_exec_cmd):
+  @mock.patch.object(update_packages_and_run_tests, 'AddLinksToCL')
+  @mock.patch.object(subprocess, 'check_output')
+  def testSuccessfullySubmittedRecipeBuilders(self, mock_cmd,
+                                              mock_add_links_to_cl):
+
+    expected_cmd = [
+        'bb', 'add', '-json', '-cl',
+        'crrev.com/c/%s' % 900, '-cl',
+        'crrev.com/c/%s' % 1200, 'some_option', 'builder1'
+    ]
+
+    bb_id = '1234'
+    create_time = '2020-04-18T00:03:53.978767Z'
+
+    mock_cmd.return_value = json.dumps({'id': bb_id, 'createTime': create_time})
+
+    chroot_path = '/some/path/to/chroot'
+    cl = 900
+    extra_cls = [1200]
+    options = ['some_option']
+    builders = ['builder1']
+
+    tests = update_packages_and_run_tests.StartRecipeBuilders(
+        cl, extra_cls, options, builders, chroot_path)
+
+    expected_tests = [{
+        'launch_time': create_time,
+        'link': 'http://ci.chromium.org/b/%s' % bb_id,
+        'buildbucket_id': bb_id,
+        'extra_cls': extra_cls,
+        'options': options,
+        'builder': builders
+    }]
+
+    self.assertEqual(tests, expected_tests)
+
+    mock_cmd.assert_called_once_with(
+        expected_cmd, cwd=chroot_path, encoding='utf-8')
+
+    mock_add_links_to_cl.assert_called_once()
+
+  @mock.patch.object(subprocess, 'check_output', return_value=None)
+  def testSuccessfullyAddedTestLinkToCL(self, mock_exec_cmd):
     chroot_path = '/abs/path/to/chroot'
 
     test_cl_number = 1000
 
-    tryjob_result = [{'link': 'https://some_tryjob_link.com'}]
+    tests = [{'link': 'https://some_tryjob_link.com'}]
 
-    update_packages_and_run_tests.AddTryjobLinkToCL(tryjob_result,
-                                                    test_cl_number, chroot_path)
+    update_packages_and_run_tests.AddLinksToCL(tests, test_cl_number,
+                                               chroot_path)
 
     expected_gerrit_message = [
         '%s/chromite/bin/gerrit' % chroot_path, 'message',
         str(test_cl_number),
-        'Started the following tryjobs:\n%s' % tryjob_result[0]['link']
+        'Started the following tests:\n%s' % tests[0]['link']
     ]
 
     mock_exec_cmd.assert_called_once_with(expected_gerrit_message)
@@ -280,8 +314,7 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase):
         '\nCq-Depend: chromium:1234, chromium:5678')
 
   # Mock ExecCommandAndCaptureOutput for the gerrit command execution.
-  @mock.patch.object(
-      subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
+  @mock.patch.object(subprocess, 'check_output', return_value=None)
   def testStartCQDryRunNoDeps(self, mock_exec_cmd):
     chroot_path = '/abs/path/to/chroot'
     test_cl_number = 1000
@@ -299,8 +332,7 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase):
     mock_exec_cmd.assert_called_once_with(expected_gerrit_message)
 
   # Mock ExecCommandAndCaptureOutput for the gerrit command execution.
-  @mock.patch.object(
-      subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
+  @mock.patch.object(subprocess, 'check_output', return_value=None)
   # test with a single deps cl.
   def testStartCQDryRunSingleDep(self, mock_exec_cmd):
     chroot_path = '/abs/path/to/chroot'
@@ -326,8 +358,7 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase):
                      mock.call(expected_gerrit_cmd_2))
 
   # Mock ExecCommandAndCaptureOutput for the gerrit command execution.
-  @mock.patch.object(
-      subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
+  @mock.patch.object(subprocess, 'check_output', return_value=None)
   def testStartCQDryRunMultipleDep(self, mock_exec_cmd):
     chroot_path = '/abs/path/to/chroot'
     test_cl_number = 1000
@@ -359,8 +390,7 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase):
                      mock.call(expected_gerrit_cmd_3))
 
   # Mock ExecCommandAndCaptureOutput for the gerrit command execution.
-  @mock.patch.object(
-      subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
+  @mock.patch.object(subprocess, 'check_output', return_value=None)
   # test with no reviewers.
   def testAddReviewersNone(self, mock_exec_cmd):
     chroot_path = '/abs/path/to/chroot'
@@ -372,8 +402,7 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase):
     self.assertTrue(mock_exec_cmd.not_called)
 
   # Mock ExecCommandAndCaptureOutput for the gerrit command execution.
-  @mock.patch.object(
-      subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None)
+  @mock.patch.object(subprocess, 'check_output', return_value=None)
   # test with multiple reviewers.
   def testAddReviewersMultiple(self, mock_exec_cmd):
     chroot_path = '/abs/path/to/chroot'