Add a binhost-pre-cq and run it on overlay / chromite changes.

BUG=chromium:443254
TEST=new unit tests, trybot job of the new builder

Change-Id: Ia34740c033bbc04c55f28f32a778da065a6fc184
Reviewed-on: https://chromium-review.googlesource.com/261415
Reviewed-by: David James <davidjames@chromium.org>
Commit-Queue: David James <davidjames@chromium.org>
Trybot-Ready: David James <davidjames@chromium.org>
Tested-by: David James <davidjames@chromium.org>

9 files changed, 128 insertions, 27 deletions

diff --git a/COMMIT-QUEUE.ini b/COMMIT-QUEUE.ini
new file mode 100644
index 000000000..baecb5bc5
--- /dev/null
+++ b/COMMIT-QUEUE.ini
@@ -0,0 +1,11 @@
+# Copyright 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Per-project Commit Queue settings.
+# Documentation: http://goo.gl/5J7oND
+
+[GENERAL]
+
+# Run the default configs plus the binhost-pre-cq.
+pre-cq-configs: default binhost-pre-cq
diff --git a/cbuildbot/builders/simple_builders.py b/cbuildbot/builders/simple_builders.py
index 51b708c56..3c4ff362e 100644
--- a/cbuildbot/builders/simple_builders.py
+++ b/cbuildbot/builders/simple_builders.py
@@ -228,6 +228,7 @@ class SimpleBuilder(generic_builders.Builder):
     self.RunSetupBoard()
     self._RunStage(chrome_stages.SyncChromeStage)
     self._RunStage(chrome_stages.PatchChromeStage)
+    self._RunStage(artifact_stages.BinhostTestStage)
 
     # Prepare stages to run in background.  If child_configs exist then
     # run each of those here, otherwise use default config.
diff --git a/cbuildbot/cbuildbot_config.py b/cbuildbot/cbuildbot_config.py
index cadcc1e38..a17504119 100755
--- a/cbuildbot/cbuildbot_config.py
+++ b/cbuildbot/cbuildbot_config.py
@@ -646,9 +646,13 @@ _settings = dict(
   postsync_reexec=True,
 
 # create_delta_sysroot -- Create delta sysroot during ArchiveStage. Disabled by
-#                          default.
+#                         default.
   create_delta_sysroot=False,
 
+# binhost_test -- Run the binhost_test stage. Only makes sense for builders that
+#                 have no boards.
+  binhost_test=False,
+
 # TODO(sosa): Collapse to one option.
 # ====================== Dev installer prebuilts options =======================
 
@@ -1334,6 +1338,7 @@ internal_chromium_pfq = internal.derive(
 internal_chromium_pfq.add_config('master-chromium-pfq',
   boards=[],
   master=True,
+  binhost_test=True,
   push_overlays=constants.BOTH_OVERLAYS,
   afdo_update_ebuild=True,
   chrome_sdk=False,
@@ -2149,6 +2154,7 @@ _config.add_group('test-ap-group',
 internal_paladin.add_config('master-paladin',
   boards=[],
   master=True,
+  binhost_test=True,
   push_overlays=constants.BOTH_OVERLAYS,
   description='Commit Queue master (all others are slaves)',
 
@@ -2398,6 +2404,11 @@ _CreateConfigsForBoards(pre_cq, _all_boards, 'pre-cq')
 _CreateConfigsForBoards(no_vmtest_pre_cq, _all_boards, 'no-vmtest-pre-cq')
 _CreateConfigsForBoards(compile_only_pre_cq, _all_boards, 'compile-only-pre-cq')
 
+no_vmtest_pre_cq.add_config(constants.BINHOST_PRE_CQ,
+  boards=[],
+  binhost_test=True,
+)
+
 # TODO(davidjames): Add peach_pit, nyan, and beaglebone to pre-cq.
 # TODO(davidjames): Update daisy_spring to build images again.
 _config.add_group('mixed-a-pre-cq',
diff --git a/cbuildbot/cbuildbot_config_unittest.py b/cbuildbot/cbuildbot_config_unittest.py
index 81d60928f..9e5992c22 100644
--- a/cbuildbot/cbuildbot_config_unittest.py
+++ b/cbuildbot/cbuildbot_config_unittest.py
@@ -813,3 +813,9 @@ class OverrideForTrybotTest(cros_test_lib.TestCase):
     self.assertFalse(redundant,
                      "Manual waterfall configs are automatically included: "
                      "%s" % (sorted(redundant),))
+
+  def testBinhostTest(self):
+    """Builders with the binhost_test setting shouldn't have boards."""
+    for config in cbuildbot_config.config.values():
+      if config.binhost_test:
+        self.assertEqual(config.boards, [])
diff --git a/cbuildbot/config_dump.json b/cbuildbot/config_dump.json
index 80e891ca6..0604f680b 100644
--- a/cbuildbot/config_dump.json
+++ b/cbuildbot/config_dump.json
@@ -10,6 +10,7 @@
         "binhost_base_url": null,
         "binhost_bucket": null,
         "binhost_key": null,
+        "binhost_test": false,
         "board_replace": false,
         "boards": null,
         "branch": false,
@@ -6370,6 +6371,29 @@
             "smoke_suite"
         ]
     },
+    "binhost-pre-cq": {
+        "archive": false,
+        "binhost_test": true,
+        "boards": [],
+        "build_packages_in_background": true,
+        "build_type": "binary",
+        "chrome_sdk_build_chrome": false,
+        "chroot_replace": true,
+        "cpe_export": false,
+        "debug_symbols": false,
+        "description": "Verifies compilation, building an image, and unit tests if supported.",
+        "doc": "http://www.chromium.org/chromium-os/build/builder-overview#TOC-Pre-CQ",
+        "health_alert_recipients": [
+            "chromeos-infra-eng@grotations.appspotmail.com"
+        ],
+        "important": true,
+        "manifest_version": true,
+        "name": "binhost-pre-cq",
+        "pre_cq": true,
+        "trybot_list": true,
+        "upload_standalone_images": false,
+        "vm_tests": []
+    },
     "bobcat-chrome-pfq": {
         "boards": [
             "bobcat"
@@ -30510,6 +30534,7 @@
     "master-chromium-pfq": {
         "active_waterfall": "chromeos",
         "afdo_update_ebuild": true,
+        "binhost_test": true,
         "boards": [],
         "build_type": "chrome",
         "chrome_rev": "latest_release",
@@ -30528,6 +30553,7 @@
     },
     "master-paladin": {
         "active_waterfall": "chromeos",
+        "binhost_test": true,
         "boards": [],
         "build_type": "paladin",
         "chrome_sdk": true,
diff --git a/cbuildbot/constants.py b/cbuildbot/constants.py
index 34e95b4d9..3aa8cac72 100644
--- a/cbuildbot/constants.py
+++ b/cbuildbot/constants.py
@@ -805,6 +805,7 @@ SHERIFF_TYPE_TO_URL = {
 CQ_MASTER = 'master-paladin'
 CANARY_MASTER = 'master-release'
 PFQ_MASTER = 'master-chromium-pfq'
+BINHOST_PRE_CQ = 'binhost-pre-cq'
 
 
 # Email validation regex. Not quite fully compliant with RFC 2822, but good
diff --git a/cbuildbot/stages/artifact_stages.py b/cbuildbot/stages/artifact_stages.py
index 85e32841d..bbfceb821 100644
--- a/cbuildbot/stages/artifact_stages.py
+++ b/cbuildbot/stages/artifact_stages.py
@@ -724,6 +724,8 @@ class DevInstallerPrebuiltsStage(UploadPrebuiltsStage):
 class BinhostTestStage(generic_stages.ForgivingBuilderStage):
   """Stage that verifies Chrome prebuilts."""
 
+  config_name = 'binhost_test'
+
   def PerformStage(self):
     # Verify our binhosts.
     commands.RunBinhostTest(self._build_root)
diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py
index 4596638db..5f0dff6c0 100644
--- a/cbuildbot/stages/sync_stages.py
+++ b/cbuildbot/stages/sync_stages.py
@@ -1033,15 +1033,18 @@ class PreCQLauncherStage(SyncStage):
     )
     cros_build_lib.PrintBuildbotLink(' | '.join(items), patch.url)
 
-
-  def VerificationsForChange(self, change):
+  def _ConfiguredVerificationsForChange(self, change):
     """Determine which configs to test |change| with.
 
+    This method returns only the configs that are asked for by the config
+    file. It does not include special-case logic for adding additional bots
+    based on the type of the repository (see VerificationsForChange for that).
+
     Args:
       change: GerritPatch instance to get configs-to-test for.
 
     Returns:
-      A list of configs.
+      A set of configs to test.
     """
     configs_to_test = None
     try:
@@ -1060,16 +1063,40 @@ class PreCQLauncherStage(SyncStage):
       cros_build_lib.Error('%s has malformed config file', change,
                            exc_info=True)
 
-    return configs_to_test or constants.PRE_CQ_DEFAULT_CONFIGS
+    return set(configs_to_test or constants.PRE_CQ_DEFAULT_CONFIGS)
+
+  def VerificationsForChange(self, change):
+    """Determine which configs to test |change| with.
+
+    Args:
+      change: GerritPatch instance to get configs-to-test for.
+
+    Returns:
+      A set of configs to test.
+    """
+    configs_to_test = set(self._ConfiguredVerificationsForChange(change))
+
+    # Add the BINHOST_PRE_CQ to any changes that affect an overlay.
+    if '/overlays/' in change.project:
+      configs_to_test.add(constants.BINHOST_PRE_CQ)
+
+    return configs_to_test
 
   def _ParsePreCQOption(self, pre_cq_option):
     """Gets a valid config list, or None, from |pre_cq_option|."""
-    if (pre_cq_option and
-        pre_cq_option.split() and
-        all(c in cbuildbot_config.config for c in pre_cq_option.split())):
-      return pre_cq_option.split()
-    else:
-      return None
+    if pre_cq_option and pre_cq_option.split():
+      configs_to_test = set(pre_cq_option.split())
+
+      # Replace 'default' with the default configs.
+      if 'default' in configs_to_test:
+        configs_to_test.discard('default')
+        configs_to_test.update(constants.PRE_CQ_DEFAULT_CONFIGS)
+
+      # Verify that all of the configs are valid.
+      if all(c in cbuildbot_config.config for c in configs_to_test):
+        return configs_to_test
+
+    return None
 
   def ScreenChangeForPreCQ(self, change):
     """Record which pre-cq tryjobs to test |change| with.
diff --git a/cbuildbot/stages/sync_stages_unittest.py b/cbuildbot/stages/sync_stages_unittest.py
index db7695828..0718f5aab 100644
--- a/cbuildbot/stages/sync_stages_unittest.py
+++ b/cbuildbot/stages/sync_stages_unittest.py
@@ -468,36 +468,52 @@ class PreCQLauncherStageTest(MasterCQSyncTestCase):
     return_string = ' '.join(configs_to_test)
     self.PatchObject(triage_lib, 'GetOptionForChange',
                      return_value=return_string)
-    self.assertEqual(self.sync_stage.VerificationsForChange(change),
-                     configs_to_test)
+    self.assertItemsEqual(self.sync_stage.VerificationsForChange(change),
+                          configs_to_test)
 
   def testVerificationsForChangeMalformedConfigFile(self):
     change = MockPatch()
     self.PatchObject(triage_lib, 'GetOptionForChange',
                      side_effect=ConfigParser.Error)
-    self.assertEqual(self.sync_stage.VerificationsForChange(change),
-                     constants.PRE_CQ_DEFAULT_CONFIGS)
+    self.assertItemsEqual(self.sync_stage.VerificationsForChange(change),
+                          constants.PRE_CQ_DEFAULT_CONFIGS)
 
   def testVerificationsForChangeNoSuchConfig(self):
     change = MockPatch()
     self.PatchObject(triage_lib, 'GetOptionForChange',
                      return_value='this_config_does_not_exist')
-    self.assertEqual(self.sync_stage.VerificationsForChange(change),
-                     constants.PRE_CQ_DEFAULT_CONFIGS)
+    self.assertItemsEqual(self.sync_stage.VerificationsForChange(change),
+                          constants.PRE_CQ_DEFAULT_CONFIGS)
 
   def testVerificationsForChangeEmptyField(self):
     change = MockPatch()
     self.PatchObject(triage_lib, 'GetOptionForChange',
                      return_value=' ')
-    self.assertEqual(self.sync_stage.VerificationsForChange(change),
-                     constants.PRE_CQ_DEFAULT_CONFIGS)
+    self.assertItemsEqual(self.sync_stage.VerificationsForChange(change),
+                          constants.PRE_CQ_DEFAULT_CONFIGS)
 
   def testVerificationsForChangeNoneField(self):
     change = MockPatch()
     self.PatchObject(triage_lib, 'GetOptionForChange',
                      return_value=None)
-    self.assertEqual(self.sync_stage.VerificationsForChange(change),
-                     constants.PRE_CQ_DEFAULT_CONFIGS)
+    self.assertItemsEqual(self.sync_stage.VerificationsForChange(change),
+                          constants.PRE_CQ_DEFAULT_CONFIGS)
+
+  def testOverlayVerifications(self):
+    change = MockPatch(project='chromiumos/overlays/chromiumos-overlay')
+    self.PatchObject(triage_lib, 'GetOptionForChange',
+                     return_value=None)
+    configs = constants.PRE_CQ_DEFAULT_CONFIGS + [constants.BINHOST_PRE_CQ]
+    self.assertItemsEqual(self.sync_stage.VerificationsForChange(change),
+                          configs)
+
+  def testRequestedDefaultVerifications(self):
+    change = MockPatch()
+    self.PatchObject(triage_lib, 'GetOptionForChange',
+                     return_value='default x86-zgb-pre-cq')
+    configs = constants.PRE_CQ_DEFAULT_CONFIGS + ['x86-zgb-pre-cq']
+    self.assertItemsEqual(self.sync_stage.VerificationsForChange(change),
+                          configs)
 
   def testVerificationsForChangeFromInvalidCommitMessage(self):
     change = MockPatch(commit_message="""First line.
@@ -507,8 +523,8 @@ pre-cq-configs: insect-pre-cq
 """)
     self.PatchObject(triage_lib, 'GetOptionForChange',
                      return_value='lumpy-pre-cq')
-    self.assertEqual(self.sync_stage.VerificationsForChange(change),
-                     ['lumpy-pre-cq'])
+    self.assertItemsEqual(self.sync_stage.VerificationsForChange(change),
+                          ['lumpy-pre-cq'])
 
   def testVerificationsForChangeFromCommitMessage(self):
     change = MockPatch(commit_message="""First line.
@@ -518,8 +534,8 @@ pre-cq-configs: stumpy-pre-cq
 """)
     self.PatchObject(triage_lib, 'GetOptionForChange',
                      return_value='lumpy-pre-cq')
-    self.assertEqual(self.sync_stage.VerificationsForChange(change),
-                     ['stumpy-pre-cq'])
+    self.assertItemsEqual(self.sync_stage.VerificationsForChange(change),
+                          ['stumpy-pre-cq'])
 
   def testMultiVerificationsForChangeFromCommitMessage(self):
     change = MockPatch(commit_message="""First line.
@@ -530,8 +546,8 @@ pre-cq-configs: link-pre-cq
 """)
     self.PatchObject(triage_lib, 'GetOptionForChange',
                      return_value='lumpy-pre-cq')
-    self.assertEqual(self.sync_stage.VerificationsForChange(change),
-                     ['stumpy-pre-cq', 'link-pre-cq'])
+    self.assertItemsEqual(self.sync_stage.VerificationsForChange(change),
+                          ['stumpy-pre-cq', 'link-pre-cq'])
 
   def _PrepareChangesWithPendingVerifications(self, verifications=None):
     """Prepare changes and pending verifications for them.