sdk_stages: Create a single toolchain overlay for each toolchain combo.

The overlay generating stage now does things a bit differently:

* It generates a single overlay for each unique combination of
  toolchains that may be required by any board we care about. As part of
  that, it now scans the complete set of known boards, as reported by
  the site_config.

* Overlay tarballs are named after their toolchains content, rather than
  the board for which they were generated.

This also renames the stage (now SDKPackageToolchainOverlaysStage) to
reflect its new semantics.

CQ-DEPEND=CL:275150
BUG=brillo:918
TEST=Unit tests
TEST=External trybots (failed on early patchsets)

Change-Id: I634131fe1c97feb748861dc93403867d1884ee11
Reviewed-on: https://chromium-review.googlesource.com/275005
Reviewed-by: Gilad Arnold <garnold@chromium.org>
Commit-Queue: Gilad Arnold <garnold@chromium.org>
Tested-by: Gilad Arnold <garnold@chromium.org>

2 files changed, 54 insertions, 23 deletions

diff --git a/cbuildbot/stages/sdk_stages.py b/cbuildbot/stages/sdk_stages.py
index 668fc3611..ef53c7da8 100644
--- a/cbuildbot/stages/sdk_stages.py
+++ b/cbuildbot/stages/sdk_stages.py
@@ -19,6 +19,7 @@ from chromite.lib import osutils
 from chromite.lib import perf_uploader
 from chromite.lib import portage_util
 from chromite.lib import retry_util
+from chromite.lib import toolchain
 
 
 # Version of the Manifest file being generated for SDK artifacts. Should be
@@ -36,7 +37,8 @@ PACKAGE_EXCLUDED_PATHS = (
 
 # Names of various packaged artifacts.
 SDK_TARBALL_NAME = 'built-sdk.tar.xz'
-BOARD_OVERLAY_TARBALL_TEMPLATE = 'built-sdk-overlay-%(board)s.tar.xz'
+TOOLCHAINS_OVERLAY_TARBALL_TEMPLATE = \
+    'built-sdk-overlay-toolchains-%(toolchains)s.tar.xz'
 
 
 def SdkPerfPath(buildroot):
@@ -193,12 +195,13 @@ class SDKPackageStage(generic_stages.BuilderStage):
                          self._run.bot_id)
 
 
-class SDKPackageBoardToolchainsStage(generic_stages.BuilderStage):
+class SDKPackageToolchainOverlaysStage(generic_stages.BuilderStage):
   """Stage that creates and packages per-board toolchain overlays."""
 
   def __init__(self, builder_run, version=None, **kwargs):
     self.sdk_version = version
-    super(SDKPackageBoardToolchainsStage, self).__init__(builder_run, **kwargs)
+    super(SDKPackageToolchainOverlaysStage, self).__init__(builder_run,
+                                                           **kwargs)
 
   def PerformStage(self):
     chroot_dir = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR)
@@ -209,12 +212,23 @@ class SDKPackageBoardToolchainsStage(generic_stages.BuilderStage):
                                       constants.SDK_OVERLAYS_OUTPUT)
     osutils.RmDir(overlay_output_dir, ignore_missing=True, sudo=True)
     osutils.SafeMakedirs(overlay_output_dir, mode=0o777, sudo=True)
-    overlay_tarball_template = os.path.join(overlay_output_dir,
-                                            BOARD_OVERLAY_TARBALL_TEMPLATE)
-
-    for board in self._boards:
-      overlay_tarball_path = overlay_tarball_template % {'board': board}
-      with osutils.TempDir(prefix='toolchain-overlay-%s.' % board,
+    overlay_tarball_template = os.path.join(
+        overlay_output_dir, TOOLCHAINS_OVERLAY_TARBALL_TEMPLATE)
+
+    # Generate an overlay tarball for each unique toolchain combination.
+    toolchains_generated = set()
+    for board in self._run.site_config.GetBoards():
+      try:
+        toolchains = '-'.join(sorted(
+            toolchain.GetToolchainsForBoard(board).iterkeys()))
+      except portage_util.MissingOverlayException:
+        # The board overlay may not exist, e.g. on external builders.
+        continue
+
+      if toolchains in toolchains_generated:
+        continue
+
+      with osutils.TempDir(prefix='toolchains-overlay-%s.' % toolchains,
                            base_dir=tmp_dir, sudo_rm=True) as overlay_dir:
         # NOTE: We let MountOverlayContext remove the mount point created by
         # the TempDir context below, because it has built-in retries for rmdir
@@ -231,8 +245,10 @@ class SDKPackageBoardToolchainsStage(generic_stages.BuilderStage):
                                     enter_chroot=True, sudo=True,
                                     extra_env=self._portage_extra_env)
 
-        # Create toolchain overlay tarball.
-        CreateTarball(overlay_dir, overlay_tarball_path)
+        CreateTarball(overlay_dir,
+                      overlay_tarball_template % {'toolchains': toolchains})
+
+      toolchains_generated.add(toolchains)
 
 
 class SDKTestStage(generic_stages.BuilderStage):
diff --git a/cbuildbot/stages/sdk_stages_unittest.py b/cbuildbot/stages/sdk_stages_unittest.py
index f3f932482..c51a90a7a 100644
--- a/cbuildbot/stages/sdk_stages_unittest.py
+++ b/cbuildbot/stages/sdk_stages_unittest.py
@@ -17,6 +17,7 @@ from chromite.lib import cros_build_lib
 from chromite.lib import osutils
 from chromite.lib import perf_uploader
 from chromite.lib import portage_util
+from chromite.lib import toolchain
 
 
 class SDKBuildToolchainsStageTest(
@@ -177,7 +178,7 @@ class SDKPackageStageTest(generic_stages_unittest.AbstractStageTestCase):
     self.assertEqual(kwargs['revision'], 123456)
 
 
-class SDKPackageBoardToolchainsStageTest(
+class SDKPackageToolchainOverlaysStageTest(
     generic_stages_unittest.AbstractStageTestCase):
   """Tests board toolchain overlay installation and packaging."""
 
@@ -214,28 +215,42 @@ class SDKPackageBoardToolchainsStageTest(
       osutils.Touch(os.path.join(merged_dir, board + '.tmp'))
 
   def ConstructStage(self):
-    return sdk_stages.SDKPackageBoardToolchainsStage(self._run)
+    return sdk_stages.SDKPackageToolchainOverlaysStage(self._run)
 
   def testTarballCreation(self):
     """Tests that tarballs are created for all board toolchains."""
     self._Prepare('chromiumos-sdk')
     self.RunStage()
 
-    # Check that a tarball was created correctly for all boards.
-    for board in self._boards:
-      overlay_tarball = os.path.join(self.build_root,
-                                     constants.DEFAULT_CHROOT_DIR,
-                                     constants.SDK_OVERLAYS_OUTPUT,
-                                     'built-sdk-overlay-%s.tar.xz' % board)
+    # Check that a tarball was created correctly for all toolchain sets.
+    all_toolchain_combos = set()
+    for board in self._run.site_config.GetBoards():
+      try:
+        all_toolchain_combos.add(
+            '-'.join(sorted(toolchain.GetToolchainsForBoard(board).iterkeys())))
+      except portage_util.MissingOverlayException:
+        pass
+
+    for toolchains in all_toolchain_combos:
+      overlay_tarball = os.path.join(
+          self.build_root, constants.DEFAULT_CHROOT_DIR,
+          constants.SDK_OVERLAYS_OUTPUT,
+          'built-sdk-overlay-toolchains-%s.tar.xz' % toolchains)
       output = cros_build_lib.RunCommand(
           ['tar', '-I', 'xz', '-tvf', overlay_tarball],
           capture_output=True).output.splitlines()
-      # First line is './', use it as an anchor, count the chars, and strip as
-      # much from all other lines.
+      # First line is './', use it as an anchor, count the chars, and strip
+      # as much from all other lines.
       stripchars = len(output[0]) - 1
       tar_lines = [x[stripchars:] for x in output[1:]]
-      # Check that the overlay tarball contains the marker file only.
-      self.assertListEqual(['/%s.tmp' % board], tar_lines)
+      # Check that the overlay tarball contains a marker file only, and that
+      # the board recorded by this marker file indeed uses the toolchains for
+      # which the tarball was built.
+      self.assertEqual(1, len(tar_lines))
+      board = tar_lines[0].lstrip('/')[:-len('.tmp')]
+      board_toolchains = '-'.join(sorted(
+          toolchain.GetToolchainsForBoard(board).iterkeys()))
+      self.assertEqual(toolchains, board_toolchains)
 
 
 class SDKTestStageTest(generic_stages_unittest.AbstractStageTestCase):