brillo build: Normalize the set of packages to build.

When building a blueprint, we should build:
* The packages listed as main packages by all the bricks listed in the
  blueprint.
* The main packages listed by the BSP.
* The implicit dependencies of the core system (target-os,
  target-os-dev, target-os-test). This is needed to be able to build a
  developer-friendly image (test image currently).

BUG=brillo:1004
TEST=`brillo build --blueprint <..>` builds target-os{,-dev,-test}.
TEST=`brillo image --blueprint <..> test` works.
TEST=unittests.

CQ-DEPEND=CL:269825

Change-Id: Ia4d0579a742a58b9ce241ec50a44ccb644aed7f7
Reviewed-on: https://chromium-review.googlesource.com/269826
Trybot-Ready: Bertrand Simonnet <bsimonnet@chromium.org>
Tested-by: Bertrand Simonnet <bsimonnet@chromium.org>
Reviewed-by: Steve Fung <stevefung@chromium.org>
Commit-Queue: Bertrand Simonnet <bsimonnet@chromium.org>

7 files changed, 78 insertions, 21 deletions

diff --git a/cli/cros/cros_build.py b/cli/cros/cros_build.py
index 77452fe65..1653cb7a6 100644
--- a/cli/cros/cros_build.py
+++ b/cli/cros/cros_build.py
@@ -60,10 +60,7 @@ To just build a single package:
       self.blueprint = blueprint_lib.Blueprint(self.options.blueprint)
 
       if not self.build_pkgs:
-        self.build_pkgs = []
-        for locator in (self.blueprint.GetBricks()[0], self.blueprint.GetBSP()):
-          for brick in brick_lib.Brick(locator).BrickStack():
-            self.build_pkgs.extend(brick.MainPackages())
+        self.build_pkgs = self.blueprint.GetPackages()
     elif self.options.brick or self.curr_brick_locator:
       self.brick = brick_lib.Brick(self.options.brick
                                    or self.curr_brick_locator)
diff --git a/cli/cros/cros_image.py b/cli/cros/cros_image.py
index dbeb95fe9..66cf51c4c 100644
--- a/cli/cros/cros_image.py
+++ b/cli/cros/cros_image.py
@@ -171,11 +171,7 @@ class ImageCommand(command.CliCommand):
 
     if self.options.blueprint:
       blueprint = blueprint_lib.Blueprint(self.options.blueprint)
-      packages = []
-      for b in blueprint.GetBricks():
-        packages.extend(brick_lib.Brick(b).MainPackages())
-      if blueprint.GetBSP():
-        packages.extend(brick_lib.Brick(blueprint.GetBSP()).MainPackages())
+      packages = blueprint.GetPackages(with_implicit=False)
 
       cmd.append('--extra_packages=%s' % ' '.join(packages))
       #TODO(stevefung): Support multiple sysroots (brbug.com/676)
diff --git a/lib/blueprint_lib.py b/lib/blueprint_lib.py
index a4edb6d00..7cd96c2c9 100644
--- a/lib/blueprint_lib.py
+++ b/lib/blueprint_lib.py
@@ -16,6 +16,16 @@ from chromite.lib import workspace_lib
 BRICKS_FIELD = 'bricks'
 BSP_FIELD = 'bsp'
 
+# Those packages are implicitly built for all blueprints.
+# - target-os is needed to build any image.
+# - target-os-dev and target-os-test are needed to build a developer friendly
+#   image. They should not be included in any production images.
+_IMPLICIT_PACKAGES = (
+    'virtual/target-os',
+    'virtual/target-os-dev',
+    'virtual/target-os-test',
+)
+
 
 class BlueprintNotFoundError(Exception):
   """The blueprint does not exist."""
@@ -122,3 +132,22 @@ class Blueprint(object):
         brick_map[b.brick_locator] = b
 
     return brick_map.values()
+
+  def GetPackages(self, with_implicit=True):
+    """Returns the list of packages needed by this blueprint.
+
+    This includes the main packages for the bricks and the bsp of this
+    blueprint. We don't add the main packages of the bricks dependencies to
+    allow inheriting a brick without inheriting its required packages.
+
+    Args:
+      with_implicit: If True, include packages that are implicitly required by
+        the core system.
+    """
+    packages = []
+    for locator in self.GetBricks() + [self.GetBSP()]:
+      packages.extend(brick_lib.Brick(locator).MainPackages())
+
+    if with_implicit:
+      packages.extend(_IMPLICIT_PACKAGES)
+    return packages
diff --git a/lib/blueprint_lib_unittest.py b/lib/blueprint_lib_unittest.py
index 9079f689a..d4c238742 100644
--- a/lib/blueprint_lib_unittest.py
+++ b/lib/blueprint_lib_unittest.py
@@ -63,6 +63,30 @@ class BlueprintLibTest(cros_test_lib.WorkspaceTestCase):
                                                         brick_b.brick_locator])
     self.assertEqual(3, len(blueprint.GetUsedBricks()))
 
+  def testGetPackages(self):
+    """Tests that we can get the needed packages for a given blueprint."""
+    self.CreateBrick('foo', main_package='app-misc/foopkg')
+    self.CreateBrick('bar', main_package='app-misc/barpkg')
+    self.CreateBrick('foobar', main_package='app-misc/foobarpkg',
+                     dependencies=['//foo', '//bar'])
+
+    self.CreateBrick('hello', main_package='app-misc/hello')
+
+    self.CreateBrick('mybsp', main_package='app-misc/bspbonjour')
+
+    blueprint = self.CreateBlueprint(name='//myblueprint',
+                                     bricks=['//hello', '//foobar'],
+                                     bsp='//mybsp')
+    packages = blueprint.GetPackages(with_implicit=False)
+    self.assertEqual(
+        set(('app-misc/foobarpkg', 'app-misc/hello', 'app-misc/bspbonjour')),
+        set(packages))
+
+    packages = blueprint.GetPackages(with_implicit=True)
+    self.assertTrue('virtual/target-os' in packages)
+    self.assertTrue('virtual/target-os-dev' in packages)
+    self.assertTrue('virtual/target-os-test' in packages)
+
   def testBlueprintAlreadyExists(self):
     """Tests creating a blueprint where one already exists."""
     self.CreateBrick('//foo')
diff --git a/lib/chroot_util.py b/lib/chroot_util.py
index 359d66ce2..3dde021af 100644
--- a/lib/chroot_util.py
+++ b/lib/chroot_util.py
@@ -155,7 +155,13 @@ def InitializeSysroots(blueprint):
     blueprint: a blueprint_lib.Blueprint object.
   """
   bsp = brick_lib.Brick(blueprint.GetBSP())
-  bricks = [brick_lib.Brick(b) for b in blueprint.GetBricks()]
+
+  # Create the brick stack.
+  # Removing duplicates while preserving a sane behaviour is hard:
+  # brbug.com/1029.
+  brick_stack = []
+  for brick_locator in blueprint.GetBricks():
+    brick_stack.extend(brick_lib.Brick(brick_locator).BrickStack())
 
   # Regenerate the portage configuration for all bricks used by this blueprint.
   for b in blueprint.GetUsedBricks():
@@ -165,7 +171,6 @@ def InitializeSysroots(blueprint):
 
   sysroot = sysroot_lib.Sysroot(sysroot_path)
   sysroot.CreateSkeleton()
-  # TODO(bsimonnet): Add support for multiple bricks. brbug.com/803
-  sysroot.WriteConfig(sysroot.GenerateBrickConfig(bricks[0], bsp))
+  sysroot.WriteConfig(sysroot.GenerateBrickConfig(brick_stack, bsp))
   sysroot.GeneratePortageConfig()
   sysroot.UpdateToolchain()
diff --git a/lib/sysroot_lib.py b/lib/sysroot_lib.py
index 0c25b8cbc..55929bd08 100644
--- a/lib/sysroot_lib.py
+++ b/lib/sysroot_lib.py
@@ -277,15 +277,16 @@ class Sysroot(object):
     header = "# Created by cros_sysroot_utils from --board=%s." % board
     return '\n'.join((header, _DictToKeyValue(config)))
 
-  def GenerateBrickConfig(self, brick, bsp=None):
-    """Generates the configuration for a given brick.
+  def GenerateBrickConfig(self, bricks, bsp=None):
+    """Generates the configuration for a given brick stack and bsp.
 
     Args:
-      brick: brick to generate the configuration for.
+      bricks: The brick stack, expanded, excluding the bsp.
       bsp: BSP to use.
     """
     config = {}
-    brick_list = brick.BrickStack()
+
+    brick_list = bricks
     if bsp:
       brick_list = bsp.BrickStack() + brick_list
 
@@ -293,8 +294,13 @@ class Sysroot(object):
     portdir_overlay_list = ([_CHROMIUMOS_OVERLAY, _ECLASS_OVERLAY] +
                             board_overlay_list)
 
-    base_brick = bsp or brick
-    toolchains = toolchain.GetToolchainsForBrick(base_brick.brick_locator)
+    # If the bsp is not set use the highest priority brick. This is meant to
+    # preserve support for building with --brick.
+    # TODO(bsimonnet): remove this once we remove support for --brick
+    # (brbug.com/916).
+    bsp = bsp or bricks[-1]
+
+    toolchains = toolchain.GetToolchainsForBrick(bsp.brick_locator)
     config['CHOST'] = toolchain.FilterToolchains(toolchains, 'default',
                                                  True).keys()[0]
     config['ARCH'] = toolchain.GetArchForTarget(config['CHOST'])
@@ -305,8 +311,7 @@ class Sysroot(object):
     config['ROOT'] = self.path + '/'
     config['PKG_CONFIG'] = self._WrapperPath('pkg-config')
 
-    header = ("# Created by cros_sysroot_utils from --brick=%s."
-              % brick.brick_locator)
+    header = '# Autogenerated by chromite.lib.sysroot_lib.'
     return '\n'.join((header, _DictToKeyValue(config)))
 
   def WriteConfig(self, config):
diff --git a/scripts/cros_sysroot_utils.py b/scripts/cros_sysroot_utils.py
index 1125a43e9..a4d3fbf95 100644
--- a/scripts/cros_sysroot_utils.py
+++ b/scripts/cros_sysroot_utils.py
@@ -85,7 +85,8 @@ def main(argv):
     sysroot.CreateAllWrappers(opts.friendlyname)
   elif opts.command == 'generate-config':
     if opts.brick:
-      config = sysroot.GenerateBrickConfig(brick_lib.Brick(opts.brick))
+      config = sysroot.GenerateBrickConfig(
+          brick_lib.Brick(opts.brick).BrickStack())
     else:
       config = sysroot.GenerateBoardConfig(opts.board)