Cleans up magic "root" in image build.

Replaces putting metadata in <target cache> and user
artifacts in <target cache>/root with placing the former
in <target cache>/metadata and the latter in <target cache>/build_root
(I figured might as well reduce potential for accidental aliasing
by also moving where metadata was kept). Both these paths are no longer
magical, instead they come from methods of the spec.config.

Also cleaned up metadata/selinux paths so they're no longer magical.

BUG: 28588412
Change-Id: Iad0edf525a2687c5f8f1973f3455fbe22e1faba6
TEST: unit tests pass, bdk build image and root sync work.

11 files changed, 204 insertions, 168 deletions

diff --git a/cli/lib/commands/v2/build/image.py b/cli/lib/commands/v2/build/image.py
index c6e1401..f8c4a89 100644
--- a/cli/lib/commands/v2/build/image.py
+++ b/cli/lib/commands/v2/build/image.py
@@ -45,19 +45,16 @@ class Image(clicommand.Command):
 
         image_build.AddPlatformOsPacks(spec, platform)
 
-        target_dir = spec.config.cache_dir_for_target(target)
         try:
             mountpoint = os.path.join(os.path.sep, args.type)
-            image_build.CreateTargetCache(spec, target, target_dir, mountpoint,
-                                          update=True)
+            image_build.CreateTargetCache(spec, target, mountpoint, update=True)
         except dependency.Error as e:
             print 'Dependencies unmet for target {}: {}'.format(target.name, e)
             return 1
 
         try:
-            ret = image_build.BuildImage(args.type, platform, target_dir,
-                                         spec.config.output_dir)
-        except image_build.Error as e:
+            ret = image_build.BuildImage(args.type, target, spec.config)
+        except (image_build.Error, IOError) as e:
             print 'Error building image for target {} at {}: {}'.format(
                 target.name, target.origin, e)
             return 1
diff --git a/cli/lib/commands/v2/root/sync.py b/cli/lib/commands/v2/root/sync.py
index 38bff47..23447b3 100644
--- a/cli/lib/commands/v2/root/sync.py
+++ b/cli/lib/commands/v2/root/sync.py
@@ -73,12 +73,10 @@ class Sync(clicommand.Command):
 
         image_build.AddPlatformOsPacks(spec, platform)
 
-        target_dir = spec.config.cache_dir_for_target(target)
         mountpoint = os.path.join(os.path.sep, _SYNC_DIR)
         try:
-            image_build.CreateTargetCache(spec, target, target_dir,
-                                          mountpoint=mountpoint,
-                                          update=True)
+            image_build.CreateTargetCache(
+                spec, target, mountpoint=mountpoint, update=True)
         except dependency.Error as e:
             print('Dependencies unmet for target {}: {}'.format(target.name, e))
             return 1
@@ -101,11 +99,11 @@ class Sync(clicommand.Command):
             for command in ('root', 'remount'):
                 adb_tool.run(device_args + [command])
 
-        # Point adb to the user artifact sysroot for syncing. This means changes
+        # Point adb to the user artifact cache for syncing. This means changes
         # to the platform won't be pushed by `bdk sync`, which is good because
         # platform changes should probably require a full image flash.
-        adb_tool.environment['ANDROID_PRODUCT_OUT'] = os.path.join(
-            target_dir, 'root')
+        adb_tool.environment['ANDROID_PRODUCT_OUT'] = (
+            spec.config.artifact_cache_for_target(target))
 
         # adb does it's own parsing rather than using a library, so we have to
         # be pretty careful to specify args the way it expects. Usage must be:
diff --git a/cli/lib/core/image_build.py b/cli/lib/core/image_build.py
index 4e721b6..03e5c7c 100644
--- a/cli/lib/core/image_build.py
+++ b/cli/lib/core/image_build.py
@@ -18,7 +18,6 @@
 """Functions for building an image."""
 
 
-import fileinput
 import os
 import shutil
 import subprocess
@@ -36,6 +35,12 @@ IMAGE_TYPE_ODM = 'odm'
 IMAGE_TYPE_SYSTEM = 'system'
 IMAGE_TYPES = [IMAGE_TYPE_ODM, IMAGE_TYPE_SYSTEM]
 
+# Some metadata files.
+FILE_CONTEXTS = policy.FC
+FILE_CONTEXTS_BIN = policy.FC + '.bin'
+IMAGE_INFO = 'image_info.txt'
+SEPOLICY = 'sepolicy'
+
 
 class Error(error.Error):
     """General build failure."""
@@ -49,15 +54,17 @@ class ImageTypeError(Error):
     """Raise when an unknown image type is seen."""
 
 
-def _CheckBuildImagePaths(cache_dir, output_dir, product_out, host_tools,
-                          build_tools):
+def _CheckBuildImagePaths(artifact_cache, metadata_cache, output_dir,
+                          product_out, host_tools, build_tools):
     """Checks that all paths necessary for an image build meet expectations.
 
     Does not validate the contents of any paths.
     Will create an empty output_dir if it doesn't already exist.
 
     Args:
-        cache_dir: Directory where user files and system files are merged.
+        artifact_cache: Directory where user files and system files are merged.
+            (This is normally per-target, not the global cache dir.)
+        metadata_cache: Directory where image build meta-data is stored.
             (This is normally per-target, not the global cache dir.)
         output_dir: Directory where image build output should be placed.
         product_out: Directory where the built platform can be found.
@@ -67,8 +74,12 @@ def _CheckBuildImagePaths(cache_dir, output_dir, product_out, host_tools,
     Raises:
         PathError: An expected directory cannot be found or isn't a dir.
     """
-    if not os.path.isdir(cache_dir):
-        raise PathError('Cache dir "{}" is not a directory.'.format(cache_dir))
+    if not os.path.isdir(artifact_cache):
+        raise PathError('Artifact cache dir "{}" is not a directory.'.format(
+            artifact_cache))
+    if not os.path.isdir(metadata_cache):
+        raise PathError('Metadata cache dir "{}" is not a directory.'.format(
+            metadata_cache))
     if os.path.isfile(output_dir):
         raise PathError('Can not create output dir "{}", is a file'.format(
             output_dir))
@@ -118,9 +129,8 @@ def AddPlatformOsPacks(spec, platform):
     spec.add_packs(os_packs)
 
 
-def CreateTargetCache(spec, target, cache_dir, mountpoint='/', update=True,
-                      verbose=True):
-    """Copies all files specified in the target to a single cache dir.
+def CreateTargetCache(spec, target, mountpoint='/', update=True, verbose=True):
+    """Copies all files specified in the target to a single artifact cache dir.
 
     This call will create a submap from the parent spec and then use the
     resulting copy destinations to populate the supplied path as if it were
@@ -130,10 +140,13 @@ def CreateTargetCache(spec, target, cache_dir, mountpoint='/', update=True,
     own image (not /system), so this will create that cache ignoring any
     Copy() nodes owned by a Pack that is prefixed by the target os or board.
 
+    The cache will be created in
+    |spec|.config.artifact_cache_for_target(|target|), with metadata in
+    |spec|.config.metadata_cache_for_target(|target|).
+
     Args:
-        spec: project.ProjectSpec global specification
-        target: project.target.Target to cache
-        cache_dir: path to place the root/ of copied files.
+        spec: project.ProjectSpec global specification.
+        target: project.target.Target to cache.
         mountpoint: optional path prefix to exclusively cache.
         update: optionally, only replaces files that have changed.
         verbose: (optional). If True, print information about what's
@@ -147,9 +160,10 @@ def CreateTargetCache(spec, target, cache_dir, mountpoint='/', update=True,
         PathError: If a required path is missing.
     """
     target_map = target.create_submap(spec.packmap)
+    artifact_cache = spec.config.artifact_cache_for_target(target)
+    metadata_cache = spec.config.metadata_cache_for_target(target)
 
-    cache_root = os.path.join(cache_dir, 'root')
-    cache = sysroot.Sysroot(cache_root, copy_newer_only=update)
+    cache = sysroot.Sysroot(artifact_cache, copy_newer_only=update)
     fs_config_files = {}
     fs_config_dirs = {}
     file_context = set()
@@ -241,8 +255,8 @@ def CreateTargetCache(spec, target, cache_dir, mountpoint='/', update=True,
     all_files = set()
     # Walk the tree as quickly as possible with os.walk.
     # Use set differencing to compute the unmanaged entries.
-    for root, _, files in os.walk(cache_root):
-        dst_path = root[len(cache_root):] or os.path.sep
+    for root, _, files in os.walk(artifact_cache):
+        dst_path = root[len(artifact_cache):] or os.path.sep
         # Don't touch unmanaged files outside of the given mountpoint.
         if not dst_path.startswith(mountpoint):
             continue
@@ -255,9 +269,9 @@ def CreateTargetCache(spec, target, cache_dir, mountpoint='/', update=True,
         print 'Removing unmanaged paths:\n {}'.format(
             '\n '.join(sorted(unmanaged_file.union(unmanaged_path))))
     for f in unmanaged_file:
-        os.remove(os.path.join(cache_root, f.lstrip(os.path.sep)))
+        os.remove(os.path.join(artifact_cache, f.lstrip(os.path.sep)))
     for p in reversed(sorted(unmanaged_path)):
-        os.rmdir(os.path.join(cache_root, p.lstrip(os.path.sep)))
+        os.rmdir(os.path.join(artifact_cache, p.lstrip(os.path.sep)))
 
     # Sort the list from most specific to least specific.
     dsts = sorted(fs_config_dirs.keys(), key=lambda k: k.count(os.path.sep),
@@ -268,8 +282,8 @@ def CreateTargetCache(spec, target, cache_dir, mountpoint='/', update=True,
 
     # Create fs_config_{files, dirs} and file_context build metadata
     # files outside of the sysroot to avoid project spec conflicts.
-    if not os.path.isdir(os.path.join(cache_dir, 'etc')):
-        os.makedirs(os.path.join(cache_dir, 'etc'))
+    if not os.path.isdir(os.path.join(metadata_cache, 'etc')):
+        os.makedirs(os.path.join(metadata_cache, 'etc'))
     # List of (base file name, generated content) to make code sharing easier.
     metadata_files = [
         (('etc', 'fs_config_dirs'), sorted_dirs),
@@ -278,62 +292,64 @@ def CreateTargetCache(spec, target, cache_dir, mountpoint='/', update=True,
     # Walk the base names, create the file, then append any pre-existing
     # content.
     for name, data in metadata_files:
-        data_file = os.path.join(cache_dir, *name)
-        tgt_file = os.path.join(cache_root, 'system', *name)
+        data_file = os.path.join(metadata_cache, *name)
+        tgt_file = os.path.join(artifact_cache, 'system', *name)
         with open(data_file, 'w') as f:
             f.write(''.join(data))
             if os.path.isfile(tgt_file):
                 with open(tgt_file, 'r') as extra:
                     shutil.copyfileobj(extra, f)
 
-    with open(os.path.join(cache_dir, 'file_contexts'), 'w') as f:
+    with open(os.path.join(metadata_cache, FILE_CONTEXTS), 'w') as f:
         f.write('\n'.join(file_context))
 
     return uncached
 
 
-def _CreateBuildProps(build_root, product_out, image_type, info_file):
+def _CreateBuildProps(file_contexts_bin, product_out, image_type,
+                      info_file):
+    info_contents = ''
     if image_type == IMAGE_TYPE_ODM:
         build_props = {
-            'mount_point': 'odm',
+            'mount_point' : image_type,
             'fs_type': 'ext4',
             # TODO(b/27854052): Get the size in here.
             'partition_size': '134217728',
             'extfs_sparse_flag': '-s',
             'skip_fsck': 'true',
-            'selinux_fc': build_root.Path('root', 'file_contexts.bin'),
+            'selinux_fc': file_contexts_bin,
         }
-        build_root.WriteFile(info_file, '\n'.join(
-            ['{}={}'.format(k, v) for k, v in build_props.iteritems()]))
+        info_contents = '\n'.join(
+            ['{}={}'.format(k, v) for k, v in build_props.iteritems()])
     elif image_type == IMAGE_TYPE_SYSTEM:
         # Add and update a copy of system_image_info.txt
-        build_root.AddFile(os.path.join(product_out,
-                                        'obj',
-                                        'PACKAGING',
-                                        'systemimage_intermediates',
-                                        'system_image_info.txt'), info_file,
-                           overwrite=True)
-        for line in fileinput.input(build_root.Path(info_file), inplace=True):
-            # We just want to change the SELinux file contexts.
-            if line.startswith('selinux_fc'):
-                line = 'selinux_fc=' + build_root.Path('root',
-                                                       'file_contexts.bin')
-            print line.rstrip()
+        # TODO: add image_type system.
+        with open(os.path.join(product_out,
+                               'obj',
+                               'PACKAGING',
+                               'systemimage_intermediates',
+                               'system_image_info.txt')) as f:
+            for line in f:
+                if line.startswith('selinux_fc'):
+                    line = 'selinux_fc=' + file_contexts_bin + '\n'
+                info_contents += line
     else:
-        build_root.WriteFile(info_file, 'mount_point={}'.format(image_type))
+        info_contents = 'mount_point={}'.format(image_type)
+    with open(info_file, 'w') as f:
+        f.write(info_contents)
 
 
-def BuildImage(image_type, platform, cache_dir, output_dir):
+def BuildImage(image_type, target, config):
     """Builds the image specified by image_type.
 
     Caller should validate target OS before calling.
 
     Args:
         image_type: The type of the image. One of IMAGE_TYPES.
-        platform: project.platform.Platform to build an image of.
-        cache_dir: Directory to build the image in. This path
-             may be prepared by prior callers. E.g., CreateTargetCache().
-        output_dir: Directory to place built .img file in.
+        target: project.target.Target to build an image of.
+        config: project.config.Config containing details of where the
+            target is cached and output should go. Parts of this may
+            be prepared by prior callers. E.g., CreateTargetCache().
 
     Returns:
         The image build exit code.
@@ -343,34 +359,42 @@ def BuildImage(image_type, platform, cache_dir, output_dir):
         util.HostUnsupportedArchError: The host is an unsupported architecture.
         ImageTypeError: An invalid parameter value has been supplied for
             image_type.
+        IOError: An error occurs writing a file.
     """
     if image_type not in IMAGE_TYPES:
         raise ImageTypeError('image_type must be one of {}: {}'.format(
             IMAGE_TYPES, image_type))
     # Set some useful variables.
+    platform = target.platform
     build_tools = platform.os_path('build', 'tools', 'releasetools')
     product_out = platform.product_out_cache
     host_arch = util.GetHostArch()
     host_tools = os.path.join(platform.build_cache, 'host', host_arch, 'bin')
-    output_file = os.path.join(output_dir, '{}.img'.format(image_type))
+    output_file = os.path.join(config.output_dir, '{}.img'.format(image_type))
+    artifact_cache = config.artifact_cache_for_target(target)
+    metadata_cache = config.metadata_cache_for_target(target)
     ret = 1
 
     # Check that all required inputs and tools exist.
-    _CheckBuildImagePaths(cache_dir, output_dir, product_out, host_tools,
-                          build_tools)
+    _CheckBuildImagePaths(artifact_cache, metadata_cache, config.output_dir,
+                          product_out, host_tools, build_tools)
     # Check to ensure there are entries in root/image-mountpoint.
-    if not os.path.exists(os.path.join(cache_dir, 'root', image_type)):
+    if not os.path.exists(os.path.join(artifact_cache, image_type)):
         print ('Warning: The specification does not define any destinations '
                'for this image type: {}'.format(image_type))
 
-    build_root = sysroot.Sysroot(cache_dir, copy_newer_only=True)
-
     # Build 'filecontexts.bin' and 'sepolicy' SELinux files.
+    sepolicy = os.path.join(metadata_cache, SEPOLICY)
+    file_contexts_bin = os.path.join(metadata_cache, FILE_CONTEXTS_BIN)
     with platform.linked():
-        policy.BuildSepolicy(platform, cache_dir)
-        policy.BuildFileContexts(platform, cache_dir)
+        policy.BuildSepolicy(platform, sepolicy)
+        # Metadata cache holds a file_contexts file for the user artifacts.
+        policy.BuildFileContexts(platform, sepolicy, file_contexts_bin,
+                                 additional_context_dir=metadata_cache)
+
+    image_info = os.path.join(metadata_cache, IMAGE_INFO)
+    _CreateBuildProps(file_contexts_bin, product_out, image_type, image_info)
 
-    _CreateBuildProps(build_root, product_out, image_type, 'image_info.txt')
 
     # Build an image from the build root.
     additional_path = host_tools + os.pathsep + build_tools
@@ -382,10 +406,9 @@ def BuildImage(image_type, platform, cache_dir, output_dir):
     # Repoint product out so that changes made locally are reflected
     # for the build tooling.    This primarily enables the discovery
     # of etc/fs_config_{files, dirs}.
-    product_out = build_root.Path()
     ret = subprocess.call(['build_image.py',
-                           build_root.Path('root', image_type),
-                           build_root.Path('image_info.txt'),
+                           os.path.join(artifact_cache, image_type),
+                           image_info,
                            output_file,
-                           product_out])
+                           metadata_cache])
     return ret
diff --git a/cli/lib/core/image_build_unittest.py b/cli/lib/core/image_build_unittest.py
index 09bdb8d..1807520 100644
--- a/cli/lib/core/image_build_unittest.py
+++ b/cli/lib/core/image_build_unittest.py
@@ -25,6 +25,7 @@ from bsp import device_stub
 from core import image_build
 from core import util_stub
 from environment import sysroot_stub
+from project import config_stub
 from project import dependency
 from project import pack
 from project import packmap_stub
@@ -40,7 +41,8 @@ class TestData(object):
     _BSP_VERSION = '0.1'
     _BUILD_TYPE = 'user'
     _BRILLO_VERSION = '9.9'
-    _CACHE_DIR = '/product/artifacts'
+    _ARTIFACT_CACHE_DIR = '/product/artifacts'
+    _METADATA_CACHE_DIR = '/product/metadata'
     _OS_DIR = '/bing/bong'
     _PLATFORM_DIR = '/foo/bar'
     _PRODUCT_DIR = '/bip/boop'
@@ -57,7 +59,6 @@ class BuildImageBase(TestData):
     def setUp(self):
         self.stub_os = stubs.StubOs()
 
-        self.stub_fileinput = stubs.StubFileinput()
         self.stub_open = stubs.StubOpen(self.stub_os)
         self.stub_selinux = policy_stub.StubPolicy()
         self.stub_shutil = stubs.StubShutil(self.stub_os)
@@ -65,7 +66,6 @@ class BuildImageBase(TestData):
         self.stub_sysroot_generator = sysroot_stub.StubSysrootGenerator()
         self.stub_util = util_stub.StubUtil(os_version=self._BRILLO_VERSION)
 
-        image_build.fileinput = self.stub_fileinput
         image_build.os = self.stub_os
         image_build.open = self.stub_open.open
         image_build.policy = self.stub_selinux
@@ -82,6 +82,10 @@ class BuildImageBase(TestData):
             product_out_cache=self._PRODUCT_DIR,
             os_dir=self._OS_DIR)
         self.target = target_stub.StubTarget(platform=self.platform)
+        self.config = config_stub.StubConfig(
+            artifact_cache=self._ARTIFACT_CACHE_DIR,
+            metadata_cache=self._METADATA_CACHE_DIR,
+            output_dir=self._IMAGE_OUT_DIR)
         self.product_out_dir = None
         self.image_type = None
 
@@ -103,7 +107,8 @@ class BuildImageBase(TestData):
         product_out_dir = self.platform.product_out_cache
         self.product_out_dir = product_out_dir
         self.stub_os.path.should_be_dir = [
-            product_out_dir, self._IMAGE_OUT_DIR, self._CACHE_DIR,
+            product_out_dir, self._IMAGE_OUT_DIR, self._ARTIFACT_CACHE_DIR,
+            self._METADATA_CACHE_DIR,
             self.stub_os.path.join(platform_dir, 'host',
                                    self.stub_util.GetHostArch(),
                                    'bin'),
@@ -112,18 +117,20 @@ class BuildImageBase(TestData):
             self.stub_os.path.should_be_dir)
 
         # Willing to make an image_build root in the cache dir.
-        self.stub_os.should_makedirs = [self._CACHE_DIR]
-        # Image_Build root should get system_image_info.txt, and modify it.
-        self.stub_fileinput.should_touch = [self.stub_os.path.join(
-            self._CACHE_DIR, 'image_info.txt')]
+        self.stub_os.should_makedirs = [] # [self._CACHE_DIR]
+        prebuilt_intermediates = self.stub_os.path.join(
+            self._PRODUCT_DIR, 'obj', 'PACKAGING', 'systemimage_intermediates',
+            'system_image_info.txt')
+        self.stub_os.path.should_exist.append(prebuilt_intermediates)
+        self.stub_open.files[prebuilt_intermediates] = stubs.StubFile()
 
         expected_call = [
             'build_image.py',
-            self.stub_os.path.join(self._CACHE_DIR, 'root', image_type),
-            self.stub_os.path.join(self._CACHE_DIR, 'image_info.txt'),
+            self.stub_os.path.join(self._ARTIFACT_CACHE_DIR, image_type),
+            self.stub_os.path.join(self._METADATA_CACHE_DIR, 'image_info.txt'),
             self.stub_os.path.join(
                 self._IMAGE_OUT_DIR, '{}.img'.format(image_type)),
-            self._CACHE_DIR]
+            self._METADATA_CACHE_DIR]
 
         return expected_call
 
@@ -143,8 +150,8 @@ class BaseTests(object):
             command = self.stub_subprocess.AddCommand()
             self.assertEqual(
                 0,
-                image_build.BuildImage(self.image_type, self.platform,
-                                       self._CACHE_DIR, self._IMAGE_OUT_DIR))
+                image_build.BuildImage(self.image_type, self.target,
+                                       self.config))
 
             command.AssertCallWas(expected_call)
 
@@ -152,10 +159,8 @@ class BaseTests(object):
             """Tests that the make exit code is returned."""
             self.stub_subprocess.AddCommand(ret_code=1)
             self.SetupForImageImage_Build()
-            self.assertEqual(
-                1,
-                image_build.BuildImage(self.image_type, self.platform,
-                                       self._CACHE_DIR, self._IMAGE_OUT_DIR))
+            self.assertEqual(1, image_build.BuildImage(
+                self.image_type, self.target, self.config))
 
         def test_image_missing_paths(self):
             self.SetupForImageImage_Build()
@@ -169,15 +174,13 @@ class BaseTests(object):
                 if directory != self._IMAGE_OUT_DIR:
                     with self.assertRaises(image_build.PathError):
                         image_build.BuildImage(
-                            self.image_type, self.platform,
-                            self._CACHE_DIR, self._IMAGE_OUT_DIR)
+                            self.image_type, self.target, self.config)
                 else:
                     self.stub_os.should_makedirs.append(self._IMAGE_OUT_DIR)
                     self.assertEqual(
                         0,
-                        image_build.BuildImage(self.image_type, self.platform,
-                                               self._CACHE_DIR,
-                                               self._IMAGE_OUT_DIR))
+                        image_build.BuildImage(self.image_type, self.target,
+                                               self.config))
                     # Put it back in.
                 self.stub_os.path.should_be_dir.append(directory)
 
@@ -188,8 +191,8 @@ class BaseTests(object):
             # Shouldn't reach the linking point.
             self.device.should_link_version = None
             with self.assertRaises(self.stub_util.HostUnsupportedArchError):
-                image_build.BuildImage(self.image_type, self.platform,
-                                       self._CACHE_DIR, self._IMAGE_OUT_DIR)
+                image_build.BuildImage(self.image_type, self.target,
+                                       self.config)
 
 
 class BuildUnsupportedImageTest(unittest.TestCase):
@@ -200,7 +203,7 @@ class BuildUnsupportedImageTest(unittest.TestCase):
         with self.assertRaises(image_build.ImageTypeError):
             # pylint: disable=protected-access
             image_build.BuildImage(bt.image_type, bt.target,
-                                   bt._CACHE_DIR, bt._IMAGE_OUT_DIR)
+                                   bt.config)
 
 
 class BuildOdmImageTest(BaseTests.BuildImageBaseTest):
@@ -241,9 +244,13 @@ class BuildSystemImageTest(BaseTests.BuildImageBaseTest):
 
 class CreateTargetCacheTest(TestData, unittest.TestCase):
     def setUp(self):
-        self.spec = project_spec_stub.StubProjectSpec()
+        self.artifact_cache_dir = '/base/path/build_root'
+        self.metadata_cache_dir = '/base/path/metadata'
+        self.config = config_stub.StubConfig(
+            artifact_cache=self.artifact_cache_dir,
+            metadata_cache=self.metadata_cache_dir)
+        self.spec = project_spec_stub.StubProjectSpec(config=self.config)
         self.target = None
-        self.cache_dir = '/base/path'
 
         p = pack.Pack('my_project', 'main')
         self.copy = pack.Copy(p)
@@ -260,18 +267,25 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
         image_build.open = self.stub_open.open
         image_build.sysroot = self.stub_sysroot_generator
 
+        metadata_split = self.metadata_cache_dir.split('/')
         self.metadata_files = [
-            self.stub_os.path.join(self.cache_dir, 'etc'),
-            self.cache_dir,
-            self.stub_os.path.sep +
-            self.cache_dir.split(self.stub_os.path.sep)[1],
-            self.stub_os.path.join(self.cache_dir, 'etc', 'fs_config_dirs'),
-            self.stub_os.path.join(self.cache_dir, 'etc', 'fs_config_files'),
-            self.stub_os.path.join(self.cache_dir, 'file_contexts'),
+            self.stub_os.path.join(self.metadata_cache_dir, 'etc')]
+        # mkdirs will recursively create the rest of the dir.
+        while len(metadata_split) > 1:
+            self.metadata_files.append('/'.join(metadata_split))
+            metadata_split.pop()
+        self.metadata_files += ['/'.join(metadata_split[:i+2]) for i in
+                                reversed(range(len(metadata_split)-1))]
+        self.metadata_files += [
+            self.stub_os.path.join(self.metadata_cache_dir, 'etc',
+                                   'fs_config_dirs'),
+            self.stub_os.path.join(self.metadata_cache_dir, 'etc',
+                                   'fs_config_files'),
+            self.stub_os.path.join(self.metadata_cache_dir, 'file_contexts'),
         ]
         # This always happens unless it fails early.
         self.stub_os.should_makedirs += [
-            self.stub_os.path.join(self.cache_dir, 'etc')]
+            self.stub_os.path.join(self.metadata_cache_dir, 'etc')]
 
     def tearDown(self):
         # Make sure all files are copied.
@@ -284,7 +298,7 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
         target = target_stub.StubTarget(
             submap_raises=dependency.Error('dep error'))
         with self.assertRaises(dependency.Error):
-            image_build.CreateTargetCache(self.spec, target, self.cache_dir)
+            image_build.CreateTargetCache(self.spec, target)
 
     def test_skip_os(self):
         p = pack.Pack('brillo.{}'.format(self._BRILLO_VERSION), 'some_os_stuff')
@@ -292,8 +306,7 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
             destinations={'/system/bin/servicemanager': [pack.Copy(p)]})
         target = target_stub.StubTarget(submaps=[simple_map])
         # sysroot stub will not be touched because the OS will be under /system.
-        image_build.CreateTargetCache(self.spec, target, self.cache_dir,
-                                      mountpoint='/odm')
+        image_build.CreateTargetCache(self.spec, target, mountpoint='/odm')
 
     def test_with_os(self):
         p = pack.Pack('brillo.{}'.format(self._BRILLO_VERSION), 'some_os_stuff')
@@ -308,8 +321,7 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
             self.stub_os.path.dirname(cpy.dst)[1:]]
         self.stub_sysroot_generator.should_add_file = [(cpy.src,
                                                         cpy.dst.lstrip('/'))]
-        image_build.CreateTargetCache(self.spec, target, self.cache_dir,
-                                      mountpoint='/')
+        image_build.CreateTargetCache(self.spec, target, mountpoint='/')
 
     def test_skip_nonmountpoint(self):
         self.copy.dst = '/system/bin/xyzzy.txt'
@@ -326,8 +338,7 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
             self.stub_os.path.dirname(odm_copy.dst)[1:]]
         self.stub_sysroot_generator.should_add_file = [(odm_copy.src,
                                                         'odm/bin/xyzzy.txt')]
-        image_build.CreateTargetCache(self.spec, target, self.cache_dir,
-                                      mountpoint='/odm')
+        image_build.CreateTargetCache(self.spec, target, mountpoint='/odm')
 
     def test_copy_file_src_file_dst(self):
         self.copy.dst = '/system/bin/xyzzy.txt'
@@ -340,16 +351,17 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
             self.stub_os.path.dirname(self.copy.dst)[1:]]
         self.stub_sysroot_generator.should_add_file = [(self.copy.src,
                                                         'system/bin/xyzzy.txt')]
-        image_build.CreateTargetCache(self.spec, target, self.cache_dir)
+        image_build.CreateTargetCache(self.spec, target)
         # Ensure the selabel was copied.
         file_contexts = self.stub_open.files[
-            self.stub_os.path.join(self.cache_dir, 'file_contexts')].contents
+            self.stub_os.path.join(self.metadata_cache_dir,
+                                   'file_contexts')].contents
         self.assertRegexpMatches(
             '\n'.join(file_contexts),
             '{}($|\n)'.format(self.copy.acl.file_context()))
         # Ensure the ACLs were copied too.
         fs_config_files = self.stub_open.files[
-            self.stub_os.path.join(self.cache_dir, 'etc',
+            self.stub_os.path.join(self.metadata_cache_dir, 'etc',
                                    'fs_config_files')].contents
         self.assertRegexpMatches(
             '\n'.join(fs_config_files),
@@ -397,24 +409,23 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
         ]
 
         # Needed for ensuring fs_config_files appending works.
-        existing = self.stub_os.path.join(self.cache_dir, 'root', 'system',
+        existing = self.stub_os.path.join(self.artifact_cache_dir, 'system',
                                           'etc', 'fs_config_files')
         self.stub_os.path.should_exist.append(existing)
         system_fs_config_files = 'abc123'
         self.stub_open.files[existing] = stubs.StubFile(system_fs_config_files)
 
-        existing = self.stub_os.path.join(self.cache_dir, 'root', 'system',
+        existing = self.stub_os.path.join(self.artifact_cache_dir, 'system',
                                           'etc', 'fs_config_dirs')
         self.stub_os.path.should_exist.append(existing)
         system_fs_config_dirs = '321bca'
         self.stub_open.files[existing] = stubs.StubFile(system_fs_config_dirs)
 
-        image_build.CreateTargetCache(self.spec, target, self.cache_dir,
-                                      mountpoint='/')
+        image_build.CreateTargetCache(self.spec, target, mountpoint='/')
         # Ensure the ACLs were copied and the system fs_config_* are appended
         # to the build-specific changes.
         fs_config_files = self.stub_open.files[
-            self.stub_os.path.join(self.cache_dir, 'etc',
+            self.stub_os.path.join(self.metadata_cache_dir, 'etc',
                                    'fs_config_files')].contents
         self.assertEqual(
             ['%\x00\x00\x01\xd2\x04\xe1\x10\x00\x00\x00\x00\x00\x00\x00\x00'
@@ -424,7 +435,7 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
         # Ensure that no directory ACLs were implicitly placed on
         # system or system/bin/.
         fs_config_dirs = self.stub_open.files[
-            self.stub_os.path.join(self.cache_dir, 'etc',
+            self.stub_os.path.join(self.metadata_cache_dir, 'etc',
                                    'fs_config_dirs')].contents
         self.assertEqual(fs_config_dirs, ['', system_fs_config_dirs])
 
@@ -441,7 +452,7 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
             self.stub_os.path.dirname(self.copy.dst)[1:]]
         self.stub_sysroot_generator.should_add_glob = [
             (self.copy.src, 'system/bin/', False)]
-        image_build.CreateTargetCache(self.spec, target, self.cache_dir)
+        image_build.CreateTargetCache(self.spec, target)
 
     def test_copy_glob_recursive(self):
         self.copy.src = '/my/bins/*'
@@ -456,7 +467,7 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
             self.stub_os.path.dirname(self.copy.dst)[1:]]
         self.stub_sysroot_generator.should_add_glob = [
             (self.copy.src, 'system/bin/', True)]
-        image_build.CreateTargetCache(self.spec, target, self.cache_dir)
+        image_build.CreateTargetCache(self.spec, target)
 
     def test_copy_dir(self):
         self.copy.src = '/my/etc/'
@@ -471,7 +482,7 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
             self.stub_os.path.dirname(self.copy.dst)[1:]]
         self.stub_sysroot_generator.should_add_dir = [
             (self.copy.src, 'system/etc/', False)]
-        image_build.CreateTargetCache(self.spec, target, self.cache_dir)
+        image_build.CreateTargetCache(self.spec, target)
 
     def test_copy_dir_recurse(self):
         self.copy.src = '/my/etc/'
@@ -486,7 +497,7 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
             self.stub_os.path.dirname(self.copy.dst)[1:]]
         self.stub_sysroot_generator.should_add_dir = [
             (self.copy.src, 'system/etc/', True)]
-        image_build.CreateTargetCache(self.spec, target, self.cache_dir)
+        image_build.CreateTargetCache(self.spec, target)
 
 
     def test_copy_invalid_combos(self):
@@ -506,7 +517,7 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
                 destinations={self.copy.dst: [self.copy]})
             target = target_stub.StubTarget(submaps=[simple_map])
             with self.assertRaises(image_build.PathError):
-                image_build.CreateTargetCache(self.spec, target, self.cache_dir)
+                image_build.CreateTargetCache(self.spec, target)
 
     def test_unmanaged_file(self):
         self.copy.dst = '/system/bin/xyzzy.txt'
@@ -519,9 +530,9 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
         self.stub_sysroot_generator.should_add_file = [(self.copy.src,
                                                         'system/bin/xyzzy.txt')]
         self.stub_os.path.should_exist.append(
-            '/base/path/root/system/foozle/blag')
-        image_build.CreateTargetCache(self.spec, target, self.cache_dir,
-                                      verbose=False)
+            self.stub_os.path.join(self.artifact_cache_dir, 'system', 'foozle',
+                                   'blag'))
+        image_build.CreateTargetCache(self.spec, target, verbose=False)
         # The metadata files should've been created.
         expect_files = self.metadata_files
         self.assertEqual(self.stub_os.path.should_exist, expect_files)
@@ -534,11 +545,11 @@ class CreateTargetCacheTest(TestData, unittest.TestCase):
         target = target_stub.StubTarget(submaps=[simple_map])
         self.stub_sysroot_generator.should_makedirs = [
             self.stub_os.path.dirname(self.copy.dst)[1:]]
-        self.stub_sysroot_generator.should_add_file = [(self.copy.src,
-                                                        'system/bin/xyzzy.txt')]
+        self.stub_sysroot_generator.should_add_file = [
+            (self.copy.src, 'system/bin/xyzzy.txt')]
         self.stub_os.path.should_exist.append(
-            '/base/path/root/system/foozle/barzle/blag.mal')
-        image_build.CreateTargetCache(self.spec, target, self.cache_dir,
-                                      verbose=False)
+            self.stub_os.path.join(self.artifact_cache_dir, 'system',
+                                   'foozle', 'barzle', 'blag.mal'))
+        image_build.CreateTargetCache(self.spec, target, verbose=False)
         expect_files = self.metadata_files
         self.assertEqual(self.stub_os.path.should_exist, expect_files)
diff --git a/cli/lib/environment/sysroot.py b/cli/lib/environment/sysroot.py
index 347d2af..7d962fd 100644
--- a/cli/lib/environment/sysroot.py
+++ b/cli/lib/environment/sysroot.py
@@ -92,22 +92,20 @@ class Sysroot(object):
         src_st = os.lstat(src)
         os.utime(real_dest, (src_st.st_atime, src_st.st_mtime + 1))
 
-    def AddFile(self, src, dest='', overwrite=False):
+    def AddFile(self, src, dest=''):
         """Copies a file from src to dest.
 
         Args:
             src: the file to copy.
             dest: where to copy to. If a directory, uses the filename from src.
                 The dest directory must already exist.
-            overwrite: (optional) If True, always overwrite, even if src is
-                older than dest.
         Raises:
             IOError if the copy fails.
         """
 
         try:
             tgt = self.Path(dest)
-            if overwrite or self._src_is_newer(src, tgt):
+            if self._src_is_newer(src, tgt):
                 shutil.copy2(src, tgt)
                 self._copy_times(src, tgt)
         except (IOError, OSError) as e:
diff --git a/cli/lib/environment/sysroot_stub.py b/cli/lib/environment/sysroot_stub.py
index 059d552..17048fb 100644
--- a/cli/lib/environment/sysroot_stub.py
+++ b/cli/lib/environment/sysroot_stub.py
@@ -53,8 +53,7 @@ class StubSysroot(object):
         else:
             raise IOError('not supposed to write {0}'.format(path))
 
-    # pylint: disable=unused-argument
-    def AddFile(self, src, dest='', overwrite=False):
+    def AddFile(self, src, dest=''):
         if not (src, dest) in self.should_add_file:
             raise IOError('Not supposed to add file {0} to {1}'.format(src,
                                                                        dest))
diff --git a/cli/lib/project/config.py b/cli/lib/project/config.py
index a4316c5..9af01f7 100644
--- a/cli/lib/project/config.py
+++ b/cli/lib/project/config.py
@@ -89,8 +89,15 @@ class Config(object):
                 '<default target="{}"/></config>').format(
                     self._output_dir, self._cache_dir, self._default_target)
 
-    def cache_dir_for_target(self, target):
-        return os.path.join(self.cache_dir, '{}.cache'.format(target.name))
+    def cache_path_for_target(self, target, *path):
+        return os.path.join(self.cache_dir, '{}.cache'.format(target.name),
+                            *path)
+
+    def artifact_cache_for_target(self, target):
+        return self.cache_path_for_target(target, 'build_root')
+
+    def metadata_cache_for_target(self, target):
+        return self.cache_path_for_target(target, 'metadata')
 
     @classmethod
     def from_element(cls, node):
diff --git a/cli/lib/project/config_stub.py b/cli/lib/project/config_stub.py
index ce2dc22..641064f 100644
--- a/cli/lib/project/config_stub.py
+++ b/cli/lib/project/config_stub.py
@@ -20,6 +20,16 @@
 
 class StubConfig(object):
 
-    def __init__(self, default_target='', origin='test_config_origin'):
+    def __init__(self, default_target='', origin='test_config_origin',
+                 artifact_cache='', metadata_cache='', output_dir=''):
         self.default_target = default_target
         self.origin = origin
+        self.output_dir = output_dir
+        self._artifact_cache = artifact_cache
+        self._metadata_cache = metadata_cache
+
+    def artifact_cache_for_target(self, _target):
+        return self._artifact_cache
+
+    def metadata_cache_for_target(self, _target):
+        return self._metadata_cache
diff --git a/cli/lib/project/project_spec_stub.py b/cli/lib/project/project_spec_stub.py
index 445de21..b51c814 100644
--- a/cli/lib/project/project_spec_stub.py
+++ b/cli/lib/project/project_spec_stub.py
@@ -24,13 +24,13 @@ from project import config_stub
 class StubProjectSpec(object):
 
     def __init__(self, targets=None, file_default_target='', file_targets=None,
-                 should_read_file=None, packmap=None):
+                 should_read_file=None, packmap=None, config=None):
         self.targets = targets or {}
         self.file_default_target = file_default_target
         self.file_targets = file_targets or {}
         self.should_read_file = should_read_file or []
         self.packmap = packmap
-        self.config = None
+        self.config = config or config_stub.StubConfig()
 
     def add_packs(self, packs_obj):
         self.packmap.update(packs_obj)
@@ -39,7 +39,7 @@ class StubProjectSpec(object):
         if not f in self.should_read_file:
             raise IOError('Not supposed to read from file "{}"'.format(f))
         self.targets = self.file_targets
-        self.config = config_stub.StubConfig(self.file_default_target)
+        self.config.default_target = self.file_default_target
         self.should_read_file.remove(f)
         return self
 
diff --git a/cli/lib/selinux/policy.py b/cli/lib/selinux/policy.py
index 7e6f4f6..7d1177a 100644
--- a/cli/lib/selinux/policy.py
+++ b/cli/lib/selinux/policy.py
@@ -65,8 +65,7 @@ SEPOLICY_BUILD_FILES = [
 ]
 
 FC = 'file_contexts'
-
-SEPOLICY_DIRS_FILE = "sepolicy_dirs.txt"
+SEPOLICY_DIRS_FILE = 'sepolicy_dirs.txt'
 
 
 def _ExpandSepolicyPaths(base, dirs, files):
@@ -153,7 +152,7 @@ def LoadCachedSepolicyDirs(platform):
     return board_sepolicy_dirs
 
 
-def BuildSepolicy(platform, cache_dir):
+def BuildSepolicy(platform, output_path):
     """Builds the main 'sepolicy' SELinux policy file.
 
     This needs to be built before attempting to build
@@ -177,19 +176,17 @@ def BuildSepolicy(platform, cache_dir):
                    '-D target_build_variant=%s' % platform.build_type]
         _RunM4(policy_inputs, policy_conf, m4_opts)
 
-        sepolicy_path = os.path.join(cache_dir, 'root', 'sepolicy')
         runner.run('checkpolicy',
-                   ('-M -c %d -o %s %s' % (POLICYVERS, sepolicy_path,
+                   ('-M -c %d -o %s %s' % (POLICYVERS, output_path,
                                            policy_conf)).split())
     finally:
         shutil.rmtree(interm_dir, ignore_errors=True)
 
 
-def BuildFileContexts(platform, cache_dir):
-    """Builds the 'file_contexts.bin' SELinux policy file.
+def BuildFileContexts(platform, sepolicy_path, output_path,
+                      additional_context_dir=None):
+    """Builds the file_contexts bin SELinux policy file."""
 
-    This requires a valid 'sepolicy' file in |cache_dir|.
-    """
     interm_dir = tempfile.mkdtemp()
     runner = tool.HostToolRunner(platform)
     try:
@@ -201,28 +198,22 @@ def BuildFileContexts(platform, cache_dir):
                                         'file_contexts.device.sorted.tmp')
         fc_concat = os.path.join(interm_dir, 'file_contexts.concat.tmp')
 
-        target_root_out = os.path.join(cache_dir, 'root')
-        filecontextsdotbin = os.path.join(target_root_out, 'file_contexts.bin')
-
         main_sepolicy_dir = os.path.join(os_path, 'system', 'sepolicy')
         main_fc = os.path.join(main_sepolicy_dir, FC)
 
         _RunM4([main_fc], fc_local)
         board_sepolicy_dirs = LoadCachedSepolicyDirs(platform)
 
-        if os.access(os.path.join(cache_dir, FC), os.R_OK):
-            # There's a 'file_contexts' file in |cache_dir| that we need to
-            # include.
-            board_sepolicy_dirs.append(cache_dir)
+        if additional_context_dir:
+            board_sepolicy_dirs.append(additional_context_dir)
 
         _RunM4(_ExpandSepolicyPaths(os_path, board_sepolicy_dirs, [FC]),
                fc_device)
-        sepolicy_path = os.path.join(target_root_out, 'sepolicy')
         runner.run('checkfc', ['-e', sepolicy_path, fc_device])
         runner.run('fc_sort', [fc_device, fc_device_sorted])
 
         _RunM4([fc_local, fc_device_sorted], fc_concat)
         runner.run('checkfc', ['-e', sepolicy_path, fc_concat])
-        runner.run('sefcontext_compile', ['-o', filecontextsdotbin, fc_concat])
+        runner.run('sefcontext_compile', ['-o', output_path, fc_concat])
     finally:
         shutil.rmtree(interm_dir, ignore_errors=True)
diff --git a/cli/lib/selinux/policy_stub.py b/cli/lib/selinux/policy_stub.py
index c94afd7..4e58ae6 100644
--- a/cli/lib/selinux/policy_stub.py
+++ b/cli/lib/selinux/policy_stub.py
@@ -21,8 +21,10 @@
 class StubPolicy(object):
     """Stubs the selinux.policy module."""
 
-    def BuildSepolicy(self, _platform, _cache_dir):
+    def BuildSepolicy(self, _platform, _output_file):
         pass
 
-    def BuildFileContexts(self, _platform, _cache_dir):
+    # pylint: disable=unused-argument
+    def BuildFileContexts(self, _platform, _sepolicy, _output_file,
+                          additional_context_dir=None):
         pass