path: root/cli/lib/core/image_build.py

#
# Copyright (C) 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


"""Functions for building an image."""


import os
import shutil
import subprocess

from core import util
from environment import sysroot
import error
from project import pack
from project import packs
from selinux import policy


# Allowed image types for BuildImage.
IMAGE_TYPE_ODM = 'odm'
IMAGE_TYPE_SYSTEM = 'system'
IMAGE_TYPES = [IMAGE_TYPE_ODM, IMAGE_TYPE_SYSTEM]

# Some metadata files.
FILE_CONTEXTS = policy.FC
FILE_CONTEXTS_BIN = policy.FC + '.bin'
IMAGE_INFO = 'image_info.txt'
SEPOLICY = 'sepolicy'


class Error(error.Error):
    """General build failure."""


class PathError(Error):
    """Raised when a provided path does not meet expectations."""


class ImageTypeError(Error):
    """Raise when an unknown image type is seen."""


def _CheckBuildImagePaths(artifact_cache, metadata_cache, output_dir,
                          product_out, host_tools, build_tools):
    """Checks that all paths necessary for an image build meet expectations.

    Does not validate the contents of any paths.
    Will create an empty output_dir if it doesn't already exist.

    Args:
        artifact_cache: Directory where user files and system files are merged.
            (This is normally per-target, not the global cache dir.)
        metadata_cache: Directory where image build meta-data is stored.
            (This is normally per-target, not the global cache dir.)
        output_dir: Directory where image build output should be placed.
        product_out: Directory where the built platform can be found.
        host_tools: Directory where host tools can be found.
        build_tools: Directory where build tools can be found.

    Raises:
        PathError: An expected directory cannot be found or isn't a dir.
    """
    if not os.path.isdir(artifact_cache):
        raise PathError('Artifact cache dir "{}" is not a directory.'.format(
            artifact_cache))
    if not os.path.isdir(metadata_cache):
        raise PathError('Metadata cache dir "{}" is not a directory.'.format(
            metadata_cache))
    if os.path.isfile(output_dir):
        raise PathError('Can not create output dir "{}", is a file'.format(
            output_dir))
    if not os.path.isdir(product_out):
        raise PathError(
            'Could not find platform build output "{}". '
            'Use `bdk build platform` to build it.'.format(
                product_out))
    if not os.path.isdir(host_tools):
        # Probably tampered with/not built.
        raise PathError(
            'Could not find host tools directory ({}). '
            'Use `bdk build platform` to build it.'.format(host_tools))
    if not os.path.isdir(build_tools):
        # This should only be missing if it was tampered with.
        raise PathError(
            'Could not find build tools directory ({}). '
            'You may need to re-download the OS version.'.format(
                build_tools))

    if not os.path.isdir(output_dir):
        os.makedirs(output_dir)


def AddPlatformOsPacks(spec, platform):
    """Adds platform OS packs to the spec.

    Args:
        spec: project.project_spec.ProjectSpec to add the OS packs to.
        platform: project.platform.Platform object to get OS info from.
    """
    product_out = platform.product_out_cache
    os_packs = packs.Packs()
    os_packs.namespace = platform.os_namespace
    os_core = pack.Pack(os_packs.namespace, 'generated_system')
    os_core.add_provides('os.core')
    base_copy = pack.Copy(os_core)
    base_copy.src = os.path.join(product_out, 'system')
    base_copy.src_type = pack.CopyType.DIR
    base_copy.recurse = True
    base_copy.dst = '/system'
    base_copy.dst_type = pack.CopyType.DIR
    # Do not generate fs_config_* or file_context_* for this pack.
    base_copy.acl.override_build = False
    os_core.add_copy(base_copy)
    os_packs.add_pack(os_core)
    spec.add_packs(os_packs)


def CreateTargetCache(spec, target, mountpoint='/', update=True, verbose=True):
    """Copies all files specified in the target to a single artifact cache dir.

    This call will create a submap from the parent spec and then use the
    resulting copy destinations to populate the supplied path as if it were
    the root of the destination.

    In the future, non-os and non-board packs will likely end up in their
    own image (not /system), so this will create that cache ignoring any
    Copy() nodes owned by a Pack that is prefixed by the target os or board.

    The cache will be created in
    |spec|.config.artifact_cache_for_target(|target|), with metadata in
    |spec|.config.metadata_cache_for_target(|target|).

    Args:
        spec: project.ProjectSpec global specification.
        target: project.target.Target to cache.
        mountpoint: optional path prefix to exclusively cache.
        update: optionally, only replaces files that have changed.
        verbose: (optional). If True, print information about what's
            happening. Default True.

    Returns:
        dict of uncached { dst_path => copy }

    Raises:
        project.dependency.Error: If there is an unfulfilled dependency.
        PathError: If a required path is missing.
    """
    target_map = target.create_submap(spec.packmap)
    artifact_cache = spec.config.artifact_cache_for_target(target)
    metadata_cache = spec.config.metadata_cache_for_target(target)

    cache = sysroot.Sysroot(artifact_cache, copy_newer_only=update)
    fs_config_files = {}
    fs_config_dirs = {}
    file_context = set()
    uncached = {}

    # pack.Copy._reconcile_paths() *should* make these always what is seen here.
    # TODO(wad): Add a <link> node.
    allowed_copies = [
        (pack.CopyType.FILE, pack.CopyType.FILE, False),
        (pack.CopyType.GLOB, pack.CopyType.DIR, False),
        (pack.CopyType.GLOB, pack.CopyType.DIR, True),
        (pack.CopyType.DIR, pack.CopyType.DIR, False),
        (pack.CopyType.DIR, pack.CopyType.DIR, True),
    ]

    final_files = {}
    for destination, copy in target_map.copy_destinations.iteritems():
        copy = copy[0]  # deduping has already occurred.

        # Currently, skip unknown mountpoints.
        if not destination.startswith(mountpoint):
            uncached[destination] = copy
            continue

        # All destinations are required to be absolute.
        # Make them relative so they play nicely with Sysroot().
        destination = destination.lstrip('/')

        # Proactively check allowed combinations so that the error
        # handling isn't interleaved with logic.
        if (copy.src_type, copy.dst_type, copy.recurse) not in allowed_copies:
            # TODO(wad): Should these be FileABugError()s?
            raise PathError(
                '{}: impossible copy reached: {}'.format(copy.pack.origin,
                                                         copy))

        # TODO(arihc): Will os packs ever draw in files from BSPs
        #     (other than build output files)? If so, need to link BSP to OS.
        files = set()
        try:
            if copy.src_type == pack.CopyType.FILE:  # FILE -> FILE
                cache.Makedirs(os.path.dirname(destination))
                if os.path.islink(copy.src):
                    # TODO(wad): Resolve how to skip permission setting on
                    #     symlinks.
                    cache.AddSymlink(copy.src, destination)
                else:
                    cache.AddFile(copy.src, destination)
                files.add(destination)
            elif copy.src_type == pack.CopyType.GLOB:  # GLOB -> DIR
                files.update(cache.AddGlob(copy.src, destination,
                                           recurse=copy.recurse))
            elif copy.src_type == pack.CopyType.DIR:  # DIR -> DIR
                files.update(cache.AddDir(copy.src, destination,
                                          recurse=copy.recurse, symlinks=True))
                fs_config_dirs[destination] = copy.acl.fs_config(binary=True)
            for f in files:
                # Even though the fs_config_files format support globs, the
                # final list of files is known at this point.
                fs_config_files[f] = copy.acl.fs_config(path=f, binary=True)
                if copy.acl.selabel:
                    file_context.update([copy.acl.file_context(path=f)])
                final_files[os.path.sep + f] = copy

        except IOError as e:
            # Annotate the error with the line that where the <copy> is defined.
            raise PathError('{}: {}'.format(copy.origin, e))

    # Walk the cache tree and remove any unmanaged dirs. First
    # we compute all allowed subpaths from the list of copied files, then
    # walk the cache tree and remove any unknown files and directories.
    final_dirs = set('/')
    for dst, copy in final_files.iteritems():
        # Final files is always a full path so let's drop the file.
        dst = os.path.dirname(dst)
        while dst != os.path.sep:
            # TODO(b/27848879) Add <dir> node set-acl support.
            tmp_acl = pack.Copy(None, dst).acl
            tmp_acl.perms = '0555'  # Default to rx for dirs.
            tmp_acl.override_build = copy.acl.override_build
            if dst not in fs_config_dirs:
                fs_config_dirs[dst] = '{}'.format(
                    tmp_acl.fs_config(binary=True))
            # The default root map covers file_context for now.
            final_dirs.add(dst)
            dst = os.path.dirname(dst)

    all_paths = set()
    all_files = set()
    # Walk the tree as quickly as possible with os.walk.
    # Use set differencing to compute the unmanaged entries.
    for root, _, files in os.walk(artifact_cache):
        dst_path = root[len(artifact_cache):] or os.path.sep
        # Don't touch unmanaged files outside of the given mountpoint.
        if not dst_path.startswith(mountpoint):
            continue
        all_paths.add(dst_path)
        all_files.update(set([os.path.join(dst_path, f) for f in files]))
    unmanaged_file = all_files.difference(final_files.keys())
    unmanaged_path = all_paths.difference(final_dirs)

    if verbose and (len(unmanaged_file) or len(unmanaged_path)):
        print 'Removing unmanaged paths:\n {}'.format(
            '\n '.join(sorted(unmanaged_file.union(unmanaged_path))))
    for f in unmanaged_file:
        os.remove(os.path.join(artifact_cache, f.lstrip(os.path.sep)))
    for p in reversed(sorted(unmanaged_path)):
        os.rmdir(os.path.join(artifact_cache, p.lstrip(os.path.sep)))

    # Sort the list from most specific to least specific.
    dsts = sorted(fs_config_dirs.keys(), key=lambda k: k.count(os.path.sep),
                  reverse=True)
    sorted_dirs = list()
    for dst in dsts:
        sorted_dirs.append(fs_config_dirs[dst])

    # Create fs_config_{files, dirs} and file_context build metadata
    # files outside of the sysroot to avoid project spec conflicts.
    if not os.path.isdir(os.path.join(metadata_cache, 'etc')):
        os.makedirs(os.path.join(metadata_cache, 'etc'))
    # List of (base file name, generated content) to make code sharing easier.
    metadata_files = [
        (('etc', 'fs_config_dirs'), sorted_dirs),
        (('etc', 'fs_config_files'), fs_config_files.values()),
    ]
    # Walk the base names, create the file, then append any pre-existing
    # content.
    for name, data in metadata_files:
        data_file = os.path.join(metadata_cache, *name)
        tgt_file = os.path.join(artifact_cache, 'system', *name)
        with open(data_file, 'w') as f:
            f.write(''.join(data))
            if os.path.isfile(tgt_file):
                with open(tgt_file, 'r') as extra:
                    shutil.copyfileobj(extra, f)

    with open(os.path.join(metadata_cache, FILE_CONTEXTS), 'w') as f:
        f.write('\n'.join(file_context))

    return uncached


def _CreateBuildProps(file_contexts_bin, product_out, image_type,
                      info_file):
    info_contents = ''
    if image_type == IMAGE_TYPE_ODM:
        build_props = {
            'mount_point' : image_type,
            'fs_type': 'ext4',
            # TODO(b/27854052): Get the size in here.
            'partition_size': '134217728',
            'extfs_sparse_flag': '-s',
            'skip_fsck': 'true',
            'selinux_fc': file_contexts_bin,
        }
        info_contents = '\n'.join(
            ['{}={}'.format(k, v) for k, v in build_props.iteritems()])
    elif image_type == IMAGE_TYPE_SYSTEM:
        # Add and update a copy of system_image_info.txt
        # TODO: add image_type system.
        with open(os.path.join(product_out,
                               'obj',
                               'PACKAGING',
                               'systemimage_intermediates',
                               'system_image_info.txt')) as f:
            for line in f:
                if line.startswith('selinux_fc'):
                    line = 'selinux_fc=' + file_contexts_bin + '\n'
                info_contents += line
    else:
        info_contents = 'mount_point={}'.format(image_type)
    with open(info_file, 'w') as f:
        f.write(info_contents)


def BuildImage(image_type, target, config):
    """Builds the image specified by image_type.

    Caller should validate target OS before calling.

    Args:
        image_type: The type of the image. One of IMAGE_TYPES.
        target: project.target.Target to build an image of.
        config: project.config.Config containing details of where the
            target is cached and output should go. Parts of this may
            be prepared by prior callers. E.g., CreateTargetCache().

    Returns:
        The image build exit code.

    Raises:
        PathError: An expected directory cannot be found or isn't a dir.
        util.HostUnsupportedArchError: The host is an unsupported architecture.
        ImageTypeError: An invalid parameter value has been supplied for
            image_type.
        IOError: An error occurs writing a file.
    """
    if image_type not in IMAGE_TYPES:
        raise ImageTypeError('image_type must be one of {}: {}'.format(
            IMAGE_TYPES, image_type))
    # Set some useful variables.
    platform = target.platform
    build_tools = platform.os_path('build', 'tools', 'releasetools')
    product_out = platform.product_out_cache
    host_arch = util.GetHostArch()
    host_tools = os.path.join(platform.build_cache, 'host', host_arch, 'bin')
    output_file = os.path.join(config.output_dir, '{}.img'.format(image_type))
    artifact_cache = config.artifact_cache_for_target(target)
    metadata_cache = config.metadata_cache_for_target(target)
    ret = 1

    # Check that all required inputs and tools exist.
    _CheckBuildImagePaths(artifact_cache, metadata_cache, config.output_dir,
                          product_out, host_tools, build_tools)
    # Check to ensure there are entries in root/image-mountpoint.
    if not os.path.exists(os.path.join(artifact_cache, image_type)):
        print ('Warning: The specification does not define any destinations '
               'for this image type: {}'.format(image_type))

    # Build 'filecontexts.bin' and 'sepolicy' SELinux files.
    sepolicy = os.path.join(metadata_cache, SEPOLICY)
    file_contexts_bin = os.path.join(metadata_cache, FILE_CONTEXTS_BIN)
    with platform.linked():
        policy.BuildSepolicy(platform, sepolicy)
        # Metadata cache holds a file_contexts file for the user artifacts.
        policy.BuildFileContexts(platform, sepolicy, file_contexts_bin,
                                 additional_context_dir=metadata_cache)

    image_info = os.path.join(metadata_cache, IMAGE_INFO)
    _CreateBuildProps(file_contexts_bin, product_out, image_type, image_info)


    # Build an image from the build root.
    additional_path = host_tools + os.pathsep + build_tools
    if 'PATH' in os.environ:
        os.environ['PATH'] += os.pathsep + additional_path
    else:
        os.environ['PATH'] = additional_path

    # Repoint product out so that changes made locally are reflected
    # for the build tooling.    This primarily enables the discovery
    # of etc/fs_config_{files, dirs}.
    ret = subprocess.call(['build_image.py',
                           os.path.join(artifact_cache, image_type),
                           image_info,
                           output_file,
                           metadata_cache])
    return ret