path: root/cli/lib/core/image_build_unittest.py

#
# Copyright (C) 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


"""Unit tests for image_build.py."""


import copy
import unittest

from core import image_build
from core import util_stub
from environment import sysroot_stub
from project import config_stub
from project import dependency
from project import pack
from project import packmap_stub
from project import platform_stub
from project import project_spec_stub
from project import target_stub
from test import stubs
from selinux import policy_stub


class TestData(object):
    _BSP = 'test_board'
    _BSP_VERSION = '0.1'
    _BUILD_TYPE = 'user'
    _BRILLO_VERSION = '9.9'
    _ARTIFACT_CACHE_DIR = '/product/artifacts'
    _METADATA_CACHE_DIR = '/product/metadata'
    _OS_ROOT = '/bing/bong'
    _PLATFORM_DIR = '/foo/bar'
    _PRODUCT_DIR = '/bip/boop'
    _IMAGE_OUT_DIR = '/baz/pop'


class BuildImageBase(TestData):
    """Base image unit test setup.

    Separate the setup from the default tests, so that
    BuildUnsupportedImageTest() can instantiate this object in its test.
    """

    def setUp(self):
        self.stub_os = stubs.StubOs()

        self.stub_open = stubs.StubOpen(self.stub_os)
        self.stub_selinux = policy_stub.StubPolicy()
        self.stub_shutil = stubs.StubShutil(self.stub_os)
        self.stub_subprocess = stubs.StubSubprocess()
        self.stub_sysroot_generator = sysroot_stub.StubSysrootGenerator()
        self.stub_util = util_stub.StubUtil(os_version=self._BRILLO_VERSION)

        image_build.os = self.stub_os
        image_build.open = self.stub_open.open
        image_build.policy = self.stub_selinux
        image_build.shutil = self.stub_shutil
        image_build.subprocess = self.stub_subprocess
        image_build.sysroot = self.stub_sysroot_generator
        image_build.util = self.stub_util

        self.platform = platform_stub.StubPlatform(
            os_version=self._BRILLO_VERSION,
            build_cache=self._PLATFORM_DIR,
            product_out_cache=self._PRODUCT_DIR,
            os_root=self._OS_ROOT, should_link=True)
        self.target = target_stub.StubTarget(platform=self.platform)
        self.config = config_stub.StubConfig(
            artifact_cache=self._ARTIFACT_CACHE_DIR,
            metadata_cache=self._METADATA_CACHE_DIR,
            output_dir=self._IMAGE_OUT_DIR)
        self.product_out_dir = None
        self.image_type = None

    def SetupForImageImage_Build(self, image_type='subclassme'):
        """Helper: establishes pre-reqs for successful image image_build.

        Returns the expected subprocess call.
        """
        if image_type is 'subclassme':
            raise unittest.SkipTest('not in subclass')
        self.image_type = image_type

        # Must be linux
        self.stub_os.SetUname(('linux', '', '', '', ''))

        # Must have copy dir, product out dir, image out dir,
        # host tools, and image_build tools.
        platform_dir = self.platform.build_cache
        product_out_dir = self.platform.product_out_cache
        self.product_out_dir = product_out_dir
        self.stub_os.path.should_be_dir = [
            product_out_dir, self._IMAGE_OUT_DIR, self._ARTIFACT_CACHE_DIR,
            self._METADATA_CACHE_DIR,
            self.stub_os.path.join(platform_dir, 'host',
                                   self.stub_util.GetHostArch(),
                                   'bin'),
            self.platform.os.path('build', 'tools', 'releasetools')]
        self.stub_os.path.should_exist += copy.deepcopy(
            self.stub_os.path.should_be_dir)

        # Willing to make an image_build root in the cache dir.
        self.stub_os.should_makedirs = [] # [self._CACHE_DIR]
        prebuilt_intermediates = self.stub_os.path.join(
            self._PRODUCT_DIR, 'obj', 'PACKAGING', 'systemimage_intermediates',
            'system_image_info.txt')
        self.stub_os.path.should_exist.append(prebuilt_intermediates)
        self.stub_open.files[prebuilt_intermediates] = stubs.StubFile()

        expected_call = [
            'build_image.py',
            self.stub_os.path.join(self._ARTIFACT_CACHE_DIR, image_type),
            self.stub_os.path.join(self._METADATA_CACHE_DIR, 'image_info.txt'),
            self.stub_os.path.join(
                self._IMAGE_OUT_DIR, '{}.img'.format(image_type)),
            self._METADATA_CACHE_DIR]

        return expected_call


class BaseTests(object):
    """Container for the base image tests.

    Put the base test class within a subclass.  Otherwise the unit test
    discover function picks them up as actual tests but marks them as skipped.
    """

    class BuildImageBaseTest(BuildImageBase, unittest.TestCase):
        """Base set of tests to run for each image type."""

        def test_image_success(self):
            expected_call = self.SetupForImageImage_Build()
            command = self.stub_subprocess.AddCommand()
            self.assertEqual(
                0,
                image_build.BuildImage(self.image_type, self.target,
                                       self.config))

            command.AssertCallWas(expected_call)

        def test_image_return_exit_code(self):
            """Tests that the make exit code is returned."""
            self.stub_subprocess.AddCommand(ret_code=1)
            self.SetupForImageImage_Build()
            self.assertEqual(1, image_build.BuildImage(
                self.image_type, self.target, self.config))

        def test_image_missing_paths(self):
            self.SetupForImageImage_Build()
            original_dirs = copy.deepcopy(self.stub_os.path.should_be_dir)
            for directory in original_dirs:
                self.stub_subprocess.AddCommand()
                # Take a dir out.
                self.stub_os.path.should_be_dir.remove(directory)
                self.stub_os.path.should_exist.remove(directory)
                # Check that we fail on all except the output dir.
                if directory != self._IMAGE_OUT_DIR:
                    with self.assertRaises(image_build.PathError):
                        image_build.BuildImage(
                            self.image_type, self.target, self.config)
                else:
                    self.stub_os.should_makedirs.append(self._IMAGE_OUT_DIR)
                    self.assertEqual(
                        0,
                        image_build.BuildImage(self.image_type, self.target,
                                               self.config))
                    # Put it back in.
                self.stub_os.path.should_be_dir.append(directory)

        def test_image_bad_host(self):
            self.SetupForImageImage_Build()
            self.stub_util.arch_is_supported = False
            self.stub_subprocess.AddCommand()
            # Shouldn't reach the linking point.
            self.platform.should_link = False
            with self.assertRaises(self.stub_util.HostUnsupportedArchError):
                image_build.BuildImage(self.image_type, self.target,
                                       self.config)


class BuildUnsupportedImageTest(unittest.TestCase):
    def test_unsupported_type(self):
        bt = BuildImageBase()
        bt.setUp()
        bt.SetupForImageImage_Build('unknown')
        with self.assertRaises(image_build.ImageTypeError):
            # pylint: disable=protected-access
            image_build.BuildImage(bt.image_type, bt.target,
                                   bt.config)


class BuildOdmImageTest(BaseTests.BuildImageBaseTest):

    def SetupForImageImage_Build(self, image_type='odm'):
        """Helper: establishes pre-reqs for successful image image_build.

        Returns the expected subprocess call.
        """
        call = super(BuildOdmImageTest, self).SetupForImageImage_Build('odm')
        self.stub_sysroot_generator.should_write += ['image_info.txt']
        return call


class BuildSystemImageTest(BaseTests.BuildImageBaseTest):

    def SetupForImageImage_Build(self, image_type='system'):
        """Helper: establishes pre-reqs for successful image image_build.

        Returns the expected subprocess call.
        """
        expected_call = super(BuildSystemImageTest,
                              self).SetupForImageImage_Build('system')
        # Make sure the built system is copied to system.
        self.stub_sysroot_generator.should_add_dir += [
            (self.stub_os.path.join(self.product_out_dir, 'system'),
             'root/system', True)
        ]
        # Image_Build root should get system_image_info.txt, and modify it.
        self.stub_sysroot_generator.should_add_file = [
            (self.stub_os.path.join(self.product_out_dir, 'obj',
                                    'PACKAGING',
                                    'systemimage_intermediates',
                                    'system_image_info.txt'),
             'image_info.txt')]
        return expected_call


class CreateTargetCacheTest(TestData, unittest.TestCase):
    def setUp(self):
        self.artifact_cache_dir = '/base/path/build_root'
        self.metadata_cache_dir = '/base/path/metadata'
        self.config = config_stub.StubConfig(
            artifact_cache=self.artifact_cache_dir,
            metadata_cache=self.metadata_cache_dir)
        self.spec = project_spec_stub.StubProjectSpec(config=self.config)
        self.target = None

        p = pack.Pack('my_project', 'main')
        self.copy = pack.Copy(p)
        self.copy.src = '/some/source.txt'
        self.copy.src_type = pack.CopyType.FILE

        self.stub_sysroot_generator = sysroot_stub.StubSysrootGenerator()
        self.stub_os = stubs.StubOs()
        self.stub_open = stubs.StubOpen(self.stub_os)
        self.stub_glob = stubs.StubGlob(self.stub_os)

        image_build.glob = self.stub_glob
        image_build.os = self.stub_os
        image_build.open = self.stub_open.open
        image_build.sysroot = self.stub_sysroot_generator

        metadata_split = self.metadata_cache_dir.split('/')
        self.metadata_files = [
            self.stub_os.path.join(self.metadata_cache_dir, 'etc')]
        # mkdirs will recursively create the rest of the dir.
        while len(metadata_split) > 1:
            self.metadata_files.append('/'.join(metadata_split))
            metadata_split.pop()
        self.metadata_files += ['/'.join(metadata_split[:i+2]) for i in
                                reversed(range(len(metadata_split)-1))]
        self.metadata_files += [
            self.stub_os.path.join(self.metadata_cache_dir, 'etc',
                                   'fs_config_dirs'),
            self.stub_os.path.join(self.metadata_cache_dir, 'etc',
                                   'fs_config_files'),
            self.stub_os.path.join(self.metadata_cache_dir, 'file_contexts'),
        ]
        # This always happens unless it fails early.
        self.stub_os.should_makedirs += [
            self.stub_os.path.join(self.metadata_cache_dir, 'etc')]

    def tearDown(self):
        # Make sure all files are copied.
        for gen in self.stub_sysroot_generator.sysroots:
            self.assertEqual(gen.should_add_file, [])
            self.assertEqual(gen.should_add_dir, [])
            self.assertEqual(gen.should_add_glob, [])

    def test_dependency_error(self):
        target = target_stub.StubTarget(
            submap_raises=dependency.Error('dep error'))
        with self.assertRaises(dependency.Error):
            image_build.CreateTargetCache(self.spec, target)

    def test_skip_os(self):
        p = pack.Pack('brillo.{}'.format(self._BRILLO_VERSION), 'some_os_stuff')
        simple_map = packmap_stub.StubPackMap(
            destinations={'/system/bin/servicemanager': [pack.Copy(p)]})
        target = target_stub.StubTarget(submaps=[simple_map])
        # sysroot stub will not be touched because the OS will be under /system.
        image_build.CreateTargetCache(self.spec, target, mountpoint='/odm')

    def test_with_os(self):
        p = pack.Pack('brillo.{}'.format(self._BRILLO_VERSION), 'some_os_stuff')
        cpy = pack.Copy(p, dst='/system/bin/servicemanager',
                        dst_type=pack.CopyType.FILE, src='/tmp/a_file',
                        src_type=pack.CopyType.FILE)
        cpy.override_build = False
        simple_map = packmap_stub.StubPackMap(
            destinations={cpy.dst: [cpy]})
        target = target_stub.StubTarget(submaps=[simple_map])
        self.stub_sysroot_generator.should_makedirs = [
            self.stub_os.path.dirname(cpy.dst)[1:]]
        self.stub_sysroot_generator.should_add_file = [(cpy.src,
                                                        cpy.dst.lstrip('/'))]
        image_build.CreateTargetCache(self.spec, target, mountpoint='/')

    def test_skip_nonmountpoint(self):
        self.copy.dst = '/system/bin/xyzzy.txt'
        self.copy.dst_type = pack.CopyType.FILE
        odm_copy = pack.Copy(self.copy.pack)
        odm_copy.dst = '/odm/bin/xyzzy.txt'
        odm_copy.dst_type = pack.CopyType.FILE
        odm_copy.src = self.copy.src
        odm_copy.src_type = self.copy.src_type
        simple_map = packmap_stub.StubPackMap(
            destinations={self.copy.dst: [self.copy], odm_copy.dst: [odm_copy]})
        target = target_stub.StubTarget(submaps=[simple_map])
        self.stub_sysroot_generator.should_makedirs = [
            self.stub_os.path.dirname(odm_copy.dst)[1:]]
        self.stub_sysroot_generator.should_add_file = [(odm_copy.src,
                                                        'odm/bin/xyzzy.txt')]
        image_build.CreateTargetCache(self.spec, target, mountpoint='/odm')

    def test_copy_file_src_file_dst(self):
        self.copy.dst = '/system/bin/xyzzy.txt'
        self.copy.dst_type = pack.CopyType.FILE
        self.copy.acl.selabel = 'u:object_r:system_foo:s0'
        simple_map = packmap_stub.StubPackMap(
            destinations={self.copy.dst: [self.copy]})
        target = target_stub.StubTarget(submaps=[simple_map])
        self.stub_sysroot_generator.should_makedirs = [
            self.stub_os.path.dirname(self.copy.dst)[1:]]
        self.stub_sysroot_generator.should_add_file = [(self.copy.src,
                                                        'system/bin/xyzzy.txt')]
        image_build.CreateTargetCache(self.spec, target)
        # Ensure the selabel was copied.
        file_contexts = self.stub_open.files[
            self.stub_os.path.join(self.metadata_cache_dir,
                                   'file_contexts')].contents
        self.assertRegexpMatches(
            '\n'.join(file_contexts),
            '{}($|\n)'.format(self.copy.acl.file_context()))
        # Ensure the ACLs were copied too.
        fs_config_files = self.stub_open.files[
            self.stub_os.path.join(self.metadata_cache_dir, 'etc',
                                   'fs_config_files')].contents
        self.assertRegexpMatches(
            '\n'.join(fs_config_files),
            '{}($|\n)'.format(self.copy.acl.fs_config(binary=True)))

    def test_copy_system_with_custom_deep_file(self):
        # When copying a "system" pack, we rely on two sources of
        # ACL setting:
        # - compiled in fs_config values
        # - includes etc/fs_config_{files,dirs}
        # This test ensures that neither are overridden.

        # A custom file.
        self.copy.dst = '/system/bin/xyzzy.txt'
        self.copy.dst_type = pack.CopyType.FILE
        self.copy.acl.user = 1234
        self.copy.acl.group = 4321

        p = pack.Pack('brillo.{}'.format(self._BRILLO_VERSION), 'some_os_stuff')
        p.add_provides('os.core')
        p.add_copy(pack.Copy(
            p, dst='/system/bin/servicemanager', dst_type=pack.CopyType.FILE,
            src='/tmp/a_file', src_type=pack.CopyType.FILE))
        copy_servicemanager = p.copies[-1]
        copy_servicemanager.acl.override_build = False
        p.add_copy(pack.Copy(
            p, dst='/system/etc/fs_config_files', dst_type=pack.CopyType.FILE,
            src='/tmp/fs_config_files', src_type=pack.CopyType.FILE))
        copy_fs_config_files = p.copies[-1]
        copy_fs_config_files.acl.override_build = False
        p.add_copy(pack.Copy(
            p, dst='/system/etc/fs_config_dirs', dst_type=pack.CopyType.FILE,
            src='/tmp/fs_config_dirs', src_type=pack.CopyType.FILE))
        p.copies[-1].acl.override_build = False

        copies = p.copies + [self.copy]
        simple_map = packmap_stub.StubPackMap(
            destinations={copy.dst: [copy] for copy in copies})
        target = target_stub.StubTarget(submaps=[simple_map])
        self.stub_sysroot_generator.should_makedirs = [
            self.stub_os.path.dirname(copy_servicemanager.dst)[1:],
            self.stub_os.path.dirname(copy_fs_config_files.dst)[1:]]
        self.stub_sysroot_generator.should_add_file = [
            (copy_.src, copy_.dst.lstrip('/')) for copy_ in copies
        ]

        # Needed for ensuring fs_config_files appending works.
        existing = self.stub_os.path.join(self.artifact_cache_dir, 'system',
                                          'etc', 'fs_config_files')
        self.stub_os.path.should_exist.append(existing)
        system_fs_config_files = 'abc123'
        self.stub_open.files[existing] = stubs.StubFile(system_fs_config_files)

        existing = self.stub_os.path.join(self.artifact_cache_dir, 'system',
                                          'etc', 'fs_config_dirs')
        self.stub_os.path.should_exist.append(existing)
        system_fs_config_dirs = '321bca'
        self.stub_open.files[existing] = stubs.StubFile(system_fs_config_dirs)

        image_build.CreateTargetCache(self.spec, target, mountpoint='/')
        # Ensure the ACLs were copied and the system fs_config_* are appended
        # to the build-specific changes.
        fs_config_files = self.stub_open.files[
            self.stub_os.path.join(self.metadata_cache_dir, 'etc',
                                   'fs_config_files')].contents
        self.assertEqual(
            ['%\x00\x00\x01\xd2\x04\xe1\x10\x00\x00\x00\x00\x00\x00\x00\x00'
             'system/bin/xyzzy.txt\x00',
             system_fs_config_files],
            fs_config_files)
        # Ensure that no directory ACLs were implicitly placed on
        # system or system/bin/.
        fs_config_dirs = self.stub_open.files[
            self.stub_os.path.join(self.metadata_cache_dir, 'etc',
                                   'fs_config_dirs')].contents
        self.assertEqual(fs_config_dirs, ['', system_fs_config_dirs])

    def test_copy_glob(self):
        self.copy.src = '/my/bins/*'
        self.copy.src_type = pack.CopyType.GLOB
        self.copy.recurse = False
        self.copy.dst = '/system/bin/'
        self.copy.dst_type = pack.CopyType.DIR
        simple_map = packmap_stub.StubPackMap(
            destinations={self.copy.dst: [self.copy]})
        target = target_stub.StubTarget(submaps=[simple_map])
        self.stub_sysroot_generator.should_makedirs = [
            self.stub_os.path.dirname(self.copy.dst)[1:]]
        self.stub_sysroot_generator.should_add_glob = [
            (self.copy.src, 'system/bin/', False)]
        image_build.CreateTargetCache(self.spec, target)

    def test_copy_glob_recursive(self):
        self.copy.src = '/my/bins/*'
        self.copy.src_type = pack.CopyType.GLOB
        self.copy.recurse = True
        self.copy.dst = '/system/bin/'
        self.copy.dst_type = pack.CopyType.DIR
        simple_map = packmap_stub.StubPackMap(
            destinations={self.copy.dst: [self.copy]})
        target = target_stub.StubTarget(submaps=[simple_map])
        self.stub_sysroot_generator.should_makedirs = [
            self.stub_os.path.dirname(self.copy.dst)[1:]]
        self.stub_sysroot_generator.should_add_glob = [
            (self.copy.src, 'system/bin/', True)]
        image_build.CreateTargetCache(self.spec, target)

    def test_copy_dir(self):
        self.copy.src = '/my/etc/'
        self.copy.src_type = pack.CopyType.DIR
        self.copy.recurse = False
        self.copy.dst = '/system/etc/'
        self.copy.dst_type = pack.CopyType.DIR
        simple_map = packmap_stub.StubPackMap(
            destinations={self.copy.dst: [self.copy]})
        target = target_stub.StubTarget(submaps=[simple_map])
        self.stub_sysroot_generator.should_makedirs = [
            self.stub_os.path.dirname(self.copy.dst)[1:]]
        self.stub_sysroot_generator.should_add_dir = [
            (self.copy.src, 'system/etc/', False)]
        image_build.CreateTargetCache(self.spec, target)

    def test_copy_dir_recurse(self):
        self.copy.src = '/my/etc/'
        self.copy.src_type = pack.CopyType.DIR
        self.copy.recurse = True
        self.copy.dst = '/system/etc/'
        self.copy.dst_type = pack.CopyType.DIR
        simple_map = packmap_stub.StubPackMap(
            destinations={self.copy.dst: [self.copy]})
        target = target_stub.StubTarget(submaps=[simple_map])
        self.stub_sysroot_generator.should_makedirs = [
            self.stub_os.path.dirname(self.copy.dst)[1:]]
        self.stub_sysroot_generator.should_add_dir = [
            (self.copy.src, 'system/etc/', True)]
        image_build.CreateTargetCache(self.spec, target)


    def test_copy_invalid_combos(self):
        combos = [
            (pack.CopyType.DIR, pack.CopyType.FILE, False),
            (pack.CopyType.DIR, pack.CopyType.FILE, True),
            (pack.CopyType.FILE, pack.CopyType.GLOB, False),
            (pack.CopyType.FILE, pack.CopyType.GLOB, True),
            (pack.CopyType.GLOB, pack.CopyType.FILE, False),
            (pack.CopyType.GLOB, pack.CopyType.FILE, True),
        ]
        for tup in combos:
            self.copy.src_type = tup[0]
            self.copy.dst_type = tup[1]
            self.copy.dst = '/system/bin/xyzzy.txt'
            simple_map = packmap_stub.StubPackMap(
                destinations={self.copy.dst: [self.copy]})
            target = target_stub.StubTarget(submaps=[simple_map])
            with self.assertRaises(image_build.PathError):
                image_build.CreateTargetCache(self.spec, target)

    def test_unmanaged_file(self):
        self.copy.dst = '/system/bin/xyzzy.txt'
        self.copy.dst_type = pack.CopyType.FILE
        simple_map = packmap_stub.StubPackMap(
            destinations={self.copy.dst: [self.copy]})
        target = target_stub.StubTarget(submaps=[simple_map])
        self.stub_sysroot_generator.should_makedirs = [
            self.stub_os.path.dirname(self.copy.dst)[1:]]
        self.stub_sysroot_generator.should_add_file = [(self.copy.src,
                                                        'system/bin/xyzzy.txt')]
        self.stub_os.path.should_exist.append(
            self.stub_os.path.join(self.artifact_cache_dir, 'system', 'foozle',
                                   'blag'))
        image_build.CreateTargetCache(self.spec, target, verbose=False)
        # The metadata files should've been created.
        expect_files = self.metadata_files
        self.assertEqual(self.stub_os.path.should_exist, expect_files)

    def test_unmanaged_dir_dir_file(self):
        self.copy.dst = '/system/bin/xyzzy.txt'
        self.copy.dst_type = pack.CopyType.FILE
        simple_map = packmap_stub.StubPackMap(
            destinations={self.copy.dst: [self.copy]})
        target = target_stub.StubTarget(submaps=[simple_map])
        self.stub_sysroot_generator.should_makedirs = [
            self.stub_os.path.dirname(self.copy.dst)[1:]]
        self.stub_sysroot_generator.should_add_file = [
            (self.copy.src, 'system/bin/xyzzy.txt')]
        self.stub_os.path.should_exist.append(
            self.stub_os.path.join(self.artifact_cache_dir, 'system',
                                   'foozle', 'barzle', 'blag.mal'))
        image_build.CreateTargetCache(self.spec, target, verbose=False)
        expect_files = self.metadata_files
        self.assertEqual(self.stub_os.path.should_exist, expect_files)