diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2021-07-15 02:09:46 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2021-07-15 02:09:46 +0000 |
commit | dd478a81d32977231ba9b00997f5f2e8590908d7 (patch) | |
tree | 4d7852d76974dbe8f7708ba392bf7e328697caac | |
parent | dba92766b742da430fb6534e35b6bd7959e8514a (diff) | |
parent | dc4c38f9f2586ec3dc0ccd611caa589b6452603e (diff) | |
download | treble-dd478a81d32977231ba9b00997f5f2e8590908d7.tar.gz |
Snap for 7550930 from dc4c38f9f2586ec3dc0ccd611caa589b6452603e to mainline-resolv-releaseandroid-mainline-12.0.0_r94android-mainline-12.0.0_r80android-mainline-12.0.0_r65android-mainline-12.0.0_r52android-mainline-12.0.0_r35android-mainline-12.0.0_r16android-mainline-12.0.0_r124android-mainline-12.0.0_r108android12-mainline-resolv-release
Change-Id: I64d4abe5d4d4c725d9a83e18d2228d109ff67bd3
67 files changed, 8182 insertions, 513 deletions
diff --git a/Android.bp b/Android.bp new file mode 100644 index 0000000..66e486e --- /dev/null +++ b/Android.bp @@ -0,0 +1,31 @@ +// +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["tools_treble_license"], +} + +// Added automatically by a large-scale-change +// See: http://go/android-license-faq +license { + name: "tools_treble_license", + visibility: [":__subpackages__"], + license_kinds: [ + "SPDX-license-identifier-Apache-2.0", + ], + license_text: [ + "LICENSE", + ], +} diff --git a/build/Android.bp b/build/Android.bp index 85ca3e9..7d1731e 100644 --- a/build/Android.bp +++ b/build/Android.bp @@ -12,9 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "tools_treble_license" + // to get the below license kinds: + // SPDX-license-identifier-Apache-2.0 + default_applicable_licenses: ["tools_treble_license"], +} + python_defaults { name: "treble_build_default", - pkg_path: "treble", + pkg_path: "treble/build", version: { py2: { enabled: false, @@ -25,18 +34,36 @@ python_defaults { }, } +python_library_host { + name: "treble_build_lib", + defaults: ["treble_build_default"], + srcs: [ + "sandbox/build_android_sandboxed.py", + "sandbox/config.py", + "sandbox/nsjail.py", + "sandbox/overlay.py", + "sandbox/rbe.py", + ], +} + python_test_host { name: "treble_build_test", main: "test.py", defaults: ["treble_build_default"], srcs: [ - "sandbox/nsjail.py", + "sandbox/build_android_sandboxed_test.py", + "sandbox/config_test.py", "sandbox/nsjail_test.py", - "sandbox/overlay.py", "sandbox/overlay_test.py", + "sandbox/rbe_test.py", "test.py", "sample_test.py", ], + libs: [ + "treble_build_lib", + ], test_config: "test.xml", - test_suites: ["general-tests"], + test_options: { + unit_test: true, + }, } diff --git a/build/TEST_MAPPING b/build/TEST_MAPPING deleted file mode 100644 index 9d49b72..0000000 --- a/build/TEST_MAPPING +++ /dev/null @@ -1,8 +0,0 @@ -{ - "presubmit" : [ - { - "name" : "treble_build_test", - "host" : true - } - ] -} diff --git a/build/sandbox/build_android_sandboxed.py b/build/sandbox/build_android_sandboxed.py new file mode 100644 index 0000000..f6a1b57 --- /dev/null +++ b/build/sandbox/build_android_sandboxed.py @@ -0,0 +1,212 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Builds an Android target in a secure sandbox.""" + +import argparse +import os +from . import config +from . import nsjail +from . import rbe + +_DEFAULT_COMMAND_WRAPPER = \ + '/src/tools/treble/build/sandbox/build_android_target.sh' + + +def build(build_target, variant, nsjail_bin, chroot, dist_dir, build_id, + max_cpus, build_goals, config_file=None, + command_wrapper=_DEFAULT_COMMAND_WRAPPER, use_rbe=False, + readonly_bind_mount=None, env=[]): + """Builds an Android target in a secure sandbox. + + Args: + build_target: A string with the name of the build target. + variant: A string with the build variant. + nsjail_bin: A string with the path to the nsjail binary. + chroot: A string with the path to the chroot of the NsJail sandbox. + dist_dir: A string with the path to the Android dist directory. + build_id: A string with the Android build identifier. + max_cpus: An integer with maximum number of CPUs. + build_goals: A list of strings with the goals and options to provide to the + build command. + config_file: A string path to an overlay configuration file. + command_wrapper: A string path to the command wrapper. + use_rbe: If true, will attempt to use RBE for the build. + readonly_bind_mount: A string path to a path to be mounted as read-only. + env: An array of environment variables to define in the NsJail sandbox in the + `var=val` syntax. + + Returns: + A list of commands that were executed. Each command is a list of strings. + """ + if config_file: + cfg = config.Config(config_file) + android_target = cfg.get_build_config_android_target(build_target) + if cfg.has_tag(build_target, 'skip'): + print('Warning: skipping build_target "{}" due to tag being set'.format(build_target)) + return [] + else: + android_target = build_target + + # All builds are required to run with the root of the + # Android source tree as the current directory. + source_dir = os.getcwd() + command = [ + command_wrapper, + '%s-%s' % (android_target, variant), + '/src', + 'make', + '-j', + ] + build_goals + + readonly_bind_mounts = [] + if readonly_bind_mount: + readonly_bind_mounts = [readonly_bind_mount] + + extra_nsjail_args = [] + cleanup = lambda: None + nsjail_wrapper = [] + if use_rbe: + cleanup = rbe.setup(env) + env = rbe.prepare_env(env) + extra_nsjail_args.extend(rbe.get_extra_nsjail_args()) + readonly_bind_mounts.extend(rbe.get_readonlybind_mounts()) + nsjail_wrapper = rbe.get_nsjail_bin_wrapper() + + ret = nsjail.run( + nsjail_bin=nsjail_bin, + chroot=chroot, + overlay_config=config_file, + source_dir=source_dir, + command=command, + build_target=build_target, + dist_dir=dist_dir, + build_id=build_id, + max_cpus=max_cpus, + extra_nsjail_args=extra_nsjail_args, + readonly_bind_mounts=readonly_bind_mounts, + env=env, + nsjail_wrapper=nsjail_wrapper) + + cleanup() + + return ret + + +def arg_parser(): + """Returns an ArgumentParser for sanboxed android builds.""" + # Use the top level module docstring for the help description + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--build_target', + help='The build target.') + parser.add_argument( + '--variant', default='userdebug', help='The Android build variant.') + parser.add_argument( + '--nsjail_bin', + required=True, + help='Path to NsJail binary.') + parser.add_argument( + '--chroot', + required=True, + help='Path to the chroot to be used for building the Android ' + 'platform. This will be mounted as the root filesystem in the ' + 'NsJail sandbox.') + parser.add_argument( + '--config_file', + required=True, + help='Path to the overlay configuration file.') + parser.add_argument( + '--command_wrapper', + default=_DEFAULT_COMMAND_WRAPPER, + help='Path to the command wrapper. ' + 'Defaults to \'%s\'.' % _DEFAULT_COMMAND_WRAPPER) + parser.add_argument( + '--readonly_bind_mount', + help='Path to the a path to be mounted as readonly inside the secure ' + 'build sandbox.') + parser.add_argument( + '--env', '-e', + type=str, + default=[], + action='append', + help='Specify an environment variable to the NSJail sandbox. Can be specified ' + 'muliple times. Syntax: var_name=value') + parser.add_argument( + '--dist_dir', + help='Path to the Android dist directory. This is where ' + 'Android platform release artifacts will be written.') + parser.add_argument( + '--build_id', + help='Build identifier what will label the Android platform ' + 'release artifacts.') + parser.add_argument( + '--max_cpus', + type=int, + help='Limit of concurrent CPU cores that the NsJail sanbox ' + 'can use.') + parser.add_argument( + '--context', + action='append', + default=[], + help='One or more contexts used to select build goals from the ' + 'configuration.') + parser.add_argument( + '--use_rbe', + action='store_true', + help='Executes the build on RBE') + return parser + + +def parse_args(parser): + """Parses command line arguments. + + Returns: + A dict of all the arguments parsed. + """ + # Convert the Namespace object to a dict + return vars(parser.parse_args()) + + +def main(): + args = parse_args(arg_parser()) + + # The --build_target argument could not be required + # using the standard 'required' argparse option because + # the argparser is reused by merge_android_sandboxed.py which + # does not require --build_target. + if args['build_target'] is None: + raise ValueError('--build_target is required.') + + cfg = config.Config(args['config_file']) + build_goals = cfg.get_build_goals(args['build_target'], set(args['context'])) + + build( + build_target=args['build_target'], + variant=args['variant'], + nsjail_bin=args['nsjail_bin'], + chroot=args['chroot'], + config_file=args['config_file'], + command_wrapper=args['command_wrapper'], + readonly_bind_mount=args['readonly_bind_mount'], + env=args['env'], + dist_dir=args['dist_dir'], + build_id=args['build_id'], + max_cpus=args['max_cpus'], + use_rbe=args['use_rbe'], + build_goals=build_goals) + + +if __name__ == '__main__': + main() diff --git a/build/sandbox/build_android_sandboxed_test.py b/build/sandbox/build_android_sandboxed_test.py new file mode 100644 index 0000000..d4848c4 --- /dev/null +++ b/build/sandbox/build_android_sandboxed_test.py @@ -0,0 +1,216 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test build_android_sandboxed.""" + +import os +import tempfile +import unittest +from . import build_android_sandboxed + + +class BuildAndroidSandboxedTest(unittest.TestCase): + + def testBasic(self): + build_android_sandboxed.nsjail.__file__ = '/' + os.chdir('/') + commands = build_android_sandboxed.build( + 'target_name', + 'userdebug', + nsjail_bin='/bin/true', + chroot='/chroot', + dist_dir='/dist_dir', + build_id='0', + max_cpus=1, + build_goals=['droid', 'dist']) + + self.assertEqual( + commands, + [ + '/bin/true', + '--env', 'USER=nobody', + '--config', '/nsjail.cfg', + '--env', 'BUILD_NUMBER=0', + '--max_cpus=1', + '--env', 'DIST_DIR=/dist', + '--bindmount', '/:/src', + '--bindmount', '/dist_dir:/dist', + '--', + '/src/tools/treble/build/sandbox/build_android_target.sh', + 'target_name-userdebug', + '/src', + 'make', '-j', 'droid', 'dist', + ] + ) + + def testBuildCommand(self): + build_android_sandboxed.nsjail.__file__ = '/' + os.chdir('/') + commands = build_android_sandboxed.build( + 'target_name', + 'userdebug', + nsjail_bin='/bin/true', + command_wrapper='/command/wrapper', + chroot='/chroot', + dist_dir='/dist_dir', + build_id='0', + max_cpus=1, + build_goals=['droid', 'dist']) + + self.assertEqual( + commands, + [ + '/bin/true', + '--env', 'USER=nobody', + '--config', '/nsjail.cfg', + '--env', 'BUILD_NUMBER=0', + '--max_cpus=1', + '--env', 'DIST_DIR=/dist', + '--bindmount', '/:/src', + '--bindmount', '/dist_dir:/dist', + '--', + '/command/wrapper', + 'target_name-userdebug', + '/src', + 'make', '-j', 'droid', 'dist', + ] + ) + + def testUser(self): + build_android_sandboxed.nsjail.__file__ = '/' + os.chdir('/') + commands = build_android_sandboxed.build( + 'target_name', + 'user', + nsjail_bin='/bin/true', + chroot='/chroot', + dist_dir='/dist_dir', + build_id='0', + max_cpus=1, + build_goals=['droid', 'dist']) + + self.assertEqual( + commands, + [ + '/bin/true', + '--env', 'USER=nobody', + '--config', '/nsjail.cfg', + '--env', 'BUILD_NUMBER=0', + '--max_cpus=1', + '--env', 'DIST_DIR=/dist', + '--bindmount', '/:/src', + '--bindmount', '/dist_dir:/dist', + '--', + '/src/tools/treble/build/sandbox/build_android_target.sh', + 'target_name-user', + '/src', + 'make', '-j', 'droid', 'dist', + ] + ) + + def testExtraBuildGoals(self): + build_android_sandboxed.nsjail.__file__ = '/' + os.chdir('/') + commands = build_android_sandboxed.build( + 'target_name', + 'userdebug', + nsjail_bin='/bin/true', + chroot='/chroot', + dist_dir='/dist_dir', + build_id='0', + max_cpus=1, + build_goals=['droid', 'dist', 'extra_build_target']) + + self.assertEqual( + commands, + [ + '/bin/true', + '--env', 'USER=nobody', + '--config', '/nsjail.cfg', + '--env', 'BUILD_NUMBER=0', + '--max_cpus=1', + '--env', 'DIST_DIR=/dist', + '--bindmount', '/:/src', + '--bindmount', '/dist_dir:/dist', + '--', + '/src/tools/treble/build/sandbox/build_android_target.sh', + 'target_name-userdebug', + '/src', + 'make', '-j', 'droid', 'dist', + 'extra_build_target' + ] + ) + + def testSkipBuildTag(self): + TEST_CONFIG_XML = """<config> + <target name="target_skip" tags="skip"> + <build_config> + <goal name="droid"/> + </build_config> + </target> + </config> + """ + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write(TEST_CONFIG_XML) + test_config.flush() + build_android_sandboxed.nsjail.__file__ = '/' + os.chdir('/') + skip_commands = build_android_sandboxed.build( + 'target_skip', + 'userdebug', + nsjail_bin='/bin/true', + chroot='/chroot', + dist_dir='/dist_dir', + build_id='0', + max_cpus=1, + build_goals=[], + config_file=test_config.name) + self.assertFalse(skip_commands) + + def testEnv(self): + build_android_sandboxed.nsjail.__file__ = '/' + os.chdir('/') + commands = build_android_sandboxed.build( + 'target_name', + 'userdebug', + nsjail_bin='/bin/true', + chroot='/chroot', + dist_dir='/dist_dir', + build_id='0', + max_cpus=1, + build_goals=['droid', 'dist'], + env=['first_env_var=first_value', 'second_env_var=second_value']) + + self.assertEqual( + commands, + [ + '/bin/true', + '--env', 'USER=nobody', + '--config', '/nsjail.cfg', + '--env', 'BUILD_NUMBER=0', + '--max_cpus=1', + '--env', 'DIST_DIR=/dist', + '--bindmount', '/:/src', + '--bindmount', '/dist_dir:/dist', + '--env', 'first_env_var=first_value', + '--env', 'second_env_var=second_value', + '--', + '/src/tools/treble/build/sandbox/build_android_target.sh', + 'target_name-userdebug', + '/src', + 'make', '-j', 'droid', 'dist', + ] + ) + +if __name__ == '__main__': + unittest.main() diff --git a/build/sandbox/build_android_target.sh b/build/sandbox/build_android_target.sh new file mode 100755 index 0000000..23e05ea --- /dev/null +++ b/build/sandbox/build_android_target.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +readonly ANDROID_TARGET=$1 +readonly BUILD_DIR=$2 +shift +shift +readonly BUILD_COMMAND="$@" + +if [[ -z "${ANDROID_TARGET}" ]]; then + echo "error: Android target not set" + exit 1 +fi + +if [[ -z "${BUILD_DIR}" ]]; then + echo "error: Build directory not set" + exit 1 +fi + +if [[ -z "${BUILD_COMMAND}" ]]; then + echo "error: Build command not set" + exit 1 +fi + +# If there is an error, exit right away instead of continuing. For example, +# lunch could fail. If so, there is no point in continuing the build. + +set -e + +echo "build_android_target.sh: source build/envsetup.sh" +source build/envsetup.sh +echo "build_android_target.sh: lunch $ANDROID_TARGET" +lunch "$ANDROID_TARGET" +echo "build_android_target.sh: cd $BUILD_DIR" +cd "$BUILD_DIR" + +# However, the build command itself cannot use set -e. I haven't figured this +# out yet, but something in the build command causes early exit for some +# targets. + +set +e + +echo "build_android_target.sh: $BUILD_COMMAND" +$BUILD_COMMAND +BUILD_COMMAND_EXIT_VALUE=$? + +# Collect RBE metrics if enabled +if [[ -n "${USE_RBE}" && -n "${RBE_DIR}" ]]; then + echo "build_android_target.sh: $RBE_DIR/dumpstats" + $RBE_DIR/dumpstats +fi + +exit $BUILD_COMMAND_EXIT_VALUE diff --git a/build/sandbox/config.py b/build/sandbox/config.py new file mode 100644 index 0000000..26bccbd --- /dev/null +++ b/build/sandbox/config.py @@ -0,0 +1,513 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Parses config file and provides various ways of using it.""" + +import xml.etree.ElementTree as ET +import collections + +# The config file must be in XML with a structure as descibed below. +# +# The top level config element shall contain one or more "target" child +# elements. Each of these may contain one or more build_config child elements. +# The build_config child elements will inherit the properties of the target +# parent. +# +# Each "target" and "build_config" may contain the following: +# +# Attributes: +# +# name: The name of the target. +# +# android_target: The name of the android target used with lunch +# +# allow_readwrite_all: "true" if the full source folder shall be mounted as +# read/write. It should be accompanied by a comment with the bug describing +# why it was required. +# +# tags: A comma-separated list of strings to be associated with the target +# and any of its nested build_targets. You can use a tag to associate +# information with a target in your configuration file, and retrieve that +# information using the get_tags API or the has_tag API. +# +# Child elements: +# +# config: A generic name-value configuration element. +# +# Attributes: +# name: Name of the configuration +# value: Value of the configuration +# +# overlay: An overlay to be mounted while building the target. +# +# Attributes: +# +# name: The name of the overlay. +# +# Child elements: +# +# replacement_path: An overlay path that supersedes any conflicts +# after it. +# +# Properties: +# +# name: The name of the replacement path. This path will will +# superced the same path for any subsequent conflicts. If two +# overlays have the same replacement path an error will occur. +# +# +# view: A map (optionally) specifying a filesystem view mapping for each +# target. +# +# Attributes: +# +# name: The name of the view. +# +# allow_readwrite: A folder to mount read/write +# inside the Android build nsjail. Each allowed read-write entry should be +# accompanied by a bug that indicates why it was required and tracks the +# progress to a fix. +# +# Attributes: +# +# path: The path to be allowed read-write mounting. +# +# build_config: A list of goals to be used while building the target. +# +# Attributes: +# +# name: The name of the build config. Defaults to the target name +# if not set. +# +# Child elements: +# +# goal: A build goal. +# +# Properties: +# +# name: The name of the build goal. The build tools pass the name +# attribute as a parameter to make. This can have a value like +# "droid" or "VAR=value". +# +# contexts: A comma-separated list of the contexts in which this +# goal applies. If this attribute is missing or blank, the goal +# applies to all contexts. Otherwise, it applies only in the +# requested contexts (see get_build_goals). + +Overlay = collections.namedtuple('Overlay', ['name', 'replacement_paths']) + +class BuildConfig(object): + """Represents configuration of a build_target. + + Attributes: + name: name of the build_target used to pull the configuration. + android_target: The name of the android target used with lunch. + tags: List of tags associated with the build target config + build_goals: List of goals to be used while building the target. + overlays: List of overlays to be mounted. + views: A list of (source, destination) string path tuple to be mounted. + See view nodes in XML. + allow_readwrite_all: If true, mount source tree as rw. + allow_readwrite: List of directories to be mounted as rw. + allowed_projects_file: a string path name of a file with a containing + allowed projects. + configurations: a map of name to value configurations + """ + + def __init__(self, + name, + android_target, + tags=frozenset(), + build_goals=(), + overlays=(), + views=(), + allow_readwrite_all=False, + allow_readwrite=(), + allowed_projects_file=None, + configurations=None): + super().__init__() + self.name = name + self.android_target = android_target + self.tags = tags + self.build_goals = list(build_goals) + self.overlays = list(overlays) + self.views = list(views) + self.allow_readwrite_all = allow_readwrite_all + self.allow_readwrite = list(allow_readwrite) + self.allowed_projects_file = allowed_projects_file + self.configurations = configurations or {} + + def validate(self): + """Run tests to validate build configuration""" + if not self.name: + raise ValueError('Error build_config must have a name.') + # Validate that a build config does not contain an overlay with + # conflicting replacement paths. + if len(self.overlays) > 1 and set.intersection( + *[o.replacement_paths for o in self.overlays]): + raise ValueError( + 'Error build_config overlays have conflicting replacement_paths.') + + @classmethod + def from_config(cls, config_elem, fs_view_map, base_config=None): + """Creates a BuildConfig from a config XML element and an optional + base_config. + + Args: + config_elem: the config XML node element to build the configuration + fs_view_map: A map of view names to list of tuple(source, destination) + paths. + base_config: the base BuildConfig to use + + Returns: + A build config generated from the config element and the base + configuration if provided. + """ + if base_config is None: + # Build a base_config with required elements from the new config_elem + name = config_elem.get('name') + base_config = cls( + name=name, android_target=config_elem.get('android_target', name)) + + return cls( + android_target=config_elem.get('android_target', + base_config.android_target), + name=config_elem.get('name', base_config.name), + allowed_projects_file=config_elem.get( + 'allowed_projects_file', base_config.allowed_projects_file), + build_goals=_get_build_config_goals(config_elem, + base_config.build_goals), + tags=_get_config_tags(config_elem, base_config.tags), + overlays=_get_overlays(config_elem, base_config.overlays), + allow_readwrite=_get_allow_readwrite(config_elem, + base_config.allow_readwrite), + views=_get_views(config_elem, fs_view_map, base_config.views), + allow_readwrite_all=_get_allowed_readwrite_all( + config_elem, base_config.allow_readwrite_all), + configurations=_get_configurations(config_elem, + base_config.configurations) + ) + + +def _get_configurations(config_elem, base): + configs = dict(base) + configs.update({ + config.get('name'): config.get('value') + for config in config_elem.findall('config') + }) + return configs + + +def _get_build_config_goals(config_elem, base=None): + """Retrieves goals from build_config or target. + + Args: + config_elem: A build_config or target xml element. + base: Initial list of goals to prepend to the list + + Returns: + A list of tuples where the first element of the tuple is the build goal + name, and the second is a list of the contexts to which this goal applies. + """ + + return base + [(goal.get('name'), set(goal.get('contexts').split(',')) + if goal.get('contexts') else None) + for goal in config_elem.findall('goal')] + + +def _get_config_tags(config_elem, base=frozenset()): + """Retrieves tags from build_config or target. + + Args: + config_elem: A build_config or target xml element. + base: Initial list of tags to seed the set + + Returns: + A set of tags for a build_config. + """ + tags = config_elem.get('tags') + return base.union(set(tags.split(',')) if tags else set()) + + +def _get_allowed_readwrite_all(config_elem, default=False): + """Determines if build_config or target is set to allow readwrite for all + source paths. + + Args: + config_elem: A build_config or target xml element. + default: Value to use if element doesn't contain the + allow_readwrite_all attribute. + + Returns: + True if build config is set to allow readwrite for all sorce paths + """ + value = config_elem.get('allow_readwrite_all') + return value == 'true' if value else default + + +def _get_overlays(config_elem, base=None): + """Retrieves list of overlays from build_config or target. + + Args: + config_elem: A build_config or target xml element. + base: Initial list of overlays to prepend to the list + + Returns: + A list of tuples of overlays and replacement paths to mount for a build_config or target. + """ + overlays = [] + for overlay in config_elem.findall('overlay'): + overlays.append( + Overlay( + name=overlay.get('name'), + replacement_paths=set([ + path.get('path') for path in overlay.findall('replacement_path') + ]))) + return base + overlays + +def _get_views(config_elem, fs_view_map, base=None): + """Retrieves list of views from build_config or target. + + Args: + config_elem: A build_config or target xml element. + base: Initial list of views to prepend to the list + + Returns: + A list of (source, destination) string path tuple to be mounted. See view + nodes in XML. + """ + return base + [fs for o in config_elem.findall('view') + for fs in fs_view_map[o.get('name')]] + + +def _get_allow_readwrite(config_elem, base=None): + """Retrieves list of directories to be mounted rw from build_config or + target. + + Args: + config_elem: A build_config or target xml element. + base: Initial list of rw directories to prepend to the list + + Returns: + A list of directories to be mounted rw. + """ + return (base + + [o.get('path') for o in config_elem.findall('allow_readwrite')]) + + +def _get_fs_view_map(config): + """Retrieves the map of filesystem views. + + Args: + config: An XML Element that is the root of the config XML tree. + + Returns: + A dict of filesystem views keyed by view name. A filesystem view is a + list of (source, destination) string path tuples. + """ + # A valid config file is not required to include FS Views, only overlay + # targets. + return { + view.get('name'): [(path.get('source'), path.get('destination')) + for path in view.findall('path') + ] for view in config.findall('view') + } + + +def _get_build_config_map(config): + """Retrieves a map of all build config. + + Args: + config: An XML Element that is the root of the config XML tree. + + Returns: + A dict of BuildConfig keyed by build_target. + """ + fs_view_map = _get_fs_view_map(config) + build_config_map = {} + for target_config in config.findall('target'): + base_target = BuildConfig.from_config(target_config, fs_view_map) + + for build_config in target_config.findall('build_config'): + build_target = BuildConfig.from_config(build_config, fs_view_map, + base_target) + build_target.validate() + build_config_map[build_target.name] = build_target + + return build_config_map + + +class Config: + """Presents an API to the static XML configuration.""" + + def __init__(self, config_filename): + """Initializes a Config instance from the specificed filename + + This method parses the XML content of the file named by config_filename + into internal data structures. You can then use various methods to query + the static config. + + Args: + config_filename: The name of the file from which to load the config. + """ + + tree = ET.parse(config_filename) + config = tree.getroot() + self._build_config_map = _get_build_config_map(config) + + def get_available_build_targets(self): + """Return a list of available build targets.""" + return sorted(self._build_config_map.keys()) + + def get_tags(self, build_target): + """Given a build_target, return the (possibly empty) set of tags.""" + return self._build_config_map[build_target].tags + + def has_tag(self, build_target, tag): + """Return true if build_target has tag. + + Args: + build_target: A string build_target to be queried. + tag: A string tag that this target may have. + + Returns: + If the build_target has the tag, True. Otherwise, False. + """ + return tag in self._build_config_map[build_target].tags + + def get_allowed_projects_file(self, build_target): + """Given a build_target, return a string with the allowed projects file.""" + return self._build_config_map[build_target].allowed_projects_file + + def get_build_config_android_target(self, build_target): + """Given a build_target, return an android_target. + + Generally a build_target maps directory to the android_target of the same + name, but they can differ. In a config.xml file, the name attribute of a + target element is the android_target (which is used for lunch). The name + attribute (if any) of a build_config element is the build_target. If a + build_config element does not have a name attribute, then the build_target + is the android_target. + + Args: + build_target: A string build_target to be queried. + + Returns: + A string android_target that can be used for lunch. + """ + return self._build_config_map[build_target].android_target + + def get_build_goals(self, build_target, contexts=frozenset()): + """Given a build_target and a context, return a list of build goals. + + For a given build_target, we may build in a variety of contexts. For + example we might build in continuous integration, or we might build + locally, or other contexts defined by the configuration file and scripts + that use it. The contexts parameter is a set of strings that specify the + contexts for which this function should retrieve goals. + + In the configuration file, each goal has a contexts attribute, which + specifies the contexts to which the goal applies. We treat a goal with no + contexts attribute as applying to all contexts. + + Example: + + <build_config> + <goal name="droid"/> + <goal name="dist" contexts="ota"/> + </build_config> + + Here we have the goal "droid", which matches all contexts, and the goal + "dist", which matches the "ota" context. Invoking this method with the + set(['ota']) would return ['droid', 'dist']. + + Args: + build_target: A string build_target to be queried. + context: A set of contexts for which to retrieve goals. + + Returns: + A list of strings, where each string is a goal to be passed to make. + """ + + build_goals = [] + for goal, build_contexts in self._build_config_map[ + build_target].build_goals: + if not build_contexts: + build_goals.append(goal) + elif build_contexts.intersection(contexts): + build_goals.append(goal) + + return build_goals + + def get_rw_allowlist_map(self): + """Return read-write allowlist map. + + Returns: + A dict of string lists of keyed by target name. Each value in the dict is + a list of allowed read-write paths corresponding to the target. + """ + return {b.name: b.allow_readwrite for b in self._build_config_map.values()} + + def get_allow_readwrite_all(self, build_target): + """Return True if the target should mount all its source as read-write. + + Args: + build_target: A string build_target to be queried. + + Returns: + True if the target should mount all its source as read-write. + """ + return self._build_config_map[build_target].allow_readwrite_all + + def get_overlay_map(self): + """Return the overlay map. + + Returns: + A dict of keyed by target name. Each value in the dict is a list of + overlay names corresponding to the target. + """ + return { + b.name : [o.name for o in b.overlays + ] for b in self._build_config_map.values() + } + + + def get_fs_view_map(self): + """Return the filesystem view map. + Returns: + A dict of filesystem views keyed by target name. A filesystem view is a + list of (source, destination) string path tuples. + """ + return {b.name : b.views for b in self._build_config_map.values()} + + + def get_build_config(self, build_target): + return self._build_config_map[build_target] + + +def factory(config_filename): + """Create an instance of a Config class. + + Args: + config_filename: The name of the file from which to load the config. This + can be None, which results in this function returning None. + + Returns: + If config_filename is None, returns None. Otherwise, a new instance of a + Config class containing the configuration parsed from config_filename. + """ + if config_filename is None: + return None + + return Config(config_filename) diff --git a/build/sandbox/config_test.py b/build/sandbox/config_test.py new file mode 100644 index 0000000..002c625 --- /dev/null +++ b/build/sandbox/config_test.py @@ -0,0 +1,307 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test config.""" + +import tempfile +import unittest +from . import config + +_TEST_CONFIG_XML = """<config> + <target name="android_target_1"> + <build_config> + <goal name="droid"/> + <goal name="dist"/> + </build_config> + </target> + <target name="android_target_2" tags="cool,hot"> + <config name="fmc_framework_images" value="image1,image2"/> + <config name="fmc_misc_info_keys" value="misc_info_keys.txt"/> + <goal name="common_goal"/> + <build_config tags="warm"> + <goal name="droid"/> + <goal name="dist"/> + <goal name="goal_for_android_target_2"/> + </build_config> + <build_config name="build_target_2" tags="dry"> + <config name="fmc_framework_images" value="bt1,bt2"/> + <config name="fmc_misc_info_keys" value="misc_info_keys_2.txt"/> + <goal name="droid"/> + <goal name="VAR=a"/> + </build_config> + </target> + <target name="android_target_3" tags=""> + <build_config> + <goal name="droid"/> + </build_config> + </target> + <target name="some_target" android_target="android_target_4"> + <goal name="droid"/> + <build_config> + </build_config> + </target> + <target name="no_goals_target"> + <build_config/> + </target> +</config> +""" + +_TEST_CONTEXTS_CONFIG_XML = """<config> + <target name="test_target"> + <build_config> + + <!-- no contexts attribute: always selected --> + <goal name="droid"/> + + <!-- empty contexts attribute: always selected --> + <goal name="always" contexts=""/> + + <!-- selected if ci context requested --> + <goal name="dist" contexts="ci"/> + + <!-- selected if x context requested --> + <goal name="VAR=value" contexts="x"/> + + <!-- selected if ci or x context requested --> + <goal name="extra_goal" contexts="ci,x"/> + + </build_config> + </target> +</config> +""" + +class ConfigTest(unittest.TestCase): + """unittest for Config.""" + + def testConfigFilenameNone(self): + cfg = config.factory(None) + self.assertIsNone(cfg) + + def testAvailableBuildTargets(self): + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write(_TEST_CONFIG_XML) + test_config.flush() + cfg = config.factory(test_config.name) + self.assertListEqual( + cfg.get_available_build_targets(), + # Sorted; not in document order. + [ + 'android_target_1', + 'android_target_2', + 'android_target_3', + 'build_target_2', + 'no_goals_target', + 'some_target', + ]) + + def testBuildTargetTags(self): + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write(_TEST_CONFIG_XML) + test_config.flush() + cfg = config.factory(test_config.name) + + self.assertSetEqual(cfg.get_tags('android_target_1'), set()) + self.assertSetEqual( + cfg.get_tags('android_target_2'), set(['cool', 'hot', 'warm'])) + self.assertSetEqual( + cfg.get_tags('build_target_2'), set(['cool', 'hot', 'dry'])) + self.assertSetEqual(cfg.get_tags('android_target_3'), set()) + + self.assertFalse(cfg.has_tag('android_target_1', 'cool')) + self.assertFalse(cfg.has_tag('android_target_1', 'hot')) + self.assertFalse(cfg.has_tag('android_target_1', 'meh')) + + self.assertTrue(cfg.has_tag('android_target_2', 'cool')) + self.assertTrue(cfg.has_tag('android_target_2', 'hot')) + self.assertFalse(cfg.has_tag('android_target_2', 'meh')) + + self.assertTrue(cfg.has_tag('build_target_2', 'cool')) + self.assertTrue(cfg.has_tag('build_target_2', 'hot')) + self.assertFalse(cfg.has_tag('build_target_2', 'meh')) + + self.assertFalse(cfg.has_tag('android_target_3', 'cool')) + self.assertFalse(cfg.has_tag('android_target_3', 'hot')) + self.assertFalse(cfg.has_tag('android_target_3', 'meh')) + + def testBuildTargetToAndroidTarget(self): + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write(_TEST_CONFIG_XML) + test_config.flush() + cfg = config.factory(test_config.name) + + # Test that build_target android_target_1 -> android_target_1. + self.assertEqual( + cfg.get_build_config_android_target('android_target_1'), + 'android_target_1') + + # Test that build_target android_target_2 -> android_target_2. + self.assertEqual( + cfg.get_build_config_android_target('android_target_2'), + 'android_target_2') + + # Test that build_target build_target_2 -> android_target_2. + self.assertEqual( + cfg.get_build_config_android_target('build_target_2'), + 'android_target_2') + + # Test overriding android_target property + self.assertEqual( + cfg.get_build_config_android_target('some_target'), + 'android_target_4') + + def testBuildTargetToBuildGoals(self): + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write(_TEST_CONFIG_XML) + test_config.flush() + cfg = config.factory(test_config.name) + + # Test that build_target android_target_1 has goals droid and dist. + self.assertEqual( + cfg.get_build_goals('android_target_1'), + ['droid', 'dist']) + + # Test that build_target android_target_2 has goals droid, dist, and + # goal_for_android_target_2. + self.assertEqual( + cfg.get_build_goals('android_target_2'), + ['common_goal', 'droid', 'dist', 'goal_for_android_target_2']) + + # Test that build_target build_target_2 has goals droid and VAR=a. + self.assertEqual( + cfg.get_build_goals('build_target_2'), + ['common_goal', 'droid', 'VAR=a']) + + # Test empty goals + self.assertEqual(cfg.get_build_goals('no_goals_target'),[]) + + def testBuildTargetToBuildGoalsWithContexts(self): + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write(_TEST_CONTEXTS_CONFIG_XML) + test_config.flush() + cfg = config.factory(test_config.name) + + # Test that when contexts is the default (empty), we select only the + # "always" goals. + + build_goals = cfg.get_build_goals('test_target') + self.assertEqual(build_goals, ['droid', 'always']) + + # Test that when contexts is explicitly empty, we select only the + # "always" goals. + + build_goals = cfg.get_build_goals('test_target', set()) + self.assertEqual(build_goals, ['droid', 'always']) + + # Similarly, test that when contexts is doesn't match any goal_contexts, + # we select only the "always" goals. + + build_goals = cfg.get_build_goals('test_target', set('no_matchy')) + self.assertEqual(build_goals, ['droid', 'always']) + + # Test that when contexts is set(['x']), we select the "always" goals and + # the x goals. + + build_goals = cfg.get_build_goals('test_target', set(['x'])) + + self.assertEqual( + build_goals, + ['droid', 'always', 'VAR=value', 'extra_goal']) + + # Test that when requested_contexts is set(['ci', 'x']), we select the + # "always" goals, the ci goals, and the x goals. + + build_goals = cfg.get_build_goals('test_target', set(['ci', 'x'])) + + self.assertEqual( + build_goals, + ['droid', 'always', 'dist', 'VAR=value', 'extra_goal']) + + def testAllowReadWriteAll(self): + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write("""<?xml version="1.0" encoding="UTF-8" ?> + <config> + <target name="target_allowed" allow_readwrite_all="true"> + <build_config> + <goal name="droid"/> + </build_config> + <allow_readwrite_all/> + </target> + <target name="target_not_allowed"> + <build_config> + <goal name="droid"/> + </build_config> + </target> + <target name="target_also_not_allowed" allow_readwrite_all="false"> + <build_config> + <goal name="droid"/> + </build_config> + </target> + </config>""") + test_config.flush() + cfg = config.factory(test_config.name) + + self.assertTrue(cfg.get_allow_readwrite_all('target_allowed')) + + self.assertFalse(cfg.get_allow_readwrite_all('target_not_allowed')) + + self.assertFalse(cfg.get_allow_readwrite_all('target_also_not_allowed')) + + def testAllowedProjectsFile(self): + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write( + '<?xml version="1.0" encoding="UTF-8" ?>' + '<config>' + ' <target name="target_name">' + ' <build_config allowed_projects_file="path/to/default/build/config/allowed_projects.xml">' + ' <goal name="build_goal"/>' + ' </build_config>' + ' <build_config name="has_allowed_projects_file" allowed_projects_file="path/to/named/build/config/allowed_projects.xml">' + ' <goal name="build_goal"/>' + ' </build_config>' + ' <build_config name="no_allowed_projects_file">' + ' <goal name="build_goal"/>' + ' </build_config>' + ' </target>' + '</config>') + test_config.flush() + cfg = config.factory(test_config.name) + + self.assertEqual( + cfg.get_allowed_projects_file('target_name'), + 'path/to/default/build/config/allowed_projects.xml') + self.assertEqual( + cfg.get_allowed_projects_file('has_allowed_projects_file'), + 'path/to/named/build/config/allowed_projects.xml') + self.assertIsNone( + cfg.get_allowed_projects_file('no_allowed_projects_file')) + + def testMergeConfig(self): + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write(_TEST_CONFIG_XML) + test_config.flush() + cfg = config.factory(test_config.name) + + bc_at2 = cfg.get_build_config('android_target_2') + self.assertDictEqual(bc_at2.configurations, { + 'fmc_framework_images': 'image1,image2', + 'fmc_misc_info_keys': 'misc_info_keys.txt' + }) + + bc_bt2 = cfg.get_build_config('build_target_2') + self.assertDictEqual(bc_bt2.configurations, { + 'fmc_framework_images': 'bt1,bt2', + 'fmc_misc_info_keys': 'misc_info_keys_2.txt' + }) + +if __name__ == '__main__': + unittest.main() diff --git a/build/sandbox/nsjail.cfg b/build/sandbox/nsjail.cfg index 03ac8ca..509ecc6 100644 --- a/build/sandbox/nsjail.cfg +++ b/build/sandbox/nsjail.cfg @@ -74,6 +74,15 @@ mount { is_bind: false } +# Some tools need /dev/shm to created a named semaphore. Use a new tmpfs to +# limit access to the external environment. +mount { + dst: "/dev/shm" + fstype: "tmpfs" + rw: true + is_bind: false +} + # Map the working User ID to a username # Some tools like Java need a valid username mount { diff --git a/build/sandbox/nsjail.py b/build/sandbox/nsjail.py index 1d2872e..c388d0b 100644 --- a/build/sandbox/nsjail.py +++ b/build/sandbox/nsjail.py @@ -25,6 +25,7 @@ import collections import os import re import subprocess +from . import config from .overlay import BindMount from .overlay import BindOverlay @@ -38,53 +39,19 @@ _META_MOUNT_POINT = '/meta' _CHROOT_MOUNT_POINTS = [ 'bin', 'sbin', - 'etc/alternatives', 'etc/default' 'etc/perl', + 'etc/alternatives', 'etc/default', 'etc/perl', 'etc/ssl', 'etc/xml', 'lib', 'lib32', 'lib64', 'libx32', 'usr', ] -def load_rw_whitelist(rw_whitelist_config): - """Loads a read/write whitelist configuration file. - - The read/write whitelist configuration file is a text file that contains a - list of source_dir relative paths which should be mounted read/write inside - the build sandbox. Empty lines and lines begnning with a comment marker ('#') - will be ignored. An empty whitelist implies that all source paths are mounted - read-only. An empty rw_whitelist_config argument implies that all source - paths are mounted read/write. - - Args: - rw_whitelist_config: A string path to a read/write whitelist file. - - Returns: - A set of whitelist path strings. - """ - if not rw_whitelist_config: - return None - - if not os.path.exists(rw_whitelist_config): - return None - - ret = set() - with open(rw_whitelist_config, 'r') as f: - for p in f.read().splitlines(): - p = p.strip() - if not p or p.startswith('#'): - continue - ret.add(p) - - return ret - def run(command, - android_target, + build_target, nsjail_bin, chroot, overlay_config=None, - rw_whitelist_config=None, source_dir=os.getcwd(), - out_dirname_for_whiteout=None, dist_dir=None, build_id=None, out_dir = None, @@ -98,22 +65,20 @@ def run(command, dry_run=False, quiet=False, env=[], + nsjail_wrapper=[], stdout=None, - stderr=None): + stderr=None, + allow_network=False): """Run inside an NsJail sandbox. Args: command: A list of strings with the command to run. - android_target: A string with the name of the target to be prepared + build_target: A string with the name of the build target to be prepared inside the container. nsjail_bin: A string with the path to the nsjail binary. chroot: A string with the path to the chroot. overlay_config: A string path to an overlay configuration file. - rw_whitelist_config: A string path to a read/write whitelist configuration file. source_dir: A string with the path to the Android platform source. - out_dirname_for_whiteout: The optional name of the folder within - source_dir that is the Android build out folder *as seen from outside - the Docker container*. dist_dir: A string with the path to the dist directory. build_id: A string with the build identifier. out_dir: An optional path to the Android build out folder. @@ -130,11 +95,13 @@ def run(command, quiet: If true, the function will not display the command and will pass -quiet argument to nsjail env: An array of environment variables to define in the jail in the `var=val` syntax. + nsjail_wrapper: A list of strings used to wrap the nsjail command. stdout: the standard output for all printed messages. Valid values are None, a file descriptor or file object. A None value means sys.stdout is used. stderr: the standard error for all printed messages. Valid values are None, a file descriptor or file object, and subprocess.STDOUT (which indicates that all stderr should be redirected to stdout). A None value means sys.stderr is used. + allow_network: allow access to host network Returns: A list of strings with the command executed. @@ -143,13 +110,11 @@ def run(command, nsjail_command = get_command( command=command, - android_target=android_target, + build_target=build_target, nsjail_bin=nsjail_bin, chroot=chroot, - overlay_config=overlay_config, - rw_whitelist_config=rw_whitelist_config, + cfg=config.factory(overlay_config), source_dir=source_dir, - out_dirname_for_whiteout=out_dirname_for_whiteout, dist_dir=dist_dir, build_id=build_id, out_dir=out_dir, @@ -161,7 +126,9 @@ def run(command, readonly_bind_mounts=readonly_bind_mounts, extra_nsjail_args=extra_nsjail_args, quiet=quiet, - env=env) + env=env, + nsjail_wrapper=nsjail_wrapper, + allow_network=allow_network) run_command( nsjail_command=nsjail_command, @@ -174,13 +141,11 @@ def run(command, return nsjail_command def get_command(command, - android_target, + build_target, nsjail_bin, chroot, - overlay_config=None, - rw_whitelist_config=None, + cfg=None, source_dir=os.getcwd(), - out_dirname_for_whiteout=None, dist_dir=None, build_id=None, out_dir = None, @@ -192,21 +157,19 @@ def get_command(command, readonly_bind_mounts=[], extra_nsjail_args=[], quiet=False, - env=[]): + env=[], + nsjail_wrapper=[], + allow_network=False): """Get command to run nsjail sandbox. Args: command: A list of strings with the command to run. - android_target: A string with the name of the target to be prepared + build_target: A string with the name of the build target to be prepared inside the container. nsjail_bin: A string with the path to the nsjail binary. chroot: A string with the path to the chroot. - overlay_config: A string path to an overlay configuration file. - rw_whitelist_config: A string path to a read/write whitelist configuration file. + cfg: A config.Config instance or None. source_dir: A string with the path to the Android platform source. - out_dirname_for_whiteout: The optional name of the folder within - source_dir that is the Android build out folder *as seen from outside - the Docker container*. dist_dir: A string with the path to the dist directory. build_id: A string with the build identifier. out_dir: An optional path to the Android build out folder. @@ -220,6 +183,7 @@ def get_command(command, quiet: If true, the function will not display the command and will pass -quiet argument to nsjail env: An array of environment variables to define in the jail in the `var=val` syntax. + allow_network: allow access to host network Returns: A list of strings with the command to execute. @@ -248,7 +212,7 @@ def get_command(command, raise ValueError('error: the provided meta_android_dir is not a path' 'relative to meta_root_dir.') - nsjail_command = [nsjail_bin, + nsjail_command = nsjail_wrapper + [nsjail_bin, '--env', 'USER=nobody', '--config', config_file] @@ -271,8 +235,6 @@ def get_command(command, nsjail_command.append('--quiet') whiteout_list = set() - if out_dirname_for_whiteout: - whiteout_list.add(os.path.join(source_dir, out_dirname_for_whiteout)) if out_dir and ( os.path.dirname(out_dir) == source_dir) and ( os.path.basename(out_dir) != 'out'): @@ -280,36 +242,34 @@ def get_command(command, if not os.path.exists(out_dir): os.makedirs(out_dir) - rw_whitelist = load_rw_whitelist(rw_whitelist_config) - - # Apply the overlay for the selected Android target to the source - # directory if an overlay configuration was provided - if overlay_config and os.path.exists(overlay_config): - overlay = BindOverlay(android_target, + # Apply the overlay for the selected Android target to the source directory + # from the supplied config.Config instance (which may be None). + if cfg is not None: + overlay = BindOverlay(build_target, source_dir, - overlay_config, + cfg, whiteout_list, _SOURCE_MOUNT_POINT, - rw_whitelist) + quiet=quiet) bind_mounts = overlay.GetBindMounts() else: bind_mounts = collections.OrderedDict() - bind_mounts[_SOURCE_MOUNT_POINT] = BindMount(source_dir, False) + bind_mounts[_SOURCE_MOUNT_POINT] = BindMount(source_dir, False, False) if out_dir: - bind_mounts[_OUT_MOUNT_POINT] = BindMount(out_dir, False) + bind_mounts[_OUT_MOUNT_POINT] = BindMount(out_dir, False, False) if dist_dir: - bind_mounts[_DIST_MOUNT_POINT] = BindMount(dist_dir, False) + bind_mounts[_DIST_MOUNT_POINT] = BindMount(dist_dir, False, False) nsjail_command.extend([ '--env', 'DIST_DIR=%s'%_DIST_MOUNT_POINT ]) if meta_root_dir: - bind_mounts[_META_MOUNT_POINT] = BindMount(meta_root_dir, False) - bind_mounts[os.path.join(_META_MOUNT_POINT, meta_android_dir)] = BindMount(source_dir, False) + bind_mounts[_META_MOUNT_POINT] = BindMount(meta_root_dir, False, False) + bind_mounts[os.path.join(_META_MOUNT_POINT, meta_android_dir)] = BindMount(source_dir, False, False) if out_dir: - bind_mounts[os.path.join(_META_MOUNT_POINT, meta_android_dir, 'out')] = BindMount(out_dir, False) + bind_mounts[os.path.join(_META_MOUNT_POINT, meta_android_dir, 'out')] = BindMount(out_dir, False, False) for bind_destination, bind_mount in bind_mounts.items(): if bind_mount.readonly: @@ -340,6 +300,11 @@ def get_command(command, for var in env: nsjail_command.extend(['--env', var]) + if allow_network: + nsjail_command.extend(['--disable_clone_newnet', + '--bindmount_ro', + '/etc/resolv.conf']) + nsjail_command.extend(extra_nsjail_args) nsjail_command.append('--') @@ -411,9 +376,6 @@ def parse_args(): '--overlay_config', help='Path to the overlay configuration file.') parser.add_argument( - '--rw_whitelist_config', - help='Path to the read/write whitelist configuration file.') - parser.add_argument( '--source_dir', default=os.getcwd(), help='Path to Android platform source to be mounted as /src.') @@ -432,24 +394,12 @@ def parse_args(): 'the Android build. This path must be relative to meta_root_dir. ' 'Defaults to \'%s\'' % _DEFAULT_META_ANDROID_DIR) parser.add_argument( - '--out_dirname_for_whiteout', - help='The optional name of the folder within source_dir that is the ' - 'Android build out folder *as seen from outside the Docker ' - 'container*.') - parser.add_argument( - '--whiteout', - action='append', - default=[], - help='Optional glob filter of directories to add to the whiteout. The ' - 'directories will not appear in the container. ' - 'Can be specified multiple times.') - parser.add_argument( '--command', default=_DEFAULT_COMMAND, help='Command to run after entering the NsJail.' 'If not set then an interactive Bash shell will be launched') parser.add_argument( - '--android_target', + '--build_target', required=True, help='Android target selected for building') parser.add_argument( @@ -502,6 +452,10 @@ def parse_args(): action='append', help='Specify an environment variable to the NSJail sandbox. Can be specified ' 'muliple times. Syntax: var_name=value') + parser.add_argument( + '--allow_network', action='store_true', + help='If provided, allow access to the host network. WARNING: Using this ' + 'flag exposes the network inside jail. Use only when needed.') return parser.parse_args() def run_with_args(args): @@ -518,11 +472,9 @@ def run_with_args(args): run(chroot=args.chroot, nsjail_bin=args.nsjail_bin, overlay_config=args.overlay_config, - rw_whitelist_config=args.rw_whitelist_config, source_dir=args.source_dir, command=args.command.split(), - android_target=args.android_target, - out_dirname_for_whiteout=args.out_dirname_for_whiteout, + build_target=args.build_target, dist_dir=args.dist_dir, build_id=args.build_id, out_dir=args.out_dir, @@ -534,7 +486,8 @@ def run_with_args(args): readonly_bind_mounts=args.bindmount_ro, dry_run=args.dry_run, quiet=args.quiet, - env=args.env) + env=args.env, + allow_network=args.allow_network) def main(): run_with_args(parse_args()) diff --git a/build/sandbox/nsjail_test.py b/build/sandbox/nsjail_test.py index cfe4303..a73bbdb 100644 --- a/build/sandbox/nsjail_test.py +++ b/build/sandbox/nsjail_test.py @@ -32,7 +32,7 @@ class NsjailTest(unittest.TestCase): chroot='/chroot', source_dir='/source_dir', command=['/bin/bash'], - android_target='target_name', + build_target='target_name', dry_run=True) self.assertEqual( commands, @@ -53,7 +53,7 @@ class NsjailTest(unittest.TestCase): chroot='/chroot', source_dir='/source_dir', command=['/bin/bash'], - android_target='target_name', + build_target='target_name', dry_run=True, meta_root_dir='/meta/dir', meta_android_dir='/android/dir') @@ -65,7 +65,7 @@ class NsjailTest(unittest.TestCase): chroot='/chroot', source_dir='/source_dir', command=['/bin/bash'], - android_target='target_name', + build_target='target_name', stdout=out) out.seek(0) stdout = out.read() @@ -81,7 +81,7 @@ class NsjailTest(unittest.TestCase): chroot='/chroot', source_dir='/source_dir', command=['/bin/bash'], - android_target='target_name') + build_target='target_name') def testDist(self): commands = nsjail.run( @@ -89,7 +89,7 @@ class NsjailTest(unittest.TestCase): chroot='/chroot', source_dir='/source_dir', command=['/bin/bash'], - android_target='target_name', + build_target='target_name', dist_dir='/dist_dir', dry_run=True) self.assertEqual( @@ -111,7 +111,7 @@ class NsjailTest(unittest.TestCase): chroot='/chroot', source_dir='/source_dir', command=['/bin/bash'], - android_target='target_name', + build_target='target_name', build_id='0', dry_run=True) self.assertEqual( @@ -132,7 +132,7 @@ class NsjailTest(unittest.TestCase): chroot='/chroot', source_dir='/source_dir', command=['/bin/bash'], - android_target='target_name', + build_target='target_name', max_cpus=1, dry_run=True) self.assertEqual( @@ -153,7 +153,7 @@ class NsjailTest(unittest.TestCase): chroot='/chroot', source_dir='/source_dir', command=['/bin/bash'], - android_target='target_name', + build_target='target_name', max_cpus=1, dry_run=True, env=['foo=bar', 'spam=eggs']) diff --git a/build/sandbox/overlay.py b/build/sandbox/overlay.py index ad298a6..873817e 100644 --- a/build/sandbox/overlay.py +++ b/build/sandbox/overlay.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Mounts all the projects required by a selected Android target. +"""Mounts all the projects required by a selected Build target. For details on how filesystem overlays work see the filesystem overlays section of the README.md. @@ -27,8 +27,10 @@ import os import subprocess import tempfile import xml.etree.ElementTree as ET +from . import config -BindMount = collections.namedtuple('BindMount', ['source_dir', 'readonly']) +BindMount = collections.namedtuple( + 'BindMount', ['source_dir', 'readonly', 'allows_replacement']) class BindOverlay(object): @@ -57,11 +59,14 @@ class BindOverlay(object): path: A string path to be checked. Returns: - A string of the conflicting path in the bind mounts. - None if there was no conflict found. + A tuple containing a string of the conflicting path in the bind mounts and + whether or not to allow this path to supersede any conflicts. + None, False if there was no conflict found. """ conflict_path = None + allows_replacement = False for bind_destination, bind_mount in self._bind_mounts.items(): + allows_replacement = bind_mount.allows_replacement # Check if the path is a subdir or the bind destination if path == bind_destination: conflict_path = bind_mount.source_dir @@ -75,21 +80,30 @@ class BindOverlay(object): conflict_path = path_in_source break - return conflict_path + return conflict_path, allows_replacement - def _AddOverlay(self, overlay_dir, intermediate_work_dir, skip_subdirs, - destination_dir, rw_whitelist): + def _AddOverlay(self, source_dir, overlay_dir, intermediate_work_dir, + skip_subdirs, allowed_projects, destination_dir, + allowed_read_write, contains_read_write, + is_replacement_allowed): """Adds a single overlay directory. Args: + source_dir: A string with the path to the Android platform source. overlay_dir: A string path to the overlay directory to apply. intermediate_work_dir: A string path to the intermediate work directory used as the base for constructing the overlay filesystem. skip_subdirs: A set of string paths to skip from overlaying. + allowed_projects: If not None, any .git project path not in this list + is excluded from overlaying. destination_dir: A string with the path to the source with the overlays applied to it. - rw_whitelist: An optional set of source paths to bind mount with - read/write access. + allowed_read_write: A function returns true if the path input should + be allowed read/write access. + contains_read_write: A function returns true if the path input contains + a sub-path that should be allowed read/write access. + is_replacement_allowed: A function returns true if the path can replace a + subsequent path. """ # Traverse the overlay directory twice # The first pass only process git projects @@ -110,15 +124,18 @@ class BindOverlay(object): current_dir_destination = os.path.normpath( os.path.join(destination_dir, current_dir_relative)) - if '.git' in subdirs: + if '.git' in subdirs or '.git' in files or '.bindmount' in files: # The current dir is a git project # so just bind mount it del subdirs[:] - if rw_whitelist is None or current_dir_origin in rw_whitelist: - self._AddBindMount(current_dir_origin, current_dir_destination, False) - else: - self._AddBindMount(current_dir_origin, current_dir_destination, True) + if '.bindmount' in files or (not allowed_projects or + os.path.relpath(current_dir_origin, source_dir) in allowed_projects): + self._AddBindMount( + current_dir_origin, current_dir_destination, + False if allowed_read_write(current_dir_origin) else True, + is_replacement_allowed( + os.path.basename(overlay_dir), current_dir_relative)) current_dir_ancestor = current_dir_origin while current_dir_ancestor and current_dir_ancestor not in dirs_with_git_projects: @@ -132,7 +149,7 @@ class BindOverlay(object): del subdirs[:] continue - if '.git' in subdirs: + if '.git' in subdirs or '.git' in files or '.bindmount' in files: del subdirs[:] continue @@ -140,37 +157,52 @@ class BindOverlay(object): current_dir_destination = os.path.normpath( os.path.join(destination_dir, current_dir_relative)) + bindCurrentDir = True + + # Directories with git projects can't be bind mounted + # because git projects are individually mounted if current_dir_origin in dirs_with_git_projects: - # Symbolic links to subdirectories - # have to be copied to the intermediate work directory. - # We can't bind mount them because bind mounts deference - # symbolic links, and the build system filters out any - # directory symbolic links. - for subdir in subdirs: - subdir_origin = os.path.join(current_dir_origin, subdir) - if os.path.islink(subdir_origin): - if subdir_origin not in skip_subdirs: - subdir_destination = os.path.join(intermediate_work_dir, - current_dir_relative, subdir) - self._CopyFile(subdir_origin, subdir_destination) - - # bind each file individually then keep travesting - for file in files: - file_origin = os.path.join(current_dir_origin, file) - file_destination = os.path.join(current_dir_destination, file) - if rw_whitelist is None or file_origin in rw_whitelist: - self._AddBindMount(file_origin, file_destination, False) - else: - self._AddBindMount(file_origin, file_destination, True) + bindCurrentDir = False - else: - # The current dir does not have any git projects to it can be bind - # mounted wholesale + # A directory that contains read-write paths should only + # ever be bind mounted if the directory itself is read-write + if contains_read_write(current_dir_origin) and not allowed_read_write(current_dir_origin): + bindCurrentDir = False + + if bindCurrentDir: + # The current dir can be bind mounted wholesale del subdirs[:] - if rw_whitelist is None or current_dir_origin in rw_whitelist: + if allowed_read_write(current_dir_origin): self._AddBindMount(current_dir_origin, current_dir_destination, False) else: self._AddBindMount(current_dir_origin, current_dir_destination, True) + continue + + # If we've made it this far then we're going to process + # each file and subdir individually + + for subdir in subdirs: + subdir_origin = os.path.join(current_dir_origin, subdir) + # Symbolic links to subdirectories + # have to be copied to the intermediate work directory. + # We can't bind mount them because bind mounts dereference + # symbolic links, and the build system filters out any + # directory symbolic links. + if os.path.islink(subdir_origin): + if subdir_origin not in skip_subdirs: + subdir_destination = os.path.join(intermediate_work_dir, + current_dir_relative, subdir) + self._CopyFile(subdir_origin, subdir_destination) + + # bind each file individually then keep traversing + for file in files: + file_origin = os.path.join(current_dir_origin, file) + file_destination = os.path.join(current_dir_destination, file) + if allowed_read_write(file_origin): + self._AddBindMount(file_origin, file_destination, False) + else: + self._AddBindMount(file_origin, file_destination, True) + def _AddArtifactDirectories(self, source_dir, destination_dir, skip_subdirs): """Add directories that were not synced as workspace source. @@ -203,13 +235,14 @@ class BindOverlay(object): if os.path.exists(repo_origin): repo_destination = os.path.normpath( os.path.join(destination_dir, '.repo')) - self._AddBindMount(repo_origin, repo_destination, False) + self._AddBindMount(repo_origin, repo_destination, True) skip_subdirs.add(repo_origin) return skip_subdirs def _AddOverlays(self, source_dir, overlay_dirs, destination_dir, - skip_subdirs, rw_whitelist): + skip_subdirs, allowed_projects, allowed_read_write, + contains_read_write, is_replacement_allowed): """Add the selected overlay directories. Args: @@ -219,8 +252,14 @@ class BindOverlay(object): destination_dir: A string with the path to the source where the overlays will be applied. skip_subdirs: A set of string paths to be skipped from overlays. - rw_whitelist: An optional set of source paths to bind mount with - read/write access. + allowed_projects: If not None, any .git project path not in this list + is excluded from overlaying. + allowed_read_write: A function returns true if the path input should + be allowed read/write access. + contains_read_write: A function returns true if the path input contains + a sub-path that should be allowed read/write access. + is_replacement_allowed: A function returns true if the path can replace a + subsequent path. """ # Create empty intermediate workdir @@ -235,7 +274,8 @@ class BindOverlay(object): # depth first traversal algorithm. # # The algorithm described works under the condition that the overlaid file - # systems do not have conflicting projects. + # systems do not have conflicting projects or that the conflict path is + # specifically called-out as a replacement path. # # The results of attempting to overlay two git projects on top # of each other are unpredictable and may push the limits of bind mounts. @@ -243,11 +283,17 @@ class BindOverlay(object): skip_subdirs.add(os.path.join(source_dir, 'overlays')) for overlay_dir in overlay_dirs: - self._AddOverlay(overlay_dir, intermediate_work_dir, - skip_subdirs, destination_dir, rw_whitelist) + self._AddOverlay(source_dir, overlay_dir, intermediate_work_dir, + skip_subdirs, allowed_projects, destination_dir, + allowed_read_write, contains_read_write, + is_replacement_allowed) - def _AddBindMount(self, source_dir, destination_dir, readonly=False): + def _AddBindMount(self, + source_dir, + destination_dir, + readonly=False, + allows_replacement=False): """Adds a bind mount for the specified directory. Args: @@ -258,18 +304,21 @@ class BindOverlay(object): it will be created. readonly: A flag to indicate whether this path should be bind mounted with read-only access. + allow_replacement: A flag to indicate whether this path is allowed to replace a + conflicting path. """ - conflict_path = self._FindBindMountConflict(destination_dir) - if conflict_path: + conflict_path, replacement = self._FindBindMountConflict(destination_dir) + if conflict_path and not replacement: raise ValueError("Project %s could not be overlaid at %s " "because it conflicts with %s" % (source_dir, destination_dir, conflict_path)) - - if len(self._bind_mounts) >= self.MAX_BIND_MOUNTS: - raise ValueError("Bind mount limit of %s reached" % self.MAX_BIND_MOUNTS) - - self._bind_mounts[destination_dir] = BindMount( - source_dir=source_dir, readonly=readonly) + elif not conflict_path: + if len(self._bind_mounts) >= self.MAX_BIND_MOUNTS: + raise ValueError("Bind mount limit of %s reached" % self.MAX_BIND_MOUNTS) + self._bind_mounts[destination_dir] = BindMount( + source_dir=source_dir, + readonly=readonly, + allows_replacement=allows_replacement) def _CopyFile(self, source_path, dest_path): """Copies a file to the specified destination. @@ -295,28 +344,116 @@ class BindOverlay(object): """ return self._bind_mounts + def _GetReadWriteFunction(self, build_config, source_dir): + """Returns a function that tells you how to mount a path. + + Args: + build_config: A config.BuildConfig instance of the build target to be + prepared. + source_dir: A string with the path to the Android platform source. + + Returns: + A function that takes a string path as an input and returns + True if the path should be mounted read-write or False if + the path should be mounted read-only. + """ + + # The read/write allowlist provides paths relative to the source dir. It + # needs to be updated with absolute paths to make lookup possible. + rw_allowlist = {os.path.join(source_dir, p) for p in build_config.allow_readwrite} + + def AllowReadWrite(path): + return build_config.allow_readwrite_all or path in rw_allowlist + + return AllowReadWrite + + def _GetContainsReadWriteFunction(self, build_config, source_dir): + """Returns a function that tells you if a directory contains a read-write dir + + Args: + build_config: A config.BuildConfig instance of the build target to be + prepared. + source_dir: A string with the path to the Android platform source. + + Returns: + A function that takes a string path as an input and returns + True if the path contains a read-write path + """ + + # Get all dirs with allowed read-write + # and all their ancestor directories + contains_rw = set() + for path in build_config.allow_readwrite: + while path not in ["", "/"]: + # The read/write allowlist provides paths relative to the source dir. It + # needs to be updated with absolute paths to make lookup possible. + contains_rw.add(os.path.join(source_dir, path)) + path = os.path.dirname(path) + + def ContainsReadWrite(path): + return build_config.allow_readwrite_all or path in contains_rw + + return ContainsReadWrite + + def _GetAllowedProjects(self, build_config): + """Returns a set of paths that are allowed to contain .git projects. + + Args: + build_config: A config.BuildConfig instance of the build target to be + prepared. + + Returns: + If the target has an allowed projects file: a set of paths. Any .git + project path not in this set should be excluded from overlaying. + Otherwise: None + """ + if not build_config.allowed_projects_file: + return None + allowed_projects = ET.parse(build_config.allowed_projects_file) + paths = set() + for child in allowed_projects.getroot().findall("project"): + paths.add(child.attrib.get("path", child.attrib["name"])) + return paths + + def _IsReplacementAllowedFunction(self, build_config): + """Returns a function to determin if a given path is replaceable. + + Args: + build_config: A config.BuildConfig instance of the build target to be + prepared. + + Returns: + A function that takes an overlay name and string path as input and + returns True if the path is replaceable. + """ + def is_replacement_allowed_func(overlay_name, path): + for overlay in build_config.overlays: + if overlay_name == overlay.name and path in overlay.replacement_paths: + return True + return False + + return is_replacement_allowed_func + def __init__(self, - target, + build_target, source_dir, - config_file, + cfg, whiteout_list = [], destination_dir=None, - rw_whitelist=None): + quiet=False): """Inits Overlay with the details of what is going to be overlaid. Args: - target: A string with the name of the target to be prepared. + build_target: A string with the name of the build target to be prepared. source_dir: A string with the path to the Android platform source. - config_file: A string path to the XML config file. + cfg: A config.Config instance. whiteout_list: A list of directories to hide from the build system. destination_dir: A string with the path where the overlay filesystem will be created. If none is provided, the overlay filesystem will be applied directly on top of source_dir. - rw_whitelist: An optional set of source paths to bind mount with - read/write access. If none is provided, all paths will be mounted with - read/write access. If the set is empty, all paths will be mounted - read-only. + quiet: A boolean that, when True, suppresses debug output. """ + self._quiet = quiet if not destination_dir: destination_dir = source_dir @@ -330,121 +467,35 @@ class BindOverlay(object): # seems appropriate skip_subdirs = set(whiteout_list) - # The read/write whitelist provids paths relative to the source dir. It - # needs to be updated with absolute paths to make lookup possible. - if rw_whitelist: - rw_whitelist = {os.path.join(source_dir, p) for p in rw_whitelist} + build_config = cfg.get_build_config(build_target) + + allowed_read_write = self._GetReadWriteFunction(build_config, source_dir) + contains_read_write = self._GetContainsReadWriteFunction(build_config, source_dir) + allowed_projects = self._GetAllowedProjects(build_config) + is_replacement_allowed = self._IsReplacementAllowedFunction(build_config) overlay_dirs = [] - overlay_map = get_overlay_map(config_file) - for overlay_dir in overlay_map[target]: - overlay_dir = os.path.join(source_dir, 'overlays', overlay_dir) + for overlay in build_config.overlays: + overlay_dir = os.path.join(source_dir, 'overlays', overlay.name) overlay_dirs.append(overlay_dir) self._AddOverlays( - source_dir, overlay_dirs, destination_dir, skip_subdirs, rw_whitelist) + source_dir, overlay_dirs, destination_dir, + skip_subdirs, allowed_projects, allowed_read_write, contains_read_write, + is_replacement_allowed) # If specified for this target, create a custom filesystem view - fs_view_map = get_fs_view_map(config_file) - if target in fs_view_map: - for path_relative_from, path_relative_to in fs_view_map[target]: - path_from = os.path.join(source_dir, path_relative_from) - if os.path.isfile(path_from) or os.path.isdir(path_from): - path_to = os.path.join(destination_dir, path_relative_to) - if rw_whitelist is None or path_from in rw_whitelist: - self._AddBindMount(path_from, path_to, False) - else: - self._AddBindMount(path_from, path_to, True) + for path_relative_from, path_relative_to in build_config.views: + path_from = os.path.join(source_dir, path_relative_from) + if os.path.isfile(path_from) or os.path.isdir(path_from): + path_to = os.path.join(destination_dir, path_relative_to) + if allowed_read_write(path_from): + self._AddBindMount(path_from, path_to, False) else: - raise ValueError("Path '%s' must be a file or directory" % path_from) + self._AddBindMount(path_from, path_to, True) + else: + raise ValueError("Path '%s' must be a file or directory" % path_from) self._overlay_dirs = overlay_dirs - print('Applied overlays ' + ' '.join(self._overlay_dirs)) - - def __del__(self): - """Cleans up Overlay. - """ - if self._overlay_dirs: - print('Stripped out overlay ' + ' '.join(self._overlay_dirs)) - -def get_config(config_file): - """Parses the overlay configuration file. - - Args: - config_file: A string path to the XML config file. - - Returns: - A root config XML Element. - None if there is no config file. - """ - config = None - if os.path.exists(config_file): - tree = ET.parse(config_file) - config = tree.getroot() - return config - -def get_overlay_map(config_file): - """Retrieves the map of overlays for each target. - - Args: - config_file: A string path to the XML config file. - - Returns: - A dict of keyed by target name. Each value in the - dict is a list of overlay names corresponding to - the target. - """ - overlay_map = {} - config = get_config(config_file) - # The presence of the config file is optional - if config: - for target in config.findall('target'): - name = target.get('name') - overlay_list = [o.get('name') for o in target.findall('overlay')] - overlay_map[name] = overlay_list - # A valid configuration file is required - # to have at least one overlay target - if not overlay_map: - raise ValueError('Error: the overlay configuration file ' - 'is missing at least one overlay target') - - return overlay_map - -def get_fs_view_map(config_file): - """Retrieves the map of filesystem views for each target. - - Args: - config_file: A string path to the XML config file. - - Returns: - A dict of filesystem views keyed by target name. - A filesystem view is a list of (source, destination) - string path tuples. - """ - fs_view_map = {} - config = get_config(config_file) - - # The presence of the config file is optional - if config: - # A valid config file is not required to - # include FS Views, only overlay targets - views = {} - for view in config.findall('view'): - name = view.get('name') - paths = [] - for path in view.findall('path'): - paths.append(( - path.get('source'), - path.get('destination'))) - views[name] = paths - - for target in config.findall('target'): - target_name = target.get('name') - view_paths = [] - for view in target.findall('view'): - view_paths.extend(views[view.get('name')]) - - if view_paths: - fs_view_map[target_name] = view_paths - - return fs_view_map + if not self._quiet: + print('Applied overlays ' + ' '.join(self._overlay_dirs)) diff --git a/build/sandbox/overlay_test.py b/build/sandbox/overlay_test.py index 8e93cf7..088dda6 100644 --- a/build/sandbox/overlay_test.py +++ b/build/sandbox/overlay_test.py @@ -23,6 +23,7 @@ import shutil import subprocess import tempfile import unittest +from . import config from . import overlay import re @@ -32,9 +33,44 @@ class BindOverlayTest(unittest.TestCase): def setUp(self): self.source_dir = tempfile.mkdtemp() self.destination_dir = tempfile.mkdtemp() + # + # base_dir/ + # base_project/ + # .git + # no_git_dir/ + # no_git_subdir1/ + # no_git_file1 + # no_git_subdir2/ + # no_git_file2 + # overlays/ + # unittest1/ + # from_dir/ + # .git/ + # upper_subdir/ + # lower_subdir/ + # from_unittest1/ + # .git/ + # from_file + # unittest2/ + # upper_subdir/ + # lower_subdir/ + # from_unittest2/ + # .git/ + # no_git_dir2/ + # no_git_subdir1/ + # no_git_subdir2/ + # .bindmount + # os.mkdir(os.path.join(self.source_dir, 'base_dir')) os.mkdir(os.path.join(self.source_dir, 'base_dir', 'base_project')) os.mkdir(os.path.join(self.source_dir, 'base_dir', 'base_project', '.git')) + os.mkdir(os.path.join(self.source_dir, 'no_git_dir')) + os.mkdir(os.path.join(self.source_dir, 'no_git_dir', 'no_git_subdir1')) + open(os.path.join(self.source_dir, + 'no_git_dir', 'no_git_subdir1', 'no_git_file1'), 'a').close() + os.mkdir(os.path.join(self.source_dir, 'no_git_dir', 'no_git_subdir2')) + open(os.path.join(self.source_dir, + 'no_git_dir', 'no_git_subdir2', 'no_git_file2'), 'a').close() os.mkdir(os.path.join(self.source_dir, 'overlays')) os.mkdir(os.path.join(self.source_dir, 'overlays', 'unittest1')) @@ -75,6 +111,15 @@ class BindOverlayTest(unittest.TestCase): 'overlays', 'unittest2', 'upper_subdir', 'lower_subdir', 'from_unittest2', '.git')) + os.mkdir(os.path.join(self.source_dir, 'overlays', 'no_git_dir2')) + os.mkdir(os.path.join(self.source_dir, + 'overlays', 'no_git_dir2', 'no_git_subdir1')) + os.mkdir(os.path.join(self.source_dir, + 'overlays', 'no_git_dir2', 'no_git_subdir2')) + open(os.path.join(self.source_dir, + 'overlays', 'no_git_dir2', 'no_git_subdir2', '.bindmount'), + 'a').close() + def tearDown(self): shutil.rmtree(self.source_dir) @@ -85,19 +130,54 @@ class BindOverlayTest(unittest.TestCase): '<config>' ' <target name="unittest">' ' <overlay name="unittest1"/>' + ' <build_config>' + ' <goal name="goal_name"/>' + ' </build_config>' ' </target>' '</config>' ) test_config.flush() o = overlay.BindOverlay( - config_file=test_config.name, - target='unittest', + cfg=config.factory(test_config.name), + build_target='unittest', source_dir=self.source_dir) self.assertIsNotNone(o) bind_mounts = o.GetBindMounts() bind_source = os.path.join(self.source_dir, 'overlays/unittest1/from_dir') bind_destination = os.path.join(self.source_dir, 'from_dir') - self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, False)) + self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, True, False)) + self.assertIn(os.path.join(self.source_dir, 'base_dir', 'base_project'), bind_mounts) + + def testValidTargetOverlayBindsAllowedProjects(self): + with tempfile.NamedTemporaryFile('w+t') as test_config, \ + tempfile.NamedTemporaryFile('w+t') as test_allowed_projects: + test_config.write( + '<?xml version="1.0" encoding="UTF-8" ?>' + '<config>' + ' <target name="unittest">' + ' <overlay name="unittest1"/>' + ' <build_config allowed_projects_file="%s">' + ' <goal name="goal_name"/>' + ' </build_config>' + ' </target>' + '</config>' % test_allowed_projects.name + ) + test_config.flush() + test_allowed_projects.write( + '<?xml version="1.0" encoding="UTF-8" ?>' + '<manifest>' + ' <project name="from_dir" path="overlays/unittest1/from_dir"/>' + '</manifest>' + ) + test_allowed_projects.flush() + o = overlay.BindOverlay( + cfg=config.factory(test_config.name), + build_target='unittest', + source_dir=self.source_dir) + self.assertIsNotNone(o) + bind_mounts = o.GetBindMounts() + self.assertIn(os.path.join(self.source_dir, 'from_dir'), bind_mounts) + self.assertNotIn(os.path.join(self.source_dir, 'base_dir', 'base_project'), bind_mounts) def testMultipleOverlays(self): with tempfile.NamedTemporaryFile('w+t') as test_config: @@ -107,27 +187,30 @@ class BindOverlayTest(unittest.TestCase): ' <target name="unittest">' ' <overlay name="unittest1"/>' ' <overlay name="unittest2"/>' + ' <build_config>' + ' <goal name="goal_name"/>' + ' </build_config>' ' </target>' '</config>' ) test_config.flush() o = overlay.BindOverlay( - config_file=test_config.name, - target='unittest', + cfg=config.factory(test_config.name), + build_target='unittest', source_dir=self.source_dir) self.assertIsNotNone(o) bind_mounts = o.GetBindMounts() bind_source = os.path.join(self.source_dir, 'overlays/unittest1/upper_subdir/lower_subdir/from_unittest1') bind_destination = os.path.join(self.source_dir, 'upper_subdir/lower_subdir/from_unittest1') - self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, False)) + self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, True, False)) bind_source = os.path.join(self.source_dir, 'overlays/unittest2/upper_subdir/lower_subdir/from_unittest2') bind_destination = os.path.join(self.source_dir, 'upper_subdir/lower_subdir/from_unittest2') - self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, False)) + self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, True, False)) - def testMultipleOverlaysWithWhitelist(self): + def testMultipleOverlaysWithAllowlist(self): with tempfile.NamedTemporaryFile('w+t') as test_config: test_config.write( '<?xml version="1.0" encoding="UTF-8" ?>' @@ -135,14 +218,17 @@ class BindOverlayTest(unittest.TestCase): ' <target name="unittest">' ' <overlay name="unittest1"/>' ' <overlay name="unittest2"/>' + ' <allow_readwrite path="overlays/unittest1/upper_subdir/lower_subdir/from_unittest1"/>' + ' <build_config>' + ' <goal name="goal_name"/>' + ' </build_config>' ' </target>' '</config>' ) test_config.flush() - rw_whitelist = set('overlays/unittest1/uppser_subdir/lower_subdir/from_unittest1') o = overlay.BindOverlay( - config_file=test_config.name, - target='unittest', + cfg=config.factory(test_config.name), + build_target='unittest', source_dir=self.source_dir) self.assertIsNotNone(o) bind_mounts = o.GetBindMounts() @@ -151,12 +237,48 @@ class BindOverlayTest(unittest.TestCase): bind_destination = os.path.join(self.source_dir, 'upper_subdir/lower_subdir/from_unittest1') self.assertEqual( bind_mounts[bind_destination], - overlay.BindMount(source_dir=bind_source, readonly=False)) + overlay.BindMount(source_dir=bind_source, readonly=False, allows_replacement=False)) bind_source = os.path.join(self.source_dir, 'overlays/unittest2/upper_subdir/lower_subdir/from_unittest2') bind_destination = os.path.join(self.source_dir, 'upper_subdir/lower_subdir/from_unittest2') - self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, False)) + self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, True, False)) + + def testAllowReadWriteNoGitDir(self): + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write( + '<?xml version="1.0" encoding="UTF-8" ?>' + '<config>' + ' <target name="unittest">' + ' <overlay name="unittest1"/>' + ' <overlay name="unittest2"/>' + ' <allow_readwrite path="no_git_dir/no_git_subdir1"/>' + ' <build_config>' + ' <goal name="goal_name"/>' + ' </build_config>' + ' </target>' + '</config>' + ) + test_config.flush() + o = overlay.BindOverlay( + cfg=config.factory(test_config.name), + build_target='unittest', + source_dir=self.source_dir) + self.assertIsNotNone(o) + bind_mounts = o.GetBindMounts() + bind_source = os.path.join(self.source_dir, + 'no_git_dir/no_git_subdir1') + bind_destination = os.path.join(self.source_dir, 'no_git_dir/no_git_subdir1') + self.assertIn(bind_destination, bind_mounts) + self.assertEqual( + bind_mounts[bind_destination], + overlay.BindMount(source_dir=bind_source, readonly=False, allows_replacement=False)) + bind_source = os.path.join(self.source_dir, + 'no_git_dir/no_git_subdir2') + bind_destination = os.path.join(self.source_dir, + 'no_git_dir/no_git_subdir2') + self.assertIn(bind_destination, bind_mounts) + self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, True, False)) def testValidOverlaidDir(self): with tempfile.NamedTemporaryFile('w+t') as test_config: @@ -165,20 +287,23 @@ class BindOverlayTest(unittest.TestCase): '<config>' ' <target name="unittest">' ' <overlay name="unittest1"/>' + ' <build_config>' + ' <goal name="goal_name"/>' + ' </build_config>' ' </target>' '</config>' ) test_config.flush() o = overlay.BindOverlay( - config_file=test_config.name, - target='unittest', + cfg=config.factory(test_config.name), + build_target='unittest', source_dir=self.source_dir, destination_dir=self.destination_dir) self.assertIsNotNone(o) bind_mounts = o.GetBindMounts() bind_source = os.path.join(self.source_dir, 'overlays/unittest1/from_dir') bind_destination = os.path.join(self.destination_dir, 'from_dir') - self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, False)) + self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, True, False)) def testValidFilesystemViewDirectoryBind(self): with tempfile.NamedTemporaryFile('w+t') as test_config: @@ -187,6 +312,9 @@ class BindOverlayTest(unittest.TestCase): '<config>' ' <target name="unittest">' ' <view name="unittestview"/>' + ' <build_config>' + ' <goal name="goal_name"/>' + ' </build_config>' ' </target>' ' <view name="unittestview">' ' <path source="overlays/unittest1/from_dir" ' @@ -196,14 +324,14 @@ class BindOverlayTest(unittest.TestCase): ) test_config.flush() o = overlay.BindOverlay( - config_file=test_config.name, - target='unittest', + cfg=config.factory(test_config.name), + build_target='unittest', source_dir=self.source_dir) self.assertIsNotNone(o) bind_mounts = o.GetBindMounts() bind_source = os.path.join(self.source_dir, 'overlays/unittest1/from_dir') bind_destination = os.path.join(self.source_dir, 'to_dir') - self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, False)) + self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, True, False)) def testValidFilesystemViewFileBind(self): with tempfile.NamedTemporaryFile('w+t') as test_config: @@ -212,6 +340,9 @@ class BindOverlayTest(unittest.TestCase): '<config>' ' <target name="unittest">' ' <view name="unittestview"/>' + ' <build_config>' + ' <goal name="goal_name"/>' + ' </build_config>' ' </target>' ' <view name="unittestview">' ' <path source="overlays/unittest1/from_file" ' @@ -221,14 +352,14 @@ class BindOverlayTest(unittest.TestCase): ) test_config.flush() o = overlay.BindOverlay( - config_file=test_config.name, - target='unittest', + cfg=config.factory(test_config.name), + build_target='unittest', source_dir=self.source_dir) self.assertIsNotNone(o) bind_mounts = o.GetBindMounts() bind_source = os.path.join(self.source_dir, 'overlays/unittest1/from_file') bind_destination = os.path.join(self.source_dir, 'to_file') - self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, False)) + self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, True, False)) def testInvalidTarget(self): with tempfile.NamedTemporaryFile('w+t') as test_config: @@ -237,16 +368,74 @@ class BindOverlayTest(unittest.TestCase): '<config>' ' <target name="unittest">' ' <overlay name="unittest1"/>' + ' <build_config>' + ' <goal name="goal_name"/>' + ' </build_config>' ' </target>' '</config>' ) test_config.flush() with self.assertRaises(KeyError): overlay.BindOverlay( - config_file=test_config.name, - target='unknown', + cfg=config.factory(test_config.name), + build_target='unknown', source_dir=self.source_dir) + def testExplicitBindMount(self): + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write( + '<?xml version="1.0" encoding="UTF-8" ?>' + '<config>' + ' <target name="target_name">' + ' <overlay name="no_git_dir2"/>' + ' <build_config>' + ' <goal name="goal_name"/>' + ' </build_config>' + ' </target>' + '</config>' + ) + test_config.flush() + o = overlay.BindOverlay( + cfg=config.factory(test_config.name), + build_target='target_name', + source_dir=self.source_dir) + self.assertIsNotNone(o) + bind_mounts = o.GetBindMounts() + + bind_source = os.path.join(self.source_dir, 'overlays/no_git_dir2/no_git_subdir1') + bind_destination = os.path.join(self.source_dir, 'no_git_subdir1') + self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, True, False)) + + bind_source = os.path.join(self.source_dir, 'overlays/no_git_dir2/no_git_subdir2') + bind_destination = os.path.join(self.source_dir, 'no_git_subdir2') + self.assertEqual(bind_mounts[bind_destination], overlay.BindMount(bind_source, True, False)) + + def testReplacementPath(self): + with tempfile.NamedTemporaryFile('w+t') as test_config: + test_config.write( + '<?xml version="1.0" encoding="UTF-8" ?>' + '<config>' + ' <target name="unittest">' + ' <overlay name="unittest1">' + ' <replacement_path path="from_dir"/>' + ' </overlay>' + ' <build_config>' + ' <goal name="goal_name"/>' + ' </build_config>' + ' </target>' + '</config>' + ) + test_config.flush() + o = overlay.BindOverlay( + cfg=config.factory(test_config.name), + build_target='unittest', + source_dir=self.source_dir) + self.assertIsNotNone(o) + bind_mounts = o.GetBindMounts() + bind_source = os.path.join(self.source_dir, 'overlays/unittest1/from_dir') + bind_destination = os.path.join(self.source_dir, 'from_dir') + self.assertEqual(bind_mounts[bind_destination], + overlay.BindMount(bind_source, True, True)) if __name__ == '__main__': unittest.main() diff --git a/build/sandbox/rbe.py b/build/sandbox/rbe.py new file mode 100644 index 0000000..fba368f --- /dev/null +++ b/build/sandbox/rbe.py @@ -0,0 +1,195 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for RBE-enabled builds.""" + +import os +import random +import subprocess +import tempfile + +# These are the environment variables that control RBE usage with the +# --use_rbe flag. If defined on the environment, the values will be +# propagated to the build; otherwise, those defaults will be used. +TOOLS_DIR = 'prebuilts/remoteexecution-client/latest' +_RBE_ENV = { + 'USE_RBE': 'true', + 'RBE_DIR': TOOLS_DIR, + 'NINJA_REMOTE_NUM_JOBS': '500', + 'FLAG_log_dir': 'out', + 'FLAG_server_address': 'unix:///tmp/reproxy_%s.sock' % random.randint(0,100000), + 'FLAG_exec_root': '/src', + 'FLAG_invocation_id': 'treble-%s' % random.randint(0,100000), + 'RBE_use_application_default_credentials': 'true', + 'RBE_reproxy_wait_seconds': '20', + 'RBE_output_dir': 'out', + 'RBE_proxy_log_dir': 'out', + 'RBE_cpp_dependency_scanner_plugin': os.path.join(TOOLS_DIR, 'dependency_scanner_go_plugin.so'), + 'RBE_re_proxy': os.path.join(TOOLS_DIR, 'reproxy'), + 'RBE_JAVAC': 'true', + 'RBE_D8': 'true', + 'RBE_R8': 'true', +} + + +def get_nsjail_bin_wrapper(): + """Returns the command executed in a closed network namespace.""" + return ['netns-exec', 'rbe-closed-ns'] + + +def env_array_to_dict(env_array): + """Converts an env var array to a dict. + + Args: + env: An array of environment variables in the `var=val` syntax. + + Returns: + A dict of string values keyed by string names. + """ + env_dict = {} + for var in env_array: + var = var.split('=') + name = var[0] + value = var[1] + env_dict[name] = value + return env_dict + +def prepare_env(env): + """Prepares an env dict for enabling RBE. + + Checks that all environment variables required to be set + by the user are defined and sets some default + values for optional environment variables + + Args: + env: An array of environment variables in the `var=val` syntax. + + Returns: + An array of environment variables in the `var=val` syntax. + """ + # Start with the default values + prepared_env = _RBE_ENV.copy() + + # Host environment variables take precedence over defaults. + for k,v in os.environ.items(): + if k.startswith('RBE_'): + prepared_env[k] = v + + # Input parameter variables take precedence over everything else + prepared_env.update(env_array_to_dict(env)) + + if 'RBE_instance' not in prepared_env: + raise EnvironmentError('The RBE_instance environment ' + 'variables must be defined') + + if 'RBE_service' not in prepared_env: + raise EnvironmentError('The RBE_service environment ' + 'variables must be defined') + + return ['%s=%s' % (k,v) for k,v in prepared_env.items()] + + +def get_readonlybind_mounts(): + """Returns a dictionary of readonly bind mounts""" + creds_file = '.config/gcloud/application_default_credentials.json' + # Bind the gcloud credentials file, if present, to authenticate. + source_creds_file = os.path.join(os.getenv('HOME'), creds_file) + dest_creds_file = os.path.join('/tmp', creds_file) + if not os.path.exists(source_creds_file): + raise IOError('Required credentials file not found: ' + source_creds_file) + return ['%s:%s' % (source_creds_file, dest_creds_file)] + + +def get_extra_nsjail_args(): + """Returns a dictionary of extra nsjail.run arguments for RBE.""" + # The nsjail should be invoked in a closed network namespace. + return ['--disable_clone_newnet'] + + +def setup(env, build_log=subprocess.DEVNULL): + """Prerequisite for having RBE enabled for the build. + + Calls RBE http proxy in a separate network namespace. + + Args: + env: An array of environment variables in the `var=val` syntax. + build_log: a file handle to write executed commands to. + + Returns: + A cleanup function to be called after the build is done. + """ + env_dict = env_array_to_dict(env) + + # Create the RBE http proxy allowlist file. + if 'RBE_service' in env_dict: + rbe_service = env_dict['RBE_service'] + else: + rbe_service = os.getenv('RBE_service') + if not rbe_service: + raise EnvironmentError('The RBE_service environment ' + 'variables must be defined') + if ':' in rbe_service: + rbe_service = rbe_service.split(':', 1)[0] + rbe_allowlist = [ + rbe_service, + 'oauth2.googleapis.com', + 'accounts.google.com', + ] + with open('/tmp/rbe_allowlist.txt', 'w+') as t: + for w in rbe_allowlist: + t.write(w + '\n') + + # Restart RBE http proxy. + script_dir = os.path.dirname(os.path.abspath(__file__)) + proxy_kill_command = ['killall', 'tinyproxy'] + port = 8000 + random.randint(0,1000) + new_conf_contents = '' + with open(os.path.join(script_dir, 'rbe_http_proxy.conf'), 'r') as base_conf: + new_conf_contents = base_conf.read() + with tempfile.NamedTemporaryFile(prefix='rbe_http_proxy_', mode='w', delete=False) as new_conf: + new_conf.write(new_conf_contents) + new_conf.write('\nPort %i\n' % port) + new_conf.close() + env.append("RBE_HTTP_PROXY=10.1.2.1:%i" % port) + + proxy_command = [ + 'netns-exec', 'rbe-open-ns', 'tinyproxy', '-c', new_conf.name, '-d'] + rbe_proxy_log = tempfile.NamedTemporaryFile(prefix='tinyproxy_', delete=False) + if build_log != subprocess.DEVNULL: + print('RBE http proxy restart commands:', file=build_log) + print(' '.join(proxy_kill_command), file=build_log) + print('cd ' + script_dir, file=build_log) + print(' '.join(proxy_command) + ' &> ' + rbe_proxy_log.name + ' &', + file=build_log) + subprocess.call( + proxy_kill_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + rbe_proxy = subprocess.Popen( + proxy_command, + cwd=script_dir, + stdout=rbe_proxy_log, + stderr=rbe_proxy_log) + + def cleanup(): + """Should be called after an RBE build is done.""" + if build_log != subprocess.DEVNULL: + print('RBE http proxy kill command:', file=build_log) + print(' '.join(proxy_kill_command), file=build_log) + rbe_proxy.terminate() + # TODO(diegowilson): Calling wait() sometimes dead locks. + # Not sure if it's a tinyproxy bug or the issue described in the wait() documentation + # https://docs.python.org/2/library/subprocess.html#subprocess.Popen.wait + # rbe_proxy.wait() + rbe_proxy_log.close() + + return cleanup diff --git a/build/sandbox/rbe_action.py b/build/sandbox/rbe_action.py new file mode 100644 index 0000000..9748fdf --- /dev/null +++ b/build/sandbox/rbe_action.py @@ -0,0 +1,83 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Runs a single action remotely with RBE.""" + +import argparse +import os +import rbe +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--command', + default='echo RBE check successful.', + help='Command to run remotely with RBE.') + parser.add_argument( + '--print', '-p', + action='store_true', + help='Prints the executed commands') + args = parser.parse_args() + + env = [] + cleanup = rbe.setup(env, sys.stdout if args.print else subprocess.DEVNULL) + src_root = os.path.normpath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../..')) + env = rbe.env_array_to_dict(rbe.prepare_env(env)) + env['PATH'] = os.getenv('PATH') + for d in ['FLAG_log_dir', 'RBE_output_dir', 'RBE_proxy_log_dir']: + env[d] = '/tmp' # We want the logs in /tmp instead of out. + try: + # Bootstrap the RBE proxy. + bootstrap_cmd = rbe.get_nsjail_bin_wrapper() + \ + [os.path.join(rbe.TOOLS_DIR, 'bootstrap')] + shell_env = ' '.join(['%s=%s' % (k,v) for k, v in env.items()]) + if args.print: + print('Bootstrap RBE reproxy:') + print('cd ' + src_root) + print('%s %s' % (shell_env, ' '.join(bootstrap_cmd))) + subprocess.check_call( + bootstrap_cmd, env=env, cwd=src_root, stdout=subprocess.DEVNULL) + # Execute the remote command. + rewrapper_cmd = rbe.get_nsjail_bin_wrapper() + [ + os.path.join(rbe.TOOLS_DIR, 'rewrapper'), + '--platform=container-image=docker://gcr.io/androidbuild-re-dockerimage/android-build-remoteexec-image@sha256:582efb38f0c229ea39952fff9e132ccbe183e14869b39888010dacf56b360d62', \ + '--labels=type=tool', + '--exec_strategy=remote', + '--dial_timeout=5s', + '--exec_root=' + src_root, + '--', + ] + args.command.split() + if args.print: + print('Run remote command with RBE:') + print('%s %s' % (shell_env, ' '.join(rewrapper_cmd))) + subprocess.check_call(rewrapper_cmd, env=env, cwd=src_root) + finally: + # Shut down the RBE proxy. + if args.print: + print('RBE proxy shutdown:') + print('killall reproxy') + subprocess.call( + ['killall', 'reproxy'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + cleanup() + + +if __name__ == '__main__': + main() diff --git a/build/sandbox/rbe_http_proxy.conf b/build/sandbox/rbe_http_proxy.conf new file mode 100644 index 0000000..474858d --- /dev/null +++ b/build/sandbox/rbe_http_proxy.conf @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# IP to listen on. +Listen 10.1.2.1 + +# IP on which external connections are created. +Bind 10.1.3.2 + +# Allow only requests from this IP. +Allow 127.0.0.1 +Allow 10.1.1.1 + +# Allow only requests on these ports. +ConnectPort 443 +ConnectPort 80 + +# Allow these many parallel connections and underlying server connections. +MaxClients 10000 +StartServers 10 +MinSpareServers 10 + +# Deny requests that don't match the domain allowlist defined below. +FilterDefaultDeny Yes + +# Domain allowslist file, generated in code. +Filter "/tmp/rbe_allowlist.txt" + +# Port to listen on. To be added by rbe.py. diff --git a/build/sandbox/rbe_test.py b/build/sandbox/rbe_test.py new file mode 100644 index 0000000..51a7b79 --- /dev/null +++ b/build/sandbox/rbe_test.py @@ -0,0 +1,36 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test rbe.""" + +import unittest +from . import rbe + +class RBETest(unittest.TestCase): + """unittest for RBE.""" + + def testExtraNsjailArgs(self): + self.assertEqual(rbe.get_extra_nsjail_args()[0], '--disable_clone_newnet') + + def testEnv(self): + env = rbe.prepare_env(env=["RBE_instance=test_instance", "RBE_service=test_service"]) + self.assertIn('USE_RBE=true', env) + self.assertIn('RBE_instance=test_instance', env) + self.assertIn('RBE_JAVAC=true', env) + self.assertIn('RBE_D8=true', env) + self.assertIn('RBE_R8=true', env) + + +if __name__ == '__main__': + unittest.main() diff --git a/build/sandbox/sample_config.xml b/build/sandbox/sample_config.xml new file mode 100644 index 0000000..dbbd412 --- /dev/null +++ b/build/sandbox/sample_config.xml @@ -0,0 +1,35 @@ +<?xml version = "1.0" encoding = "UTF-8" ?> +<!-- +Defines sample build configuration file. +--> +<config> + <target name="aosp_cf_x86_phone_default" android_target="aosp_cf_x86_phone" + allow_readwrite_all="true"> + <!-- Target elements can define elements and attributes that are inherited + by build_config child elements. --> + <goal name="droid"/> + <build_config> + <!-- build_config name will inherit the name aosp_cf_x86_phone_default + and append dist to the goal list. --> + <goal name="dist"/> + </build_config> + <build_config name="aosp_cf_x86_phone_no_dist" tags="skip"> + </build_config> + <build_config name="aosp_cf_x86_phone_ro" allow_readwrite_all="false"> + <!-- This build_config will override allow_readwrite_all attribute. --> + </build_config> + <build_config name="aosp_cf_x86_tv" android_target="aosp_cf_x86_tv"> + <!-- This build will use aosp_cf_x86_tv as the lunch target. --> + </build_config> + </target> + <target name="aosp_car_arm64"> + <!-- If android_target isn't provided target will use name as default + android_target. --> + <goal name="droid"/> + <goal name="dist"/> + <build_config> + <!-- build_config will inherit the name and android_target: + aosp_car_arm64. --> + </build_config> + </target> +</config> diff --git a/fetcher/Android.bp b/fetcher/Android.bp new file mode 100644 index 0000000..787d0b7 --- /dev/null +++ b/fetcher/Android.bp @@ -0,0 +1,49 @@ +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "tools_treble_license" + // to get the below license kinds: + // SPDX-license-identifier-Apache-2.0 + default_applicable_licenses: ["tools_treble_license"], +} + +python_defaults { + name: "fetcher-defaults", + version: { + py2: { + enabled: false, + embedded_launcher: false, + }, + py3: { + enabled: true, + embedded_launcher: false, + }, + }, +} + + +python_library_host { + name: "fetcher-lib", + defaults: ["fetcher-defaults"], + srcs: [ + "fetcher_lib.py" + ], + pkg_path: "treble/fetcher", + libs: [ + "py-google-api-python-client", + "py-oauth2client", + "py-six", + ], +} + +python_binary_host { + name: "fetcher", + main: "fetcher.py", + defaults: ["fetcher-defaults"], + srcs: [ + "fetcher.py", + ], + libs: [ + "fetcher-lib", + ], +} diff --git a/fetcher/fetcher.py b/fetcher/fetcher.py new file mode 100644 index 0000000..b78d316 --- /dev/null +++ b/fetcher/fetcher.py @@ -0,0 +1,66 @@ +"""Fetches artifacts from Android Build.""" +import argparse +import os + +from treble.fetcher import fetcher_lib + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--json_keyfile', + help='JSON keyfile containing credentials. ' + '(Default: Use default credential file)') + parser.add_argument( + '--target', required=True, help='The target name to download from.') + parser.add_argument( + '--artifact', + action='append', + default=[], + help='The name of the artifact to download. ' + 'Can be specified multiple times.') + parser.add_argument( + '--regex', + action='append', + default=[], + help='A regex pattern to compare to the names of the artifact to ' + 'download. Can be specified multiple times.') + + parser.add_argument( + '--out_dir', + default='out/artifacts/', + help='Path to store fetched artifact to.') + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + '--branch', help='Download from the latest build of this branch.') + group.add_argument('--build_id', help='Download from the specified build.') + + args = parser.parse_args() + client = fetcher_lib.create_client_from_json_keyfile( + json_keyfile_name=args.json_keyfile) + + build_id = fetcher_lib.get_latest_build_id( + client=client, branch=args.branch, + target=args.target) if args.branch else args.build_id + + for artifact in args.artifact: + fetcher_lib.fetch_artifact( + client=client, + build_id=build_id, + target=args.target, + resource_id=artifact, + dest=os.path.join(args.out_dir, artifact)) + + for re in args.regex: + fetcher_lib.fetch_artifacts( + client=client, + build_id=build_id, + target=args.target, + pattern=re, + out_dir=args.out_dir) + + +if __name__ == '__main__': + main() diff --git a/fetcher/fetcher_lib.py b/fetcher/fetcher_lib.py new file mode 100644 index 0000000..0ec0173 --- /dev/null +++ b/fetcher/fetcher_lib.py @@ -0,0 +1,284 @@ +"""Provides helper functions for fetching artifacts.""" + +import io +import os +import re +import sys +import sysconfig +import time + +# This is a workaround to put '/usr/lib/python3.X' ahead of googleapiclient +# Using embedded_launcher won't work since py3-cmd doesn't contain _ssl module. +if sys.version_info.major == 3: + sys.path.insert(0, os.path.dirname(sysconfig.get_paths()['purelib'])) + +# pylint: disable=import-error,g-bad-import-order,g-import-not-at-top +import apiclient +from googleapiclient.discovery import build +from six.moves import http_client + +import httplib2 +from oauth2client.service_account import ServiceAccountCredentials + +_SCOPE_URL = 'https://www.googleapis.com/auth/androidbuild.internal' +_DEF_JSON_KEYFILE = '.config/gcloud/application_default_credentials.json' + + +# 20 MB default chunk size -- used in Buildbot +_DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024 + +# HTTP errors -- used in Builbot +_DEFAULT_MASKED_ERRORS = [404] +_DEFAULT_RETRIED_ERRORS = [503] +_DEFAULT_RETRIES = 10 + + +def _create_http_from_p12(robot_credentials_file, robot_username): + """Creates a credentialed HTTP object for requests. + + Args: + robot_credentials_file: The path to the robot credentials file. + robot_username: A string containing the username of the robot account. + + Returns: + An authorized httplib2.Http object. + """ + try: + credentials = ServiceAccountCredentials.from_p12_keyfile( + service_account_email=robot_username, + filename=robot_credentials_file, + scopes=_SCOPE_URL) + except AttributeError: + raise ValueError('Machine lacks openssl or pycrypto support') + http = httplib2.Http() + return credentials.authorize(http) + + +def _simple_execute(http_request, + masked_errors=None, + retried_errors=None, + retry_delay_seconds=5, + max_tries=_DEFAULT_RETRIES): + """Execute http request and return None on specified errors. + + Args: + http_request: the apiclient provided http request + masked_errors: list of errors to return None on + retried_errors: list of erros to retry the request on + retry_delay_seconds: how many seconds to sleep before retrying + max_tries: maximum number of attmpts to make request + + Returns: + The result on success or None on masked errors. + """ + if not masked_errors: + masked_errors = _DEFAULT_MASKED_ERRORS + if not retried_errors: + retried_errors = _DEFAULT_RETRIED_ERRORS + + last_error = None + for _ in range(max_tries): + try: + return http_request.execute() + except http_client.errors.HttpError as e: + last_error = e + if e.resp.status in masked_errors: + return None + elif e.resp.status in retried_errors: + time.sleep(retry_delay_seconds) + else: + # Server Error is server error + raise e + + # We've gone through the max_retries, raise the last error + raise last_error # pylint: disable=raising-bad-type + + +def create_client(http): + """Creates an Android build api client from an authorized http object. + + Args: + http: An authorized httplib2.Http object. + + Returns: + An authorized android build api client. + """ + return build(serviceName='androidbuildinternal', version='v2beta1', http=http) + + +def create_client_from_json_keyfile(json_keyfile_name=None): + """Creates an Android build api client from a json keyfile. + + Args: + json_keyfile_name: The location of the keyfile, if None is provided use + default location. + + Returns: + An authorized android build api client. + """ + if not json_keyfile_name: + json_keyfile_name = os.path.join(os.getenv('HOME'), _DEF_JSON_KEYFILE) + + credentials = ServiceAccountCredentials.from_json_keyfile_name( + filename=json_keyfile_name, scopes=_SCOPE_URL) + http = httplib2.Http() + credentials.authorize(http) + return create_client(http) + + +def create_client_from_p12(robot_credentials_file, robot_username): + """Creates an Android build api client from a config file. + + Args: + robot_credentials_file: The path to the robot credentials file. + robot_username: A string containing the username of the robot account. + + Returns: + An authorized android build api client. + """ + http = _create_http_from_p12(robot_credentials_file, robot_username) + return create_client(http) + + +def fetch_artifact(client, build_id, target, resource_id, dest): + """Fetches an artifact. + + Args: + client: An authorized android build api client. + build_id: AB build id + target: the target name to download from + resource_id: the resource id of the artifact + dest: path to store the artifact + """ + out_dir = os.path.dirname(dest) + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + dl_req = client.buildartifact().get_media( + buildId=build_id, + target=target, + attemptId='latest', + resourceId=resource_id) + + print('Fetching %s to %s...' % (resource_id, dest)) + with io.FileIO(dest, mode='wb') as fh: + downloader = apiclient.http.MediaIoBaseDownload( + fh, dl_req, chunksize=_DEFAULT_CHUNK_SIZE) + done = False + while not done: + status, done = downloader.next_chunk(num_retries=_DEFAULT_RETRIES) + print('Fetching...' + str(status.progress() * 100)) + + print('Done Fetching %s to %s' % (resource_id, dest)) + + +def get_build_list(client, **kwargs): + """Get a list of builds from the android build api that matches parameters. + + Args: + client: An authorized android build api client. + **kwargs: keyworded arguments to pass to build api. + + Returns: + Response from build api. + """ + build_request = client.build().list(**kwargs) + + return _simple_execute(build_request) + + +def list_artifacts(client, regex, **kwargs): + """List artifacts from the android build api that matches parameters. + + Args: + client: An authorized android build api client. + regex: Regular expression pattern to match artifact name. + **kwargs: keyworded arguments to pass to buildartifact.list api. + + Returns: + List of matching artifact names. + """ + matching_artifacts = [] + kwargs.setdefault('attemptId', 'latest') + regex = re.compile(regex) + req = client.buildartifact().list(**kwargs) + while req: + result = _simple_execute(req) + if result and 'artifacts' in result: + for a in result['artifacts']: + if regex.match(a['name']): + matching_artifacts.append(a['name']) + req = client.buildartifact().list_next(req, result) + return matching_artifacts + + +def fetch_artifacts(client, out_dir, target, pattern, build_id): + """Fetches target files artifacts matching patterns. + + Args: + client: An authorized instance of an android build api client for making + requests. + out_dir: The directory to store the fetched artifacts to. + target: The target name to download from. + pattern: A regex pattern to match to artifacts filename. + build_id: The Android Build id. + """ + if not os.path.exists(out_dir): + os.makedirs(out_dir) + + # Build a list of needed artifacts + artifacts = list_artifacts( + client=client, + regex=pattern, + buildId=build_id, + target=target) + + for artifact in artifacts: + fetch_artifact( + client=client, + build_id=build_id, + target=target, + resource_id=artifact, + dest=os.path.join(out_dir, artifact)) + + +def get_latest_build_id(client, branch, target): + """Get the latest build id. + + Args: + client: An authorized instance of an android build api client for making + requests. + branch: The branch to download from + target: The target name to download from. + Returns: + The build id. + """ + build_response = get_build_list( + client=client, + branch=branch, + target=target, + maxResults=1, + successful=True, + buildType='submitted') + + if not build_response: + raise ValueError('Unable to determine latest build ID!') + + return build_response['builds'][0]['buildId'] + + +def fetch_latest_artifacts(client, out_dir, target, pattern, branch): + """Fetches target files artifacts matching patterns from the latest build. + + Args: + client: An authorized instance of an android build api client for making + requests. + out_dir: The directory to store the fetched artifacts to. + target: The target name to download from. + pattern: A regex pattern to match to artifacts filename + branch: The branch to download from + """ + build_id = get_latest_build_id( + client=client, branch=branch, target=target) + + fetch_artifacts(client, out_dir, target, pattern, build_id) diff --git a/hacksaw/README.md b/hacksaw/README.md new file mode 100644 index 0000000..58bbb60 --- /dev/null +++ b/hacksaw/README.md @@ -0,0 +1,180 @@ +# hacksaw + +**HACK** in a **S**peedy **A**ccess **W**orkspace + +## What is Hacksaw? + +If you have a large multi-gigabyte codebase spread out through multiple git projects it can take a long time branch off a clean workspace. Hacksaw is a tool that +lets you split off a clean workspace in seconds. It does so by only copying git projects that you +explicitly select to be edited. All other projects are read-only bind mounts. This lets you build without cloning the full codebase to a new location! + +## How much faster is it, really? + +Lets look at some performance numbers for creating a hacksaw workspace using as a codebase the AOSP master branch as of 2020-8-4. The machine used was a c2-standard-60 Google Cloud Platform VM with 60 vCPUs and 240 GiB of RAM. Each action was performed at least 10 times then averaged out. + +* Create a new Hacksaw workspace + + Time: 0.4 sec + + Disk usage: 7.9 MiB + +* Remove a Hacksaw workspace with no edits or build artifacts. + + Time: 0.6 sec + +* Create a new Hacksaw workspace and edit build/make project. + + Time: 0.6 sec + + Disk usage: 18 MiB + +* Create a new Hacksaw workspace and edit frameworks/base project. + + Time: 7.5 sec + + Disk usage: 1.3 GiB + +As you can see, the time it takes to set up a new hacksaw workspace is proportional to +the git projects checked out for editing. Contrast that with how long it takes +to create a workspace using a full repo sync with a local +mirror. + +* Create a new full repo workspace [using a fresh local mirror](https://source.android.com/setup/build/downloading#using-a-local-mirror) + + Time: 12 min 32 sec + + Disk usage: 88 GiB + +* Remove a full repo workspace with no build artifacts + + Time: 28 seconds + +## Can you give me an example? + +``` +$ mkdir ~/aosp +$ cd ~/aosp +$ repo init -u https://android.googlesource.com/platform/manifest +... +$ repo sync --quiet --current-branch --no-tags --no-clone-bundle --jobs=$(nproc) +... +$ hacksaw codebase add aosp ~/aosp +Added codebase aosp +$ hacksaw codebase default aosp +Default codebase set to aosp +$ hacksaw workspace new big-feature +Composing................................................................. +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +........................................... +Workspace composed +Created big-feature at ~/hacksaw/big-feature +$ hacksaw edit ~/hacksaw/big-feature/tools/treble +Created branch big-feature on project ~/hacksaw/big-feature/tools/treble +$ hacksaw workspace new quick-fix +Composing................................................................. +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +.......................................................................... +........................................... +Workspace composed +Created big-feature at ~/hacksaw/quick-fix +$ hacksaw edit ~/hacksaw/quick-fix/tools/treble +Created branch quick-fix on project ~/hacksaw/quick-fix/tools/treble +``` + +## How do I install it? + +Building hacksaw requires [golang to be installed](https://golang.org/doc/install). +To install the hacksaw client run the following: + +``` +go get android.googlesource.com/platform/tools/treble.git/hacksaw/cmd/hacksaw +``` + +This will install hacksaw to ~/go/bin/hacksaw. You may choose to copy that +to a location in your path. For example: + +``` +sudo cp ~/go/bin/hacksaw /usr/local/bin +sudo chmod 755 /usr/local/bin/hacksaw +``` + +## How do I make sure that creating a hacksaw workspace is fast? + +Hacksaw creates bind mounts for all git projects in a codebase. It then +copies **everything** else. Make sure you remove all build artifacts from a +codebase before create a workspace, otherwise it may spend a long time copying +them. + +## How do I run it with sudo? + +Commands that mount and unmount will require sudo. That includes commands like + +* `hacksaw workspace new` +* `hacksaw edit` +* `hacksaw workspace remove` + +Other commmands like `hacksaw workspace list` or `hacksaw add codebase` do not +mount or unmount so do not require sudo. + +If you would like to avoid using sudo you may install hacksawd as explained below. + +## How do I run it without sudo? + +Hacksawd is a privileged system daemon whose only job is to manage bind mounts. +The provided install script will install to your system + +``` +go get android.googlesource.com/platform/tools/treble.git/hacksaw/cmd/hacksawd +sudo cp ~/go/bin/hacksawd /usr/local/bin +sudo chmod 755 /usr/local/bin/hacksawd +sudo ~/go/src/android.googlesource.com/platform/tools/treble.git/hacksaw/scripts/install-service.sh +``` + +The installation scripts creates a new "hacksaw" group and adds you to it. You +will need to log out and log back in for the group changes to take effect. After that you should be able to run any hacksaw command without sudo. + +If you wish to uninstall the service then run: + +``` +sudo ~/go/src/android.googlesource.com/platform/tools/treble.git/hacksaw/scripts/uninstall-service.sh +sudo rm /usr/local/bin/hacksawd +``` +## How do I sync? + +You sync your codebases using `repo sync`. All updates will be propagated to workspaces. +Except for projects that you are currently editing. Those will require you to `git pull` +manually in the workspace project. + +## How does hacksaw work? + +Hacksaw uses read-only bind mounts to create project references from +a workspace to a codebase. When you mark a project for editing then +its read-only bind mount gets replaced by a writable Git worktree. + +![Workspace diagram](images/workspace-diagram.png) + + +## What are the known issues? + +* Some repo commands don't work yet. Namely: `repo start` and `repo upload`. + So at the moment you can only upload to Gerrit [using git + push](https://gerrit-review.googlesource.com/Documentation/user-upload.html#_git_push). +* Failing to create a workspace is not rolled back. +* Editing nested projects is not supported yet. So if you have a git project + that contains other git projects you will get some unexpected behaviour. +* Git submodules are not supported yet, but the tool is designed with + future git submodule support in mind. +* Syncing a codebase does update the existing projects in all attached + workspaces but it does not remove or add new projects. Perhaps there + should be a new "workspace sync" command for that? + +## Where can I get more help? + +You can ask hacksaw-users@googlegroups.com by [joining the group](https://groups.google.com/forum/#!forum/hacksaw-users).
\ No newline at end of file diff --git a/hacksaw/bind/bind.go b/hacksaw/bind/bind.go new file mode 100644 index 0000000..ac94b41 --- /dev/null +++ b/hacksaw/bind/bind.go @@ -0,0 +1,22 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bind + +type PathBinder interface { + BindReadOnly(source string, destination string) error + BindReadWrite(source string, destination string) error + Unbind(destination string) error + List() ([]string, error) +} diff --git a/hacksaw/bind/local.go b/hacksaw/bind/local.go new file mode 100644 index 0000000..5679590 --- /dev/null +++ b/hacksaw/bind/local.go @@ -0,0 +1,99 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bind + +import ( + "fmt" + "path/filepath" + "syscall" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/mount" +) + +//localBinder executes PathBinder calls locally +type localBinder struct { + mounter mount.Mounter +} + +func NewLocalPathBinder() PathBinder { + var p localBinder + p.mounter = mount.NewSystemMounter() + return &p +} + +func NewFakePathBinder() PathBinder { + var p localBinder + p.mounter = mount.NewFakeMounter() + return &p +} + +func (p localBinder) checkValidPath(inPath string) error { + for dir := filepath.Dir(inPath); dir != "." && dir != "/"; dir = filepath.Dir(dir) { + // Only allow mounts in hacksaw path + if filepath.Base(dir) == "hacksaw" { + return nil + } + } + return fmt.Errorf("Not allowed to bind mount path %s because it's outside a hacksaw workspace", inPath) +} + +func (p localBinder) BindReadOnly(source string, destination string) error { + // TODO: check valid path considering sym links + source, err := filepath.EvalSymlinks(source) + if err != nil { + return err + } + destination, err = filepath.EvalSymlinks(destination) + if err != nil { + return err + } + err = p.mounter.Mount(source, destination, + "bind", syscall.MS_BIND, "") + if err != nil { + return err + } + err = p.mounter.Mount(source, destination, + "bind", syscall.MS_REMOUNT|syscall.MS_BIND|syscall.MS_RDONLY, "") + return err +} + +func (p localBinder) BindReadWrite(source string, destination string) error { + // TODO: check valid path considering sym links + source, err := filepath.EvalSymlinks(source) + if err != nil { + return err + } + destination, err = filepath.EvalSymlinks(destination) + if err != nil { + return err + } + err = p.mounter.Mount(source, destination, + "bind", syscall.MS_BIND, "") + return err +} + +func (p localBinder) Unbind(destination string) error { + // TODO: check valid path considering sym links + destination, err := filepath.EvalSymlinks(destination) + if err != nil { + return err + } + err = p.mounter.Unmount(destination, syscall.MNT_DETACH) + return err +} + +func (p localBinder) List() ([]string, error) { + return p.mounter.List() +} diff --git a/hacksaw/bind/remoteclient.go b/hacksaw/bind/remoteclient.go new file mode 100644 index 0000000..122c261 --- /dev/null +++ b/hacksaw/bind/remoteclient.go @@ -0,0 +1,103 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bind + +import ( + "errors" + "net/rpc" +) + +//RemoteBinderClient sends client requests to RPC +type RemoteBinderClient struct { + rpcClient *rpc.Client + SocketPath string +} + +func NewRemoteBindClient(socketPath string) PathBinder { + var r RemoteBinderClient + r.SocketPath = socketPath + return &r +} + +func (r *RemoteBinderClient) initRpcClient() error { + if r.rpcClient != nil { + return nil + } + var err error + r.rpcClient, err = rpc.DialHTTP("unix", r.SocketPath) + return err +} + +func (r *RemoteBinderClient) BindReadOnly(source string, destination string) error { + args := BindReadOnlyArgs{source, destination} + var reply BindReadOnlyReply + if err := r.initRpcClient(); err != nil { + return err + } + if err := r.rpcClient.Call("Server.BindReadOnly", &args, &reply); err != nil { + return err + } + if reply.Err != "" { + return errors.New(reply.Err) + } + return nil +} + +func (r *RemoteBinderClient) BindReadWrite(source string, destination string) error { + args := BindReadWriteArgs{source, destination} + var reply BindReadWriteReply + if err := r.initRpcClient(); err != nil { + return err + } + if err := r.rpcClient.Call("Server.BindReadWrite", &args, &reply); err != nil { + return err + } + if reply.Err != "" { + return errors.New(reply.Err) + } + return nil +} + +func (r *RemoteBinderClient) Unbind(destination string) error { + args := UnbindArgs{destination} + var reply UnbindReply + if err := r.initRpcClient(); err != nil { + return err + } + if err := r.rpcClient.Call("Server.Unbind", &args, &reply); err != nil { + return err + } + if reply.Err != "" { + return errors.New(reply.Err) + } + return nil +} + +func (r *RemoteBinderClient) List() ([]string, error) { + var args ListArgs + var reply ListReply + if err := r.initRpcClient(); err != nil { + return nil, err + } + if err := r.rpcClient.Call("Server.List", &args, &reply); err != nil { + return nil, err + } + + if reply.Err != "" { + return nil, errors.New(reply.Err) + } + + return reply.BindList, nil +} diff --git a/hacksaw/bind/remoteserver.go b/hacksaw/bind/remoteserver.go new file mode 100644 index 0000000..c059635 --- /dev/null +++ b/hacksaw/bind/remoteserver.go @@ -0,0 +1,102 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bind + +//Server receives RPC requests +type Server struct { + priv PathBinder +} + +func NewServer(binder PathBinder) *Server { + var s Server + s.priv = binder + return &s +} + +type BindReadOnlyArgs struct { + Source string + Destination string +} + +type BindReadOnlyReply struct { + // Errors types cannot be passed as-is in RPC so they + // must be converted to plain strings. + // Details at https://github.com/golang/go/issues/23340 + Err string +} + +func (s Server) BindReadOnly(args *BindReadOnlyArgs, reply *BindReadOnlyReply) error { + if err := s.priv.BindReadOnly(args.Source, args.Destination); err != nil { + reply.Err = err.Error() + } + return nil +} + +type BindReadWriteArgs struct { + Source string + Destination string +} + +type BindReadWriteReply struct { + // Errors types cannot be passed as-is in RPC so they + // must be converted to plain strings. + // Details at https://github.com/golang/go/issues/23340 + Err string +} + +func (s Server) BindReadWrite(args *BindReadWriteArgs, reply *BindReadWriteReply) error { + if err := s.priv.BindReadWrite(args.Source, args.Destination); err != nil { + reply.Err = err.Error() + } + return nil +} + +type UnbindArgs struct { + Destination string +} + +type UnbindReply struct { + // Errors types cannot be passed as-is in RPC so they + // must be converted to plain strings. + // Details at https://github.com/golang/go/issues/23340 + Err string +} + +func (s Server) Unbind(args *UnbindArgs, reply *UnbindReply) error { + if err := s.priv.Unbind(args.Destination); err != nil { + reply.Err = err.Error() + } + return nil +} + +type ListArgs struct { +} + +type ListReply struct { + BindList []string + // Errors types cannot be passed as-is in RPC so they + // must be converted to plain strings. + // Details at https://github.com/golang/go/issues/23340 + Err string +} + +func (s Server) List(args *ListArgs, reply *ListReply) error { + bindList, err := s.priv.List() + if err != nil { + reply.Err = err.Error() + } + reply.BindList = bindList + return nil +} diff --git a/hacksaw/bind/remoteserver_test.go b/hacksaw/bind/remoteserver_test.go new file mode 100644 index 0000000..6f1eabd --- /dev/null +++ b/hacksaw/bind/remoteserver_test.go @@ -0,0 +1,111 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bind + +import ( + "io/ioutil" + "os" + "path" + "reflect" + "testing" +) + +func TestServerBind(t *testing.T) { + mountTempDir, err := ioutil.TempDir("", "mount") + if err != nil { + t.Error(err) + } + fakeBinder := NewFakePathBinder() + server := NewServer(fakeBinder) + roSourceDir := path.Join(mountTempDir, "path/to/readonly/source") + if err = os.MkdirAll(roSourceDir, os.ModePerm); err != nil { + t.Error(err) + } + roDestDir := path.Join(mountTempDir, "path/to/hacksaw/readonly/destination") + if err = os.MkdirAll(roDestDir, os.ModePerm); err != nil { + t.Error(err) + } + bindROArgs := BindReadOnlyArgs{ + Source: roSourceDir, + Destination: roDestDir, + } + var bindROReply BindReadOnlyReply + if err := server.BindReadOnly(&bindROArgs, &bindROReply); err != nil { + t.Error(err) + } + if bindROReply.Err != "" { + t.Error(bindROReply.Err) + } + rwSourceDir := path.Join(mountTempDir, "path/to/readwrite/source") + if err = os.MkdirAll(rwSourceDir, os.ModePerm); err != nil { + t.Error(err) + } + rwDestDir := path.Join(mountTempDir, "path/to/hacksaw/readwrite/destination") + if err = os.MkdirAll(rwDestDir, os.ModePerm); err != nil { + t.Error(err) + } + bindRWArgs := BindReadWriteArgs{ + Source: rwSourceDir, + Destination: rwDestDir, + } + var bindRWReply BindReadWriteReply + if err := server.BindReadWrite(&bindRWArgs, &bindRWReply); err != nil { + t.Error(err) + } + if bindRWReply.Err != "" { + t.Error(bindRWReply.Err) + } + var listArgs ListArgs + var listReply ListReply + err = server.List(&listArgs, &listReply) + if err != nil { + t.Error(err) + } + if listReply.Err != "" { + t.Error(listReply.Err) + } + expectedList := []string{ + roDestDir, + rwDestDir, + } + if !reflect.DeepEqual(listReply.BindList, expectedList) { + t.Errorf("Bind list %v is different than expected bind %v", + listReply.BindList, expectedList) + } + unbindArgs := UnbindArgs{ + Destination: rwDestDir, + } + var unbindReply UnbindReply + if err := server.Unbind(&unbindArgs, &unbindReply); err != nil { + t.Error(err) + } + if unbindReply.Err != "" { + t.Error(unbindReply.Err) + } + err = server.List(&listArgs, &listReply) + if err != nil { + t.Error(err) + } + if listReply.Err != "" { + t.Error(listReply.Err) + } + expectedList = []string{ + roDestDir, + } + if !reflect.DeepEqual(listReply.BindList, expectedList) { + t.Errorf("Bind list %v is different than expected bind %v", + listReply.BindList, expectedList) + } +} diff --git a/hacksaw/client/client.go b/hacksaw/client/client.go new file mode 100644 index 0000000..4db9f2c --- /dev/null +++ b/hacksaw/client/client.go @@ -0,0 +1,62 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package client implements the hacksaw cli client +package client + +import ( + "io/ioutil" + "os" + "path" + "reflect" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/bind" + "android.googlesource.com/platform/tools/treble.git/hacksaw/config" +) + +func ensureConfigFileExists(cfgPath string) error { + _, err := os.Stat(cfgPath) + if os.IsNotExist(err) { + //continue to init config file + } else { + return err + } + + //init config file + if err = os.MkdirAll(path.Dir(cfgPath), os.ModePerm); err != nil { + return err + } + return ioutil.WriteFile(cfgPath, []byte("{}"), os.ModePerm) +} + +func HandleCommand(workspaceTopDir string, pathBinder bind.PathBinder, args []string) error { + cfgPath := path.Join(workspaceTopDir, "config.json") + if err := ensureConfigFileExists(cfgPath); err != nil { + return err + } + cfg := config.GetConfig() + if err := cfg.ReadConfigFromFile(cfgPath); err != nil { + return err + } + //Save a copy of the config to detect changes + savedCfg := cfg.Copy() + cmd := NewCommand(pathBinder, workspaceTopDir) + if err := cmd.Handle(args); err != nil { + return err + } + if reflect.DeepEqual(savedCfg, cfg) { + return nil + } + return cfg.WriteConfigToFile(cfgPath) +} diff --git a/hacksaw/client/client_test.go b/hacksaw/client/client_test.go new file mode 100644 index 0000000..90437c2 --- /dev/null +++ b/hacksaw/client/client_test.go @@ -0,0 +1,36 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "io/ioutil" + "path" + "testing" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/bind" +) + +func TestClientHelp(t *testing.T) { + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + wsTopDir := path.Join(wsTempDir, "hacksaw") + pathBinder := bind.NewFakePathBinder() + args := []string{"hacksaw", "help"} + if err = HandleCommand(wsTopDir, pathBinder, args); err != nil { + t.Error(err) + } +} diff --git a/hacksaw/client/command.go b/hacksaw/client/command.go new file mode 100644 index 0000000..b06c2d4 --- /dev/null +++ b/hacksaw/client/command.go @@ -0,0 +1,247 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "fmt" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/bind" + "android.googlesource.com/platform/tools/treble.git/hacksaw/codebase" + "android.googlesource.com/platform/tools/treble.git/hacksaw/workspace" +) + +const usage = `Usage: hacksaw <command> <options> + +Commands: + help: show this message + codebase add <name> <path>: Add codebase to hacksaw's list. + codebase list: List all codebases that hacksaw knows about. + codebase default <name>: Change the default codebase. + codebase remove <name>: Remove a codebase from hacksaw's list. + workspace new <workspace_name> <codebase_name>: Create a new workspace + from a codebase. The codebase is optional if a default codebase + has already been set. + workspace recreate <name>: Recreate an existing workspace. + This will recreate any read-only bind mounts which may be necessary + when a machine is rebooted. + workspace list: List all known workspaces. + workspace remove <name>: Remove a workspace. + edit <path>: Make a workspace path editable by checking out the parent git project.` + +type Command struct { + workspace workspace.Workspace +} + +func NewCommand(bm bind.PathBinder, wtd string) Command { + return Command{workspace.New(bm, wtd)} +} + +func (c Command) addCodebase(args []string) error { + if len(args) < 5 { + return fmt.Errorf("Codebase name and path are required\n"+ + "Usage: %s %s %s <codebase_name> <path>", + args[0], args[1], args[2]) + } + name := args[3] + path := args[4] + _, err := codebase.Add(name, path) + if err != nil { + return err + } + fmt.Println("Added codebase", name) + return err +} + +func (c Command) defaultCodebase(args []string) error { + if len(args) < 4 { + return fmt.Errorf("Codebase name is required\n"+ + "Usage: %s %s %s <codebase_name>", + args[0], args[1], args[2]) + } + name := args[3] + if err := codebase.SetDefault(name); err != nil { + return err + } + fmt.Println("Default codebase set to", name) + return nil +} + +func (c Command) listCodebases() { + def := codebase.Default() + if def == "" { + def = "None" + } + fmt.Println("Default codebase:") + fmt.Println("\t", def) + + list := codebase.List() + fmt.Println("Codebases:") + for name, path := range list { + fmt.Println("\t", name, path) + } +} + +func (c Command) removeCodebase(args []string) error { + if len(args) < 4 { + return fmt.Errorf("Codebase name required\n"+ + "Usage: %s %s %s <codebase>", + args[0], args[1], args[2]) + } + name := args[3] + _, err := codebase.Remove(name) + if err != nil { + return err + } + fmt.Println("Removed codebase", name) + return nil +} + +func (c Command) createWorkspace(args []string) error { + var codebaseName string + defaultCodebase := codebase.Default() + switch len(args) { + case 4: + if defaultCodebase == "" { + return fmt.Errorf("Codebase name is required\n"+ + "Usage: %s %s %s <name> <codebase>", + args[0], args[1], args[2]) + } else { + codebaseName = defaultCodebase + } + case 5: + codebaseName = args[4] + default: + return fmt.Errorf("Unexpected number of arguments\n"+ + "Usage: %s %s %s <name> <codebase>", + args[0], args[1], args[2]) + } + + workspaceName := args[3] + dir, err := c.workspace.Create(workspaceName, codebaseName) + if err != nil { + return err + } + fmt.Println("Created", workspaceName, "at", dir) + return nil +} + +func (c Command) recreateWorkspace(args []string) error { + if len(args) < 4 { + return fmt.Errorf("Workspace name is required\n"+ + "Usage: %s %s %s <name>", + args[0], args[1], args[2]) + } + + workspaceName := args[3] + dir, err := c.workspace.Recreate(workspaceName) + if err != nil { + return err + } + fmt.Println("Recreated", workspaceName, "at", dir) + return nil +} + +func (c Command) listWorkspaces() { + list := c.workspace.List() + fmt.Println("Workspaces:") + for name, codebase := range list { + fmt.Println("\t", name, codebase) + } +} + +func (c Command) removeWorkspace(args []string) error { + if len(args) < 4 { + return fmt.Errorf("Workspace name required\n"+ + "Usage: %s %s %s <name>", + args[0], args[1], args[2]) + } + name := args[3] + _, err := c.workspace.Remove(name) + if err != nil { + return err + } + fmt.Println("Removed workspace", name) + return nil +} + +func (c Command) editProject(args []string) error { + if len(args) < 3 { + return fmt.Errorf("Edit path required\n"+ + "Usage: %s %s <path>", + args[0], args[1]) + } + path := args[2] + branchName, projectPath, err := c.workspace.Edit(path) + if err != nil { + return err + } + fmt.Printf("Created branch %s on project %s\n", + branchName, projectPath) + return nil +} + +//Handle parses all command line arguments +func (c Command) Handle(args []string) error { + if len(args) < 2 || args[1] == "help" { + fmt.Println(usage) + return nil + } + + command := args[1] + + switch command { + case "codebase", "cb": + if len(args) < 3 { + return fmt.Errorf("Not enough arguments for codebase command") + } + subcommand := args[2] + switch subcommand { + case "add": + return c.addCodebase(args) + case "remove", "rm": + return c.removeCodebase(args) + case "default", "def": + return c.defaultCodebase(args) + case "list", "ls": + c.listCodebases() + return nil + default: + return fmt.Errorf("Command \"%s %s\" not found", command, subcommand) + } + case "workspace", "ws": + if len(args) < 3 { + return fmt.Errorf("Not enough arguments for workspace command") + } + subcommand := args[2] + switch subcommand { + case "new": + return c.createWorkspace(args) + case "recreate": + return c.recreateWorkspace(args) + case "remove", "rm": + return c.removeWorkspace(args) + case "list", "ls": + c.listWorkspaces() + return nil + default: + return fmt.Errorf("Command \"%s %s\" not found", command, subcommand) + } + case "edit": + return c.editProject(args) + default: + return fmt.Errorf("Command \"%s\" not found", command) + } + return nil +} diff --git a/hacksaw/client/command_test.go b/hacksaw/client/command_test.go new file mode 100644 index 0000000..ccdaf9d --- /dev/null +++ b/hacksaw/client/command_test.go @@ -0,0 +1,395 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "io/ioutil" + "os" + "os/exec" + "path" + "testing" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/bind" + "android.googlesource.com/platform/tools/treble.git/hacksaw/codebase" + "android.googlesource.com/platform/tools/treble.git/hacksaw/config" +) + +func TestHelpCommand(t *testing.T) { + cmd := NewCommand(bind.NewFakePathBinder(), "") + args := []string{"hacksaw", "help"} + err := cmd.Handle(args) + if err != nil { + t.Error(err) + } +} + +func TestNoCommand(t *testing.T) { + cmd := NewCommand(bind.NewFakePathBinder(), "") + args := []string{"hacksaw"} + err := cmd.Handle(args) + if err != nil { + t.Error(err) + } +} + +func TestBadCommand(t *testing.T) { + cmd := NewCommand(bind.NewFakePathBinder(), "") + args := []string{"hacksaw", "doesnotexist"} + if err := cmd.Handle(args); err == nil { + t.Errorf("Failed to handle bad command") + } +} + +func TestEmptyCodebaseCommand(t *testing.T) { + cmd := NewCommand(bind.NewFakePathBinder(), "") + args := []string{"hacksaw", "codebase"} + if err := cmd.Handle(args); err == nil { + t.Errorf("Allowed an empty codebase command") + } +} + +func TestAddCodebase(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + cmd := NewCommand(bind.NewFakePathBinder(), "") + args := []string{"hacksaw", "codebase", "add", "test-codebase", codebaseDir} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } +} + +func TestDefaultCodebase(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + gitDir := path.Join(codebaseDir, "project", ".git") + if err = os.MkdirAll(gitDir, os.ModePerm); err != nil { + t.Error(err) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + cmd := NewCommand(bind.NewFakePathBinder(), "") + args := []string{"hacksaw", "codebase", "add", "test-codebase", codebaseDir} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } + args = []string{"hacksaw", "codebase", "default", "test-codebase"} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } + + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + cmd = NewCommand(bind.NewFakePathBinder(), wsTopDir) + args = []string{"hacksaw", "workspace", "new", "test-workspace"} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } +} + +func TestListCodebases(t *testing.T) { + cmd := NewCommand(bind.NewFakePathBinder(), "") + args := []string{"hacksaw", "codebase", "list"} + if err := cmd.Handle(args); err != nil { + t.Error(err) + } +} + +func TestRemoveCodebase(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + cmd := NewCommand(bind.NewFakePathBinder(), "") + args := []string{"hacksaw", "codebase", "add", "test-codebase", codebaseDir} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } + args = []string{"hacksaw", "codebase", "remove", "test-codebase"} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } +} + +func TestEmptyWorkspaceCommand(t *testing.T) { + cmd := NewCommand(bind.NewFakePathBinder(), "") + args := []string{"hacksaw", "workspace"} + if err := cmd.Handle(args); err == nil { + t.Errorf("Allowed an empty workspace command") + } +} + +func TestCreateWorkspace(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + gitDir := path.Join(codebaseDir, "project", ".git") + if err = os.MkdirAll(gitDir, os.ModePerm); err != nil { + t.Error(err) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + cmd := NewCommand(bind.NewFakePathBinder(), wsTopDir) + args := []string{"hacksaw", "workspace", "new", "test-workspace", "test-codebase"} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } +} + +func TestRecreateWorkspace(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + gitDir := path.Join(codebaseDir, "project", ".git") + if err = os.MkdirAll(gitDir, os.ModePerm); err != nil { + t.Error(err) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + cmd := NewCommand(bind.NewFakePathBinder(), wsTopDir) + args := []string{"hacksaw", "workspace", "new", "test-workspace", "test-codebase"} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } + args = []string{"hacksaw", "workspace", "recreate", "test-workspace"} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } + args = []string{"hacksaw", "workspace", "recreate", "does-not-exist"} + if err = cmd.Handle(args); err == nil { + t.Error("Allowed to recreate an unexistant workspace") + } +} + +func TestListWorkspace(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + gitDir := path.Join(codebaseDir, "project", ".git") + if err = os.MkdirAll(gitDir, os.ModePerm); err != nil { + t.Error(err) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + cmd := NewCommand(bind.NewFakePathBinder(), wsTopDir) + + args := []string{"hacksaw", "workspace", "new", "test-workspace", "test-codebase"} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } + + args = []string{"hacksaw", "workspace", "list"} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } +} + +func TestRemoveWorkspace(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + gitDir := path.Join(codebaseDir, "project", ".git") + if err = os.MkdirAll(gitDir, os.ModePerm); err != nil { + t.Error(err) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + cmd := NewCommand(bind.NewFakePathBinder(), wsTopDir) + + args := []string{"hacksaw", "workspace", "new", "test-workspace", "test-codebase"} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } + + args = []string{"hacksaw", "workspace", "remove", "test-workspace"} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } +} + +func TestEditWorkspace(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "codebase") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + projectDir := path.Join(codebaseDir, "project") + if err = os.MkdirAll(projectDir, os.ModePerm); err != nil { + t.Error(err) + } + gitCmd := exec.Command("git", "-C", projectDir, "init") + output, err := gitCmd.CombinedOutput() + if err != nil { + t.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + gitCmd.String(), err.Error(), output) + } + gitCmd = exec.Command("git", "-C", projectDir, "commit", `--message="Initial commit"`, "--allow-empty") + output, err = gitCmd.CombinedOutput() + if err != nil { + t.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + gitCmd.String(), err.Error(), output) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + cmd := NewCommand(bind.NewFakePathBinder(), wsTopDir) + + args := []string{"hacksaw", "workspace", "new", "test-workspace", "test-codebase"} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } + wsProjectDir := path.Join(wsTopDir, "test-workspace", "project") + args = []string{"hacksaw", "edit", wsProjectDir} + if err = cmd.Handle(args); err != nil { + t.Error(err) + } +} diff --git a/hacksaw/cmd/hacksaw/main.go b/hacksaw/cmd/hacksaw/main.go new file mode 100644 index 0000000..60d1a88 --- /dev/null +++ b/hacksaw/cmd/hacksaw/main.go @@ -0,0 +1,160 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// hacksaw let's you create lightweight workspaces of large codebases +package main + +import ( + "fmt" + "net" + "net/http" + "net/rpc" + "os" + "os/exec" + "os/user" + "path/filepath" + "strconv" + "syscall" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/bind" + "android.googlesource.com/platform/tools/treble.git/hacksaw/client" +) + +const hacksawdSocketPath = "/var/run/hacksaw.sock" +const tmpSocketPath = "/tmp/hacksaw.sock" + +func getPathBinder() bind.PathBinder { + if os.Geteuid() == 0 { + // Called by root without SUDO_USER + // Most likely in a mount namepace + return bind.NewLocalPathBinder() + } + _, err := os.Stat(tmpSocketPath) + if err == nil { + return bind.NewRemoteBindClient(tmpSocketPath) + } else { + return bind.NewRemoteBindClient(hacksawdSocketPath) + } +} + +func getWorkspaceTopDir() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + // The hacksaw mount daemon requires all mounts + // to be contained in a directory named "hacksaw" + topDir := filepath.Join(home, "hacksaw") + _, err = os.Stat(topDir) + if err == nil { + // expected case + } else if os.IsNotExist(err) { + return topDir, nil + } else { + return "", err + } + topDir, err = filepath.EvalSymlinks(topDir) + if err != nil { + return "", err + } + return topDir, nil +} + +func dropPrivileges(sudoUser string, socketPath string) error { + usr, err := user.Lookup(sudoUser) + if err != nil { + return err + } + sudoUid, err := strconv.ParseUint(usr.Uid, 10, 32) + if err != nil { + return err + } + sudoGid, err := strconv.ParseUint(usr.Gid, 10, 32) + if err != nil { + return err + } + if err = os.Chown(socketPath, int(sudoUid), int(sudoGid)); err != nil { + return err + } + cmd := exec.Command(os.Args[0], os.Args[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), "HOME="+usr.HomeDir) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{ + Uid: uint32(sudoUid), + Gid: uint32(sudoGid), + }, + Setsid: true, + } + if err := cmd.Start(); err != nil { + return err + } + _, err = cmd.Process.Wait() + if err != nil { + return err + } + if err = cmd.Process.Release(); err != nil { + return err + } + return nil +} + +func createPathBinderListener(socketPath string) (net.Listener, error) { + listener, err := net.Listen("unix", socketPath) + if err != nil { + return nil, err + } + + binder := bind.NewLocalPathBinder() + server := bind.NewServer(binder) + if err = rpc.Register(server); err != nil { + return nil, err + } + rpc.HandleHTTP() + return listener, nil +} + +func handleSudoUser(sudoUser string) error { + if err := os.RemoveAll(tmpSocketPath); err != nil { + return err + } + listener, err := createPathBinderListener(tmpSocketPath) + if err != nil { + return err + } + defer os.RemoveAll(tmpSocketPath) + go http.Serve(listener, nil) + return dropPrivileges(sudoUser, tmpSocketPath) +} + +func run(args []string) error { + sudoUser := os.Getenv("SUDO_USER") + if os.Geteuid() == 0 && sudoUser != "" { + return handleSudoUser(sudoUser) + } + workspaceTopDir, err := getWorkspaceTopDir() + if err != nil { + return err + } + pathBinder := getPathBinder() + return client.HandleCommand(workspaceTopDir, pathBinder, args) +} + +func main() { + if err := run(os.Args); err != nil { + fmt.Println("Error:", err) + os.Exit(1) + } +} diff --git a/hacksaw/cmd/hacksaw/main_test.go b/hacksaw/cmd/hacksaw/main_test.go new file mode 100644 index 0000000..09669ab --- /dev/null +++ b/hacksaw/cmd/hacksaw/main_test.go @@ -0,0 +1,39 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestHelp(t *testing.T) { + homeDir, err := ioutil.TempDir("", "hacksaw_test_home") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(homeDir) + if err = os.Setenv("HOME", homeDir); err != nil { + t.Error(err) + } + args := []string{ + "hacksaw", + "help", + } + if err = run(args); err != nil { + t.Error(err) + } +} diff --git a/hacksaw/cmd/hacksawd/main.go b/hacksaw/cmd/hacksawd/main.go new file mode 100644 index 0000000..8f1ad1a --- /dev/null +++ b/hacksaw/cmd/hacksawd/main.go @@ -0,0 +1,52 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// hacksawd is a privileged daemon that manages the mounts +package main + +import ( + "net" + "net/http" + "net/rpc" + "os" + "strconv" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/bind" +) + +func main() { + if os.Getenv("LISTEN_PID") != strconv.Itoa(os.Getpid()) { + panic("Unexpected PID") + } + + if os.Getenv("LISTEN_FDS") != strconv.Itoa(1) { + panic("Unexpected number of socket fds") + } + + const socketFD = 3 + socketFile := os.NewFile(socketFD, "hacksawd.sock") + + listener, err := net.FileListener(socketFile) + if err != nil { + panic(err) + } + + binder := bind.NewLocalPathBinder() + server := bind.NewServer(binder) + if err = rpc.Register(server); err != nil { + panic(err) + } + rpc.HandleHTTP() + http.Serve(listener, nil) +} diff --git a/hacksaw/codebase/codebase.go b/hacksaw/codebase/codebase.go new file mode 100644 index 0000000..9c38b91 --- /dev/null +++ b/hacksaw/codebase/codebase.go @@ -0,0 +1,92 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package codebase let's you manage codebases +package codebase + +import ( + "fmt" + "os" + "path/filepath" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/config" +) + +// Add a codebase to the list of supported codebases +func Add(name string, path string) (*config.Config, error) { + absPath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + //check that the codebase path is valid before adding + _, err = os.Stat(absPath) + if err != nil { + return nil, err + } + cfg := config.GetConfig() + if _, ok := cfg.Codebases[name]; ok { + return cfg, fmt.Errorf("Codebase %s already exists", name) + } + cfg.Codebases[name] = absPath + return cfg, err +} + +// Remove an existing codebase +func Remove(remove string) (*config.Config, error) { + cfg := config.GetConfig() + _, ok := cfg.Codebases[remove] + if !ok { + return nil, fmt.Errorf("Codebase %s not found", remove) + } + delete(cfg.Codebases, remove) + if cfg.DefaultCodebase == remove { + cfg.DefaultCodebase = "" + } + return cfg, nil +} + +// Default gets the default codebase +func Default() string { + cfg := config.GetConfig() + def := cfg.DefaultCodebase + return def +} + +// SetDefault sets the default codebase +func SetDefault(def string) error { + cfg := config.GetConfig() + _, ok := cfg.Codebases[def] + if !ok { + return fmt.Errorf("Codebase %s not found", def) + } + cfg.DefaultCodebase = def + return nil +} + +// List all supported codebases +func List() map[string]string { + cfg := config.GetConfig() + return cfg.Codebases +} + +// GetDir retrieves the directory of a specific workspace +func GetDir(codebase string) (string, error) { + cfg := config.GetConfig() + dir, ok := cfg.Codebases[codebase] + if !ok { + return dir, fmt.Errorf("Codebase %s not found", + codebase) + } + return dir, nil +} diff --git a/hacksaw/codebase/codebase_test.go b/hacksaw/codebase/codebase_test.go new file mode 100644 index 0000000..16442b0 --- /dev/null +++ b/hacksaw/codebase/codebase_test.go @@ -0,0 +1,217 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codebase + +import ( + "io/ioutil" + "os" + "path" + "reflect" + "testing" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/config" +) + +func TestAdd(t *testing.T) { + defer config.GetConfig().Reset() + firstCodebaseDir, err := ioutil.TempDir("", "first") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(firstCodebaseDir) + outputConfig, err := Add("first-codebase", firstCodebaseDir) + if err != nil { + t.Error(err) + } + expectedConfig := config.Config{ + DefaultCodebase: "", + Codebases: map[string]string{ + "first-codebase": firstCodebaseDir, + }, + Workspaces: map[string]string{}} + if !reflect.DeepEqual(expectedConfig, *outputConfig) { + t.Errorf("Expected config %v different than output config %v", + expectedConfig, *outputConfig) + } + + secondCodebaseDir, err := ioutil.TempDir("", "second") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(secondCodebaseDir) + if outputConfig, err = Add("second-codebase", secondCodebaseDir); err != nil { + t.Error(err) + } + expectedConfig = config.Config{ + DefaultCodebase: "", + Codebases: map[string]string{ + "first-codebase": firstCodebaseDir, + "second-codebase": secondCodebaseDir, + }, + Workspaces: map[string]string{}} + if !reflect.DeepEqual(expectedConfig, *outputConfig) { + t.Errorf("Expected config %v different than output config %v", + expectedConfig, *outputConfig) + } +} + +func TestAddDuplicate(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "first") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + _, err = Add("codebase", codebaseDir) + if err != nil { + t.Error(err) + } + _, err = Add("codebase", codebaseDir) + if err == nil { + t.Error("Allowed duplicate codebase") + } +} + +func TestAddInvalidCodebase(t *testing.T) { + defer config.GetConfig().Reset() + tempDir, err := ioutil.TempDir("", "codebase") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(tempDir) + testCodebaseDir := path.Join(tempDir, "doesnotexist") + _, err = Add("test-codebase", testCodebaseDir) + if err == nil { + t.Error("Adding a codebase with an invalid directory should not be allowed") + } +} + +func TestList(t *testing.T) { + defer config.GetConfig().Reset() + firstCodebaseDir, err := ioutil.TempDir("", "first") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(firstCodebaseDir) + _, err = Add("first-codebase", firstCodebaseDir) + if err != nil { + t.Error(err) + } + secondCodebaseDir, err := ioutil.TempDir("", "second") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(secondCodebaseDir) + if _, err = Add("second-codebase", secondCodebaseDir); err != nil { + t.Error(err) + } + list := List() + expectedList := map[string]string{ + "first-codebase": firstCodebaseDir, + "second-codebase": secondCodebaseDir, + } + if !reflect.DeepEqual(expectedList, list) { + t.Errorf("Codebase list %v is different than expected list %v", + list, expectedList) + } +} + +func TestSetGetDefault(t *testing.T) { + defer config.GetConfig().Reset() + firstCodebaseDir, err := ioutil.TempDir("", "first") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(firstCodebaseDir) + _, err = Add("first-codebase", firstCodebaseDir) + if err != nil { + t.Error(err) + } + secondCodebaseDir, err := ioutil.TempDir("", "second") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(secondCodebaseDir) + _, err = Add("second-codebase", secondCodebaseDir) + if err != nil { + t.Error(err) + } + if err = SetDefault("second-codebase"); err != nil { + t.Error(err) + } + def := Default() + if def != "second-codebase" { + t.Error("Setting default codebase failed") + } +} + +func TestBadDefault(t *testing.T) { + defer config.GetConfig().Reset() + testCodebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(testCodebaseDir) + _, err = Add("test-codebase", testCodebaseDir) + if err != nil { + t.Error(err) + } + if err = SetDefault("does-not-exist"); err == nil { + t.Error("Allowed invalid default codebase") + } +} + +func TestRemove(t *testing.T) { + defer config.GetConfig().Reset() + testCodebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(testCodebaseDir) + _, err = Add("remove-me", testCodebaseDir) + if err != nil { + t.Error(err) + } + SetDefault("remove-me") + cfg, err := Remove("remove-me") + if err != nil { + t.Error(err) + } + _, ok := cfg.Codebases["remove-me"] + if ok { + t.Error("Removed codebase is still in the configuration") + } + if cfg.DefaultCodebase != "" { + t.Error("Removing the default codebase did not reset the default") + } +} + +func TestBadRemove(t *testing.T) { + defer config.GetConfig().Reset() + testCodebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(testCodebaseDir) + _, err = Add("remove-me", testCodebaseDir) + if err != nil { + t.Error(err) + } + _, err = Remove("does-not-exist") + if err == nil { + t.Error("Attempt to remove an unexistant codebase", + "should have failed") + } +} diff --git a/hacksaw/config/config.go b/hacksaw/config/config.go new file mode 100644 index 0000000..83a0ffd --- /dev/null +++ b/hacksaw/config/config.go @@ -0,0 +1,109 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package config reads and writes hacksaw configuration data to io +package config + +import ( + "encoding/json" + "io" + "io/ioutil" + "os" + "sync" +) + +type Config struct { + DefaultCodebase string + Codebases map[string]string //key: name, value: path + Workspaces map[string]string //key: name, value: codebase +} + +//Read the configuration from an io.Reader +func (c *Config) Read(input io.Reader) error { + cfgBytes, err := ioutil.ReadAll(input) + if err != nil { + return err + } + return json.Unmarshal(cfgBytes, &c) +} + +//Write the configuration to an io.Writer +func (c Config) Write(output io.Writer) error { + cfgBytes, err := json.MarshalIndent(c, "", " ") + if err != nil { + return err + } + _, err = output.Write(cfgBytes) + return err +} + +func (c *Config) ReadConfigFromFile(filePath string) error { + _, err := os.Stat(filePath) + if err != nil { + return err + } + cfgFile, err := os.Open(filePath) + if err != nil { + return err + } + defer cfgFile.Close() + err = c.Read(cfgFile) + return err +} + +func (c Config) WriteConfigToFile(filePath string) error { + cfgFile, err := os.Create(filePath) + if err != nil { + return err + } + defer cfgFile.Close() + return c.Write(cfgFile) +} + +//Config gets a copy of the config +func (c Config) Copy() Config { + cfgCopy := Config{ + DefaultCodebase: c.DefaultCodebase, + Codebases: map[string]string{}, + Workspaces: map[string]string{}} + for name, path := range c.Codebases { + cfgCopy.Codebases[name] = path + } + for name, codebase := range c.Workspaces { + cfgCopy.Workspaces[name] = codebase + } + return cfgCopy +} + +//Reset sets the config to zero values +func (c *Config) Reset() { + *c = Config{ + DefaultCodebase: "", + Codebases: map[string]string{}, + Workspaces: map[string]string{}} +} + +var singleton *Config +var once sync.Once + +//Config gets the singleton config instance +func GetConfig() *Config { + once.Do(func() { + singleton = &Config{ + DefaultCodebase: "", + Codebases: map[string]string{}, + Workspaces: map[string]string{}} + }) + return singleton +} diff --git a/hacksaw/config/config_test.go b/hacksaw/config/config_test.go new file mode 100644 index 0000000..e010022 --- /dev/null +++ b/hacksaw/config/config_test.go @@ -0,0 +1,74 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "bytes" + "reflect" + "testing" +) + +const testWriteExpectedOutput = `{ + "DefaultCodebase": "test-codebase", + "Codebases": { + "test-codebase": "/path/to/test/codebase" + }, + "Workspaces": {} +}` + +func TestWrite(t *testing.T) { + inputConfig := Config{ + DefaultCodebase: "test-codebase", + Codebases: map[string]string{ + "test-codebase": "/path/to/test/codebase", + }, + Workspaces: map[string]string{}} + var outputBuffer bytes.Buffer + if err := inputConfig.Write(&outputBuffer); err != nil { + t.Error(err) + } + expectedOutput := []byte(testWriteExpectedOutput) + if bytes.Compare(outputBuffer.Bytes(), expectedOutput) != 0 { + t.Errorf("Output %s is different that expected output %s", + string(outputBuffer.Bytes()), string(expectedOutput)) + } +} + +const testReadInput = `{ + "DefaultCodebase": "test-codebase", + "Codebases": { + "test-codebase": "/path/to/test/codebase" + }, + "Workspaces": {} +}` + +func TestRead(t *testing.T) { + inputBytes := []byte(testReadInput) + inputBuffer := bytes.NewBuffer(inputBytes) + var outputConfig Config + if err := outputConfig.Read(inputBuffer); err != nil { + t.Error(err) + } + expectedOutput := Config{ + DefaultCodebase: "test-codebase", + Codebases: map[string]string{ + "test-codebase": "/path/to/test/codebase", + }, + Workspaces: map[string]string{}} + if !reflect.DeepEqual(outputConfig, expectedOutput) { + t.Errorf("Written config %v is different than read config %v", + outputConfig, expectedOutput) + } +} diff --git a/hacksaw/git/lister.go b/hacksaw/git/lister.go new file mode 100644 index 0000000..44cb3e7 --- /dev/null +++ b/hacksaw/git/lister.go @@ -0,0 +1,19 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +type ProjectLister interface { + List(codebasePath string) ([]string, error) +} diff --git a/hacksaw/git/repo.go b/hacksaw/git/repo.go new file mode 100644 index 0000000..b7887ea --- /dev/null +++ b/hacksaw/git/repo.go @@ -0,0 +1,47 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "bufio" + "os" + "path" +) + +type repoLister struct { +} + +func NewRepoLister() ProjectLister { + var rl repoLister + return &rl +} + +func (rl repoLister) List(codebasePath string) ([]string, error) { + projectList := []string{".repo"} + listPath := path.Join(codebasePath, ".repo/project.list") + listFile, err := os.Open(listPath) + if err != nil { + return projectList, err + } + defer listFile.Close() + scanner := bufio.NewScanner(listFile) + for scanner.Scan() { + line := scanner.Text() + if line != "" { + projectList = append(projectList, line) + } + } + return projectList, err +} diff --git a/hacksaw/git/repo_test.go b/hacksaw/git/repo_test.go new file mode 100644 index 0000000..394bf44 --- /dev/null +++ b/hacksaw/git/repo_test.go @@ -0,0 +1,55 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "io/ioutil" + "os" + "path" + "reflect" + "testing" +) + +func TestList(t *testing.T) { + testCodebaseDir, err := ioutil.TempDir("", "codebase") + if err != nil { + t.Error(err) + } + if err = os.Mkdir(path.Join(testCodebaseDir, ".repo"), os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte( + "path/to/project1\n" + + "path/to/project2\n" + + "path/to/project3") + listFile := path.Join(testCodebaseDir, ".repo/project.list") + if err = ioutil.WriteFile(listFile, listContents, os.ModePerm); err != nil { + t.Error(err) + } + lister := NewRepoLister() + projectList, err := lister.List(testCodebaseDir) + if err != nil { + t.Error(err) + } + expectedList := []string{ + ".repo", + "path/to/project1", + "path/to/project2", + "path/to/project3", + } + if !reflect.DeepEqual(projectList, expectedList) { + t.Errorf("Got list %v but expected %v", projectList, expectedList) + } +} diff --git a/hacksaw/go.mod b/hacksaw/go.mod new file mode 100644 index 0000000..65a6d50 --- /dev/null +++ b/hacksaw/go.mod @@ -0,0 +1,3 @@ +module android.googlesource.com/platform/tools/treble.git/hacksaw + +go 1.15 diff --git a/hacksaw/images/workspace-diagram.png b/hacksaw/images/workspace-diagram.png Binary files differnew file mode 100644 index 0000000..cea44be --- /dev/null +++ b/hacksaw/images/workspace-diagram.png diff --git a/hacksaw/mount/fake.go b/hacksaw/mount/fake.go new file mode 100644 index 0000000..a2d059c --- /dev/null +++ b/hacksaw/mount/fake.go @@ -0,0 +1,66 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mount + +import ( + "fmt" +) + +type MountEntry struct { + Source string + FSType string + Flags uintptr + Data string +} + +type fakeMounter struct { + mountMap map[string]MountEntry //keyed by target +} + +func NewFakeMounter() *fakeMounter { + var f fakeMounter + f.mountMap = make(map[string]MountEntry) + return &f +} + +func (f *fakeMounter) Mount(source string, target string, fstype string, flags uintptr, data string) error { + //Using the target as the key prevents more + //than one source mapping to the same target + f.mountMap[target] = MountEntry{ + Source: source, + FSType: fstype, + Flags: flags, + Data: data, + } + return nil +} + +func (f *fakeMounter) Unmount(target string, flags int) error { + _, ok := f.mountMap[target] + if !ok { + return fmt.Errorf("Mount %s not found", target) + } + delete(f.mountMap, target) + return nil +} + +func (f *fakeMounter) List() ([]string, error) { + var list []string + for target, _ := range f.mountMap { + list = append(list, target) + } + + return list, nil +} diff --git a/hacksaw/mount/mount.go b/hacksaw/mount/mount.go new file mode 100644 index 0000000..f301eb0 --- /dev/null +++ b/hacksaw/mount/mount.go @@ -0,0 +1,21 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mount + +type Mounter interface { + Mount(string, string, string, uintptr, string) error + Unmount(string, int) error + List() ([]string, error) +} diff --git a/hacksaw/mount/system.go b/hacksaw/mount/system.go new file mode 100644 index 0000000..1d8bfbd --- /dev/null +++ b/hacksaw/mount/system.go @@ -0,0 +1,80 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mount + +import ( + "bufio" + "io" + "os" + "strings" + "syscall" +) + +type systemMounter struct { +} + +func NewSystemMounter() *systemMounter { + var f systemMounter + return &f +} + +func (f *systemMounter) Mount(source string, target string, fstype string, flags uintptr, data string) error { + return syscall.Mount(source, target, fstype, flags, data) +} + +func (f *systemMounter) Unmount(target string, flags int) error { + return syscall.Unmount(target, flags) +} + +func (f *systemMounter) List() ([]string, error) { + mountsFile, err := os.Open("/proc/mounts") + if err != nil { + return nil, err + } + defer mountsFile.Close() + mounts, err := f.parseMounts(mountsFile) + if err != nil { + return nil, err + } + var mountList []string + for _, mount := range mounts { + mountList = append(mountList, mount.Path) + } + return mountList, err +} + +type Mount struct { + Device string + Path string + Type string + Opts string +} + +func (f *systemMounter) parseMounts(mountSource io.Reader) ([]Mount, error) { + var mounts []Mount + scanner := bufio.NewScanner(mountSource) + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + mount := Mount{ + Device: fields[0], + Path: fields[1], + Type: fields[2], + Opts: fields[3], + } + mounts = append(mounts, mount) + } + return mounts, scanner.Err() +} diff --git a/hacksaw/scripts/hacksaw.service b/hacksaw/scripts/hacksaw.service new file mode 100644 index 0000000..513ef30 --- /dev/null +++ b/hacksaw/scripts/hacksaw.service @@ -0,0 +1,11 @@ +[Unit] +Description=Hacksaw service +ConditionPathExists=/usr/local/bin/hacksawd + +[Service] +# Runs as root by default +Type=simple +ExecStart=/usr/local/bin/hacksawd + +[Install] +RequiredBy=hacksaw.socket diff --git a/hacksaw/scripts/hacksaw.socket b/hacksaw/scripts/hacksaw.socket new file mode 100644 index 0000000..46bec93 --- /dev/null +++ b/hacksaw/scripts/hacksaw.socket @@ -0,0 +1,8 @@ +[Socket] +ListenStream=/var/run/hacksaw.sock +SocketMode=0660 +SocketUser=root +SocketGroup=hacksaw + +[Install] +WantedBy=sockets.target diff --git a/hacksaw/scripts/install-service.sh b/hacksaw/scripts/install-service.sh new file mode 100755 index 0000000..7229288 --- /dev/null +++ b/hacksaw/scripts/install-service.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e +set -v + +SCRIPT_DIR=$(dirname $(readlink --canonicalize $0)) + +# This script must be run with sudo + +# The hacksaw group is used to manage access to the +# hacksawd service +groupadd hacksaw +usermod -a -G hacksaw ${SUDO_USER} +cp ${SCRIPT_DIR}/hacksaw.service /lib/systemd/system/hacksaw.service +cp ${SCRIPT_DIR}/hacksaw.socket /lib/systemd/system/hacksaw.socket +chmod 755 /lib/systemd/system/hacksaw.* +systemctl enable hacksaw.service +systemctl enable hacksaw.socket +# hacksaw.socket starts hacksaw.service on-demand +# so hacksaw.service does not need to be explicitly +# started +systemctl start hacksaw.socket +echo "The installation scripts creates a new 'hacksaw' group and adds you to it." +echo "You will need to log out and log back in for the group changes to take effect." diff --git a/hacksaw/scripts/uninstall-service.sh b/hacksaw/scripts/uninstall-service.sh new file mode 100755 index 0000000..e617708 --- /dev/null +++ b/hacksaw/scripts/uninstall-service.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e +set -v + +SCRIPT_DIR=$(dirname $(readlink --canonicalize $0)) + +# This script must be run with sudo + +systemctl stop hacksaw.socket +systemctl stop hacksaw.service +systemctl disable hacksaw.service +systemctl disable hacksaw.socket +rm /lib/systemd/system/hacksaw.service +rm /lib/systemd/system/hacksaw.socket +deluser ${SUDO_USER} hacksaw +groupdel hacksaw diff --git a/hacksaw/workspace/compose.go b/hacksaw/workspace/compose.go new file mode 100644 index 0000000..f1cc2c8 --- /dev/null +++ b/hacksaw/workspace/compose.go @@ -0,0 +1,151 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workspace + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/bind" + "android.googlesource.com/platform/tools/treble.git/hacksaw/git" +) + +type Composer struct { + pathBinder bind.PathBinder +} + +func NewComposer(bm bind.PathBinder) Composer { + return Composer{bm} +} + +func isDirEmpty(name string) (bool, error) { + dir, err := os.Open(name) + if err != nil { + return false, err + } + defer dir.Close() + _, err = dir.Readdirnames(1) + if err == io.EOF { + return true, nil + } + return false, err +} + +//Compose a workspace from a codebase +//Returns a list of path binds in the order they +//were bound +func (m Composer) Compose(codebasePath string, workspacePath string) ([]string, error) { + lister := git.NewRepoLister() + gitProjects, err := lister.List(codebasePath) + if err != nil { + return nil, err + } + fmt.Print("Composing") + var bindList []string + //Sorting the list of projects in alphabetical + //order ensures that parent projects are bound + //before their nested child projects, which is important + //to avoid bind conflicts + sort.Strings(gitProjects) + for _, project := range gitProjects { + fmt.Print(".") //Display some progress + //skip empty project names + if project == "" { + continue + } + source := filepath.Join(codebasePath, project) + destination := filepath.Join(workspacePath, project) + if err = os.MkdirAll(destination, os.ModePerm); err != nil { + fmt.Print("\n") + return bindList, err + } + isEmpty, err := isDirEmpty(destination) + if err != nil { + return bindList, err + } + if !isEmpty { + // If the destination dir already existed and + // was not empty then assume we are recreating + // a workspace and the current path already + // existed in the workspace + continue + } + if err = m.pathBinder.BindReadOnly(source, destination); err != nil { + fmt.Print("\n") + return bindList, err + } + bindList = append(bindList, destination) + } + fmt.Print("\n") + fmt.Println("Workspace composed") + copier := NewFileCopier() + return bindList, copier.Copy(codebasePath, gitProjects, workspacePath) +} + +//Dismantle a workspace +//Returns a list of path unbinds in the order they +//were unbound +func (m Composer) Dismantle(dismantlePath string) ([]string, error) { + bindList, err := m.List(dismantlePath) + if err != nil { + return nil, err + } + //Sorting the list of binds in reverse alphabetical + //order ensures that nested child projects are unbound + //before their parent projects, which is important + //to avoid unbind conflicts + sort.Sort(sort.Reverse(sort.StringSlice(bindList))) + fmt.Print("Dismantling") + var unbindList []string + for _, bindPath := range bindList { + fmt.Print(".") //Display some progress + if err = m.pathBinder.Unbind(bindPath); err != nil { + fmt.Print("\n") + return unbindList, err + } + unbindList = append(unbindList, bindPath) + } + fmt.Print("\n") + fmt.Println("Workspace dismantled") + return unbindList, err +} + +//Unbind a project +func (m Composer) Unbind(unbindPath string) error { + return m.pathBinder.Unbind(unbindPath) +} + +//List all binds attached under a directory +func (m Composer) List(listPath string) ([]string, error) { + listPath, err := filepath.EvalSymlinks(listPath) + if err != nil { + return nil, err + } + fullBindList, err := m.pathBinder.List() + if err != nil { + return nil, err + } + var matchBindList []string + for _, bindPath := range fullBindList { + if strings.HasPrefix(bindPath+"/", listPath+"/") { + matchBindList = append(matchBindList, bindPath) + } + } + return matchBindList, err +} diff --git a/hacksaw/workspace/compose_test.go b/hacksaw/workspace/compose_test.go new file mode 100644 index 0000000..087499c --- /dev/null +++ b/hacksaw/workspace/compose_test.go @@ -0,0 +1,158 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Ve +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workspace + +import ( + "io/ioutil" + "os" + "path" + "reflect" + "testing" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/bind" +) + +func TestBasicCompose(t *testing.T) { + var err error + var codebaseDir string + if codebaseDir, err = ioutil.TempDir("", "codebase"); err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + gitDir := path.Join(codebaseDir, "project", ".git") + if err = os.MkdirAll(gitDir, os.ModePerm); err != nil { + t.Error(err) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + var wsTempDir string + if wsTempDir, err = ioutil.TempDir("", "workspace"); err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + fakePathBinder := bind.NewFakePathBinder() + wsComposer := NewComposer(fakePathBinder) + list, err := wsComposer.Compose(codebaseDir, wsTopDir) + if err != nil { + t.Error(err) + } + expectedList := []string{ + path.Join(wsTopDir, ".repo"), + path.Join(wsTopDir, "project"), + } + if !reflect.DeepEqual(expectedList, list) { + t.Errorf("Expected a list of binds %v and got %v", expectedList, list) + } + list, err = wsComposer.Dismantle(wsTopDir) + if err != nil { + t.Error(err) + } + expectedList = []string{ + path.Join(wsTopDir, "project"), + path.Join(wsTopDir, ".repo"), + } + if !reflect.DeepEqual(expectedList, list) { + t.Errorf("Expected a list of unbinds %v and got %v", expectedList, list) + } +} + +const projectListContents = `project2/subprojectb +project1 +project2 +project1/subprojecta` + +func TestBindOrder(t *testing.T) { + var err error + var codebaseDir string + if codebaseDir, err = ioutil.TempDir("", "codebase"); err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + project2Git := path.Join(codebaseDir, "project2", ".git") + if err = os.MkdirAll(project2Git, os.ModePerm); err != nil { + t.Error(err) + } + project1Git := path.Join(codebaseDir, "project1", ".git") + if err = os.MkdirAll(project1Git, os.ModePerm); err != nil { + t.Error(err) + } + subProjectAGit := path.Join(codebaseDir, "project1", "subprojecta", ".git") + if err = os.MkdirAll(subProjectAGit, os.ModePerm); err != nil { + t.Error(err) + } + subProjectBGit := path.Join(codebaseDir, "project2", "subprojectb", ".git") + if err = os.MkdirAll(subProjectBGit, os.ModePerm); err != nil { + t.Error(err) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte(projectListContents) + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + var wsTempDir string + if wsTempDir, err = ioutil.TempDir("", "workspace"); err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + fakePathBinder := bind.NewFakePathBinder() + wsComposer := NewComposer(fakePathBinder) + bindList, err := wsComposer.Compose(codebaseDir, wsTopDir) + if err != nil { + t.Error(err) + } + expectedList := []string{ + path.Join(wsTopDir, ".repo"), + path.Join(wsTopDir, "project1"), + path.Join(wsTopDir, "project1", "subprojecta"), + path.Join(wsTopDir, "project2"), + path.Join(wsTopDir, "project2", "subprojectb"), + } + if !reflect.DeepEqual(expectedList, bindList) { + t.Errorf("Expected a list of binds %v and got %v", expectedList, bindList) + } + unbindList, err := wsComposer.Dismantle(wsTopDir) + if err != nil { + t.Error(err) + } + expectedList = []string{ + path.Join(wsTopDir, "project2", "subprojectb"), + path.Join(wsTopDir, "project2"), + path.Join(wsTopDir, "project1", "subprojecta"), + path.Join(wsTopDir, "project1"), + path.Join(wsTopDir, ".repo"), + } + if !reflect.DeepEqual(expectedList, unbindList) { + t.Errorf("Expected a list of unbinds %v and got %v", expectedList, unbindList) + } +} diff --git a/hacksaw/workspace/copier.go b/hacksaw/workspace/copier.go new file mode 100644 index 0000000..f90edc8 --- /dev/null +++ b/hacksaw/workspace/copier.go @@ -0,0 +1,241 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workspace + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +type FileCopier struct { +} + +func NewFileCopier() *FileCopier { + var f FileCopier + return &f +} + +func (f FileCopier) GetIsGitProjectFunc(codebaseDir string, gitProjects []string) func(string) (bool, error) { + //Convert the git project list to a set to speed up lookups + gitProjectSet := make(map[string]struct{}) + var exists = struct{}{} + for _, project := range gitProjects { + gitProjectSet[project] = exists + } + + return func(pathToCheck string) (bool, error) { + var err error + if pathToCheck, err = filepath.Rel(codebaseDir, pathToCheck); err != nil { + return false, err + } + if _, ok := gitProjectSet[pathToCheck]; ok { + return true, err + } + return false, err + } +} + +func (f FileCopier) GetContainsGitProjectFunc(codebaseDir string, gitProjects []string) func(string) (bool, error) { + //Extract the set of dirs that contain git projects + containsGitSet := make(map[string]struct{}) + var exists = struct{}{} + for _, project := range gitProjects { + for dir := project; dir != "." && dir != "/"; dir = filepath.Dir(dir) { + containsGitSet[dir] = exists + } + } + + return func(pathToCheck string) (bool, error) { + var err error + if pathToCheck, err = filepath.Rel(codebaseDir, pathToCheck); err != nil { + return false, err + } + if _, ok := containsGitSet[pathToCheck]; ok { + return true, err + } + return false, err + } +} + +//gitProjects is relative to codebaseDir +func (f FileCopier) Copy(codebaseDir string, gitProjects []string, workspaceDir string) error { + isGitProject := f.GetIsGitProjectFunc(codebaseDir, gitProjects) + containsGitProject := f.GetContainsGitProjectFunc(codebaseDir, gitProjects) + + return filepath.Walk(codebaseDir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Copy files + if !info.IsDir() { + return f.CopyNode(info, codebaseDir, path, workspaceDir) + } + + if path == filepath.Clean(codebaseDir) { + return nil + } + + // Always skip traversal of root repo directories + if path == filepath.Join(codebaseDir, ".repo") { + return filepath.SkipDir + } + + // Skip all git projects + var isGitProj bool + if isGitProj, err = isGitProject(path); err != nil { + return err + } + if isGitProj { + return filepath.SkipDir + } + + // Copy over files + var containsGitProj bool + if containsGitProj, err = containsGitProject(path); err != nil { + return err + } + if !containsGitProj { + destPath, err := f.GetDestPath(codebaseDir, path, workspaceDir) + if err != nil { + return err + } + if err = f.CopyDirRecursive(info, path, destPath); err != nil { + return err + } + return filepath.SkipDir + } + return f.CopyNode(info, codebaseDir, path, workspaceDir) + }) +} + +func (f FileCopier) GetDestPath(codebaseDir, sourcePath, workspaceDir string) (string, error) { + if !strings.HasPrefix(sourcePath+"/", codebaseDir+"/") { + return "", fmt.Errorf("%s is not contained in %s", sourcePath, codebaseDir) + } + relPath, err := filepath.Rel(codebaseDir, sourcePath) + if err != nil { + return "", err + } + destPath := filepath.Join(workspaceDir, relPath) + return destPath, err +} + +// Copy any single file, symlink or dir non-recursively +// sourcePath must be contained in codebaseDir +func (f FileCopier) CopyNode(sourceInfo os.FileInfo, codebaseDir, sourcePath, workspaceDir string) error { + destPath, err := f.GetDestPath(codebaseDir, sourcePath, workspaceDir) + if err != nil { + return err + } + switch { + case sourceInfo.Mode()&os.ModeSymlink == os.ModeSymlink: + return f.CopySymlink(sourcePath, destPath) + case sourceInfo.Mode().IsDir(): + return f.CopyDirOnly(sourceInfo, destPath) + default: + return f.CopyFile(sourceInfo, sourcePath, destPath) + } +} + +func (f FileCopier) CopySymlink(sourcePath string, destPath string) error { + // Skip symlink if it already exists at the destination + _, err := os.Lstat(destPath) + if err == nil { + return nil + } + + target, err := os.Readlink(sourcePath) + if err != nil { + return err + } + + return os.Symlink(target, destPath) +} + +// CopyDirOnly copies a directory non-recursively +// sourcePath must be contained in codebaseDir +func (f FileCopier) CopyDirOnly(sourceInfo os.FileInfo, destPath string) error { + _, err := os.Stat(destPath) + if err == nil { + // Dir already exists, nothing to do + return err + } else if os.IsNotExist(err) { + return os.Mkdir(destPath, sourceInfo.Mode()) + } + return err +} + +// CopyFile copies a single file +// sourcePath must be contained in codebaseDir +func (f FileCopier) CopyFile(sourceInfo os.FileInfo, sourcePath, destPath string) error { + //Skip file if it already exists at the destination + _, err := os.Lstat(destPath) + if err == nil { + return nil + } + + sourceFile, err := os.Open(sourcePath) + if err != nil { + return err + } + defer sourceFile.Close() + + destFile, err := os.Create(destPath) + if err != nil { + return err + } + defer destFile.Close() + + _, err = io.Copy(destFile, sourceFile) + if err != nil { + return err + } + return os.Chmod(destPath, sourceInfo.Mode()) +} + +func (f FileCopier) CopyDirRecursive(sourceInfo os.FileInfo, sourcePath, destPath string) error { + if err := f.CopyDirOnly(sourceInfo, destPath); err != nil { + return err + } + childNodes, err := ioutil.ReadDir(sourcePath) + if err != nil { + return err + } + for _, childInfo := range childNodes { + childSourcePath := filepath.Join(sourcePath, childInfo.Name()) + childDestPath := filepath.Join(destPath, childInfo.Name()) + switch { + case childInfo.Mode()&os.ModeSymlink == os.ModeSymlink: + if err = f.CopySymlink(childSourcePath, childDestPath); err != nil { + return err + } + case childInfo.Mode().IsDir(): + if err = f.CopyDirRecursive(childInfo, childSourcePath, childDestPath); err != nil { + return err + } + default: + if err = f.CopyFile(childInfo, childSourcePath, childDestPath); err != nil { + return err + } + } + } + return err +} diff --git a/hacksaw/workspace/copier_test.go b/hacksaw/workspace/copier_test.go new file mode 100644 index 0000000..134f64e --- /dev/null +++ b/hacksaw/workspace/copier_test.go @@ -0,0 +1,136 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workspace + +import ( + "io/ioutil" + "os" + "path" + "testing" +) + +func TestCopy(t *testing.T) { + // Setup codebase + codebaseDir, err := ioutil.TempDir("", "codebase") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + project1GitDir := path.Join(codebaseDir, "project1", ".git") + if err = os.MkdirAll(project1GitDir, os.ModePerm); err != nil { + t.Error(err) + } + emptyBytes := []byte{} + project1File := path.Join(codebaseDir, "project1", "projectfile") + if err = ioutil.WriteFile(project1File, emptyBytes, os.ModePerm); err != nil { + t.Error(err) + } + symlink := path.Join(codebaseDir, "symlink") + if err = os.Symlink(path.Join(codebaseDir, "project1"), symlink); err != nil { + t.Error(err) + } + project2GitDir := path.Join(codebaseDir, "dirwithprojects", "project2", ".git") + if err = os.MkdirAll(project2GitDir, os.ModePerm); err != nil { + t.Error(err) + } + dirWithoutProjects := path.Join(codebaseDir, "dirwithoutprojects") + if err = os.Mkdir(dirWithoutProjects, os.ModePerm); err != nil { + t.Error(err) + } + projectSiblingFile := path.Join(codebaseDir, "dirwithprojects", "projectsiblingfile") + if err = ioutil.WriteFile(projectSiblingFile, emptyBytes, os.ModePerm); err != nil { + t.Error(err) + } + noProjectFile1 := path.Join(dirWithoutProjects, "noprojectfile1") + if err = ioutil.WriteFile(noProjectFile1, emptyBytes, os.ModePerm); err != nil { + t.Error(err) + } + noProjectFile2 := path.Join(dirWithoutProjects, "noprojectfile2") + if err = ioutil.WriteFile(noProjectFile2, emptyBytes, os.ModePerm); err != nil { + t.Error(err) + } + topFile := path.Join(codebaseDir, "topfile") + if err = ioutil.WriteFile(topFile, emptyBytes, os.ModePerm); err != nil { + t.Error(err) + } + gitProjects := []string{ + "project1", + "dirwithprojects/project2", + } + + // Set up workspace + workspaceDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + //This dir may already exist if the projects have been mounted + wsDirWithProjects := path.Join(workspaceDir, "dirwithprojects") + if err = os.Mkdir(wsDirWithProjects, os.ModePerm); err != nil { + t.Error(err) + } + + copier := NewFileCopier() + if err = copier.Copy(codebaseDir, gitProjects, workspaceDir); err != nil { + t.Error(err) + } + + wsTopFile := path.Join(workspaceDir, "topfile") + _, err = os.Stat(wsTopFile) + if err != nil { + t.Error(err) + } + wsNoProjectFile1 := path.Join(workspaceDir, "dirwithoutprojects", "noprojectfile1") + _, err = os.Stat(wsNoProjectFile1) + if err != nil { + t.Error(err) + } + wsNoProjectFile2 := path.Join(workspaceDir, "dirwithoutprojects", "noprojectfile2") + _, err = os.Stat(wsNoProjectFile2) + if err != nil { + t.Error(err) + } + wsProjectSiblingFile := path.Join(workspaceDir, "dirwithprojects", "projectsiblingfile") + _, err = os.Stat(wsProjectSiblingFile) + if err != nil { + t.Error(err) + } + wsSymlink := path.Join(workspaceDir, "symlink") + _, err = os.Stat(wsSymlink) + if err != nil { + t.Error(err) + } + //TODO: check why this is failing + // if linkInfo.Mode() & os.ModeSymlink != os.ModeSymlink { + // t.Error("Symlink not copied as symlink") + // } + wsProject2Dir := path.Join(workspaceDir, "dirwithprojects", "project2") + _, err = os.Stat(wsProject2Dir) + if err == nil { + t.Error("Project2 mistakenly copied") + } else if os.IsNotExist(err) { + //This error is expected + } else { + t.Error(err) + } + wsProject1Dir := path.Join(workspaceDir, "project1") + _, err = os.Stat(wsProject1Dir) + if err == nil { + t.Error("Project1 mistakenly copied") + } else if os.IsNotExist(err) { + //This error is expected + } else { + t.Error(err) + } +} diff --git a/hacksaw/workspace/workspace.go b/hacksaw/workspace/workspace.go new file mode 100644 index 0000000..43fbee6 --- /dev/null +++ b/hacksaw/workspace/workspace.go @@ -0,0 +1,329 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package workspace let's you manage workspaces +package workspace + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/bind" + "android.googlesource.com/platform/tools/treble.git/hacksaw/codebase" + "android.googlesource.com/platform/tools/treble.git/hacksaw/config" + "android.googlesource.com/platform/tools/treble.git/hacksaw/git" +) + +type Workspace struct { + composer Composer + topDir string +} + +func New(bm bind.PathBinder, topDir string) Workspace { + return Workspace{NewComposer(bm), topDir} +} + +// Create workspace +func (w Workspace) Create(workspaceName string, codebaseName string) (string, error) { + cfg := config.GetConfig() + _, ok := cfg.Codebases[codebaseName] + if !ok { + return "", fmt.Errorf("Codebase %s does not exist", codebaseName) + } + if _, ok := cfg.Workspaces[workspaceName]; ok { + return "", fmt.Errorf("Workspace %s already exists", workspaceName) + } + cfg.Workspaces[workspaceName] = codebaseName + workspaceDir, err := w.GetDir(workspaceName) + if err != nil { + return "", err + } + if err = os.MkdirAll(workspaceDir, os.ModePerm); err != nil { + return "", err + } + codebaseDir, err := codebase.GetDir(codebaseName) + if err != nil { + return "", err + } + //TODO: match the order of parameters with Create + if _, err = w.composer.Compose(codebaseDir, workspaceDir); err != nil { + return "", err + } + return workspaceDir, nil +} + +// Recreate workspace +func (w Workspace) Recreate(workspaceName string) (string, error) { + cfg := config.GetConfig() + codebaseName, ok := cfg.Workspaces[workspaceName] + if !ok { + return "", fmt.Errorf("Workspace %s does not exist", workspaceName) + } + workspaceDir, err := w.GetDir(workspaceName) + if err != nil { + return "", err + } + codebaseDir, err := codebase.GetDir(codebaseName) + if err != nil { + return "", err + } + if _, err = w.composer.Compose(codebaseDir, workspaceDir); err != nil { + return "", err + } + return workspaceDir, nil +} + +// GetDir retrieves the directory of a specific workspace +func (w Workspace) GetDir(workspaceName string) (string, error) { + cfg := config.GetConfig() + _, ok := cfg.Workspaces[workspaceName] + if !ok { + return "", fmt.Errorf("Workspace %s not found", workspaceName) + } + dir := filepath.Join(w.topDir, workspaceName) + return dir, nil +} + +// GetCodebase retrieves the codebase that a workspace belongs to +func (w Workspace) GetCodebase(workspaceName string) (string, error) { + cfg := config.GetConfig() + codebase, ok := cfg.Workspaces[workspaceName] + if !ok { + return "", fmt.Errorf("Workspace %s not found", workspaceName) + } + return codebase, nil +} + +//SetTopDir sets the directory that contains all workspaces +func (w *Workspace) SetTopDir(dir string) { + w.topDir = dir +} + +func (w Workspace) List() map[string]string { + cfg := config.GetConfig() + list := make(map[string]string) + for name, codebaseName := range cfg.Workspaces { + list[name] = codebaseName + } + return list +} + +func (w Workspace) DetachGitWorktrees(workspaceName string, unbindList []string) error { + workspaceDir, err := w.GetDir(workspaceName) + if err != nil { + return err + } + workspaceDir, err = filepath.Abs(workspaceDir) + if err != nil { + return err + } + //resolve all symlinks so it can be + //matched to mount paths + workspaceDir, err = filepath.EvalSymlinks(workspaceDir) + if err != nil { + return err + } + codebaseName, err := w.GetCodebase(workspaceName) + if err != nil { + return err + } + codebaseDir, err := codebase.GetDir(codebaseName) + if err != nil { + return err + } + lister := git.NewRepoLister() + gitProjects, err := lister.List(codebaseDir) + if err != nil { + return err + } + gitWorktrees := make(map[string]bool) + for _, project := range gitProjects { + gitWorktrees[project] = true + } + //projects that were unbound were definitely + //never git worktrees + for _, unbindPath := range unbindList { + project, err := filepath.Rel(workspaceDir, unbindPath) + if err != nil { + return err + } + if _, ok := gitWorktrees[project]; ok { + gitWorktrees[project] = false + } + } + for project, isWorktree := range gitWorktrees { + if !isWorktree { + continue + } + codebaseProject := filepath.Join(codebaseDir, project) + workspaceProject := filepath.Join(workspaceDir, project) + _, err = os.Stat(workspaceProject) + if err == nil { + //proceed to detach + } else if os.IsNotExist(err) { + //just skip if it doesn't exist + continue + } else { + return err + } + contents, err := ioutil.ReadDir(workspaceProject) + if err != nil { + return err + } + if len(contents) == 0 { + //empty directory, not even a .git + //not a wortree + continue + } + fmt.Print(".") + cmd := exec.Command("git", + "-C", codebaseProject, + "worktree", "remove", "--force", workspaceProject) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + cmd.String(), err.Error(), output) + } + cmd = exec.Command("git", + "-C", codebaseProject, + "branch", "--delete", "--force", workspaceName) + output, err = cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + cmd.String(), err.Error(), output) + } + } + return nil +} + +func (w Workspace) Remove(remove string) (*config.Config, error) { + cfg := config.GetConfig() + _, ok := cfg.Workspaces[remove] + if !ok { + return cfg, fmt.Errorf("Workspace %s not found", remove) + } + workspaceDir, err := w.GetDir(remove) + if err != nil { + return cfg, err + } + unbindList, err := w.composer.Dismantle(workspaceDir) + if err != nil { + return cfg, err + } + fmt.Print("Detaching worktrees") + if err = w.DetachGitWorktrees(remove, unbindList); err != nil { + return cfg, err + } + fmt.Print("\n") + fmt.Println("Removing files") + if err = os.RemoveAll(workspaceDir); err != nil { + return cfg, err + } + delete(cfg.Workspaces, remove) + return cfg, err +} + +func (w Workspace) Edit(editPath string) (string, string, error) { + editPath, err := filepath.Abs(editPath) + if err != nil { + return "", "", err + } + editPath, err = filepath.EvalSymlinks(editPath) + if err != nil { + return "", "", err + } + relProjectPath, err := w.getReadOnlyProjectFromPath(editPath) + if err != nil { + return "", "", err + } + workspaceName, err := w.getWorkspaceFromPath(editPath) + if err != nil { + return "", "", err + } + workspaceDir, err := w.GetDir(workspaceName) + if err != nil { + return "", "", err + } + codebaseName, err := w.GetCodebase(workspaceName) + if err != nil { + return "", "", err + } + codebaseDir, err := codebase.GetDir(codebaseName) + if err != nil { + return "", "", err + } + wsProjectPath := filepath.Join(workspaceDir, relProjectPath) + if err = w.composer.Unbind(wsProjectPath); err != nil { + return "", "", err + } + //TODO: support editing nested projects + //the command above unbinds nested child projects but + //we don't rebind them after checking out an editable project branch + cbProjectPath := filepath.Join(codebaseDir, relProjectPath) + branchName := workspaceName + cmd := exec.Command("git", + "-C", cbProjectPath, + "worktree", "add", + "-b", branchName, + wsProjectPath) + output, err := cmd.CombinedOutput() + if err != nil { + return "", "", fmt.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + cmd.String(), err.Error(), output) + } + return branchName, wsProjectPath, err +} + +func (w Workspace) getReadOnlyProjectFromPath(inPath string) (string, error) { + worspaceName, err := w.getWorkspaceFromPath(inPath) + if err != nil { + return "", err + } + workspacePath, err := w.GetDir(worspaceName) + if err != nil { + return "", err + } + bindList, err := w.composer.List(workspacePath) + if err != nil { + return "", err + } + for _, bindPath := range bindList { + if !strings.HasPrefix(inPath+"/", bindPath+"/") { + continue + } + relProjectPath, err := filepath.Rel(workspacePath, bindPath) + if err != nil { + return "", err + } + return relProjectPath, nil + } + return "", fmt.Errorf("Path %s is already editable", inPath) +} + +func (w Workspace) getWorkspaceFromPath(inPath string) (string, error) { + for workspaceName, _ := range w.List() { + dir, err := w.GetDir(workspaceName) + if err != nil { + return "", err + } + if strings.HasPrefix(inPath+"/", dir+"/") { + return workspaceName, nil + } + } + return "", fmt.Errorf("Path %s is not contained in a workspace", inPath) +} diff --git a/hacksaw/workspace/workspace_test.go b/hacksaw/workspace/workspace_test.go new file mode 100644 index 0000000..c4476ae --- /dev/null +++ b/hacksaw/workspace/workspace_test.go @@ -0,0 +1,457 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workspace + +import ( + "io/ioutil" + "os" + "os/exec" + "path" + "testing" + + "android.googlesource.com/platform/tools/treble.git/hacksaw/bind" + "android.googlesource.com/platform/tools/treble.git/hacksaw/codebase" + "android.googlesource.com/platform/tools/treble.git/hacksaw/config" +) + +func TestBasicCreate(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "codebase") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + gitDir := path.Join(codebaseDir, "project", ".git") + if err = os.MkdirAll(gitDir, os.ModePerm); err != nil { + t.Error(err) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + // The top dir must be named "hacksaw" + // otherwise the mounters will reject any + // mount requests + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + ws := New(bind.NewFakePathBinder(), wsTopDir) + if _, err = ws.Create("test-workspace", "test-codebase"); err != nil { + t.Error(err) + } + workspaceDir, err := ws.GetDir("test-workspace") + if err != nil { + t.Error(err) + } + _, err = os.Stat(workspaceDir) + if err != nil { + t.Error(err) + } +} + +func TestWorkspaceDuplicate(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "codebase") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + gitDir := path.Join(codebaseDir, "project", ".git") + if err = os.MkdirAll(gitDir, os.ModePerm); err != nil { + t.Error(err) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + // The top dir must be named "hacksaw" + // otherwise the mounters will reject any + // mount requests + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + ws := New(bind.NewFakePathBinder(), wsTopDir) + _, err = ws.Create("test-workspace", "test-codebase") + if err != nil { + t.Error(err) + } + _, err = ws.Create("test-workspace", "test-codebase") + if err == nil { + t.Error("Allowed workspace duplicate") + } +} + +func TestCreateWorkspaceFromBadCodebase(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + ws := New(bind.NewFakePathBinder(), wsTopDir) + if _, err = ws.Create("test-workspace", "does-not-exist"); err == nil { + t.Error("Allowed bad codebase") + } +} + +func TestList(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + gitDir := path.Join(codebaseDir, "project", ".git") + if err = os.MkdirAll(gitDir, os.ModePerm); err != nil { + t.Error(err) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + ws := New(bind.NewFakePathBinder(), wsTopDir) + if _, err = ws.Create("test-workspace", "test-codebase"); err != nil { + t.Error(err) + } + list := ws.List() + cb, ok := list["test-workspace"] + if !ok || cb != "test-codebase" { + t.Error("Added workspace not listed") + } +} + +func TestRemove(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "test") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + gitDir := path.Join(codebaseDir, "project", ".git") + if err = os.MkdirAll(gitDir, os.ModePerm); err != nil { + t.Error(err) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + ws := New(bind.NewFakePathBinder(), wsTopDir) + if _, err = ws.Create("test-workspace", "test-codebase"); err != nil { + t.Error(err) + } + workspaceDir, err := ws.GetDir("test-workspace") + if err != nil { + t.Error(err) + } + _, err = os.Stat(workspaceDir) + if err != nil { + t.Error(err) + } + cfg, err := ws.Remove("test-workspace") + if err != nil { + t.Error(err) + } + _, ok := cfg.Workspaces["test-codebase"] + if ok { + t.Error("Removed workspace test-codebase is still in the configuration") + } + _, err = os.Stat(workspaceDir) + if err == nil { + t.Error("Workspace test-workspace was removed but its directory remains") + } else if os.IsNotExist(err) { + // This is the expected error + } else { + t.Error(err) + } +} + +func TestEdit(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "codebase") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + projectDir := path.Join(codebaseDir, "project") + if err = os.MkdirAll(projectDir, os.ModePerm); err != nil { + t.Error(err) + } + cmd := exec.Command("git", "-C", projectDir, "init") + output, err := cmd.CombinedOutput() + if err != nil { + t.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + cmd.String(), err.Error(), output) + } + cmd = exec.Command("git", "-C", projectDir, "commit", `--message="Initial commit"`, "--allow-empty") + output, err = cmd.CombinedOutput() + if err != nil { + t.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + cmd.String(), err.Error(), output) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte("project") + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + ws := New(bind.NewFakePathBinder(), wsTopDir) + if _, err = ws.Create("test-workspace", "test-codebase"); err != nil { + t.Error(err) + } + workspaceDir, err := ws.GetDir("test-workspace") + if err != nil { + t.Error(err) + } + _, err = os.Stat(workspaceDir) + if err != nil { + t.Error(err) + } + editPath := path.Join(workspaceDir, "project") + branchName, wsProjectDir, err := ws.Edit(editPath) + if err != nil { + t.Error(err) + } + if branchName == "" { + t.Error("Editing returned an empty branch") + } + if wsProjectDir == "" { + t.Error("Editing returned an empty project path") + } + cmd = exec.Command("git", "-C", wsProjectDir, "show", branchName) + output, err = cmd.CombinedOutput() + if err != nil { + t.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + cmd.String(), err.Error(), output) + } + //Recreate workspace and try editing again + _, err = ws.Remove("test-workspace") + if err != nil { + t.Error(err) + } + _, err = ws.Create("test-workspace", "test-codebase") + if err != nil { + t.Error(err) + } + _, _, err = ws.Edit(editPath) + if err != nil { + t.Error(err) + } +} + +const projectList = `read-only-project +editable-project` + +func TestRecreate(t *testing.T) { + defer config.GetConfig().Reset() + codebaseDir, err := ioutil.TempDir("", "codebase") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(codebaseDir) + roProjectDir := path.Join(codebaseDir, "read-only-project") + if err = os.MkdirAll(roProjectDir, os.ModePerm); err != nil { + t.Error(err) + } + cmd := exec.Command("git", "-C", roProjectDir, "init") + output, err := cmd.CombinedOutput() + if err != nil { + t.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + cmd.String(), err.Error(), output) + } + cmd = exec.Command("git", "-C", roProjectDir, "commit", `--message="Initial commit"`, "--allow-empty") + output, err = cmd.CombinedOutput() + if err != nil { + t.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + cmd.String(), err.Error(), output) + } + linkPath := path.Join(codebaseDir, "symlink") + if err = os.Symlink(roProjectDir, linkPath); err != nil { + t.Error(err) + } + rwProjectDir := path.Join(codebaseDir, "editable-project") + if err = os.MkdirAll(rwProjectDir, os.ModePerm); err != nil { + t.Error(err) + } + cmd = exec.Command("git", "-C", rwProjectDir, "init") + output, err = cmd.CombinedOutput() + if err != nil { + t.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + cmd.String(), err.Error(), output) + } + cmd = exec.Command("git", "-C", rwProjectDir, "commit", `--message="Initial commit"`, "--allow-empty") + output, err = cmd.CombinedOutput() + if err != nil { + t.Errorf("Command\n%s\nfailed with the following:\n%s\n%s", + cmd.String(), err.Error(), output) + } + repoDir := path.Join(codebaseDir, ".repo") + if err = os.Mkdir(repoDir, os.ModePerm); err != nil { + t.Error(err) + } + listContents := []byte(projectList) + listPath := path.Join(repoDir, "project.list") + if err = ioutil.WriteFile(listPath, listContents, os.ModePerm); err != nil { + t.Error(err) + } + _, err = codebase.Add("test-codebase", codebaseDir) + if err != nil { + t.Error(err) + } + wsTempDir, err := ioutil.TempDir("", "workspace") + if err != nil { + t.Error(err) + } + defer os.RemoveAll(wsTempDir) + wsTopDir := path.Join(wsTempDir, "hacksaw") + if err = os.Mkdir(wsTopDir, os.ModePerm); err != nil { + t.Error(err) + } + pathBinder := bind.NewFakePathBinder() + ws := New(pathBinder, wsTopDir) + if _, err = ws.Create("test-workspace", "test-codebase"); err != nil { + t.Error(err) + } + workspaceDir, err := ws.GetDir("test-workspace") + if err != nil { + t.Error(err) + } + editPath := path.Join(workspaceDir, "editable-project") + _, _, err = ws.Edit(editPath) + if err != nil { + t.Error(err) + } + emptyFilePath := path.Join(editPath, "empty-edit") + emptyFile, err := os.Create(emptyFilePath) + if err != nil { + t.Error(err) + } + emptyFile.Close() + if _, err = ws.Recreate("test-workspace"); err != nil { + t.Error(err) + } + _, err = os.Stat(emptyFilePath) + if err != nil { + t.Error(err) + } + wsRoProjectDir := path.Join(workspaceDir, "read-only-project") + isRoPathBound := false + pathList, err := pathBinder.List() + if err != nil { + t.Error(err) + } + for _, path := range pathList { + if path == wsRoProjectDir { + isRoPathBound = true + } + } + if !isRoPathBound { + t.Error("Read only project was not mounted to the workspace") + } +} diff --git a/split/Android.bp b/split/Android.bp index 8b0c932..331354b 100644 --- a/split/Android.bp +++ b/split/Android.bp @@ -12,9 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "tools_treble_license" + // to get the below license kinds: + // SPDX-license-identifier-Apache-2.0 + default_applicable_licenses: ["tools_treble_license"], +} + python_defaults { name: "treble_split_default", - pkg_path: "treble", + pkg_path: "treble/split", version: { py2: { enabled: false, @@ -38,6 +47,31 @@ python_binary_host { data: [ "default_config.xml", ], + version: { + py3: { + embedded_launcher: true, + }, + }, +} + +python_library_host { + name: "treble_xml_diff", + defaults: ["treble_split_default"], + srcs: [ + "xml_diff.py", + ], +} + +python_binary_host { + name: "treble_manifest_diff", + main: "manifest_diff.py", + defaults: ["treble_split_default"], + srcs: [ + "manifest_diff.py", + ], + libs: [ + "treble_xml_diff", + ], } python_test_host { @@ -45,9 +79,13 @@ python_test_host { main: "test.py", defaults: ["treble_split_default"], srcs: [ + "manifest_diff.py", + "manifest_diff_test.py", "manifest_split.py", "manifest_split_test.py", "test.py", + "xml_diff.py", + "xml_diff_test.py", ], libs: [ "py-mock", diff --git a/split/README.md b/split/README.md new file mode 100644 index 0000000..db3e7de --- /dev/null +++ b/split/README.md @@ -0,0 +1,107 @@ +# Manifest Split + +## Overview + +Split manifests are Android repo manifests that contain the minimum set of +projects necessary to build a given target. If a project isn't used for building +the target, it shouldn't be in the split manifest. This smaller manifest can be +used to sync the Android source tree and build the specific target. This sync +should be faster and smaller than a sync of a full manifest because it is +syncing less projects. + +The `treble_manifest_split` tool is used to automatically create a split +manifest from a full manifest using dependency information from the source tree +and the build outputs. The tool attempts to infer as many dependencies as it +can, but some will be missed due to implicit dependencies in the build system +and source tree. This is solved by manually fine-tuning a tool configuration XML +specific to your target. + +## Example for aosp_arm64 + +### 1. Run a full build using a full manifest + +The `treble_manifest_split` tool needs the ninja build graph and deps log from a +completed build in order to have a full view of the dependency graph. While the +build graph is created at the beginning of a ninja build, the deps log is not +complete until the build finishes. + +Use standard Android build commands to build your target. + +### 2. Use the treble_manifest_split tool + +```shell +# Change to the directory where you ran the full build. +cd /path/to/android + +# Set command line variables for the Android target you are using and the build +# target that should be buildable from your split manifest. +ANDROID_TARGET=aosp_arm64-userdebug +BUILD_TARGET=droid + +# Build treble_manifest_split as a python binary. +lunch $ANDROID_TARGET +m treble_manifest_split + +# Create the split manifest using a sample config XML specific to aosp_arm64. +out/host/linux-x86/bin/treble_manifest_split \ + --manifest .repo/manifests/default.xml \ + --split-manifest split_default.xml \ + --debug-file debug.json \ + --config tools/treble/split/sample_config.xml \ + $BUILD_TARGET +``` + +### 3. Build using the split manifest + +You should test that the split manifest created by the tool can be used to build +the partial target files package. + +1. Initialize a new repo directory using the steps in + https://source.android.com/setup/build/downloading#initializing-a-repo-client. +1. Replace the `.repo/manifests/default.xml` full manifest with the + newly-generated split manifest. +1. Use standard `repo sync` commands to sync your repo. +1. Attempt a build of your target. + +### 4. Fix build errors + +Build errors may arise due to missing dependencies that were previously provided +by now-removed projects. These dependencies may be implicit in the source code, +or an explicit dependency type that is not yet able to be automatically detected +by the tool. + +1. Find the dependency source project in your full-manifest repo directory. +1. Update your config XML to manually add projects to your split manifest. + + - For example, the following line in `sample_config.xml` in this tool + directory specifies a project that should be included in the split + manifest even if the tool doesn't automatically detect that it is + necessary. + + ``` + <add_project name="platform/external/python/cpython3" /> + ``` + +1. Regenerate the split manifest using `treble_manifest_split` in your + full-manifest directory. Remember to pass the path of your config XML to the + script's `--config` flag. + +### 5. Compare built artifacts + +A successful build alone is not sufficient to have full confidence in the split +manifest. You should diff the output artifacts of the split-manifest build +against the output artifacts of the full-manifest build. + +Suggestions for viewing diffs: + +- Use an external directory diffing tool on the output directories for each + partition, such as `out/target/product/<device>/system`. +- Use `development/vndk/tools/image-diff-tool/diff.py` on output directories, + or on a zipped target-files archive if you are creating `dist` builds. + +The following may cause differences between output artifacts: + +- Non-hermetic inputs used in the module build rule, such as timestamps. Can + be fixed by removing the timestamp from the build rule. +- An implicit and optional source dependency. Can be fixed by manually adding + the project that defines the missing source. diff --git a/split/default_config.xml b/split/default_config.xml index 478bf9b..f9745b6 100644 --- a/split/default_config.xml +++ b/split/default_config.xml @@ -1,25 +1,8 @@ <?xml version="1.0" encoding="utf-8"?> <config> - <!-- Needed for repo tool --> + <!-- ============================================================= --> + <!-- Needed for the repo tool. --> <add_project name="platform/tools/repohooks" /> - <!-- Needed for early build of Soong --> - <add_project name="platform/external/golang-protobuf" /> - <!-- system/iorap: iorap.inode2filename depends on librxcpp --> - <add_project name="platform/external/Reactive-Extensions/RxCpp" /> - <!-- system/bt/gd: bluetooth_packets_python3 depends on pybind11_headers --> - <add_project name="platform/external/python/pybind11" /> - <!-- test/vts/tools/build/tasks/vts_package.mk runs 'cd' on these dirs --> - <add_project name="platform/test/vts-testcase/fuzz" /> - <add_project name="platform/test/vts-testcase/hal" /> - <add_project name="platform/test/vts-testcase/hal-trace" /> - <add_project name="platform/test/vts-testcase/kernel" /> - <add_project name="platform/test/vts-testcase/nbu" /> - <add_project name="platform/test/vts-testcase/performance" /> - <add_project name="platform/test/vts-testcase/security" /> - <add_project name="platform/test/vts-testcase/vndk" /> - <!-- bionic/libc:libc_ndk depends on libscudo --> - <add_project name="platform/external/scudo" /> - <!-- external/crosvm/gpu_display uses module type wayland_protocol_codegen --> - <add_project name="platform/external/wayland-protocols" /> + <!-- ============================================================= --> </config> diff --git a/split/manifest_diff.py b/split/manifest_diff.py new file mode 100644 index 0000000..3ba99c0 --- /dev/null +++ b/split/manifest_diff.py @@ -0,0 +1,275 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Compares two repo manifest xml files. + +Checks to see if the manifests contain same projects. And if those projects +contain the same attributes, linkfile elements and copyfile elements. +""" + +import argparse +import sys +import textwrap +from typing import Set +import xml.etree.ElementTree as ET +import dataclasses +from treble.split import xml_diff + +Element = ET.Element +Change = xml_diff.Change +ChangeMap = xml_diff.ChangeMap + +_SINGLE_NODE_ELEMENTS = ('default', 'manifest-server', 'repo-hooks', 'include') +_INDENT = (' ' * 2) + + +@dataclasses.dataclass +class ProjectChanges: + """A collection of changes between project elements. + + Attributes: + attributes: A ChangeMap of attributes changes. Keyed by attribute name. + linkfiles: A ChangeMap of linkfile elements changes. Keyed by dest. + copyfiles: A ChangeMap of copyfile elements changes. Keyed by dest. + """ + attributes: ChangeMap = dataclasses.field(default_factory=ChangeMap) + linkfiles: ChangeMap = dataclasses.field(default_factory=ChangeMap) + copyfiles: ChangeMap = dataclasses.field(default_factory=ChangeMap) + + def __bool__(self): + return bool(self.attributes) or bool(self.linkfiles) or bool(self.copyfiles) + + def __repr__(self): + if not self: + return 'No changes' + + ret_str = '' + + if self.attributes: + ret_str += 'Attributes:\n' + ret_str += textwrap.indent(str(self.attributes), _INDENT) + if self.linkfiles: + ret_str += 'Link Files:\n' + ret_str += textwrap.indent(str(self.linkfiles), _INDENT) + if self.copyfiles: + ret_str += 'Copy Files:\n' + ret_str += textwrap.indent(str(self.copyfiles), _INDENT) + + return ret_str + + +@dataclasses.dataclass +class ManifestChanges: + """A collection of changes between manifests. + + Attributes: + projects: A ChangeMap of changes to project elements. Keyed by project path. + remotes: A ChangeMap of changes to remote elements. Keyed by remote name. + other: A ChangeMap of changes to other elements. Keyed by element tag. + """ + projects: ChangeMap = dataclasses.field(default_factory=ChangeMap) + remotes: ChangeMap = dataclasses.field(default_factory=ChangeMap) + other: ChangeMap = dataclasses.field(default_factory=ChangeMap) + + def has_changes(self): + return self.projects or self.remotes or self.other + + def __repr__(self): + ret_str = 'Project Changes:\n' + ret_str += (textwrap.indent(str(self.projects) + '\n', _INDENT) + if self.projects else _INDENT + 'No changes found.\n\n') + ret_str += 'Remote Changes:\n' + ret_str += (textwrap.indent(str(self.remotes) + '\n', _INDENT) + if self.remotes else _INDENT + 'No changes found.\n\n') + ret_str += 'Other Changes:\n' + ret_str += (textwrap.indent(str(self.other) + '\n', _INDENT) + if self.other else _INDENT + 'No changes found.\n\n') + + return ret_str + + +def subelement_file_changes(tag: str, p1: Element, p2: Element) -> ChangeMap: + """Get the changes copyfile or linkfile elements between two project elements. + + Arguments: + tag: The tag of the element. + p1: the xml element for the base project. + p2: the xml element for the new roject. + + Returns: + A ChangeMap of copyfile or linkfile changes. Keyed by dest attribute. + """ + return xml_diff.compare_subelements( + tag=tag, + p1=p1, + p2=p2, + ignored_attrs=set(), + key_fn=lambda x: x.get('dest'), + diff_fn=xml_diff.attribute_changes) + + +def project_changes(p1: Element, p2: Element, + ignored_attrs: Set[str]) -> ProjectChanges: + """Get the changes between two project elements. + + Arguments: + p1: the xml element for the base project. + p2: the xml element for the new project. + ignored_attrs: a set of attribute names to ignore changes. + + Returns: + A ProjectChanges object of the changes. + """ + return ProjectChanges( + attributes=xml_diff.attribute_changes(p1, p2, ignored_attrs), + linkfiles=subelement_file_changes('linkfile', p1, p2), + copyfiles=subelement_file_changes('copyfile', p1, p2)) + + +def compare_single_node_elements(manifest_e1: Element, manifest_e2: Element, + ignored_attrs: Set[str]) -> ChangeMap: + """Get the changes between single element nodes such as <defaults> in a manifest. + + Arguments: + manifest_e1: the xml element for the base manifest. + manifest_e2: the xml element for the new manifest. + ignored_attrs: a set of attribute names to ignore changes. + + Returns: + A ChangeMap of changes. Keyed by elements tag name. + """ + changes = ChangeMap() + for tag in _SINGLE_NODE_ELEMENTS: + e1 = manifest_e1.find(tag) + e2 = manifest_e2.find(tag) + if e1 is None and e2 is None: + continue + elif e1 is None: + changes.added[tag] = xml_diff.element_string(e2) + elif e2 is None: + changes.removed[tag] = xml_diff.element_string(e1) + else: + attr_changes = xml_diff.attribute_changes(e1, e2, ignored_attrs) + if attr_changes: + changes.modified[tag] = attr_changes + return changes + + +def compare_remote_elements(manifest_e1: Element, manifest_e2: Element, + ignored_attrs: Set[str]) -> ChangeMap: + """Get the changes to remote elements between two manifests. + + Arguments: + manifest_e1: the xml element for the base manifest. + manifest_e2: the xml element for the new manifest. + ignored_attrs: a set of attribute names to ignore changes. + + Returns: + A ChangeMap of changes to remote elements. Keyed by name attribute. + """ + return xml_diff.compare_subelements( + tag='remote', + p1=manifest_e1, + p2=manifest_e2, + ignored_attrs=ignored_attrs, + key_fn=lambda x: x.get('name'), + diff_fn=xml_diff.attribute_changes) + + +def compare_project_elements(manifest_e1, manifest_e2, + ignored_attrs: Set[str]) -> ChangeMap: + """Get the changes to project elements between two manifests. + + Arguments: + manifest_e1: the xml element for the base manifest. + manifest_e2: the xml element for the new manifest. + ignored_attrs: a set of attribute names to ignore changes. + + Returns: + A ChangeMap of changes to project elements. Keyed by path/name attribute. + """ + # Ignore path attribute since it's already keyed on that value and avoid false + # detection when path == name on one element and path == None on the other. + project_ignored_attrs = ignored_attrs | set(['path']) + return xml_diff.compare_subelements( + tag='project', + p1=manifest_e1, + p2=manifest_e2, + ignored_attrs=project_ignored_attrs, + key_fn=lambda x: x.get('path', x.get('name')), + diff_fn=project_changes) + + +def compare_manifest_elements(manifest_e1, manifest_e2, + ignored_attrs: Set[str]) -> ManifestChanges: + """Get the changes between two manifests xml elements. + + Arguments: + manifest_e1: the xml element for the base manifest. + manifest_e2: the xml element for the new manifest. + ignored_attrs: a set of attribute names to ignore changes. + + Returns: + A ManifestChanges. + """ + return ManifestChanges( + projects=compare_project_elements(manifest_e1, manifest_e2, + ignored_attrs), + remotes=compare_remote_elements(manifest_e1, manifest_e2, ignored_attrs), + other=compare_single_node_elements(manifest_e1, manifest_e2, + ignored_attrs)) + + +def compare_manifest_files(manifest_a: str, manifest_b: str, + ignored_attrs: Set[str]) -> ManifestChanges: + """Get the changes between two manifests files. + + Arguments: + manifest_a: Path to the base manifest xml file. + manifest_b: Path to the manifest xml file to compare against. + ignored_attrs: a set of attribute names to ignore changes. + + Returns: + A ManifestChanges. + """ + e1 = ET.parse(manifest_a).getroot() + e2 = ET.parse(manifest_b).getroot() + return compare_manifest_elements( + manifest_e1=e1, manifest_e2=e2, ignored_attrs=ignored_attrs) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--ignored_attributes', + type=str, + help='A comma separated list of attributes to ignore when comparing ' + + 'project elements.') + parser.add_argument('manifest_a', help='Path to the base manifest xml file.') + parser.add_argument( + 'manifest_b', help='Path to the manifest xml file to compare against.') + args = parser.parse_args() + + ignored_attributes = set( + args.ignored_attributes.split(',')) if args.ignored_attributes else set() + changes = compare_manifest_files(args.manifest_a, args.manifest_b, + ignored_attributes) + + print(changes) + if changes: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/split/manifest_diff_test.py b/split/manifest_diff_test.py new file mode 100644 index 0000000..ca1e742 --- /dev/null +++ b/split/manifest_diff_test.py @@ -0,0 +1,168 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test manifest diff.""" +import unittest +import xml.etree.ElementTree as ET + +import manifest_diff + + +class ManifestDiffTest(unittest.TestCase): + + def _assertEqualCanonical(self, change1, change2): + def _canonicalize(change): + return { + identifier : ' '.join(sorted(value.split(' '))) + for identifier, value in change.items() + } + return self.assertEqual(_canonicalize(change1), _canonicalize(change2)) + + def test_project_changes(self): + p1 = ET.fromstring("""<project attr1="hello"> + <linkfile src="newfile2" dest="notneeded" /> + <linkfile src="oldfile1" dest="dest1" /> + <copyfile src="oldfile2" dest="dest2" /> + </project> + """) + p2 = ET.fromstring("""<project> + <linkfile src="newfile1" dest="dest1" /> + <copyfile src="newfile2" dest="dest2" /> + <copyfile src="somefile" dest="addedfile" /> + </project> + """) + changes = manifest_diff.project_changes(p1, p2, set()) + self.assertEqual(changes.linkfiles.added, {}) + self._assertEqualCanonical( + changes.linkfiles.removed, + {'notneeded': '<linkfile src="newfile2" dest="notneeded" />'}) + self.assertEqual( + changes.linkfiles.modified, { + 'dest1': + manifest_diff.ChangeMap(modified={ + 'src': manifest_diff.Change('oldfile1', 'newfile1') + }) + }) + self._assertEqualCanonical( + changes.copyfiles.added, + {'addedfile': '<copyfile src="somefile" dest="addedfile" />'}) + self.assertEqual(changes.copyfiles.removed, {}) + self.assertEqual( + changes.copyfiles.modified, { + 'dest2': + manifest_diff.ChangeMap(modified={ + 'src': manifest_diff.Change('oldfile2', 'newfile2') + }) + }) + self.assertEqual( + changes.attributes, + manifest_diff.ChangeMap( + added={}, removed={'attr1': 'hello'}, modified={})) + + def test_project_changes_same(self): + p1 = ET.fromstring("""<project attr1="hello"> + <linkfile src="newfile2" dest="notneeded" /> + <linkfile src="oldfile1" dest="dest1" /> + <copyfile src="oldfile2" dest="dest2" /> + </project> + """) + changes = manifest_diff.project_changes(p1, p1, set()) + self.assertFalse(changes) + + def test_compare_single_node_elements(self): + m1 = ET.fromstring("""<manifest> + <default revision='dev' remote='aosp' /> + <repo-hooks /> + </manifest> + """) + m2 = ET.fromstring("""<manifest> + <default revision='release' /> + </manifest> + """) + changes = manifest_diff.compare_single_node_elements(m1, m2, set()) + self.assertEqual(changes.added, {}) + self._assertEqualCanonical(changes.removed, {'repo-hooks': '<repo-hooks />'}) + self.assertEqual( + changes.modified, { + 'default': + manifest_diff.ChangeMap( + added={}, + removed={'remote': 'aosp'}, + modified={ + 'revision': manifest_diff.Change('dev', 'release') + }) + }) + + def test_compare_remote_elements(self): + m1 = ET.fromstring("""<manifest> + <remote revision="dev" name="aosp" fetch="https://aosp-source.com" /> + <remote name="android" fetch="https://android-source.com" attr="test" /> + <repo-hooks /> + </manifest> + """) + m2 = ET.fromstring("""<manifest> + <remote revision="dev" name="android" fetch="https://android-source.com" + attr="test2"/> + </manifest> + """) + changes = manifest_diff.compare_remote_elements(m1, m2, set()) + self.assertEqual(changes.added, {}) + self._assertEqualCanonical( + changes.removed, { + 'aosp': + '<remote revision="dev" name="aosp" fetch="https://aosp-source.com" />' + }) + self.assertEqual( + changes.modified, { + 'android': + manifest_diff.ChangeMap( + added={'revision': 'dev'}, + removed={}, + modified={'attr': manifest_diff.Change('test', 'test2')}) + }) + + def test_compare_project_elements(self): + m1 = ET.fromstring("""<manifest> + <project name="platform/project1" path="system/project1" /> + <project name="platform/project2" path="system/project2" /> + <project name="platform/project3" path="system/project3" /> + </manifest>""") + m2 = ET.fromstring("""<manifest> + <project name="platform/project1" path="system/project1" /> + <project name="system/project2" /> + <project name="platform/project4" path="system/project4" /> + </manifest>""") + changes = manifest_diff.compare_project_elements(m1, m2, set()) + self._assertEqualCanonical( + changes.added, { + 'system/project4': + '<project name="platform/project4" path="system/project4" />' + }) + self._assertEqualCanonical( + changes.removed, { + 'system/project3': + '<project name="platform/project3" path="system/project3" />' + }) + self.assertEqual( + changes.modified, { + 'system/project2': + manifest_diff.ProjectChanges( + attributes=manifest_diff.ChangeMap( + added={}, + removed={}, + modified={ + 'name': + manifest_diff.Change('platform/project2', + 'system/project2') + })) + }) diff --git a/split/manifest_split.py b/split/manifest_split.py index 89f8dc5..d5f9b95 100644 --- a/split/manifest_split.py +++ b/split/manifest_split.py @@ -32,6 +32,12 @@ options: <add_project name="vendor/my/needed/project" /> <remove_project name="vendor/my/unused/project" /> </config> + --ignore-default-config + If provided, don't include default_config.xml. + --installed-prebuilt + Specify the directory containing an installed prebuilt Android.bp file. + Supply this option zero or more times, once for each installed prebuilt + directory. --repo-list <path> Optional path to the output of the 'repo list' command. Used if the output of 'repo list' needs pre-processing before being used by @@ -44,15 +50,20 @@ options: --module-info <path> Optional path to the module-info.json file found in an out dir. If not provided, the default file is used based on the lunch environment. + --skip-module-info + If provided, skip parsing module-info.json for direct and adjacent + dependencies. Overrides --module-info option. --kati-stamp <path> Optional path to the .kati_stamp file found in an out dir. If not provided, the default file is used based on the lunch environment. + --skip-kati + If provided, skip Kati makefiles projects. Overrides --kati-stamp option. --overlay <path> Optional path(s) to treat as overlays when parsing the kati stamp file and scanning for makefiles. See the tools/treble/build/sandbox directory for more info about overlays. This flag can be passed more than once. - --debug - Print debug messages. + --debug-file <path> + If provided, debug info will be written to a JSON file at this path. -h (--help) Display this usage message and exit. """ @@ -60,15 +71,20 @@ options: from __future__ import print_function import getopt -import hashlib import json import logging import os -import pkg_resources +import pkgutil +import re import subprocess import sys +import tempfile +from typing import Dict, List, Pattern, Set, Tuple import xml.etree.ElementTree as ET +import dataclasses + + logging.basicConfig( stream=sys.stdout, level=logging.INFO, @@ -78,62 +94,132 @@ logger = logging.getLogger(os.path.basename(__file__)) # Projects determined to be needed despite the dependency not being visible # to ninja. -DEFAULT_CONFIG_PATH = pkg_resources.resource_filename(__name__, - "default_config.xml") +DEFAULT_CONFIG_XML = "default_config.xml" +# Pattern that matches a java dependency. +_JAVA_LIB_PATTERN = re.compile( + # pylint: disable=line-too-long + '^out/target/common/obj/JAVA_LIBRARIES/(.+)_intermediates/classes-header.jar$' +) -def read_config(config_file): - """Reads a config XML file to find extra projects to add or remove. - Args: - config_file: The filename of the config XML. +@dataclasses.dataclass +class PathMappingConfig: + pattern: Pattern[str] + sub: str - Returns: - A tuple of (set of remove_projects, set of add_projects) from the config. + +@dataclasses.dataclass +class ManifestSplitConfig: + """Holds the configuration for the split manifest tool. + + Attributes: + remove_projects: A Dict of project name to the config file that specified + this project, for projects that should be removed from the resulting + manifest. + add_projects: A Dict of project name to the config file that specified + this project, for projects that should be added to the resulting manifest. + path_mappings: A list of PathMappingConfigs to modify a path in the build + sandbox to the path in the manifest. """ - root = ET.parse(config_file).getroot() - remove_projects = set( - [child.attrib["name"] for child in root.findall("remove_project")]) - add_projects = set( - [child.attrib["name"] for child in root.findall("add_project")]) - return remove_projects, add_projects + remove_projects: Dict[str, str] + add_projects: Dict[str, str] + path_mappings: List[PathMappingConfig] + + @classmethod + def from_config_files(cls, config_files: List[str]): + """Reads from a list of config XML files. + + Args: + config_files: A list of config XML filenames. + + Returns: + A ManifestSplitConfig from the files. + """ + remove_projects: Dict[str, str] = {} + add_projects: Dict[str, str] = {} + path_mappings = [] + for config_file in config_files: + root = ET.parse(config_file).getroot() + + remove_projects.update({ + c.attrib["name"]: config_file for c in root.findall("remove_project") + }) + + add_projects.update( + {c.attrib["name"]: config_file for c in root.findall("add_project")}) + + path_mappings.extend([ + PathMappingConfig( + re.compile(child.attrib["pattern"]), child.attrib["sub"]) + for child in root.findall("path_mapping") + ]) + + return cls(remove_projects, add_projects, path_mappings) -def get_repo_projects(repo_list_file): - """Returns a dict of { project path : project name } using 'repo list'. +def get_repo_projects(repo_list_file, manifest, path_mappings): + """Returns a dict of { project path : project name } using the manifest. + + The path_mappings stop on the first match mapping. If the mapping results in + an empty string, that entry is removed. Args: - repo_list_file: An optional filename to read instead of calling the repo - list command. + repo_list_file: An optional filename to read instead of parsing the manifest. + manifest: The manifest object to scan for projects. + path_mappings: A list of PathMappingConfigs to modify a path in the build + sandbox to the path in the manifest. """ repo_list = [] if repo_list_file: with open(repo_list_file) as repo_list_lines: - repo_list = [line.strip() for line in repo_list_lines if line.strip()] + repo_list = [line.strip().split(" : ") for line in repo_list_lines if line.strip()] else: - repo_list = subprocess.check_output([ - "repo", - "list", - ]).decode().strip("\n").split("\n") - return dict([entry.split(" : ") for entry in repo_list]) + root = manifest.getroot() + repo_list = [(p.get("path", p.get("name")), p.get("name")) for p in root.findall("project")] + + repo_dict = {} + for entry in repo_list: + path, project = entry + for mapping in path_mappings: + if mapping.pattern.fullmatch(path): + path = mapping.pattern.sub(mapping.sub, path) + break + # If the resulting path mapping is empty, then don't add entry + if path: + repo_dict[path] = project + return repo_dict -def get_module_info(module_info_file, repo_projects): - """Returns a dict of { project name : set of modules } in each project. +class ModuleInfo: + """Contains various mappings to/from module/project""" - Args: - module_info_file: The path to a module-info.json file from a build. - repo_projects: The output of the get_repo_projects function. + def __init__(self, module_info_file, repo_projects): + """Initialize a module info instance. - Raises: - ValueError: A module from module-info.json belongs to a path not - known by the repo projects output. - """ - project_modules = {} + Builds various maps related to platform build system modules and how they + relate to each other and projects. + + Args: + module_info_file: The path to a module-info.json file from a build. + repo_projects: The output of the get_repo_projects function. + + Raises: + ValueError: A module from module-info.json belongs to a path not + known by the repo projects output. + """ + # Maps a project to the set of modules it contains. + self.project_modules = {} + # Maps a module to the project that contains it. + self.module_project = {} + # Maps a module to its class. + self.module_class = {} + # Maps a module to modules it depends on. + self.module_deps = {} - with open(module_info_file) as module_info_file: - module_info = json.load(module_info_file) + with open(module_info_file) as module_info_file: + module_info = json.load(module_info_file) def module_has_valid_path(module): return ("path" in module_info[module] and module_info[module]["path"] and @@ -153,8 +239,25 @@ def get_module_info(module_info_file, repo_projects): if not project_path: raise ValueError("Unknown module path for module %s: %s" % (module, module_info[module])) - project_modules.setdefault(repo_projects[project_path], set()).add(module) - return project_modules + repo_project = repo_projects[project_path] + self.project_modules.setdefault(repo_project, set()).add(module) + self.module_project[module] = repo_project + + def dep_from_raw_dep(raw_dep): + match = re.search(_JAVA_LIB_PATTERN, raw_dep) + return match.group(1) if match else raw_dep + + def deps_from_raw_deps(raw_deps): + return [dep_from_raw_dep(raw_dep) for raw_dep in raw_deps] + + self.module_class = { + module: module_info[module]["class"][0] + for module in module_info + } + self.module_deps = { + module: deps_from_raw_deps(module_info[module]["dependencies"]) + for module in module_info + } def get_ninja_inputs(ninja_binary, ninja_build_file, modules): @@ -165,18 +268,35 @@ def get_ninja_inputs(ninja_binary, ninja_build_file, modules): Args: ninja_binary: The path to a ninja binary. ninja_build_file: The path to a .ninja file from a build. - modules: The set of modules to scan for inputs. + modules: The list of modules to scan for inputs. """ - inputs = set( - subprocess.check_output([ - ninja_binary, - "-f", - ninja_build_file, - "-t", - "inputs", - "-d", - ] + list(modules)).decode().strip("\n").split("\n")) - return {path.strip() for path in inputs} + inputs = set() + NINJA_SHARD_LIMIT = 20000 + for i in range(0, len(modules), NINJA_SHARD_LIMIT): + modules_shard = modules[i:i + NINJA_SHARD_LIMIT] + inputs = inputs.union(set( + subprocess.check_output([ + ninja_binary, + "-f", + ninja_build_file, + "-t", + "inputs", + "-d", + ] + list(modules_shard)).decode().strip("\n").split("\n"))) + + def input_allowed(path): + path = path.strip() + if path.endswith("TEST_MAPPING") and "test_mapping" not in modules: + # Exclude projects that are only needed for TEST_MAPPING files, unless the + # user is asking to build 'test_mapping'. + return False + if path.endswith("MODULE_LICENSE_GPL"): + # Exclude projects that are included only due to having a + # MODULE_LICENSE_GPL file, if no other inputs from that project are used. + return False + return path + + return {path.strip() for path in inputs if input_allowed(path)} def get_kati_makefiles(kati_stamp_file, overlays): @@ -278,21 +398,22 @@ def scan_repo_projects(repo_projects, input_path): def get_input_projects(repo_projects, inputs): - """Returns the set of project names that contain the given input paths. + """Returns the collection of project names that contain the given input paths. Args: repo_projects: The output of the get_repo_projects function. inputs: The paths of input files used in the build, as given by the ninja inputs tool. """ - input_project_paths = [ - scan_repo_projects(repo_projects, input_path) - for input_path in inputs - if (not input_path.startswith("out/") and not input_path.startswith("/")) - ] + input_project_paths = {} + for input_path in inputs: + if not input_path.startswith("out/") and not input_path.startswith("/"): + input_project_paths.setdefault( + scan_repo_projects(repo_projects, input_path), []).append(input_path) + return { - repo_projects[project_path] - for project_path in input_project_paths + repo_projects[project_path]: inputs + for project_path, inputs in input_project_paths.items() if project_path is not None } @@ -317,28 +438,21 @@ def update_manifest(manifest, input_projects, remove_projects): return manifest -def create_manifest_sha1_element(manifest, name): - """Creates and returns an ElementTree 'hash' Element using a sha1 hash. - - Args: - manifest: The manifest ElementTree to hash. - name: The name string to give this element. - - Returns: - The ElementTree 'hash' Element. - """ - sha1_element = ET.Element("hash") - sha1_element.set("type", "sha1") - sha1_element.set("name", name) - sha1_element.set("value", - hashlib.sha1(ET.tostring(manifest.getroot())).hexdigest()) - return sha1_element +@dataclasses.dataclass +class DebugInfo: + """Simple class to store structured debug info for a project.""" + direct_input: bool = False + adjacent_input: bool = False + deps_input: bool = False + kati_makefiles: List[str] = dataclasses.field(default_factory=list) + manual_add_config: str = "" + manual_remove_config: str = "" def create_split_manifest(targets, manifest_file, split_manifest_file, config_files, repo_list_file, ninja_build_file, ninja_binary, module_info_file, kati_stamp_file, - overlays): + overlays, installed_prebuilts, debug_file): """Creates and writes a split manifest by inspecting build inputs. Args: @@ -356,123 +470,177 @@ def create_split_manifest(targets, manifest_file, split_manifest_file, kati_stamp_file: The path to a .kati_stamp file from a build. overlays: A list of paths to treat as overlays when parsing the kati stamp file. + installed_prebuilts: A list of paths for which to create "fake" repo + entries. These entries allow the tool to recognize modules that installed + rather than being sync'd via a manifest. + debug_file: If not None, the path to write JSON debug info. """ - remove_projects = set() - add_projects = set() - for config_file in config_files: - config_remove_projects, config_add_projects = read_config(config_file) - remove_projects = remove_projects.union(config_remove_projects) - add_projects = add_projects.union(config_add_projects) + debug_info = {} + + config = ManifestSplitConfig.from_config_files(config_files) + original_manifest = ET.parse(manifest_file) - repo_projects = get_repo_projects(repo_list_file) - module_info = get_module_info(module_info_file, repo_projects) + + repo_projects = get_repo_projects(repo_list_file, original_manifest, + config.path_mappings) + repo_projects.update({ip: ip for ip in installed_prebuilts}) inputs = get_ninja_inputs(ninja_binary, ninja_build_file, targets) - input_projects = get_input_projects(repo_projects, inputs) - if logger.isEnabledFor(logging.DEBUG): - for project in sorted(input_projects): - logger.debug("Direct dependency: %s", project) - logger.info("%s projects needed for targets \"%s\"", len(input_projects), - " ".join(targets)) - - kati_makefiles = get_kati_makefiles(kati_stamp_file, overlays) - kati_makefiles_projects = get_input_projects(repo_projects, kati_makefiles) - if logger.isEnabledFor(logging.DEBUG): - for project in sorted(kati_makefiles_projects.difference(input_projects)): - logger.debug("Kati makefile dependency: %s", project) - input_projects = input_projects.union(kati_makefiles_projects) - logger.info("%s projects after including Kati makefiles projects.", - len(input_projects)) + input_projects = set(get_input_projects(repo_projects, inputs).keys()) + for project in input_projects: + debug_info.setdefault(project, DebugInfo()).direct_input = True + logger.info( + "%s projects needed for Ninja-graph direct dependencies of targets \"%s\"", + len(input_projects), " ".join(targets)) + + if kati_stamp_file: + kati_makefiles = get_kati_makefiles(kati_stamp_file, overlays) + kati_makefiles_projects = get_input_projects(repo_projects, kati_makefiles) + for project, makefiles in kati_makefiles_projects.items(): + debug_info.setdefault(project, DebugInfo()).kati_makefiles = makefiles + input_projects = input_projects.union(kati_makefiles_projects.keys()) + logger.info("%s projects after including Kati makefiles projects.", + len(input_projects)) + else: + logger.info("Kati makefiles projects skipped.") - if logger.isEnabledFor(logging.DEBUG): - manual_projects = add_projects.difference(input_projects) - for project in sorted(manual_projects): - logger.debug("Manual inclusion: %s", project) - input_projects = input_projects.union(add_projects) + for project, cfile in config.add_projects.items(): + debug_info.setdefault(project, DebugInfo()).manual_add_config = cfile + for project, cfile in config.remove_projects.items(): + debug_info.setdefault(project, DebugInfo()).manual_remove_config = cfile + input_projects = input_projects.union(config.add_projects.keys()) logger.info("%s projects after including manual additions.", len(input_projects)) # Remove projects from our set of input projects before adding adjacent # modules, so that no project is added only because of an adjacent # dependency in a to-be-removed project. - input_projects = input_projects.difference(remove_projects) + input_projects = input_projects.difference(config.remove_projects.keys()) # While we still have projects whose modules we haven't checked yet, - checked_projects = set() - projects_to_check = input_projects.difference(checked_projects) + if module_info_file: + module_info = ModuleInfo(module_info_file, repo_projects) + checked_projects = set() + projects_to_check = input_projects.difference(checked_projects) + logger.info("Checking module-info dependencies for direct and adjacent modules...") + else: + logging.info("Direct and adjacent modules skipped.") + projects_to_check = None + + iteration = 0 + while projects_to_check: + iteration += 1 # check all modules in each project, modules = [] + deps_additions = set() + + def process_deps(module): + for d in module_info.module_deps[module]: + if d in module_info.module_class: + if module_info.module_class[d] == "HEADER_LIBRARIES": + hla = module_info.module_project[d] + if hla not in input_projects: + deps_additions.add(hla) + for project in projects_to_check: checked_projects.add(project) - if project not in module_info: + if project not in module_info.project_modules: continue - modules += module_info[project] + for module in module_info.project_modules[project]: + modules.append(module) + process_deps(module) + + for project in deps_additions: + debug_info.setdefault(project, DebugInfo()).deps_input = True + input_projects = input_projects.union(deps_additions) + logger.info( + "pass %d - %d projects after including HEADER_LIBRARIES dependencies", + iteration, len(input_projects)) # adding those modules' input projects to our list of projects. inputs = get_ninja_inputs(ninja_binary, ninja_build_file, modules) - adjacent_module_additions = get_input_projects(repo_projects, inputs) - if logger.isEnabledFor(logging.DEBUG): - for project in sorted( - adjacent_module_additions.difference(input_projects)): - logger.debug("Adjacent module dependency: %s", project) + adjacent_module_additions = set( + get_input_projects(repo_projects, inputs).keys()) + for project in adjacent_module_additions: + debug_info.setdefault(project, DebugInfo()).adjacent_input = True input_projects = input_projects.union(adjacent_module_additions) - logger.info("%s total projects so far.", len(input_projects)) + logger.info( + "pass %d - %d projects after including adjacent-module Ninja-graph dependencies", + iteration, len(input_projects)) projects_to_check = input_projects.difference(checked_projects) - original_manifest = ET.parse(manifest_file) - original_sha1 = create_manifest_sha1_element(original_manifest, "original") + logger.info("%s projects - complete", len(input_projects)) + split_manifest = update_manifest(original_manifest, input_projects, - remove_projects) - split_manifest.getroot().append(original_sha1) - split_manifest.getroot().append( - create_manifest_sha1_element(split_manifest, "self")) + config.remove_projects.keys()) split_manifest.write(split_manifest_file) + if debug_file: + with open(debug_file, "w") as debug_fp: + logger.info("Writing debug info to %s", debug_file) + json.dump( + debug_info, + fp=debug_fp, + sort_keys=True, + indent=2, + default=lambda info: info.__dict__) + def main(argv): try: opts, args = getopt.getopt(argv, "h", [ "help", - "debug", + "debug-file=", "manifest=", "split-manifest=", "config=", + "ignore-default-config", "repo-list=", "ninja-build=", "ninja-binary=", "module-info=", + "skip-module-info", "kati-stamp=", + "skip-kati", "overlay=", + "installed-prebuilt=", ]) except getopt.GetoptError as err: print(__doc__, file=sys.stderr) print("**%s**" % str(err), file=sys.stderr) sys.exit(2) + debug_file = None manifest_file = None split_manifest_file = None - config_files = [DEFAULT_CONFIG_PATH] + config_files = [] repo_list_file = None ninja_build_file = None module_info_file = None - ninja_binary = "ninja" + ninja_binary = "prebuilts/build-tools/linux-x86/bin/ninja" kati_stamp_file = None overlays = [] + installed_prebuilts = [] + ignore_default_config = False + skip_kati = False + skip_module_info = False for o, a in opts: if o in ("-h", "--help"): print(__doc__, file=sys.stderr) sys.exit() - elif o in ("--debug"): - logger.setLevel(logging.DEBUG) + elif o in ("--debug-file"): + debug_file = a elif o in ("--manifest"): manifest_file = a elif o in ("--split-manifest"): split_manifest_file = a elif o in ("--config"): config_files.append(a) + elif o == "--ignore-default-config": + ignore_default_config = True elif o in ("--repo-list"): repo_list_file = a elif o in ("--ninja-build"): @@ -481,10 +649,16 @@ def main(argv): ninja_binary = a elif o in ("--module-info"): module_info_file = a + elif o == "--skip-module-info": + skip_module_info = True elif o in ("--kati-stamp"): kati_stamp_file = a + elif o == "--skip-kati": + skip_kati = True elif o in ("--overlay"): overlays.append(a) + elif o in ("--installed-prebuilt"): + installed_prebuilts.append(a) else: assert False, "unknown option \"%s\"" % o @@ -500,29 +674,47 @@ def main(argv): print(__doc__, file=sys.stderr) print("**Missing required flag --split-manifest**", file=sys.stderr) sys.exit(2) - if not module_info_file: + + if skip_module_info: + if module_info_file: + logging.warning("User provided both --skip-module-info and --module-info args. Arg --module-info ignored.") + module_info_file = None + elif not module_info_file: module_info_file = os.path.join(os.environ["ANDROID_PRODUCT_OUT"], "module-info.json") - if not kati_stamp_file: + if skip_kati: + if kati_stamp_file: + logging.warning("User provided both --skip-kati and --kati-stamp args. Arg --kati-stamp ignored.") + kati_stamp_file = None + elif not kati_stamp_file: kati_stamp_file = os.path.join( os.environ["ANDROID_BUILD_TOP"], "out", ".kati_stamp-%s" % os.environ["TARGET_PRODUCT"]) + if not ninja_build_file: ninja_build_file = os.path.join( os.environ["ANDROID_BUILD_TOP"], "out", "combined-%s.ninja" % os.environ["TARGET_PRODUCT"]) - create_split_manifest( - targets=args, - manifest_file=manifest_file, - split_manifest_file=split_manifest_file, - config_files=config_files, - repo_list_file=repo_list_file, - ninja_build_file=ninja_build_file, - ninja_binary=ninja_binary, - module_info_file=module_info_file, - kati_stamp_file=kati_stamp_file, - overlays=overlays) + with tempfile.NamedTemporaryFile() as default_config_file: + if not ignore_default_config: + default_config_file.write(pkgutil.get_data(__name__, DEFAULT_CONFIG_XML)) + default_config_file.flush() + config_files.insert(0, default_config_file.name) + + create_split_manifest( + targets=args, + manifest_file=manifest_file, + split_manifest_file=split_manifest_file, + config_files=config_files, + repo_list_file=repo_list_file, + ninja_build_file=ninja_build_file, + ninja_binary=ninja_binary, + module_info_file=module_info_file, + kati_stamp_file=kati_stamp_file, + overlays=overlays, + installed_prebuilts=installed_prebuilts, + debug_file=debug_file) if __name__ == "__main__": diff --git a/split/manifest_split_test.py b/split/manifest_split_test.py index 2fdb9bb..546d3c1 100644 --- a/split/manifest_split_test.py +++ b/split/manifest_split_test.py @@ -13,9 +13,10 @@ # limitations under the License. """Test manifest split.""" -import hashlib +import json import mock import os +import re import subprocess import tempfile import unittest @@ -34,12 +35,40 @@ class ManifestSplitTest(unittest.TestCase): <add_project name="add2" /> <remove_project name="remove1" /> <remove_project name="remove2" /> + <path_mapping pattern="p1.*" sub="$0" /> </config>""") test_config.flush() - remove_projects, add_projects = manifest_split.read_config( - test_config.name) - self.assertEqual(remove_projects, set(['remove1', 'remove2'])) - self.assertEqual(add_projects, set(['add1', 'add2'])) + config = manifest_split.ManifestSplitConfig.from_config_files( + [test_config.name]) + self.assertEqual(config.remove_projects, { + 'remove1': test_config.name, + 'remove2': test_config.name + }) + self.assertEqual(config.add_projects, { + 'add1': test_config.name, + 'add2': test_config.name + }) + self.assertEqual(config.path_mappings, [ + manifest_split.PathMappingConfig(re.compile('p1.*'), '$0'), + ]) + + def test_get_repo_projects_from_manifest(self): + manifest_contents = """ + <manifest> + <project name="platform/project1" path="system/project1" /> + <project name="platform/project2" path="system/project2" /> + <project name="platform/project3" path="system/project3" /> + </manifest>""" + manifest = ET.ElementTree(ET.fromstring(manifest_contents)) + projects = manifest_split.get_repo_projects( + None, manifest, path_mappings=[]) + self.assertDictEqual( + { + 'system/project1': 'platform/project1', + 'system/project2': 'platform/project2', + 'system/project3': 'platform/project3', + }, projects) + def test_get_repo_projects(self): with tempfile.NamedTemporaryFile('w+t') as repo_list_file: @@ -47,7 +76,29 @@ class ManifestSplitTest(unittest.TestCase): system/project1 : platform/project1 system/project2 : platform/project2""") repo_list_file.flush() - repo_projects = manifest_split.get_repo_projects(repo_list_file.name) + repo_projects = manifest_split.get_repo_projects( + repo_list_file.name, None, path_mappings=[]) + self.assertEqual( + repo_projects, { + 'system/project1': 'platform/project1', + 'system/project2': 'platform/project2', + }) + + def test_get_repo_projects_with_mappings(self): + with tempfile.NamedTemporaryFile('w+t') as repo_list_file: + repo_list_file.write(""" + overlay/system/project1 : platform/project1 + system/project2 : platform/project2 + hide/this/one : platform/project3""") + repo_list_file.flush() + path_mappings = [ + manifest_split.PathMappingConfig(re.compile('^overlay/(.*)'), '\\1'), + manifest_split.PathMappingConfig(re.compile('^hide/this/one.*'), ''), + ] + + repo_projects = manifest_split.get_repo_projects(repo_list_file.name, + None, + path_mappings) self.assertEqual( repo_projects, { 'system/project1': 'platform/project1', @@ -57,34 +108,89 @@ class ManifestSplitTest(unittest.TestCase): def test_get_module_info(self): with tempfile.NamedTemporaryFile('w+t') as module_info_file: module_info_file.write("""{ - "target1a": { "path": ["system/project1"] }, - "target1b": { "path": ["system/project1"] }, - "target2": { "path": ["out/project2"] }, - "target3": { "path": ["vendor/google/project3"] } + "target1a": { "class": ["EXECUTABLES"], "path": ["system/project1"], "dependencies": ["target2"] }, + "target1b": { "class": ["EXECUTABLES"], "path": ["system/project1"], "dependencies": ["target3", "target42"] }, + "target2": { "class": ["SHARED_LIBRARIES"], "path": ["out/project2"], "dependencies": [] }, + "target3": { "class": ["SHARED_LIBRARIES"], "path": ["vendor/google/project3"], "dependencies": ["x", "y", "z"] }, + "target4a": { "class": ["APPS"], "path": ["system/project4"], "dependencies": ["out/target/common/obj/JAVA_LIBRARIES/target4b_intermediates/classes-header.jar"] }, + "target4b": { "class": ["JAVA_LIBRARIES"], "path": ["system/project4"], "dependencies": [] } }""") module_info_file.flush() repo_projects = { 'system/project1': 'platform/project1', + 'system/project4': 'platform/project4', 'vendor/google/project3': 'vendor/project3', } - module_info = manifest_split.get_module_info(module_info_file.name, - repo_projects) + module_info = manifest_split.ModuleInfo(module_info_file.name, + repo_projects) self.assertEqual( - module_info, { + module_info.project_modules, { 'platform/project1': set(['target1a', 'target1b']), + 'platform/project4': set(['target4a', 'target4b']), 'vendor/project3': set(['target3']), }) + self.assertEqual( + module_info.module_project, { + 'target1a': 'platform/project1', + 'target1b': 'platform/project1', + 'target3': 'vendor/project3', + 'target4a': 'platform/project4', + 'target4b': 'platform/project4', + }) + self.assertEqual( + module_info.module_class, { + 'target1a': 'EXECUTABLES', + 'target1b': 'EXECUTABLES', + 'target2': 'SHARED_LIBRARIES', + 'target3': 'SHARED_LIBRARIES', + 'target4a': 'APPS', + 'target4b': 'JAVA_LIBRARIES', + }) + self.assertEqual( + module_info.module_deps, { + 'target1a': ['target2'], + 'target1b': ['target3', 'target42'], + 'target2': [], + 'target3': ['x', 'y', 'z'], + 'target4a': ['target4b'], + 'target4b': [], + }) def test_get_module_info_raises_on_unknown_module_path(self): with tempfile.NamedTemporaryFile('w+t') as module_info_file: module_info_file.write("""{ - "target1": { "path": ["system/unknown/project1"] } + "target1": { "class": ["EXECUTABLES"], "path": ["system/unknown/project1"], "dependencies": [] } }""") module_info_file.flush() repo_projects = {} with self.assertRaisesRegex(ValueError, 'Unknown module path for module target1'): - manifest_split.get_module_info(module_info_file.name, repo_projects) + manifest_split.ModuleInfo(module_info_file.name, repo_projects) + + @mock.patch.object(subprocess, 'check_output', autospec=True) + def test_get_ninja_inputs(self, mock_check_output): + mock_check_output.return_value = b""" + path/to/input1 + path/to/input2 + path/to/TEST_MAPPING + path/to/MODULE_LICENSE_GPL + """ + + inputs = manifest_split.get_ninja_inputs('unused', 'unused', ['droid']) + self.assertEqual(inputs, {'path/to/input1', 'path/to/input2'}) + + @mock.patch.object(subprocess, 'check_output', autospec=True) + def test_get_ninja_inputs_includes_test_mapping(self, mock_check_output): + mock_check_output.return_value = b""" + path/to/input1 + path/to/input2 + path/to/TEST_MAPPING + """ + + inputs = manifest_split.get_ninja_inputs('unused', 'unused', + ['droid', 'test_mapping']) + self.assertEqual( + inputs, {'path/to/input1', 'path/to/input2', 'path/to/TEST_MAPPING'}) @mock.patch.object(subprocess, 'check_output', autospec=True) def test_get_kati_makefiles(self, mock_check_output): @@ -113,9 +219,7 @@ class ManifestSplitTest(unittest.TestCase): makefiles.append(symlink_dest) # Mock the output of ckati_stamp_dump: - mock_check_output.side_effect = [ - '\n'.join(makefiles).encode(), - ] + mock_check_output.return_value = '\n'.join(makefiles).encode() kati_makefiles = manifest_split.get_kati_makefiles( 'stamp-file', ['overlays/oem_overlay/']) @@ -163,8 +267,10 @@ class ManifestSplitTest(unittest.TestCase): '/tmp/absolute/path/file.java', ] self.assertEqual( - manifest_split.get_input_projects(repo_projects, inputs), - set(['platform/project1', 'platform/project2'])) + manifest_split.get_input_projects(repo_projects, inputs), { + 'platform/project1': ['system/project1/path/to/file.h'], + 'platform/project2': ['system/project2/path/to/another_file.cc'], + }) def test_update_manifest(self): manifest_contents = """ @@ -185,22 +291,16 @@ class ManifestSplitTest(unittest.TestCase): ET.tostring(projects[0]).strip().decode(), '<project name="platform/project1" path="system/project1" />') - def test_create_manifest_sha1_element(self): - manifest = ET.ElementTree(ET.fromstring('<manifest></manifest>')) - manifest_sha1 = hashlib.sha1(ET.tostring(manifest.getroot())).hexdigest() - self.assertEqual( - ET.tostring( - manifest_split.create_manifest_sha1_element( - manifest, 'test_manifest')).decode(), - '<hash name="test_manifest" type="sha1" value="%s" />' % manifest_sha1) - @mock.patch.object(subprocess, 'check_output', autospec=True) def test_create_split_manifest(self, mock_check_output): with tempfile.NamedTemporaryFile('w+t') as repo_list_file, \ tempfile.NamedTemporaryFile('w+t') as manifest_file, \ tempfile.NamedTemporaryFile('w+t') as module_info_file, \ tempfile.NamedTemporaryFile('w+t') as config_file, \ - tempfile.NamedTemporaryFile('w+t') as split_manifest_file: + tempfile.NamedTemporaryFile('w+t') as split_manifest_file, \ + tempfile.TemporaryDirectory() as temp_dir: + + os.chdir(temp_dir) repo_list_file.write(""" system/project1 : platform/project1 @@ -208,7 +308,11 @@ class ManifestSplitTest(unittest.TestCase): system/project3 : platform/project3 system/project4 : platform/project4 system/project5 : platform/project5 - system/project6 : platform/project6""") + system/project6 : platform/project6 + system/project7 : platform/project7 + system/project8 : platform/project8 + system/project9 : platform/project9 + vendor/project1 : vendor/project1""") repo_list_file.flush() manifest_file.write(""" @@ -219,16 +323,23 @@ class ManifestSplitTest(unittest.TestCase): <project name="platform/project4" path="system/project4" /> <project name="platform/project5" path="system/project5" /> <project name="platform/project6" path="system/project6" /> + <project name="platform/project7" path="system/project7" /> + <project name="platform/project8" path="system/project8" /> + <project name="platform/project9" path="system/project9" /> + <project name="vendor/project1" path="vendor/project1" /> </manifest>""") manifest_file.flush() module_info_file.write("""{ - "droid": { "path": ["system/project1"] }, - "target_a": { "path": ["out/project2"] }, - "target_b": { "path": ["system/project3"] }, - "target_c": { "path": ["system/project4"] }, - "target_d": { "path": ["system/project5"] }, - "target_e": { "path": ["system/project6"] } + "droid": { "class": ["EXECUTABLES"], "path": ["system/project1"], "dependencies": [] }, + "target_a": { "class": ["EXECUTABLES"], "path": ["out/project2"], "dependencies": ["unknown_module_a"] }, + "target_b": { "class": ["EXECUTABLES"], "path": ["system/project3"], "dependencies": ["target_f", "unknown_module_b"] }, + "target_c": { "class": ["EXECUTABLES"], "path": ["system/project4"], "dependencies": [] }, + "target_d": { "class": ["EXECUTABLES"], "path": ["system/project5"], "dependencies": [] }, + "target_e": { "class": ["EXECUTABLES"], "path": ["system/project6"], "dependencies": [] }, + "target_f": { "class": ["HEADER_LIBRARIES"], "path": ["system/project7"], "dependencies": [] }, + "target_g": { "class": ["SHARED_LIBRARIES"], "path": ["system/project8"], "dependencies": ["target_h"] }, + "target_h": { "class": ["HEADER_LIBRARIES"], "path": ["system/project9"], "dependencies": [] } }""") module_info_file.flush() @@ -253,9 +364,14 @@ class ManifestSplitTest(unittest.TestCase): system/project4/file3 """ + product_makefile = 'vendor/project1/product.mk' + os.makedirs(os.path.dirname(product_makefile)) + os.mknod(product_makefile) + kati_stamp_dump = product_makefile.encode() + mock_check_output.side_effect = [ ninja_inputs_droid, - b'', # Unused kati makefiles. This is tested in its own method. + kati_stamp_dump, ninja_inputs_target_b, ninja_inputs_target_c, ] @@ -267,11 +383,13 @@ class ManifestSplitTest(unittest.TestCase): </config>""") config_file.flush() + debug_file = os.path.join(temp_dir, 'debug.json') + manifest_split.create_split_manifest( ['droid'], manifest_file.name, split_manifest_file.name, [config_file.name], repo_list_file.name, 'build-target.ninja', 'ninja', module_info_file.name, 'unused kati stamp', - ['unused overlay']) + ['unused overlay'], [], debug_file) split_manifest = ET.parse(split_manifest_file.name) split_manifest_projects = [ child.attrib['name'] @@ -288,8 +406,217 @@ class ManifestSplitTest(unittest.TestCase): 'platform/project4', # Manual inclusion from config file 'platform/project6', + # From target_b (depends on target_f header library) + 'platform/project7', + # Inclusion from the Kati makefile stamp + 'vendor/project1', + ]) + + with open(debug_file) as debug_fp: + debug_data = json.load(debug_fp) + + # Dependency for droid, but no other adjacent modules + self.assertTrue(debug_data['platform/project1']['direct_input']) + self.assertFalse(debug_data['platform/project1']['adjacent_input']) + self.assertFalse(debug_data['platform/project1']['deps_input']) + + # Dependency for droid and an adjacent module + self.assertTrue(debug_data['platform/project3']['direct_input']) + self.assertTrue(debug_data['platform/project3']['adjacent_input']) + self.assertFalse(debug_data['platform/project3']['deps_input']) + + # Dependency only for an adjacent module + self.assertFalse(debug_data['platform/project4']['direct_input']) + self.assertTrue(debug_data['platform/project4']['adjacent_input']) + self.assertFalse(debug_data['platform/project4']['deps_input']) + + # Included via header library + self.assertFalse(debug_data['platform/project7']['direct_input']) + self.assertFalse(debug_data['platform/project7']['adjacent_input']) + self.assertTrue(debug_data['platform/project7']['deps_input']) + + # Included due to the config file + self.assertEqual( + debug_data['platform/project6']['manual_add_config'], + config_file.name) + + # Included due to the Kati makefile stamp + self.assertEqual(debug_data['vendor/project1']['kati_makefiles'][0], + product_makefile) + + @mock.patch.object(manifest_split, 'get_ninja_inputs', autospec=True) + @mock.patch.object(manifest_split, 'get_kati_makefiles', autospec=True) + @mock.patch.object(manifest_split.ModuleInfo, '__init__', autospec=True) + def test_create_split_manifest_skip_kati_module_info(self, mock_init, + mock_get_kati_makefiles, + mock_get_ninja_inputs): + with tempfile.NamedTemporaryFile('w+t') as repo_list_file, \ + tempfile.NamedTemporaryFile('w+t') as manifest_file, \ + tempfile.NamedTemporaryFile('w+t') as module_info_file, \ + tempfile.NamedTemporaryFile('w+t') as config_file, \ + tempfile.NamedTemporaryFile('w+t') as split_manifest_file, \ + tempfile.TemporaryDirectory() as temp_dir: + + os.chdir(temp_dir) + + manifest_file.write(""" + <manifest> + </manifest>""") + manifest_file.flush() + + manifest_split.create_split_manifest( + targets=['droid'], + manifest_file=manifest_file.name, + split_manifest_file=split_manifest_file.name, + config_files=[], + repo_list_file=repo_list_file.name, + ninja_build_file='build-target.ninja', + ninja_binary='ninja', + kati_stamp_file=None, + module_info_file=None, + overlays=[], + installed_prebuilts=[], + debug_file=None) + + mock_get_ninja_inputs.assert_called_with( + 'ninja', 'build-target.ninja', ['droid']) + mock_get_kati_makefiles.assert_not_called() + mock_init.assert_not_called() + + @mock.patch.object(subprocess, 'check_output', autospec=True) + def test_create_split_manifest_installed_prebuilt(self, mock_check_output): + + # The purpose of this test is to verify that create_split_manifests treats + # installed prebuilts as projects, even though the installed prebuilts are + # not in the manifest. This use case occurs when installed prebuilts + # contribute modules to the build, but the installed prebuilts themselves + # aren't sourced from the manifest. + + with tempfile.NamedTemporaryFile('w+t') as repo_list_file, \ + tempfile.NamedTemporaryFile('w+t') as manifest_file, \ + tempfile.NamedTemporaryFile('w+t') as module_info_file, \ + tempfile.NamedTemporaryFile('w+t') as split_manifest_file, \ + tempfile.TemporaryDirectory() as temp_dir: + + os.chdir(temp_dir) + + repo_list_file.write(""" + system/project1 : platform/project1 + vendor/project1 : vendor/project1""") + repo_list_file.flush() + + # Here we have small manifest that does not include "prebuilt/project3" + # or "prebuilt/project4". + + manifest_file.write(""" + <manifest> + <project name="platform/project1" path="system/project1" /> + <project name="vendor/project1" path="vendor/project1" /> + </manifest>""") + manifest_file.flush() + + # Here's the module_info.json file. It contains modules whose paths are + # "prebuilt/project3" and "prebult/project4", which are not found in the + # manifest. Normally create_split_manifest doesn't tolerate a path that + # doesn't correspond to a manifest project. However, this test verifies + # that you can use these modules if you tell create_split_manifest about + # the installed prebuilts via a parameter. + + module_info_file.write("""{ + "droid": { "class": ["EXECUTABLES"], "path": ["system/project1"], "dependencies": [] }, + "target_a": { "class": ["EXECUTABLES"], "path": ["system/project1"], "dependencies": ["target_b", "target_c"] }, + "target_b": { "class": ["SHARED_LIBRARIES"], "path": ["prebuilt/project3"], "dependencies": [] }, + "target_c": { "class": ["SHARED_LIBRARIES"], "path": ["prebuilt/project4"], "dependencies": [] } + }""") + module_info_file.flush() + + # droid needs inputs from project1 + ninja_inputs_droid = b""" + system/project1/file1 + """ + + # target_a needs inputs from prebuilt/project3 and prebuilt/project4 + ninja_inputs_target_a = b""" + prebuilt/project3/file2 + prebuilt/project4/file3 + """ + + # target_b needs inputs from prebuilt/project3 + ninja_inputs_target_b = b""" + prebuilt/project3/file4 + """ + + # target_c needs inputs from prebuilt/project4 + ninja_inputs_target_c = b""" + prebuilt/project4/file5 + """ + + product_makefile = 'vendor/project1/product.mk' + os.makedirs(os.path.dirname(product_makefile)) + os.mknod(product_makefile) + kati_stamp_dump = product_makefile.encode() + + mock_check_output.side_effect = [ + ninja_inputs_droid, + kati_stamp_dump, + ninja_inputs_target_a, + ninja_inputs_target_b, + ninja_inputs_target_c, + ] + + debug_file = os.path.join(temp_dir, 'debug.json') + + manifest_split.create_split_manifest( + targets=['droid'], + manifest_file=manifest_file.name, + split_manifest_file=split_manifest_file.name, + config_files=[], + repo_list_file=repo_list_file.name, + ninja_build_file='build-target.ninja', + ninja_binary='ninja', + module_info_file=module_info_file.name, + kati_stamp_file='unused kati stamp', + overlays=['unused overlay'], + + # This is a key part of the test. Passing these two "projects" as + # prebuilts allows create_split_manifest to recognize them as + # projects even though they are not in the manifest. + + installed_prebuilts=['prebuilt/project3', 'prebuilt/project4'], + + debug_file = debug_file) + + split_manifest = ET.parse(split_manifest_file.name) + + split_manifest_projects = [ + child.attrib['name'] + for child in split_manifest.getroot().findall('project') + ] + + # Note that the installed prebuilts do not appear in the final split + # manfiest output because they were not in the manifest to begin with. + + self.assertEqual( + split_manifest_projects, + [ + # From droid + 'platform/project1', + # Inclusion from the Kati makefile stamp + 'vendor/project1', ]) + with open(debug_file) as debug_fp: + debug_data = json.load(debug_fp) + + # Dependency for droid, but no other adjacent modules + self.assertTrue(debug_data['platform/project1']['direct_input']) + self.assertFalse(debug_data['platform/project1']['adjacent_input']) + self.assertFalse(debug_data['platform/project1']['deps_input']) + + # Included due to the Kati makefile stamp + self.assertEqual(debug_data['vendor/project1']['kati_makefiles'][0], + product_makefile) + if __name__ == '__main__': unittest.main() diff --git a/split/sample_config.xml b/split/sample_config.xml new file mode 100644 index 0000000..b9318ff --- /dev/null +++ b/split/sample_config.xml @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="utf-8"?> +<config> + <!-- treble_manifest_split config for aosp_arm64. --> + + <!-- ============================================================= --> + + <!-- Add projects that provide modules required for build success, but + are not detected automatically by the tool. --> + + <!-- Lots of modules require py3-stdlib --> + <add_project name="platform/external/python/cpython3" /> + + <!-- bionic/libc:libc_jemalloc_wrapper requires libjemalloc5 --> + <add_project name="platform/external/jemalloc_new" /> + + <!-- libcore/ojluni:ojluni-phony requires art-notices-for-framework-stubs-jar --> + <add_project name="platform/prebuilts/module_sdk/art" /> + + <!-- Provides executable "as", needed by clang for some modules. --> + <add_project name="platform/prebuilts/gcc/linux-x86/arm/arm-linux-androideabi-4.9" /> + + <!-- ============================================================= --> + + <!-- Remove some device projects that the tool included due to Kati makefile + scanning, but are not required to build aosp_arm64. --> + <remove_project name="device/generic/goldfish" /> + <remove_project name="device/generic/goldfish-opengl" /> + <remove_project name="device/google/atv" /> + <remove_project name="device/google/trout" /> + <remove_project name="device/ti/beagle-x15" /> + +</config> + diff --git a/split/xml_diff.py b/split/xml_diff.py new file mode 100644 index 0000000..81230a9 --- /dev/null +++ b/split/xml_diff.py @@ -0,0 +1,141 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A library containing functions for diffing XML elements.""" +import textwrap +from typing import Any, Callable, Dict, Set +import xml.etree.ElementTree as ET +import dataclasses + +Element = ET.Element + +_INDENT = (' ' * 2) + + +@dataclasses.dataclass +class Change: + value_from: str + value_to: str + + def __repr__(self): + return f'{self.value_from} -> {self.value_to}' + + +@dataclasses.dataclass +class ChangeMap: + """A collection of changes broken down by added, removed and modified. + + Attributes: + added: A dictionary of string identifiers to the added string. + removed: A dictionary of string identifiers to the removed string. + modified: A dictionary of string identifiers to the changed object. + """ + added: Dict[str, str] = dataclasses.field(default_factory=dict) + removed: Dict[str, str] = dataclasses.field(default_factory=dict) + modified: Dict[str, Any] = dataclasses.field(default_factory=dict) + + def __repr__(self): + ret_str = '' + if self.added: + ret_str += 'Added:\n' + for value in self.added.values(): + ret_str += textwrap.indent(str(value) + '\n', _INDENT) + if self.removed: + ret_str += 'Removed:\n' + for value in self.removed.values(): + ret_str += textwrap.indent(str(value) + '\n', _INDENT) + if self.modified: + ret_str += 'Modified:\n' + for name, value in self.modified.items(): + ret_str += textwrap.indent(name + ':\n', _INDENT) + ret_str += textwrap.indent(str(value) + '\n', _INDENT * 2) + return ret_str + + def __bool__(self): + return bool(self.added) or bool(self.removed) or bool(self.modified) + + +def element_string(e: Element) -> str: + return ET.tostring(e).decode(encoding='UTF-8').strip() + + +def attribute_changes(e1: Element, e2: Element, + ignored_attrs: Set[str]) -> ChangeMap: + """Get the changes in attributes between two XML elements. + + Arguments: + e1: the first xml element. + e2: the second xml element. + ignored_attrs: a set of attribute names to ignore changes. + + Returns: + A ChangeMap of attribute changes. Keyed by attribute name. + """ + changes = ChangeMap() + attributes = set(e1.keys()) | set(e2.keys()) + for attr in attributes: + if attr in ignored_attrs: + continue + a1 = e1.get(attr) + a2 = e2.get(attr) + if a1 == a2: + continue + elif not a1: + changes.added[attr] = a2 or '' + elif not a2: + changes.removed[attr] = a1 + else: + changes.modified[attr] = Change(value_from=a1, value_to=a2) + return changes + + +def compare_subelements( + tag: str, + p1: Element, + p2: Element, + ignored_attrs: Set[str], + key_fn: Callable[[Element], str], + diff_fn: Callable[[Element, Element, Set[str]], Any]) -> ChangeMap: + """Get the changes between subelements of two parent elements. + + Arguments: + tag: tag name for children element. + p1: the base parent xml element. + p2: the parent xml element to compare + ignored_attrs: a set of attribute names to ignore changes. + key_fn: Function that takes a subelement and returns a key + diff_fn: Function that take two subelements and a set of ignored + attributes, returns the differences + + Returns: + A ChangeMap object of the changes. + """ + changes = ChangeMap() + group1 = {} + for e1 in p1.findall(tag): + group1[key_fn(e1)] = e1 + + for e2 in p2.findall(tag): + key = key_fn(e2) + e1 = group1.pop(key, None) + if e1 is None: + changes.added[key] = element_string(e2) + else: + echange = diff_fn(e1, e2, ignored_attrs) + if echange: + changes.modified[key] = echange + + for name, e1 in group1.items(): + changes.removed[name] = element_string(e1) + + return changes diff --git a/split/xml_diff_test.py b/split/xml_diff_test.py new file mode 100644 index 0000000..aef4562 --- /dev/null +++ b/split/xml_diff_test.py @@ -0,0 +1,63 @@ +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test XML diff.""" +import unittest +import xml.etree.ElementTree as ET + +import xml_diff + + +class XmlDiffTest(unittest.TestCase): + + def test_attribute_changes(self): + e1 = ET.fromstring('<node attr1="hello" attr2="hello2" ignored="me"/>') + e2 = ET.fromstring('<node attr3="hello3" attr2="bye2"/>') + changes = xml_diff.attribute_changes(e1, e2, set(['ignored'])) + self.assertEqual( + xml_diff.ChangeMap( + added={'attr3': 'hello3'}, + removed={'attr1': 'hello'}, + modified={'attr2': xml_diff.Change('hello2', 'bye2')}), changes) + + def test_compare_subelements(self): + p1 = ET.fromstring("""<parent> + <tag1 attr="newfile2" attrkey="notneeded" /> + <tag1 attr="oldfile1" attrkey="dest1" /> + <tag2 attr="oldfile2" attrkey="dest2" /> + </parent> + """) + p2 = ET.fromstring("""<parent> + <tag1 attr="newfile1" attrkey="dest1" /> + <tag2 attr="newfile2" attrkey="dest2" /> + <tag2 attr="somefile" attrkey="addedfile" /> + </parent> + """) + + changes = xml_diff.compare_subelements( + tag='tag1', + p1=p1, + p2=p2, + ignored_attrs=set(), + key_fn=lambda x: x.get('attrkey'), + diff_fn=xml_diff.attribute_changes) + self.assertEqual(changes.added, {}) + self.assertEqual( + changes.removed, + {'notneeded': '<tag1 attr="newfile2" attrkey="notneeded" />'}) + self.assertEqual( + changes.modified, { + 'dest1': + xml_diff.ChangeMap( + modified={'attr': xml_diff.Change('oldfile1', 'newfile1')}) + }) diff --git a/vf/merge.sh b/vf/merge.sh new file mode 100755 index 0000000..20ceb6b --- /dev/null +++ b/vf/merge.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# Thin wrapper around merge_target_files for vendor-frozen targets to +# allow flag changes to be made in a presubmit-guarded change. + +set -e + +while getopts ":t:d:v:b:m:" option ; do + case "${option}" in + t) TARGET=${OPTARG} ;; + d) DIST_DIR=${OPTARG} ;; + v) VENDOR_DIR=${OPTARG} ;; + b) BUILD_ID=${OPTARG} ;; + # TODO(b/170638547) Remove the need for merge configs. + m) MERGE_CONFIG_DIR=${OPTARG} ;; + *) echo "Unexpected argument: -${OPTARG}" >&2 ;; + esac +done + +if [[ -z "${TARGET}" ]]; then + echo "error: -t target argument not set" + exit 1 +fi +if [[ -z "${DIST_DIR}" ]]; then + echo "error: -d dist dir argument not set" + exit 1 +fi +if [[ -z "${VENDOR_DIR}" ]]; then + echo "error: -v vendor dir argument not set" + exit 1 +fi +if [[ -z "${BUILD_ID}" ]]; then + echo "error: -b build id argument not set" + exit 1 +fi +if [[ -z "${MERGE_CONFIG_DIR}" ]]; then + echo "error: -m merge config dir argument not set" + exit 1 +fi + +# Move the system-only build artifacts to a separate folder +# so that the flashing tools use the merged files instead. +readonly SYSTEM_DIR=${DIST_DIR}/system_build +mkdir -p ${SYSTEM_DIR} +mv -f ${DIST_DIR}/android-info.txt ${SYSTEM_DIR} +mv -f ${DIST_DIR}/${TARGET}-*.zip ${SYSTEM_DIR} + +source build/envsetup.sh +lunch ${TARGET}-userdebug + +out/host/linux-x86/bin/merge_target_files \ + --framework-target-files ${SYSTEM_DIR}/${TARGET}-target_files*.zip \ + --vendor-target-files ${VENDOR_DIR}/*-target_files-*.zip \ + --framework-item-list ${MERGE_CONFIG_DIR}/framework_item_list.txt \ + --framework-misc-info-keys ${MERGE_CONFIG_DIR}/framework_misc_info_keys.txt \ + --vendor-item-list ${MERGE_CONFIG_DIR}/vendor_item_list.txt \ + --allow-duplicate-apkapex-keys \ + --output-target-files ${DIST_DIR}/${TARGET}-target_files-${BUILD_ID}.zip \ + --output-img ${DIST_DIR}/${TARGET}-img-${BUILD_ID}.zip \ + --output-ota ${DIST_DIR}/${TARGET}-ota-${BUILD_ID}.zip + +# Copy bootloader.img, radio.img, and android-info.txt, needed for flashing. +cp ${VENDOR_DIR}/bootloader.img ${DIST_DIR}/bootloader.img +cp ${VENDOR_DIR}/radio.img ${DIST_DIR}/radio.img +unzip -j -d ${DIST_DIR} \ + ${VENDOR_DIR}/*-target_files-*.zip \ + OTA/android-info.txt |