From 8a3b1e2de20688cb35e989705959fc165439952c Mon Sep 17 00:00:00 2001 From: Bill Peckham Date: Tue, 25 May 2021 14:40:13 -0700 Subject: Suppress really long commands For really long commands, we elide some of parameters of the command to avoid producing pages of annoying output. Test: generate a failed command, observe elision Test: test.sh Change-Id: I0b6c3bd5b04eabb9ec6918aa6558963cc966587f --- build/sandbox/nsjail.py | 12 +++++++++++- build/sandbox/nsjail_test.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/build/sandbox/nsjail.py b/build/sandbox/nsjail.py index c388d0b..4d23040 100644 --- a/build/sandbox/nsjail.py +++ b/build/sandbox/nsjail.py @@ -351,7 +351,17 @@ def run_command(nsjail_command, print(' '.join(nsjail_command), file=stdout) if not dry_run: - subprocess.check_call(nsjail_command, stdout=stdout, stderr=stderr) + try: + subprocess.check_call(nsjail_command, stdout=stdout, stderr=stderr) + except subprocess.CalledProcessError as error: + if len(error.cmd) > 13: + cmd = error.cmd[:6] + ['...elided...'] + error.cmd[-6:] + else: + cmd = error.cmd + msg = 'nsjail command %s failed with return code %d' % (cmd, error.returncode) + # Raise from None to avoid exception chaining. + raise RuntimeError(msg) from None + def parse_args(): """Parse command line arguments. diff --git a/build/sandbox/nsjail_test.py b/build/sandbox/nsjail_test.py index a73bbdb..8ea93ef 100644 --- a/build/sandbox/nsjail_test.py +++ b/build/sandbox/nsjail_test.py @@ -75,7 +75,7 @@ class NsjailTest(unittest.TestCase): self.assertEqual(stdout, expected) def testFailingJailedCommand(self): - with self.assertRaises(subprocess.CalledProcessError): + with self.assertRaises(RuntimeError): nsjail.run( nsjail_bin='/bin/false', chroot='/chroot', -- cgit v1.2.3 From 7f797aaf5d864febcdf4f8873e15adf09e2beef5 Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Mon, 28 Jun 2021 12:56:19 -0700 Subject: Allows build flags in addition to build goals. Flags are treated similar to build goals but allow users to differentiate between goals and flags, for the use case of building a new goal but keeping all flags the same. Bug: 191706110 Test: Use s in automation. Test: Use s in local builds. Test: build/test.sh Change-Id: Ic2f467350176ceeef51d06c0c6f29c8f9673f89c --- build/sandbox/build_android_sandboxed.py | 44 ++++++++++++---------- build/sandbox/config.py | 64 +++++++++++++++++++++----------- build/sandbox/config_test.py | 64 ++++++++++++++++---------------- build/sandbox/sample_config.xml | 6 +-- 4 files changed, 103 insertions(+), 75 deletions(-) diff --git a/build/sandbox/build_android_sandboxed.py b/build/sandbox/build_android_sandboxed.py index f6a1b57..e43aa0e 100644 --- a/build/sandbox/build_android_sandboxed.py +++ b/build/sandbox/build_android_sandboxed.py @@ -23,10 +23,19 @@ _DEFAULT_COMMAND_WRAPPER = \ '/src/tools/treble/build/sandbox/build_android_target.sh' -def build(build_target, variant, nsjail_bin, chroot, dist_dir, build_id, - max_cpus, build_goals, config_file=None, - command_wrapper=_DEFAULT_COMMAND_WRAPPER, use_rbe=False, - readonly_bind_mount=None, env=[]): +def build(build_target, + variant, + nsjail_bin, + chroot, + dist_dir, + build_id, + max_cpus, + build_goals, + config_file=None, + command_wrapper=_DEFAULT_COMMAND_WRAPPER, + use_rbe=False, + readonly_bind_mount=None, + env=[]): """Builds an Android target in a secure sandbox. Args: @@ -43,8 +52,8 @@ def build(build_target, variant, nsjail_bin, chroot, dist_dir, build_id, command_wrapper: A string path to the command wrapper. use_rbe: If true, will attempt to use RBE for the build. readonly_bind_mount: A string path to a path to be mounted as read-only. - env: An array of environment variables to define in the NsJail sandbox in the - `var=val` syntax. + env: An array of environment variables to define in the NsJail sandbox in + the `var=val` syntax. Returns: A list of commands that were executed. Each command is a list of strings. @@ -53,7 +62,8 @@ def build(build_target, variant, nsjail_bin, chroot, dist_dir, build_id, cfg = config.Config(config_file) android_target = cfg.get_build_config_android_target(build_target) if cfg.has_tag(build_target, 'skip'): - print('Warning: skipping build_target "{}" due to tag being set'.format(build_target)) + print('Warning: skipping build_target "{}" due to tag being set'.format( + build_target)) return [] else: android_target = build_target @@ -108,15 +118,11 @@ def arg_parser(): # Use the top level module docstring for the help description parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '--build_target', - help='The build target.') + parser.add_argument('--build_target', help='The build target.') parser.add_argument( '--variant', default='userdebug', help='The Android build variant.') parser.add_argument( - '--nsjail_bin', - required=True, - help='Path to NsJail binary.') + '--nsjail_bin', required=True, help='Path to NsJail binary.') parser.add_argument( '--chroot', required=True, @@ -131,13 +137,14 @@ def arg_parser(): '--command_wrapper', default=_DEFAULT_COMMAND_WRAPPER, help='Path to the command wrapper. ' - 'Defaults to \'%s\'.' % _DEFAULT_COMMAND_WRAPPER) + 'Defaults to \'%s\'.' % _DEFAULT_COMMAND_WRAPPER) parser.add_argument( '--readonly_bind_mount', help='Path to the a path to be mounted as readonly inside the secure ' 'build sandbox.') parser.add_argument( - '--env', '-e', + '--env', + '-e', type=str, default=[], action='append', @@ -163,9 +170,7 @@ def arg_parser(): help='One or more contexts used to select build goals from the ' 'configuration.') parser.add_argument( - '--use_rbe', - action='store_true', - help='Executes the build on RBE') + '--use_rbe', action='store_true', help='Executes the build on RBE') return parser @@ -191,6 +196,7 @@ def main(): cfg = config.Config(args['config_file']) build_goals = cfg.get_build_goals(args['build_target'], set(args['context'])) + build_flags = cfg.get_build_flags(args['build_target'], set(args['context'])) build( build_target=args['build_target'], @@ -205,7 +211,7 @@ def main(): build_id=args['build_id'], max_cpus=args['max_cpus'], use_rbe=args['use_rbe'], - build_goals=build_goals) + build_goals=build_goals + build_flags) if __name__ == '__main__': diff --git a/build/sandbox/config.py b/build/sandbox/config.py index 26bccbd..a1be52a 100644 --- a/build/sandbox/config.py +++ b/build/sandbox/config.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Parses config file and provides various ways of using it.""" import xml.etree.ElementTree as ET @@ -107,6 +106,7 @@ import collections Overlay = collections.namedtuple('Overlay', ['name', 'replacement_paths']) + class BuildConfig(object): """Represents configuration of a build_target. @@ -116,8 +116,8 @@ class BuildConfig(object): tags: List of tags associated with the build target config build_goals: List of goals to be used while building the target. overlays: List of overlays to be mounted. - views: A list of (source, destination) string path tuple to be mounted. - See view nodes in XML. + views: A list of (source, destination) string path tuple to be mounted. See + view nodes in XML. allow_readwrite_all: If true, mount source tree as rw. allow_readwrite: List of directories to be mounted as rw. allowed_projects_file: a string path name of a file with a containing @@ -130,6 +130,7 @@ class BuildConfig(object): android_target, tags=frozenset(), build_goals=(), + build_flags=(), overlays=(), views=(), allow_readwrite_all=False, @@ -141,6 +142,7 @@ class BuildConfig(object): self.android_target = android_target self.tags = tags self.build_goals = list(build_goals) + self.build_flags = list(build_flags) self.overlays = list(overlays) self.views = list(views) self.allow_readwrite_all = allow_readwrite_all @@ -161,8 +163,7 @@ class BuildConfig(object): @classmethod def from_config(cls, config_elem, fs_view_map, base_config=None): - """Creates a BuildConfig from a config XML element and an optional - base_config. + """Creates a BuildConfig from a config XML element and an optional base_config. Args: config_elem: the config XML node element to build the configuration @@ -188,6 +189,8 @@ class BuildConfig(object): 'allowed_projects_file', base_config.allowed_projects_file), build_goals=_get_build_config_goals(config_elem, base_config.build_goals), + build_flags=_get_build_config_flags(config_elem, + base_config.build_flags), tags=_get_config_tags(config_elem, base_config.tags), overlays=_get_overlays(config_elem, base_config.overlays), allow_readwrite=_get_allow_readwrite(config_elem, @@ -196,8 +199,7 @@ class BuildConfig(object): allow_readwrite_all=_get_allowed_readwrite_all( config_elem, base_config.allow_readwrite_all), configurations=_get_configurations(config_elem, - base_config.configurations) - ) + base_config.configurations)) def _get_configurations(config_elem, base): @@ -226,6 +228,13 @@ def _get_build_config_goals(config_elem, base=None): for goal in config_elem.findall('goal')] +def _get_build_config_flags(config_elem, base=None): + """See _get_build_config_goals. Gets 'flag' instead of 'goal'.""" + return base + [(goal.get('name'), set(goal.get('contexts').split(',')) + if goal.get('contexts') else None) + for goal in config_elem.findall('flag')] + + def _get_config_tags(config_elem, base=frozenset()): """Retrieves tags from build_config or target. @@ -241,13 +250,12 @@ def _get_config_tags(config_elem, base=frozenset()): def _get_allowed_readwrite_all(config_elem, default=False): - """Determines if build_config or target is set to allow readwrite for all - source paths. + """Determines if build_config or target is set to allow readwrite for all source paths. Args: config_elem: A build_config or target xml element. - default: Value to use if element doesn't contain the - allow_readwrite_all attribute. + default: Value to use if element doesn't contain the allow_readwrite_all + attribute. Returns: True if build config is set to allow readwrite for all sorce paths @@ -264,7 +272,8 @@ def _get_overlays(config_elem, base=None): base: Initial list of overlays to prepend to the list Returns: - A list of tuples of overlays and replacement paths to mount for a build_config or target. + A list of tuples of overlays and replacement paths to mount for a + build_config or target. """ overlays = [] for overlay in config_elem.findall('overlay'): @@ -276,6 +285,7 @@ def _get_overlays(config_elem, base=None): ]))) return base + overlays + def _get_views(config_elem, fs_view_map, base=None): """Retrieves list of views from build_config or target. @@ -287,13 +297,14 @@ def _get_views(config_elem, fs_view_map, base=None): A list of (source, destination) string path tuple to be mounted. See view nodes in XML. """ - return base + [fs for o in config_elem.findall('view') - for fs in fs_view_map[o.get('name')]] + return base + [ + fs for o in config_elem.findall('view') + for fs in fs_view_map[o.get('name')] + ] def _get_allow_readwrite(config_elem, base=None): - """Retrieves list of directories to be mounted rw from build_config or - target. + """Retrieves list of directories to be mounted rw from build_config or target. Args: config_elem: A build_config or target xml element. @@ -450,6 +461,18 @@ class Config: return build_goals + def get_build_flags(self, build_target, contexts=frozenset()): + """See get_build_goals. Gets flags instead of goals.""" + build_flags = [] + for flag, build_contexts in self._build_config_map[ + build_target].build_flags: + if not build_contexts: + build_flags.append(flag) + elif build_contexts.intersection(contexts): + build_flags.append(flag) + + return build_flags + def get_rw_allowlist_map(self): """Return read-write allowlist map. @@ -478,19 +501,18 @@ class Config: overlay names corresponding to the target. """ return { - b.name : [o.name for o in b.overlays - ] for b in self._build_config_map.values() + b.name: [o.name for o in b.overlays + ] for b in self._build_config_map.values() } - def get_fs_view_map(self): """Return the filesystem view map. + Returns: A dict of filesystem views keyed by target name. A filesystem view is a list of (source, destination) string path tuples. """ - return {b.name : b.views for b in self._build_config_map.values()} - + return {b.name: b.views for b in self._build_config_map.values()} def get_build_config(self, build_target): return self._build_config_map[build_target] diff --git a/build/sandbox/config_test.py b/build/sandbox/config_test.py index 002c625..139b5f4 100644 --- a/build/sandbox/config_test.py +++ b/build/sandbox/config_test.py @@ -21,7 +21,7 @@ _TEST_CONFIG_XML = """ - + @@ -30,14 +30,14 @@ _TEST_CONFIG_XML = """ - + - + @@ -67,10 +67,10 @@ _TEST_CONTEXTS_CONFIG_XML = """ - + - + @@ -80,6 +80,7 @@ _TEST_CONTEXTS_CONFIG_XML = """ """ + class ConfigTest(unittest.TestCase): """unittest for Config.""" @@ -159,32 +160,28 @@ class ConfigTest(unittest.TestCase): cfg.get_build_config_android_target('some_target'), 'android_target_4') - def testBuildTargetToBuildGoals(self): + def testBuildTargetToBuildGoalsAndFlags(self): with tempfile.NamedTemporaryFile('w+t') as test_config: test_config.write(_TEST_CONFIG_XML) test_config.flush() cfg = config.factory(test_config.name) - # Test that build_target android_target_1 has goals droid and dist. - self.assertEqual( - cfg.get_build_goals('android_target_1'), - ['droid', 'dist']) + self.assertEqual(cfg.get_build_goals('android_target_1'), ['droid']) + self.assertEqual(cfg.get_build_flags('android_target_1'), ['dist']) - # Test that build_target android_target_2 has goals droid, dist, and - # goal_for_android_target_2. self.assertEqual( cfg.get_build_goals('android_target_2'), - ['common_goal', 'droid', 'dist', 'goal_for_android_target_2']) + ['common_goal', 'droid', 'goal_for_android_target_2']) + self.assertEqual(cfg.get_build_flags('android_target_2'), ['dist']) - # Test that build_target build_target_2 has goals droid and VAR=a. self.assertEqual( - cfg.get_build_goals('build_target_2'), - ['common_goal', 'droid', 'VAR=a']) + cfg.get_build_goals('build_target_2'), ['common_goal', 'droid']) + self.assertEqual(cfg.get_build_flags('build_target_2'), ['VAR=a']) # Test empty goals - self.assertEqual(cfg.get_build_goals('no_goals_target'),[]) + self.assertEqual(cfg.get_build_goals('no_goals_target'), []) - def testBuildTargetToBuildGoalsWithContexts(self): + def testBuildTargetToBuildGoalsAndFlagsWithContexts(self): with tempfile.NamedTemporaryFile('w+t') as test_config: test_config.write(_TEST_CONTEXTS_CONFIG_XML) test_config.flush() @@ -212,19 +209,19 @@ class ConfigTest(unittest.TestCase): # the x goals. build_goals = cfg.get_build_goals('test_target', set(['x'])) + build_flags = cfg.get_build_flags('test_target', set(['x'])) - self.assertEqual( - build_goals, - ['droid', 'always', 'VAR=value', 'extra_goal']) + self.assertEqual(build_goals, ['droid', 'always', 'extra_goal']) + self.assertEqual(build_flags, ['VAR=value']) # Test that when requested_contexts is set(['ci', 'x']), we select the # "always" goals, the ci goals, and the x goals. build_goals = cfg.get_build_goals('test_target', set(['ci', 'x'])) + build_flags = cfg.get_build_flags('test_target', set(['ci', 'x'])) - self.assertEqual( - build_goals, - ['droid', 'always', 'dist', 'VAR=value', 'extra_goal']) + self.assertEqual(build_goals, ['droid', 'always', 'extra_goal']) + self.assertEqual(build_flags, ['dist', 'VAR=value']) def testAllowReadWriteAll(self): with tempfile.NamedTemporaryFile('w+t') as test_config: @@ -292,16 +289,19 @@ class ConfigTest(unittest.TestCase): cfg = config.factory(test_config.name) bc_at2 = cfg.get_build_config('android_target_2') - self.assertDictEqual(bc_at2.configurations, { - 'fmc_framework_images': 'image1,image2', - 'fmc_misc_info_keys': 'misc_info_keys.txt' - }) + self.assertDictEqual( + bc_at2.configurations, { + 'fmc_framework_images': 'image1,image2', + 'fmc_misc_info_keys': 'misc_info_keys.txt' + }) bc_bt2 = cfg.get_build_config('build_target_2') - self.assertDictEqual(bc_bt2.configurations, { - 'fmc_framework_images': 'bt1,bt2', - 'fmc_misc_info_keys': 'misc_info_keys_2.txt' - }) + self.assertDictEqual( + bc_bt2.configurations, { + 'fmc_framework_images': 'bt1,bt2', + 'fmc_misc_info_keys': 'misc_info_keys_2.txt' + }) + if __name__ == '__main__': unittest.main() diff --git a/build/sandbox/sample_config.xml b/build/sandbox/sample_config.xml index dbbd412..070749f 100644 --- a/build/sandbox/sample_config.xml +++ b/build/sandbox/sample_config.xml @@ -10,8 +10,8 @@ Defines sample build configuration file. - + and append dist to the flag list. --> + @@ -26,7 +26,7 @@ Defines sample build configuration file. - + -- cgit v1.2.3 From 96cc20c87abcc5644b85fd2f7a6dbd0ef8365485 Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Thu, 8 Jul 2021 16:35:19 +0000 Subject: Update sample config.xml file to use 64-bit aosp_cf_x86_64_phone. Change-Id: Iee1d851a2363b3d41a5488fac7da91179736d98d --- build/sandbox/sample_config.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/sandbox/sample_config.xml b/build/sandbox/sample_config.xml index 070749f..3c75217 100644 --- a/build/sandbox/sample_config.xml +++ b/build/sandbox/sample_config.xml @@ -3,19 +3,19 @@ Defines sample build configuration file. --> - - - + - + -- cgit v1.2.3 From 3cf56a2aa3b7c13d6fcc86996c39c5a0878230aa Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Mon, 12 Jul 2021 12:19:10 -0700 Subject: Create repack_gki, a standalone tool for repacking GKI images. Bug: 191162695 Test: repack_gki --kernel_version 5.10 \ --ramdisk_build_id --kernel_build_id Test: unpack_bootimg for each individual boot.img. Ensure kernel images are replaced, and adhere to the following: - allsyms boot images use the debug (allsyms) kernel Image - gz boot images use the gz kernel Image - lz4 boot images use the lz4 kernel Image Test: Repeat for boot images inside the img.zip. Test: Ensure BOOT/kernel-* inside target_files.zip match kernel build. Change-Id: I0a3287ca83cb6e7b9739bfece4cffa1081272748 --- gki/Android.bp | 50 ++++++++++++++++ gki/repack_gki.py | 137 ++++++++++++++++++++++++++++++++++++++++++ gki/repack_gki_lib.py | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 347 insertions(+) create mode 100644 gki/Android.bp create mode 100644 gki/repack_gki.py create mode 100644 gki/repack_gki_lib.py diff --git a/gki/Android.bp b/gki/Android.bp new file mode 100644 index 0000000..d5b886d --- /dev/null +++ b/gki/Android.bp @@ -0,0 +1,50 @@ +package { + // See: http://go/android-license-faq + // A large-scale-change added 'default_applicable_licenses' to import + // all of the 'license_kinds' from "tools_treble_license" + // to get the below license kinds: + // SPDX-license-identifier-Apache-2.0 + default_applicable_licenses: ["tools_treble_license"], +} + +python_defaults { + name: "repack_gki_defaults", + version: { + py2: { + enabled: false, + embedded_launcher: false, + }, + py3: { + enabled: true, + embedded_launcher: false, + }, + }, +} + +python_library_host { + name: "repack_gki_lib", + defaults: ["repack_gki_defaults"], + srcs: [ + "repack_gki_lib.py", + ], + libs: [ + "fetcher-lib", + ], + pkg_path: "treble/gki", +} + +python_binary_host { + name: "repack_gki", + main: "repack_gki.py", + defaults: ["repack_gki_defaults"], + srcs: [ + "repack_gki.py", + ], + libs: [ + "repack_gki_lib", + ], + required: [ + "mkbootimg", + "unpack_bootimg", + ], +} diff --git a/gki/repack_gki.py b/gki/repack_gki.py new file mode 100644 index 0000000..38e50fd --- /dev/null +++ b/gki/repack_gki.py @@ -0,0 +1,137 @@ +"""Repacks GKI boot images with the given kernel images.""" +import argparse +import json +import os +import shutil +import tempfile + +from treble.fetcher import fetcher_lib +from treble.gki import repack_gki_lib + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + '--json_keyfile', + help='JSON keyfile containing credentials. ' + '(Default: Use default credential file)') + parser.add_argument( + '--ramdisk_build_id', + required=True, + help='Download from the specified build.') + parser.add_argument( + '--ramdisk_target', + required=True, + help='Name of the ramdisk target from the ramdisk branch.') + parser.add_argument( + '--kernel_build_id', + required=True, + help='Download from the specified build.') + parser.add_argument( + '--kernel_target', + required=True, + help='Name of the kernel target from the kernel branch.') + parser.add_argument( + '--kernel_debug_target', + required=True, + help='Name of the kernel debug target from the kernel branch.') + parser.add_argument( + '--kernel_version', + required=True, + help='The Kernel version to use when repacking.') + parser.add_argument( + '--out_dir', required=True, help='Save output to this directory.') + + args = parser.parse_args() + client = fetcher_lib.create_client_from_json_keyfile( + json_keyfile_name=args.json_keyfile) + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + with tempfile.TemporaryDirectory() as tmp_bootimg_dir, \ + tempfile.TemporaryDirectory() as tmp_kernel_dir: + # Fetch boot images. + repack_gki_lib.fetch_bootimg( + client=client, + out_dir=tmp_bootimg_dir, + build_id=args.ramdisk_build_id, + kernel_version=args.kernel_version, + target=args.ramdisk_target, + ) + + # Fetch kernel artifacts. + kernel_dir, kernel_debug_dir = repack_gki_lib.fetch_kernel( + client=client, + out_dir=tmp_kernel_dir, + build_id=args.kernel_build_id, + kernel_target=args.kernel_target, + kernel_debug_target=args.kernel_debug_target, + ) + + # Save kernel artifacts to the out dir. + kernel_out_dir = os.path.join(args.out_dir, 'kernel', args.kernel_version) + if not os.path.exists(kernel_out_dir): + os.makedirs(kernel_out_dir) + + def copy_kernel_file(in_dir, filename, outname=None): + if not outname: + outname = filename + shutil.copy( + os.path.join(in_dir, filename), os.path.join(kernel_out_dir, outname)) + + copy_kernel_file(kernel_dir, 'System.map') + copy_kernel_file(kernel_dir, 'abi.xml') + copy_kernel_file(kernel_dir, 'abi_symbollist') + copy_kernel_file(kernel_dir, 'Image', + 'kernel-{}'.format(args.kernel_version)) + copy_kernel_file(kernel_dir, 'Image.lz4', + 'kernel-{}-lz4'.format(args.kernel_version)) + copy_kernel_file(kernel_dir, 'Image.gz', + 'kernel-{}-gz'.format(args.kernel_version)) + copy_kernel_file(kernel_debug_dir, 'System.map', 'System.map-allsyms') + copy_kernel_file(kernel_debug_dir, 'Image', + 'kernel-{}-allsyms'.format(args.kernel_version)) + copy_kernel_file(kernel_debug_dir, 'Image.lz4', + 'kernel-{}-lz4-allsyms'.format(args.kernel_version)) + copy_kernel_file(kernel_debug_dir, 'Image.gz', + 'kernel-{}-gz-allsyms'.format(args.kernel_version)) + + # Repack individual boot images using the fetched kernel artifacts, + # then save to the out dir. + repack_gki_lib.repack_bootimgs(tmp_bootimg_dir, kernel_dir, + kernel_debug_dir) + shutil.copytree(tmp_bootimg_dir, args.out_dir, dirs_exist_ok=True) + + # Repack boot images inside the img.zip and save to the out dir. + img_zip_name = [f for f in os.listdir(tmp_bootimg_dir) if '-img-' in f][0] + img_zip_path = os.path.join(tmp_bootimg_dir, img_zip_name) + repack_gki_lib.repack_img_zip(img_zip_path, kernel_dir, kernel_debug_dir, + args.kernel_version) + shutil.copy(img_zip_path, args.out_dir) + + # Replace kernels within the target_files.zip and save to the out dir. + target_files_zip_name = [ + f for f in os.listdir(tmp_bootimg_dir) if '-target_files-' in f + ][0] + target_files_zip_path = os.path.join(tmp_bootimg_dir, target_files_zip_name) + repack_gki_lib.replace_target_files_zip_kernels(target_files_zip_path, + kernel_out_dir, + args.kernel_version) + shutil.copy(target_files_zip_path, args.out_dir) + + # Copy otatools.zip from the ramdisk build, used for GKI signing. + shutil.copy(os.path.join(tmp_bootimg_dir, 'otatools.zip'), args.out_dir) + + # Write prebuilt-info.txt using the prebuilt artifact build IDs. + data = { + 'ramdisk-build-id': int(args.ramdisk_build_id), + 'kernel-build-id': int(args.kernel_build_id), + } + with open(os.path.join(kernel_out_dir, 'prebuilt-info.txt'), 'w') as f: + json.dump(data, f, indent=4) + + +if __name__ == '__main__': + main() diff --git a/gki/repack_gki_lib.py b/gki/repack_gki_lib.py new file mode 100644 index 0000000..6df90d1 --- /dev/null +++ b/gki/repack_gki_lib.py @@ -0,0 +1,160 @@ +"""Helper library for repacking GKI boot images.""" +import os +import shutil +import subprocess +import tempfile + +from treble.fetcher import fetcher_lib + + +def fetch_bootimg(client, out_dir, build_id, kernel_version, target): + """Fetches boot.img artifacts from a given build ID.""" + fetcher_lib.fetch_artifacts( + client=client, + build_id=build_id, + target=target, + pattern=r'(.*-img-.*\.zip|.*-target_files-.*\.zip|boot-debug-{version}.*\.img|boot-test-harness-{version}.*\.img|otatools.zip)' + .format(version=kernel_version), + out_dir=out_dir) + + +def fetch_kernel(client, out_dir, build_id, kernel_target, kernel_debug_target): + """Fetches kernel artifacts from a given build ID.""" + kernel_dir = os.path.join(out_dir, 'kernel') + kernel_debug_dir = os.path.join(out_dir, 'kernel_debug') + os.makedirs(kernel_dir) + os.makedirs(kernel_debug_dir) + + fetcher_lib.fetch_artifacts( + client=client, + build_id=build_id, + target=kernel_target, + pattern=r'(Image|Image.lz4|System\.map|abi.xml|abi_symbollist)', + out_dir=kernel_dir) + fetcher_lib.fetch_artifacts( + client=client, + build_id=build_id, + target=kernel_debug_target, + pattern=r'(Image|Image.lz4|System\.map)', + out_dir=kernel_debug_dir) + + print('Compressing kernels') + + def compress_kernel(kernel_path): + zipped_kernel_path = os.path.join(os.path.dirname(kernel_path), 'Image.gz') + with open(zipped_kernel_path, 'wb') as zipped_kernel: + cmd = [ + 'gzip', + '-nc', + kernel_path, + ] + print(' '.join(cmd)) + subprocess.check_call(cmd, stdout=zipped_kernel) + + compress_kernel(os.path.join(kernel_dir, 'Image')) + compress_kernel(os.path.join(kernel_debug_dir, 'Image')) + + return kernel_dir, kernel_debug_dir + + +def _replace_kernel(bootimg_path, kernel_path): + """Unpacks a boot.img, replaces the kernel, then repacks.""" + with tempfile.TemporaryDirectory() as unpack_dir: + print('Unpacking bootimg %s' % bootimg_path) + cmd = [ + 'out/host/linux-x86/bin/unpack_bootimg', + '--boot_img', + bootimg_path, + '--out', + unpack_dir, + '--format', + 'mkbootimg', + ] + print(' '.join(cmd)) + mkbootimg_args = subprocess.check_output(cmd).decode('utf-8').split(' ') + print('Copying kernel %s' % kernel_path) + shutil.copy(kernel_path, os.path.join(unpack_dir, 'kernel')) + print('Repacking with mkbootimg') + cmd = [ + 'out/host/linux-x86/bin/mkbootimg', + '--output', + bootimg_path, + ] + mkbootimg_args + print(' '.join(cmd)) + subprocess.check_call(cmd) + + +def repack_bootimgs(bootimg_dir, kernel_dir, kernel_debug_dir): + """Repacks all boot images in a given dir using the provided kernels.""" + for bootimg_path in os.listdir(bootimg_dir): + bootimg_path = os.path.join(bootimg_dir, bootimg_path) + if not bootimg_path.endswith('.img'): + continue + + kernel_name = 'Image' + if '-gz' in bootimg_path: + kernel_name = 'Image.gz' + elif '-lz4' in bootimg_path: + kernel_name = 'Image.lz4' + + kernel_path = os.path.join(kernel_dir, kernel_name) + if bootimg_path.endswith('-allsyms.img'): + kernel_path = os.path.join(kernel_debug_dir, kernel_name) + + _replace_kernel(bootimg_path, kernel_path) + + +def repack_img_zip(img_zip_path, kernel_dir, kernel_debug_dir, kernel_version): + """Repacks boot images within an img.zip archive.""" + with tempfile.TemporaryDirectory() as unzip_dir: + pattern = 'boot-{}*'.format(kernel_version) + print('Unzipping %s to repack bootimgs' % img_zip_path) + cmd = [ + 'unzip', + '-d', + unzip_dir, + img_zip_path, + pattern, + ] + print(' '.join(cmd)) + subprocess.check_call(cmd) + repack_bootimgs(unzip_dir, kernel_dir, kernel_debug_dir) + cmd = [ + 'zip', + img_zip_path, + pattern, + ] + print(' '.join(cmd)) + subprocess.check_call(cmd, cwd=unzip_dir) + + +def replace_target_files_zip_kernels(target_files_zip_path, kernel_out_dir, + kernel_version): + """Replaces the BOOT/kernel-* kernels within a target_files.zip archive.""" + with tempfile.TemporaryDirectory() as unzip_dir: + pattern = 'BOOT/kernel-{}*'.format(kernel_version) + print( + 'Unzipping %s to replace kernels in preparation for signing' % + target_files_zip_path,) + cmd = [ + 'unzip', + '-d', + unzip_dir, + target_files_zip_path, + pattern, + ] + print(' '.join(cmd)) + subprocess.check_call(cmd) + for kernel in os.listdir(kernel_out_dir): + if kernel.startswith('kernel-{}'.format(kernel_version)): + print('Copying %s' % kernel) + shutil.copy( + os.path.join(kernel_out_dir, kernel), + os.path.join(unzip_dir, 'BOOT')) + cmd = [ + 'zip', + target_files_zip_path, + pattern, + ] + print(' '.join(cmd)) + subprocess.check_call(cmd, cwd=unzip_dir) -- cgit v1.2.3 From 45940bd8bdd20c6ec4813629bb7f7c366f9ad1ce Mon Sep 17 00:00:00 2001 From: Po-Chien Hsueh Date: Wed, 14 Jul 2021 14:34:39 +0800 Subject: build_android_sandboxed.py to allow multiple --readonly_bind_mount Add action='append' to the argument readonly_bind_mount so that we can specify multiple readonly bind mount paths for the builds. Interface build_android_sandboxed.build() is also changed: - readonly_bind_mount: A string path + readonly_bind_mounts: A list of string paths Bug: 186381711 Test: Built on AB Change-Id: I587c6b6e68edee6163391d6e155213b3d993098a --- build/sandbox/build_android_sandboxed.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/build/sandbox/build_android_sandboxed.py b/build/sandbox/build_android_sandboxed.py index e43aa0e..8518074 100644 --- a/build/sandbox/build_android_sandboxed.py +++ b/build/sandbox/build_android_sandboxed.py @@ -34,7 +34,7 @@ def build(build_target, config_file=None, command_wrapper=_DEFAULT_COMMAND_WRAPPER, use_rbe=False, - readonly_bind_mount=None, + readonly_bind_mounts=[], env=[]): """Builds an Android target in a secure sandbox. @@ -51,7 +51,7 @@ def build(build_target, config_file: A string path to an overlay configuration file. command_wrapper: A string path to the command wrapper. use_rbe: If true, will attempt to use RBE for the build. - readonly_bind_mount: A string path to a path to be mounted as read-only. + readonly_bind_mounts: A list of string paths to be mounted as read-only. env: An array of environment variables to define in the NsJail sandbox in the `var=val` syntax. @@ -79,10 +79,6 @@ def build(build_target, '-j', ] + build_goals - readonly_bind_mounts = [] - if readonly_bind_mount: - readonly_bind_mounts = [readonly_bind_mount] - extra_nsjail_args = [] cleanup = lambda: None nsjail_wrapper = [] @@ -140,8 +136,11 @@ def arg_parser(): 'Defaults to \'%s\'.' % _DEFAULT_COMMAND_WRAPPER) parser.add_argument( '--readonly_bind_mount', + type=str, + default=[], + action='append', help='Path to the a path to be mounted as readonly inside the secure ' - 'build sandbox.') + 'build sandbox. Can be specified multiple times') parser.add_argument( '--env', '-e', @@ -205,7 +204,7 @@ def main(): chroot=args['chroot'], config_file=args['config_file'], command_wrapper=args['command_wrapper'], - readonly_bind_mount=args['readonly_bind_mount'], + readonly_bind_mounts=args['readonly_bind_mount'], env=args['env'], dist_dir=args['dist_dir'], build_id=args['build_id'], -- cgit v1.2.3 From c9802b3bfc7fec7b0c900e7c51b0077549dd00cb Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Fri, 30 Jul 2021 09:13:41 -0700 Subject: Include vmlinux in the fetched kernel artifacts. Bug: 195097892 Test: Create a repacked GKI build, observe vmlinux in 'kernel/' artifacts folder. Change-Id: I3356621b38609c1f7a6c5493b1f198e0697ef0b3 --- gki/repack_gki.py | 1 + gki/repack_gki_lib.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/gki/repack_gki.py b/gki/repack_gki.py index 38e50fd..3c12c56 100644 --- a/gki/repack_gki.py +++ b/gki/repack_gki.py @@ -84,6 +84,7 @@ def main(): copy_kernel_file(kernel_dir, 'System.map') copy_kernel_file(kernel_dir, 'abi.xml') copy_kernel_file(kernel_dir, 'abi_symbollist') + copy_kernel_file(kernel_dir, 'vmlinux') copy_kernel_file(kernel_dir, 'Image', 'kernel-{}'.format(args.kernel_version)) copy_kernel_file(kernel_dir, 'Image.lz4', diff --git a/gki/repack_gki_lib.py b/gki/repack_gki_lib.py index 6df90d1..f48c832 100644 --- a/gki/repack_gki_lib.py +++ b/gki/repack_gki_lib.py @@ -29,7 +29,7 @@ def fetch_kernel(client, out_dir, build_id, kernel_target, kernel_debug_target): client=client, build_id=build_id, target=kernel_target, - pattern=r'(Image|Image.lz4|System\.map|abi.xml|abi_symbollist)', + pattern=r'(Image|Image.lz4|System\.map|abi.xml|abi_symbollist|vmlinux)', out_dir=kernel_dir) fetcher_lib.fetch_artifacts( client=client, -- cgit v1.2.3 From 45ebb74b40709888614e7481ecacd143941e2711 Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Thu, 30 Sep 2021 16:35:18 -0700 Subject: Enforces gsi_* prefix when fetching gsi artifacts. This was accidentally fetching signed artifacts from ramdisk builds, in addition to the main artifacts. These were unused by repack_gki.py but were confusing customers by appearing in the build's output. Bug: 201536148 Test: repack_gki.py; observe no signed artifacts Change-Id: I0c9b38be04c51b66cfa028821de4b9b0a4f03e2f --- gki/repack_gki_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gki/repack_gki_lib.py b/gki/repack_gki_lib.py index f48c832..289a82c 100644 --- a/gki/repack_gki_lib.py +++ b/gki/repack_gki_lib.py @@ -13,7 +13,7 @@ def fetch_bootimg(client, out_dir, build_id, kernel_version, target): client=client, build_id=build_id, target=target, - pattern=r'(.*-img-.*\.zip|.*-target_files-.*\.zip|boot-debug-{version}.*\.img|boot-test-harness-{version}.*\.img|otatools.zip)' + pattern=r'(gsi_.*-img-.*\.zip|gsi_.*-target_files-.*\.zip|boot-debug-{version}.*\.img|boot-test-harness-{version}.*\.img|otatools.zip)' .format(version=kernel_version), out_dir=out_dir) -- cgit v1.2.3 From feba270c8478554e7d4585b9ec4f49d55b74f39d Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Mon, 4 Oct 2021 16:26:49 -0700 Subject: Replace 'import mock' with unittest.mock. Bug: 151436598 Test: atest treble_manifest_split_test Change-Id: Iff404ea02dc243bfadf85393006b500ee3b78b37 --- split/Android.bp | 3 --- split/manifest_split_test.py | 18 +++++++++--------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/split/Android.bp b/split/Android.bp index 331354b..f35167f 100644 --- a/split/Android.bp +++ b/split/Android.bp @@ -87,9 +87,6 @@ python_test_host { "xml_diff.py", "xml_diff_test.py", ], - libs: [ - "py-mock", - ], test_config: "test.xml", test_suites: ["general-tests"], } diff --git a/split/manifest_split_test.py b/split/manifest_split_test.py index 546d3c1..73976cc 100644 --- a/split/manifest_split_test.py +++ b/split/manifest_split_test.py @@ -14,12 +14,12 @@ """Test manifest split.""" import json -import mock import os import re import subprocess import tempfile import unittest +import unittest.mock import xml.etree.ElementTree as ET import manifest_split @@ -167,7 +167,7 @@ class ManifestSplitTest(unittest.TestCase): 'Unknown module path for module target1'): manifest_split.ModuleInfo(module_info_file.name, repo_projects) - @mock.patch.object(subprocess, 'check_output', autospec=True) + @unittest.mock.patch.object(subprocess, 'check_output', autospec=True) def test_get_ninja_inputs(self, mock_check_output): mock_check_output.return_value = b""" path/to/input1 @@ -179,7 +179,7 @@ class ManifestSplitTest(unittest.TestCase): inputs = manifest_split.get_ninja_inputs('unused', 'unused', ['droid']) self.assertEqual(inputs, {'path/to/input1', 'path/to/input2'}) - @mock.patch.object(subprocess, 'check_output', autospec=True) + @unittest.mock.patch.object(subprocess, 'check_output', autospec=True) def test_get_ninja_inputs_includes_test_mapping(self, mock_check_output): mock_check_output.return_value = b""" path/to/input1 @@ -192,7 +192,7 @@ class ManifestSplitTest(unittest.TestCase): self.assertEqual( inputs, {'path/to/input1', 'path/to/input2', 'path/to/TEST_MAPPING'}) - @mock.patch.object(subprocess, 'check_output', autospec=True) + @unittest.mock.patch.object(subprocess, 'check_output', autospec=True) def test_get_kati_makefiles(self, mock_check_output): with tempfile.TemporaryDirectory() as temp_dir: os.chdir(temp_dir) @@ -291,7 +291,7 @@ class ManifestSplitTest(unittest.TestCase): ET.tostring(projects[0]).strip().decode(), '') - @mock.patch.object(subprocess, 'check_output', autospec=True) + @unittest.mock.patch.object(subprocess, 'check_output', autospec=True) def test_create_split_manifest(self, mock_check_output): with tempfile.NamedTemporaryFile('w+t') as repo_list_file, \ tempfile.NamedTemporaryFile('w+t') as manifest_file, \ @@ -444,9 +444,9 @@ class ManifestSplitTest(unittest.TestCase): self.assertEqual(debug_data['vendor/project1']['kati_makefiles'][0], product_makefile) - @mock.patch.object(manifest_split, 'get_ninja_inputs', autospec=True) - @mock.patch.object(manifest_split, 'get_kati_makefiles', autospec=True) - @mock.patch.object(manifest_split.ModuleInfo, '__init__', autospec=True) + @unittest.mock.patch.object(manifest_split, 'get_ninja_inputs', autospec=True) + @unittest.mock.patch.object(manifest_split, 'get_kati_makefiles', autospec=True) + @unittest.mock.patch.object(manifest_split.ModuleInfo, '__init__', autospec=True) def test_create_split_manifest_skip_kati_module_info(self, mock_init, mock_get_kati_makefiles, mock_get_ninja_inputs): @@ -483,7 +483,7 @@ class ManifestSplitTest(unittest.TestCase): mock_get_kati_makefiles.assert_not_called() mock_init.assert_not_called() - @mock.patch.object(subprocess, 'check_output', autospec=True) + @unittest.mock.patch.object(subprocess, 'check_output', autospec=True) def test_create_split_manifest_installed_prebuilt(self, mock_check_output): # The purpose of this test is to verify that create_split_manifests treats -- cgit v1.2.3 From 3cd2d4cb07f2c92694bad70e7b669ed409fbd08e Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Fri, 8 Oct 2021 13:05:19 -0700 Subject: Export env vars before lunching and building. This is necessary for vars that are checked in early-parse makefiles like [device].mk or BoardConfig.mk. Bug: 201683015 Test: Use build_android_target.sh within NSJail, observe that build command vars are exported to the environment and used by the build's device.mk file. Change-Id: Ie6222ac977cb3faddaa3a8f2091cb32fb3479a02 --- build/sandbox/build_android_target.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build/sandbox/build_android_target.sh b/build/sandbox/build_android_target.sh index 23e05ea..ed3e421 100755 --- a/build/sandbox/build_android_target.sh +++ b/build/sandbox/build_android_target.sh @@ -40,6 +40,16 @@ fi set -e +BUILD_COMMAND_ARRAY=($BUILD_COMMAND) +for i in ${BUILD_COMMAND_ARRAY[@]}; +do + if [[ $i =~ ^[A-Z_][A-Z0-9_]*= ]]; + then + echo "build_android_target.sh: export $i"; + export $i; + fi; +done; + echo "build_android_target.sh: source build/envsetup.sh" source build/envsetup.sh echo "build_android_target.sh: lunch $ANDROID_TARGET" -- cgit v1.2.3 From ba8987a1bb99e35da9894afc7cae18ffc3b1a859 Mon Sep 17 00:00:00 2001 From: Jose Galmes Date: Wed, 13 Oct 2021 09:57:55 -0700 Subject: Support for running multiple commands in build_android_target.sh. Bug: b/202730785 Test: Test with a build that uses multiple comamnds to build the kernel and SoC Change-Id: I8c30bb33397d08fe987af326590124d7e2a83203 --- build/sandbox/build_android_target.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/sandbox/build_android_target.sh b/build/sandbox/build_android_target.sh index 23e05ea..fd716cc 100755 --- a/build/sandbox/build_android_target.sh +++ b/build/sandbox/build_android_target.sh @@ -54,7 +54,7 @@ cd "$BUILD_DIR" set +e echo "build_android_target.sh: $BUILD_COMMAND" -$BUILD_COMMAND +eval $BUILD_COMMAND BUILD_COMMAND_EXIT_VALUE=$? # Collect RBE metrics if enabled -- cgit v1.2.3 From 3d2b04fc36ae367d63600953ecae4e35ab217d84 Mon Sep 17 00:00:00 2001 From: Jose Galmes Date: Fri, 19 Nov 2021 08:36:40 -0800 Subject: Set RBE exec strategy to racing. Bug: 207032233 Test: Run Keystone build using RBE. Change-Id: I034dc9460b88a6b6375b454bcd2d3d7be08dc083 --- build/sandbox/rbe.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/sandbox/rbe.py b/build/sandbox/rbe.py index fba368f..6d959b8 100644 --- a/build/sandbox/rbe.py +++ b/build/sandbox/rbe.py @@ -40,6 +40,10 @@ _RBE_ENV = { 'RBE_JAVAC': 'true', 'RBE_D8': 'true', 'RBE_R8': 'true', + 'RBE_CXX_EXEC_STRATEGY' : 'racing', + 'RBE_JAVAC_EXEC_STRATEGY' : 'racing', + 'RBE_R8_EXEC_STRATEGY' : 'racing', + 'RBE_D8_EXEC_STRATEGY' : 'racing', } -- cgit v1.2.3 From 08514c337dc38956b6a34e8b4f4b4d0c79f69f6c Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Thu, 2 Dec 2021 23:53:44 +0000 Subject: Updates repack_gki.py for ABI XML rename and lack of 5.15 boot.img. android13-5.15 renamed these ABI files, and also seems to only create them on debug builds. Bug: 205746464 Bug: 209035444 Change-Id: I17a2225712e0571d80b940d16e891e0780b4c72e --- gki/repack_gki.py | 21 ++++++++++++--------- gki/repack_gki_lib.py | 8 ++++++-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/gki/repack_gki.py b/gki/repack_gki.py index 3c12c56..90b632e 100644 --- a/gki/repack_gki.py +++ b/gki/repack_gki.py @@ -82,7 +82,6 @@ def main(): os.path.join(in_dir, filename), os.path.join(kernel_out_dir, outname)) copy_kernel_file(kernel_dir, 'System.map') - copy_kernel_file(kernel_dir, 'abi.xml') copy_kernel_file(kernel_dir, 'abi_symbollist') copy_kernel_file(kernel_dir, 'vmlinux') copy_kernel_file(kernel_dir, 'Image', @@ -92,6 +91,8 @@ def main(): copy_kernel_file(kernel_dir, 'Image.gz', 'kernel-{}-gz'.format(args.kernel_version)) copy_kernel_file(kernel_debug_dir, 'System.map', 'System.map-allsyms') + copy_kernel_file(kernel_debug_dir, 'abi-generated.xml') + copy_kernel_file(kernel_debug_dir, 'abi-full-generated.xml') copy_kernel_file(kernel_debug_dir, 'Image', 'kernel-{}-allsyms'.format(args.kernel_version)) copy_kernel_file(kernel_debug_dir, 'Image.lz4', @@ -113,14 +114,16 @@ def main(): shutil.copy(img_zip_path, args.out_dir) # Replace kernels within the target_files.zip and save to the out dir. - target_files_zip_name = [ - f for f in os.listdir(tmp_bootimg_dir) if '-target_files-' in f - ][0] - target_files_zip_path = os.path.join(tmp_bootimg_dir, target_files_zip_name) - repack_gki_lib.replace_target_files_zip_kernels(target_files_zip_path, - kernel_out_dir, - args.kernel_version) - shutil.copy(target_files_zip_path, args.out_dir) + # TODO(b/209035444): GSI target_files does not yet include a 5.15 boot.img. + if args.kernel_version != '5.15': + target_files_zip_name = [ + f for f in os.listdir(tmp_bootimg_dir) if '-target_files-' in f + ][0] + target_files_zip_path = os.path.join(tmp_bootimg_dir, target_files_zip_name) + repack_gki_lib.replace_target_files_zip_kernels(target_files_zip_path, + kernel_out_dir, + args.kernel_version) + shutil.copy(target_files_zip_path, args.out_dir) # Copy otatools.zip from the ramdisk build, used for GKI signing. shutil.copy(os.path.join(tmp_bootimg_dir, 'otatools.zip'), args.out_dir) diff --git a/gki/repack_gki_lib.py b/gki/repack_gki_lib.py index 289a82c..9051a65 100644 --- a/gki/repack_gki_lib.py +++ b/gki/repack_gki_lib.py @@ -29,13 +29,13 @@ def fetch_kernel(client, out_dir, build_id, kernel_target, kernel_debug_target): client=client, build_id=build_id, target=kernel_target, - pattern=r'(Image|Image.lz4|System\.map|abi.xml|abi_symbollist|vmlinux)', + pattern=r'(Image|Image.lz4|System\.map|abi_symbollist|vmlinux)', out_dir=kernel_dir) fetcher_lib.fetch_artifacts( client=client, build_id=build_id, target=kernel_debug_target, - pattern=r'(Image|Image.lz4|System\.map)', + pattern=r'(Image|Image.lz4|System\.map|abi-generated.xml|abi-full-generated.xml)', out_dir=kernel_debug_dir) print('Compressing kernels') @@ -107,6 +107,10 @@ def repack_bootimgs(bootimg_dir, kernel_dir, kernel_debug_dir): def repack_img_zip(img_zip_path, kernel_dir, kernel_debug_dir, kernel_version): """Repacks boot images within an img.zip archive.""" with tempfile.TemporaryDirectory() as unzip_dir: + # TODO(b/209035444): 5.15 GSI boot.img is not yet available, so reuse 5.10 boot.img + # which should have an identical ramdisk. + if kernel_version == '5.15': + kernel_version = '5.10' pattern = 'boot-{}*'.format(kernel_version) print('Unzipping %s to repack bootimgs' % img_zip_path) cmd = [ -- cgit v1.2.3 From d87e2c922948990b7ee869768b5412947a410a86 Mon Sep 17 00:00:00 2001 From: Sundong Ahn Date: Fri, 21 Jan 2022 07:13:55 +0000 Subject: Prevents the UnknownApiNameOrVersion error After merged the google-api-python-client v2.34.0, "static_discovery=False" is required for discovery.build to avoid UnknownApiNameOrVersion error. Bug: 215654987 Test: presubmit build in keystone gerrit Change-Id: I20b8ce4b19796c90f948bfeb899ec48b285bec94 --- fetcher/fetcher_lib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fetcher/fetcher_lib.py b/fetcher/fetcher_lib.py index 0ec0173..9701494 100644 --- a/fetcher/fetcher_lib.py +++ b/fetcher/fetcher_lib.py @@ -103,7 +103,8 @@ def create_client(http): Returns: An authorized android build api client. """ - return build(serviceName='androidbuildinternal', version='v2beta1', http=http) + return build(serviceName='androidbuildinternal', version='v2beta1', http=http, + static_discovery=False) def create_client_from_json_keyfile(json_keyfile_name=None): -- cgit v1.2.3 From 57efc1cdc6593dac414dfdba96da4638f95bf1b4 Mon Sep 17 00:00:00 2001 From: Iavor-Valentin Iftime Date: Mon, 7 Feb 2022 19:59:22 +0000 Subject: Add argument for skipping radio.img Some targets are missing radio.img, eg. Android TV => this script fails when trying to copy radio.img. Bug: 186097910 Change-Id: I92c13e299ca481cad5b5ff41451102937ad4368b Test: tools/treble/vf/merge.sh .... -r false" --- vf/merge.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/vf/merge.sh b/vf/merge.sh index 20ceb6b..372bb0e 100755 --- a/vf/merge.sh +++ b/vf/merge.sh @@ -5,7 +5,7 @@ set -e -while getopts ":t:d:v:b:m:" option ; do +while getopts ":t:d:v:b:m:r:" option ; do case "${option}" in t) TARGET=${OPTARG} ;; d) DIST_DIR=${OPTARG} ;; @@ -13,6 +13,7 @@ while getopts ":t:d:v:b:m:" option ; do b) BUILD_ID=${OPTARG} ;; # TODO(b/170638547) Remove the need for merge configs. m) MERGE_CONFIG_DIR=${OPTARG} ;; + r) HAS_RADIO_IMG=${OPTARG} ;; *) echo "Unexpected argument: -${OPTARG}" >&2 ;; esac done @@ -37,6 +38,9 @@ if [[ -z "${MERGE_CONFIG_DIR}" ]]; then echo "error: -m merge config dir argument not set" exit 1 fi +if [[ -z "${HAS_RADIO_IMG}" ]]; then + HAS_RADIO_IMG="true" +fi # Move the system-only build artifacts to a separate folder # so that the flashing tools use the merged files instead. @@ -61,7 +65,11 @@ out/host/linux-x86/bin/merge_target_files \ # Copy bootloader.img, radio.img, and android-info.txt, needed for flashing. cp ${VENDOR_DIR}/bootloader.img ${DIST_DIR}/bootloader.img -cp ${VENDOR_DIR}/radio.img ${DIST_DIR}/radio.img +# Copy radio.img unless arg is "false" (eg. Android TV targets) +if [[ $HAS_RADIO_IMG = "true" ]]; then + cp ${VENDOR_DIR}/radio.img ${DIST_DIR}/radio.img +fi + unzip -j -d ${DIST_DIR} \ ${VENDOR_DIR}/*-target_files-*.zip \ OTA/android-info.txt -- cgit v1.2.3 From 5b94ece44266d753b19e70bf1951b3fe5e19608d Mon Sep 17 00:00:00 2001 From: Rob Seymour Date: Thu, 10 Feb 2022 16:10:03 +0000 Subject: Add configuration parameter to ignore paths from module-info file manifest_split tool will skip any modules in the module-info file whose path starts with an ignored path. User can specify paths to ignore via XML configuration: Bug: 218564451 Test: Run split manifest tool with and without ignore path configuration option Test: treble_manifest_split_test Change-Id: I1542a065118e81ef067712f1c5ad8434dce5ddb4 --- split/manifest_split.py | 25 +++++++++++++++++++------ split/manifest_split_test.py | 7 +++++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/split/manifest_split.py b/split/manifest_split.py index d5f9b95..5114f24 100644 --- a/split/manifest_split.py +++ b/split/manifest_split.py @@ -121,10 +121,12 @@ class ManifestSplitConfig: this project, for projects that should be added to the resulting manifest. path_mappings: A list of PathMappingConfigs to modify a path in the build sandbox to the path in the manifest. + ignore_paths: Set of paths to ignore when parsing module_info_file """ remove_projects: Dict[str, str] add_projects: Dict[str, str] path_mappings: List[PathMappingConfig] + ignore_paths: Set[str] @classmethod def from_config_files(cls, config_files: List[str]): @@ -139,6 +141,8 @@ class ManifestSplitConfig: remove_projects: Dict[str, str] = {} add_projects: Dict[str, str] = {} path_mappings = [] + """ Always ignore paths in out/ directory. """ + ignore_paths = set(["out/"]) for config_file in config_files: root = ET.parse(config_file).getroot() @@ -155,7 +159,10 @@ class ManifestSplitConfig: for child in root.findall("path_mapping") ]) - return cls(remove_projects, add_projects, path_mappings) + ignore_paths.update( + {c.attrib["name"]: config_file for c in root.findall("ignore_path")}) + + return cls(remove_projects, add_projects, path_mappings, ignore_paths) def get_repo_projects(repo_list_file, manifest, path_mappings): @@ -195,7 +202,7 @@ def get_repo_projects(repo_list_file, manifest, path_mappings): class ModuleInfo: """Contains various mappings to/from module/project""" - def __init__(self, module_info_file, repo_projects): + def __init__(self, module_info_file, repo_projects, ignore_paths): """Initialize a module info instance. Builds various maps related to platform build system modules and how they @@ -204,6 +211,7 @@ class ModuleInfo: Args: module_info_file: The path to a module-info.json file from a build. repo_projects: The output of the get_repo_projects function. + ignore_paths: Set of paths to ignore from module_info_file data Raises: ValueError: A module from module-info.json belongs to a path not @@ -221,14 +229,18 @@ class ModuleInfo: with open(module_info_file) as module_info_file: module_info = json.load(module_info_file) + # Check that module contains a path and the path is not in set of + # ignore paths def module_has_valid_path(module): - return ("path" in module_info[module] and module_info[module]["path"] and - not module_info[module]["path"][0].startswith("out/")) + paths = module.get("path") + if not paths: + return False + return all(not paths[0].startswith(p) for p in ignore_paths) module_paths = { module: module_info[module]["path"][0] for module in module_info - if module_has_valid_path(module) + if module_has_valid_path(module_info[module]) } module_project_paths = { module: scan_repo_projects(repo_projects, module_paths[module]) @@ -519,7 +531,8 @@ def create_split_manifest(targets, manifest_file, split_manifest_file, # While we still have projects whose modules we haven't checked yet, if module_info_file: - module_info = ModuleInfo(module_info_file, repo_projects) + module_info = ModuleInfo(module_info_file, repo_projects, + config.ignore_paths) checked_projects = set() projects_to_check = input_projects.difference(checked_projects) logger.info("Checking module-info dependencies for direct and adjacent modules...") diff --git a/split/manifest_split_test.py b/split/manifest_split_test.py index 73976cc..d9c6f76 100644 --- a/split/manifest_split_test.py +++ b/split/manifest_split_test.py @@ -121,8 +121,9 @@ class ManifestSplitTest(unittest.TestCase): 'system/project4': 'platform/project4', 'vendor/google/project3': 'vendor/project3', } + ignore_paths = set(['out/']) module_info = manifest_split.ModuleInfo(module_info_file.name, - repo_projects) + repo_projects, ignore_paths) self.assertEqual( module_info.project_modules, { 'platform/project1': set(['target1a', 'target1b']), @@ -163,9 +164,11 @@ class ManifestSplitTest(unittest.TestCase): }""") module_info_file.flush() repo_projects = {} + ignore_paths = set() with self.assertRaisesRegex(ValueError, 'Unknown module path for module target1'): - manifest_split.ModuleInfo(module_info_file.name, repo_projects) + manifest_split.ModuleInfo(module_info_file.name, repo_projects, + ignore_paths) @unittest.mock.patch.object(subprocess, 'check_output', autospec=True) def test_get_ninja_inputs(self, mock_check_output): -- cgit v1.2.3 From 230bbd7db6df95a448a1c833d7c8d032e89a37e1 Mon Sep 17 00:00:00 2001 From: Iavor-Valentin Iftime Date: Thu, 17 Feb 2022 19:33:51 +0000 Subject: Copy vendor otatools.zip to destination folder Required by sign_target_files_apks when using "--vendor_otatools" argument. Bug: 220126689 Bug: 186097910 Test: tools/treble/vf/merge.sh .... Test: sign_target_files_apks \ --vendor_otatools= \ --vendor_partitions=vendor,odm \ merged-target_files.zip \ signed-target_files.zip Change-Id: I5dffaa3e4cd122ae4d8a6a9c09eb640df41a3242 --- vf/merge.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vf/merge.sh b/vf/merge.sh index 372bb0e..e8058b0 100755 --- a/vf/merge.sh +++ b/vf/merge.sh @@ -70,6 +70,11 @@ if [[ $HAS_RADIO_IMG = "true" ]]; then cp ${VENDOR_DIR}/radio.img ${DIST_DIR}/radio.img fi +# Copy vendor otatools.zip, needed by sign_target_files_apks +if [[ -f "${VENDOR_DIR}/otatools.zip" ]]; then + cp ${VENDOR_DIR}/otatools.zip ${DIST_DIR}/otatools_vendor.zip +fi + unzip -j -d ${DIST_DIR} \ ${VENDOR_DIR}/*-target_files-*.zip \ OTA/android-info.txt -- cgit v1.2.3 From 96d45412aaa0433a0a7df907a89284d0c5fb8d49 Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Mon, 14 Mar 2022 21:53:37 +0000 Subject: Allow inferred merge configs. Only provides merge config flags to merge_target_files if this script is provided a MERGE_CONFIG_DIR. Change-Id: Idbffb2ec40b0fd64313021ac228c262a9e44b3e4 --- vf/merge.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/vf/merge.sh b/vf/merge.sh index e8058b0..8e76241 100755 --- a/vf/merge.sh +++ b/vf/merge.sh @@ -11,7 +11,6 @@ while getopts ":t:d:v:b:m:r:" option ; do d) DIST_DIR=${OPTARG} ;; v) VENDOR_DIR=${OPTARG} ;; b) BUILD_ID=${OPTARG} ;; - # TODO(b/170638547) Remove the need for merge configs. m) MERGE_CONFIG_DIR=${OPTARG} ;; r) HAS_RADIO_IMG=${OPTARG} ;; *) echo "Unexpected argument: -${OPTARG}" >&2 ;; @@ -34,10 +33,6 @@ if [[ -z "${BUILD_ID}" ]]; then echo "error: -b build id argument not set" exit 1 fi -if [[ -z "${MERGE_CONFIG_DIR}" ]]; then - echo "error: -m merge config dir argument not set" - exit 1 -fi if [[ -z "${HAS_RADIO_IMG}" ]]; then HAS_RADIO_IMG="true" fi @@ -52,16 +47,20 @@ mv -f ${DIST_DIR}/${TARGET}-*.zip ${SYSTEM_DIR} source build/envsetup.sh lunch ${TARGET}-userdebug +EXTRA_FLAGS="" +if [[ "${MERGE_CONFIG_DIR}" ]]; then + EXTRA_FLAGS+=" --framework-item-list ${MERGE_CONFIG_DIR}/framework_item_list.txt \ + --framework-misc-info-keys ${MERGE_CONFIG_DIR}/framework_misc_info_keys.txt \ + --vendor-item-list ${MERGE_CONFIG_DIR}/vendor_item_list.txt" +fi out/host/linux-x86/bin/merge_target_files \ --framework-target-files ${SYSTEM_DIR}/${TARGET}-target_files*.zip \ --vendor-target-files ${VENDOR_DIR}/*-target_files-*.zip \ - --framework-item-list ${MERGE_CONFIG_DIR}/framework_item_list.txt \ - --framework-misc-info-keys ${MERGE_CONFIG_DIR}/framework_misc_info_keys.txt \ - --vendor-item-list ${MERGE_CONFIG_DIR}/vendor_item_list.txt \ --allow-duplicate-apkapex-keys \ --output-target-files ${DIST_DIR}/${TARGET}-target_files-${BUILD_ID}.zip \ --output-img ${DIST_DIR}/${TARGET}-img-${BUILD_ID}.zip \ - --output-ota ${DIST_DIR}/${TARGET}-ota-${BUILD_ID}.zip + --output-ota ${DIST_DIR}/${TARGET}-ota-${BUILD_ID}.zip \ + ${EXTRA_FLAGS} # Copy bootloader.img, radio.img, and android-info.txt, needed for flashing. cp ${VENDOR_DIR}/bootloader.img ${DIST_DIR}/bootloader.img @@ -77,4 +76,4 @@ fi unzip -j -d ${DIST_DIR} \ ${VENDOR_DIR}/*-target_files-*.zip \ - OTA/android-info.txt + OTA/android-info.txt \ No newline at end of file -- cgit v1.2.3