diff options
Diffstat (limited to 'catapult/devil/devil/utils/update_dependencies.py')
-rwxr-xr-x | catapult/devil/devil/utils/update_dependencies.py | 218 |
1 files changed, 218 insertions, 0 deletions
diff --git a/catapult/devil/devil/utils/update_dependencies.py b/catapult/devil/devil/utils/update_dependencies.py new file mode 100755 index 00000000..df1f88b2 --- /dev/null +++ b/catapult/devil/devil/utils/update_dependencies.py @@ -0,0 +1,218 @@ +#! /usr/bin/env python +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Updates the chromium binaries used by devil. + +This currently must be called from the top-level chromium src directory. +""" + +import argparse +import collections +import json +import logging +import os +import sys + +_DEVIL_ROOT_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..')) + +sys.path.append(_DEVIL_ROOT_DIR) +from devil import base_error +from devil import devil_env +from devil.utils import cmd_helper + +_DEVICE_ARCHS = [ + { + 'cpu': 'arm', + 'platform': 'android_armeabi-v7a', + }, + { + 'cpu': 'arm64', + 'platform': 'android_arm64-v8a', + }, + { + 'cpu': 'x86', + 'platform': 'android_x86', + }, + { + 'cpu': 'x64', + 'platform': 'android_x86_64', + }, +] +_HOST_ARCH = [{ + # Host binaries use x86_64, not arm, but they build with the + # host toolchain within a target_cpu="arm" build. + 'cpu': 'arm', + 'platform': 'linux2_x86_64', +}] + +_CHROMIUM_DEPS = { + 'chromium_commands': { + 'archs': _HOST_ARCH, + 'build_path': 'lib.java/chromium_commands.dex.jar', + 'target_name': 'chromium_commands_java', + }, + 'forwarder_device': { + 'archs': _DEVICE_ARCHS, + 'build_path': 'device_forwarder', + 'target_name': 'forwarder2', + }, + 'forwarder_host': { + 'archs': _HOST_ARCH, + 'build_path': 'clang_x64/host_forwarder', + 'target_name': 'forwarder2', + }, + 'md5sum_device': { + 'archs': _DEVICE_ARCHS, + 'build_path': 'md5sum_bin', + 'target_name': 'md5sum', + }, + 'md5sum_host': { + 'archs': _HOST_ARCH, + 'build_path': 'clang_x64/md5sum_bin', + 'target_name': 'md5sum', + }, +} + + +def BuildTargetsForCpu(targets, cpu, output_dir): + logging.info('Building %s', cpu) + + gn_args = [ + 'ffmpeg_branding="Chrome"', + 'is_component_build=false', + 'is_debug=false', + 'proprietary_codecs=true', + 'symbol_level=1', + 'target_cpu="%s"' % cpu, + 'target_os="android"', + 'use_goma=true', + ] + + cmd = ['gn', 'gen', '--args=%s' % (' '.join(gn_args)), output_dir] + ec = cmd_helper.RunCmd(cmd) + if ec: + raise base_error.BaseError('%s failed with %d' % (cmd, ec)) + + ec = cmd_helper.RunCmd(['autoninja', '-C', output_dir] + targets) + if ec: + raise base_error.BaseError('building %s failed with %d' % (cpu, ec)) + + +def UpdateDependency(dependency_name, dependency_info, local_path, platform): + bucket = dependency_info['cloud_storage_bucket'] + folder = dependency_info['cloud_storage_base_folder'] + + # determine the hash + ec, sha1sum_output = cmd_helper.GetCmdStatusAndOutput(['sha1sum', local_path]) + if ec: + raise base_error.BaseError( + 'Failed to determine SHA1 for %s: %s' % (local_path, sha1sum_output)) + + dependency_sha1 = sha1sum_output.split()[0] + + # upload + remote_path = '%s_%s' % (dependency_name, dependency_sha1) + gs_dest = 'gs://%s/%s/%s' % (bucket, folder, remote_path) + ec, gsutil_output = cmd_helper.GetCmdStatusAndOutput( + ['gsutil', 'cp', local_path, gs_dest]) + if ec: + raise base_error.BaseError( + 'Failed to upload %s to %s: %s' % (remote_path, gs_dest, gsutil_output)) + + # update entry in json + file_info = dependency_info['file_info'] + if platform not in file_info: + file_info[platform] = { + 'cloud_storage_hash': '', + # the user will need to manually update the download path after + # uploading a previously unknown dependency. + 'download_path': 'FIXME', + } + file_info[platform]['cloud_storage_hash'] = dependency_sha1 + + +def UpdateChromiumDependencies(dependencies, args): + deps_by_platform = collections.defaultdict(list) + for dep_name, dep_info in _CHROMIUM_DEPS.iteritems(): + archs = dep_info.get('archs', []) + for a in archs: + deps_by_platform[(a.get('cpu'), a.get('platform'))].append( + (dep_name, dep_info.get('build_path'), dep_info.get('target_name'))) + + for arch, arch_deps in deps_by_platform.iteritems(): + targets = [target_name for _n, _b, target_name in arch_deps] + cpu, platform = arch + output_dir = os.path.join(args.chromium_src_dir, 'out-devil-deps', platform) + BuildTargetsForCpu(targets, cpu, output_dir) + + for dep_name, build_path, _ in arch_deps: + local_path = os.path.abspath(os.path.join(output_dir, build_path)) + UpdateDependency(dep_name, + dependencies.get('dependencies', {}).get(dep_name, {}), + local_path, platform) + + return dependencies + + +def UpdateGivenDependency(dependencies, args): + dep_name = args.name or os.path.basename(args.path) + if not dep_name in dependencies.get('dependencies', {}): + raise base_error.BaseError('Could not find dependency "%s" in %s' % + (dep_name, args.dependencies_json)) + + UpdateDependency(dep_name, + dependencies.get('dependencies', {}).get(dep_name, {}), + args.path, args.platform) + + return dependencies + + +def main(raw_args): + parser = argparse.ArgumentParser(description=__doc__) + + # pylint: disable=protected-access + parser.add_argument( + '--dependencies-json', + type=os.path.abspath, + default=devil_env._DEVIL_DEFAULT_CONFIG, + help='Binary dependency configuration file to update.') + # pylint: enable=protected-access + + subparsers = parser.add_subparsers() + chromium_parser = subparsers.add_parser('chromium') + chromium_parser.add_argument( + '--chromium-src-dir', + type=os.path.realpath, + default=os.getcwd(), + help='Path to chromium/src checkout root.') + chromium_parser.set_defaults(update_dependencies=UpdateChromiumDependencies) + + dependency_parser = subparsers.add_parser('dependency') + dependency_parser.add_argument('--name', help='Name of dependency to update.') + dependency_parser.add_argument( + '--path', + type=os.path.abspath, + help='Path to file to upload as new version of dependency.') + dependency_parser.add_argument( + '--platform', help='Platform of dependency to update.') + dependency_parser.set_defaults(update_dependencies=UpdateGivenDependency) + + args = parser.parse_args(raw_args) + + logging.getLogger().setLevel(logging.INFO) + + with open(args.dependencies_json) as f: + dependencies = json.load(f) + + dependencies = args.update_dependencies(dependencies, args) + + with open(args.dependencies_json, 'w') as f: + json.dump(dependencies, f, indent=2, separators=(',', ': '), sort_keys=True) + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) |