diff options
author | Pirama Arumuga Nainar <pirama@google.com> | 2016-09-19 22:19:55 -0700 |
---|---|---|
committer | Pirama Arumuga Nainar <pirama@google.com> | 2016-10-06 21:01:10 -0700 |
commit | 4967a710c84587c654b56c828382219c3937dacb (patch) | |
tree | ae0d585998e15fc175ca3e9146b21e53f290624b /build.py | |
parent | d27d7495bf70da4faffda3d8083396e0e07f2931 (diff) | |
download | clang-4967a710c84587c654b56c828382219c3937dacb.tar.gz |
Update aosp/master Clang for rebase to r275480
Bug: http://b/31320715
This merges commit ac9cc4764cf47a6c3f031687d8592e080c9f5001 from
aosp/dev.
Test: Build AOSP and run RenderScript tests (host tests for slang and
libbcc, RsTest, CTS)
Change-Id: Ic2875e5c3673c83448cd7d1013861e42947b1b55
Diffstat (limited to 'build.py')
-rwxr-xr-x | build.py | 376 |
1 files changed, 317 insertions, 59 deletions
@@ -14,23 +14,44 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from __future__ import print_function - +"""Builds the Android Clang toolchain.""" import argparse import glob +import logging import multiprocessing import os -import shutil -import subprocess +import pprint import sys import version +# Disable all the "too many/few methods/parameters" warnings and the like. +# pylint: disable=design + +# Disable lint warnings for todo comments and the like. +# pylint: disable=fixme + +# TODO: Add docstrings? +# pylint: disable=missing-docstring + + THIS_DIR = os.path.realpath(os.path.dirname(__file__)) ORIG_ENV = dict(os.environ) +class Config(object): + """Container for global configuration options.""" + + # Set True to skip all actions (log only). Controlled by --dry-run. + dry_run = False + + +def logger(): + """Returns the default logger for the module.""" + return logging.getLogger(__name__) + + def android_path(*args): return os.path.realpath(os.path.join(THIS_DIR, '../..', *args)) @@ -39,9 +60,7 @@ def build_path(*args): # Our multistage build directories will be placed under OUT_DIR if it is in # the environment. By default they will be placed under # $ANDROID_BUILD_TOP/out. - top_out = ORIG_ENV.get('OUT_DIR', android_path('out')) - if not os.path.isabs(top_out): - top_out = os.path.realpath(top_out) + top_out = ORIG_ENV.get('OUT_DIR', 'out') return os.path.join(top_out, *args) @@ -53,18 +72,71 @@ def long_version(): return '.'.join([version.major, version.minor, version.patch]) +def check_call(cmd, *args, **kwargs): + """Proxy for subprocess.check_call with logging and dry-run support.""" + import subprocess + logger().info('check_call: %s', ' '.join(cmd)) + if 'env' in kwargs: + # Rather than dump the whole environment to the terminal every time, + # just print the difference between this call and our environment. + # Note that this will not include environment that was *removed* from + # os.environ. + extra_env = dict(set(kwargs['env'].items()) - set(os.environ.items())) + if len(extra_env) > 0: + logger().info('check_call additional env:\n%s', + pprint.pformat(extra_env)) + if not Config.dry_run: + subprocess.check_call(cmd, *args, **kwargs) + + def install_file(src, dst): - print('Copying ' + src) - shutil.copy2(src, dst) + """Proxy for shutil.copy2 with logging and dry-run support.""" + import shutil + logger().info('copy %s %s', src, dst) + if not Config.dry_run: + shutil.copy2(src, dst) def install_directory(src, dst): - print('Copying ' + src) - shutil.copytree(src, dst) + """Proxy for shutil.copytree with logging and dry-run support.""" + import shutil + logger().info('copytree %s %s', src, dst) + if not Config.dry_run: + shutil.copytree(src, dst) + + +def rmtree(path): + """Proxy for shutil.rmtree with logging and dry-run support.""" + import shutil + logger().info('rmtree %s', path) + if not Config.dry_run: + shutil.rmtree(path) + + +def rename(src, dst): + """Proxy for os.rename with logging and dry-run support.""" + logger().info('rename %s %s', src, dst) + if not Config.dry_run: + os.rename(src, dst) + + +def makedirs(path): + """Proxy for os.makedirs with logging and dry-run support.""" + logger().info('makedirs %s', path) + if not Config.dry_run: + os.makedirs(path) + + +def symlink(src, dst): + """Proxy for os.symlink with logging and dry-run support.""" + logger().info('symlink %s %s', src, dst) + if not Config.dry_run: + os.symlink(src, dst) def build(out_dir, prebuilts_path=None, prebuilts_version=None, - build_all_llvm_tools=None): + build_all_clang_tools=None, build_all_llvm_tools=None, + debug_clang=None, max_jobs=multiprocessing.cpu_count()): products = ( 'aosp_arm', 'aosp_arm64', @@ -75,11 +147,13 @@ def build(out_dir, prebuilts_path=None, prebuilts_version=None, ) for product in products: build_product(out_dir, product, prebuilts_path, prebuilts_version, - build_all_llvm_tools) + build_all_clang_tools, build_all_llvm_tools, debug_clang, + max_jobs) def build_product(out_dir, product, prebuilts_path, prebuilts_version, - build_all_llvm_tools): + build_all_clang_tools, build_all_llvm_tools, debug_clang, + max_jobs): env = dict(ORIG_ENV) env['DISABLE_LLVM_DEVICE_BUILDS'] = 'true' env['DISABLE_RELOCATION_PACKER'] = 'true' @@ -91,18 +165,27 @@ def build_product(out_dir, product, prebuilts_path, prebuilts_version, env['TARGET_BUILD_VARIANT'] = 'userdebug' env['TARGET_PRODUCT'] = product + if debug_clang: + env['FORCE_BUILD_LLVM_DEBUG'] = 'true' + env['FORCE_BUILD_LLVM_DISABLE_NDEBUG'] = 'true' + overrides = [] if prebuilts_path is not None: overrides.append('LLVM_PREBUILTS_BASE={}'.format(prebuilts_path)) if prebuilts_version is not None: overrides.append('LLVM_PREBUILTS_VERSION={}'.format(prebuilts_version)) - jobs_arg = '-j{}'.format(multiprocessing.cpu_count()) - targets = ['clang-toolchain'] + # Use at least 1 and at most all available CPUs (sanitize the user input). + jobs_arg = '-j{}'.format( + max(1, min(max_jobs, multiprocessing.cpu_count()))) + + targets = ['clang-toolchain-minimal'] + if build_all_clang_tools: + targets += ['clang-toolchain-full'] if build_all_llvm_tools: targets += ['llvm-tools'] - subprocess.check_call( - ['make', jobs_arg] + overrides + targets, cwd=android_path(), env=env) + check_call(['make', jobs_arg] + overrides + targets, + cwd=android_path(), env=env) def package_toolchain(build_dir, build_name, host, dist_dir): @@ -113,9 +196,9 @@ def package_toolchain(build_dir, build_name, host, dist_dir): # Remove any previously installed toolchain so it doesn't pollute the # build. if os.path.exists(install_host_dir): - shutil.rmtree(install_host_dir) + rmtree(install_host_dir) - install_toolchain(build_dir, install_dir, host) + install_toolchain(build_dir, install_dir, host, True) version_file_path = os.path.join(install_dir, 'AndroidVersion.txt') with open(version_file_path, 'w') as version_file: @@ -124,26 +207,35 @@ def package_toolchain(build_dir, build_name, host, dist_dir): tarball_name = package_name + '-' + host package_path = os.path.join(dist_dir, tarball_name) + '.tar.bz2' - print('Packaging ' + package_path) + logger().info('Packaging %s', package_path) args = [ 'tar', '-cjC', install_host_dir, '-f', package_path, package_name ] - subprocess.check_call(args) + check_call(args) + + +def install_minimal_toolchain(build_dir, install_dir, host, strip): + install_built_host_files(build_dir, install_dir, host, strip, minimal=True) + install_headers(build_dir, install_dir, host) + install_sanitizers(build_dir, install_dir, host) -def install_toolchain(build_dir, install_dir, host): - install_built_host_files(build_dir, install_dir, host) +def install_toolchain(build_dir, install_dir, host, strip): + install_built_host_files(build_dir, install_dir, host, strip) + install_compiler_wrapper(install_dir, host) install_sanitizer_scripts(install_dir) install_scan_scripts(install_dir) install_analyzer_scripts(install_dir) install_headers(build_dir, install_dir, host) install_profile_rt(build_dir, install_dir, host) install_sanitizers(build_dir, install_dir, host) + install_sanitizer_tests(build_dir, install_dir, host) + install_libomp(build_dir, install_dir, host) install_license_files(install_dir) install_repo_prop(install_dir) -def install_built_host_files(build_dir, install_dir, host): +def get_built_host_files(host, minimal): is_windows = host.startswith('windows') is_darwin = host.startswith('darwin-x86') bin_ext = '.exe' if is_windows else '' @@ -159,6 +251,17 @@ def install_built_host_files(build_dir, install_dir, host): 'bin/clang' + bin_ext, 'bin/clang++' + bin_ext, ] + if not is_windows: + built_files.extend(['lib64/libc++' + lib_ext]) + + if minimal: + return built_files + + built_files.extend([ + 'bin/clang-format' + bin_ext, + 'bin/clang-tidy' + bin_ext, + ]) + if is_windows: built_files.extend([ 'bin/clang_32' + bin_ext, @@ -169,16 +272,20 @@ def install_built_host_files(build_dir, install_dir, host): 'bin/llvm-as' + bin_ext, 'bin/llvm-dis' + bin_ext, 'bin/llvm-link' + bin_ext, - 'lib64/libc++' + lib_ext, + 'bin/llvm-symbolizer' + bin_ext, 'lib64/libLLVM' + lib_ext, 'lib64/LLVMgold' + lib_ext, ]) + return built_files + +def install_built_host_files(build_dir, install_dir, host, strip, minimal=None): + built_files = get_built_host_files(host, minimal) for built_file in built_files: dirname = os.path.dirname(built_file) install_path = os.path.join(install_dir, dirname) if not os.path.exists(install_path): - os.makedirs(install_path) + makedirs(install_path) built_path = os.path.join(build_dir, 'host', host, built_file) install_file(built_path, install_path) @@ -186,15 +293,15 @@ def install_built_host_files(build_dir, install_dir, host): file_name = os.path.basename(built_file) # Only strip bin files (not libs) on darwin. - if not is_darwin or built_file.startswith('bin/'): - subprocess.check_call( - ['strip', os.path.join(install_path, file_name)]) + is_darwin = host.startswith('darwin-x86') + if strip and (not is_darwin or built_file.startswith('bin/')): + check_call(['strip', os.path.join(install_path, file_name)]) def install_sanitizer_scripts(install_dir): script_path = android_path( 'external/compiler-rt/lib/asan/scripts/asan_device_setup') - shutil.copy2(script_path, os.path.join(install_dir, 'bin')) + install_file(script_path, os.path.join(install_dir, 'bin')) def install_analyzer_scripts(install_dir): @@ -222,16 +329,16 @@ def install_analyzer_scripts(install_dir): for arch, target in arch_target_pairs: arch_path = os.path.join(install_dir, 'bin', arch) - os.makedirs(arch_path) + makedirs(arch_path) analyzer_file_path = os.path.join(arch_path, 'analyzer') - print('Creating ' + analyzer_file_path) + logger().info('Creating %s', analyzer_file_path) with open(analyzer_file_path, 'w') as analyzer_file: analyzer_file.write( analyzer_text.format(clang_suffix='', target=target)) analyzerpp_file_path = os.path.join(arch_path, 'analyzer++') - print('Creating ' + analyzerpp_file_path) + logger().info('Creating %s', analyzerpp_file_path) with open(analyzerpp_file_path, 'w') as analyzerpp_file: analyzerpp_file.write( analyzer_text.format(clang_suffix='++', target=target)) @@ -239,7 +346,7 @@ def install_analyzer_scripts(install_dir): def install_scan_scripts(install_dir): tools_install_dir = os.path.join(install_dir, 'tools') - os.makedirs(tools_install_dir) + makedirs(tools_install_dir) tools = ('scan-build', 'scan-view') tools_dir = android_path('external/clang/tools') for tool in tools: @@ -260,7 +367,7 @@ def install_headers(build_dir, install_dir, host): headers_src = android_path('external/clang/lib/Headers') headers_dst = os.path.join( install_dir, 'lib64/clang', short_version(), 'include') - os.makedirs(headers_dst) + makedirs(headers_dst) for header in os.listdir(headers_src): if not should_copy(header): continue @@ -275,14 +382,14 @@ def install_headers(build_dir, install_dir, host): 'libclangBasic_intermediates/include/clang/Basic/arm_neon.h') install_file(arm_neon_h, headers_dst) - os.symlink(short_version(), - os.path.join(install_dir, 'lib64/clang', long_version())) + symlink(short_version(), + os.path.join(install_dir, 'lib64/clang', long_version())) def install_profile_rt(build_dir, install_dir, host): lib_dir = os.path.join( install_dir, 'lib64/clang', short_version(), 'lib/linux') - os.makedirs(lib_dir) + makedirs(lib_dir) install_target_profile_rt(build_dir, lib_dir) @@ -325,6 +432,33 @@ def install_host_profile_rt(build_dir, host, lib_dir): install_file(built_lib, os.path.join(lib_dir, lib_name)) +def install_libomp(build_dir, install_dir, host): + # libomp is not built for Darwin + if host == 'darwin-x86': + return + + lib_dir = os.path.join( + install_dir, 'lib64/clang', short_version(), 'lib/linux') + if not os.path.isdir(lib_dir): + makedirs(lib_dir) + + product_to_arch = { + 'generic': 'arm', + 'generic_arm64': 'arm64', + 'generic_x86': 'x86', + 'generic_x86_64': 'x86_64', + } + + for product, arch in product_to_arch.items(): + module = 'libomp-' + arch + product_dir = os.path.join(build_dir, 'target/product', product) + shared_libs = os.path.join(product_dir, 'obj/SHARED_LIBRARIES') + built_lib = os.path.join( + shared_libs, + '{}_intermediates/PACKED/{}.so'.format(module, module)) + install_file(built_lib, os.path.join(lib_dir, module + '.so')) + + def install_sanitizers(build_dir, install_dir, host): headers_src = android_path('external/compiler-rt/include/sanitizer') clang_lib = os.path.join(install_dir, 'lib64/clang', short_version()) @@ -332,23 +466,61 @@ def install_sanitizers(build_dir, install_dir, host): lib_dst = os.path.join(clang_lib, 'lib/linux') install_directory(headers_src, headers_dst) + if not os.path.exists(lib_dst): + makedirs(lib_dst) + if host == 'linux-x86': install_host_sanitizers(build_dir, host, lib_dst) - # Tuples of (product, arch, libdir) + # Tuples of (product, arch) product_to_arch = ( - ('generic', 'arm', 'lib'), - ('generic_arm64', 'aarch64', 'lib64'), - ('generic_x86', 'i686', 'lib'), + ('generic', 'arm'), + ('generic_arm64', 'aarch64'), + ('generic_x86', 'i686'), + ('generic_mips', 'mips'), + ('generic_mips64', 'mips64'), ) - for product, arch, libdir in product_to_arch: + sanitizers = ('asan', 'ubsan_standalone') + + for product, arch in product_to_arch: + for sanitizer in sanitizers: + module = 'libclang_rt.{}-{}-android'.format(sanitizer, arch) + product_dir = os.path.join(build_dir, 'target/product', product) + lib_dir = os.path.join(product_dir, 'obj/SHARED_LIBRARIES', + '{}_intermediates'.format(module)) + lib_name = '{}.so'.format(module) + built_lib = os.path.join(lib_dir, 'PACKED', lib_name) + install_file(built_lib, lib_dst) + + +# Also install the asan_test binaries. We need to do this because the +# platform sources for compiler-rt are potentially different from our +# toolchain sources. The only way to ensure that this test builds +# correctly is to make it a prebuilt based on our latest toolchain +# sources. Note that this is only created/compiled by the previous +# stage (usually stage1) compiler. We are not doing a subsequent +# compile with our stage2 binaries to construct any further +# device-targeted objects. +def install_sanitizer_tests(build_dir, install_dir, host): + # Tuples of (product, arch) + product_to_arch = ( + ('generic', 'arm'), + ('generic_arm64', 'aarch64'), + ('generic_x86', 'i686'), + ('generic_mips', 'mips'), + ('generic_mips64', 'mips64'), + ) + + for product, arch in product_to_arch: product_dir = os.path.join(build_dir, 'target/product', product) - system_dir = os.path.join(product_dir, 'system') - system_lib_dir = os.path.join(system_dir, libdir) - lib_name = 'libclang_rt.asan-{}-android.so'.format(arch) - built_lib = os.path.join(system_lib_dir, lib_name) - install_file(built_lib, lib_dst) + test_module = 'asan_test' + test_dir = os.path.join(product_dir, 'obj/EXECUTABLES', + '{}_intermediates'.format(test_module)) + built_test = os.path.join(test_dir, 'PACKED', test_module) + test_dst = os.path.join(install_dir, 'test', arch, 'bin') + makedirs(test_dst) + install_file(built_test, test_dst) def install_host_sanitizers(build_dir, host, lib_dst): @@ -381,11 +553,13 @@ def install_host_sanitizers(build_dir, host, lib_dst): def install_license_files(install_dir): projects = ( 'clang', + 'clang-tools-extra', 'compiler-rt', 'libcxx', 'libcxxabi', 'libunwind_llvm', 'llvm', + 'openmp_llvm' ) notices = [] @@ -406,7 +580,7 @@ def install_repo_prop(install_dir): dist_dir = os.environ.get('DIST_DIR') if dist_dir is not None: dist_repo_prop = os.path.join(dist_dir, file_name) - shutil.copy(dist_repo_prop, install_dir) + install_file(dist_repo_prop, install_dir) else: out_file = os.path.join(install_dir, file_name) with open(out_file, 'w') as prop_file: @@ -414,14 +588,48 @@ def install_repo_prop(install_dir): 'repo', 'forall', '-c', 'echo $REPO_PROJECT $(git rev-parse HEAD)', ] - subprocess.check_call(cmd, stdout=prop_file) + check_call(cmd, stdout=prop_file) + + +def install_compiler_wrapper(install_dir, host): + is_windows = host.startswith('windows') + bin_ext = '.exe' if is_windows else '' + + built_files = [ + 'bin/clang' + bin_ext, + 'bin/clang++' + bin_ext, + ] + + if is_windows: + built_files.extend([ + 'bin/clang_32' + bin_ext, + ]) + + wrapper_dir = android_path('external/clang') + wrapper = os.path.join(wrapper_dir, 'compiler_wrapper') + + for built_file in built_files: + old_file = os.path.join(install_dir, built_file) + new_file = os.path.join(install_dir, built_file + ".real") + rename(old_file, new_file) + install_file(wrapper, old_file) def parse_args(): parser = argparse.ArgumentParser() + parser.add_argument('-j', action='store', dest='jobs', type=int, + default=multiprocessing.cpu_count(), + help='Specify number of executed jobs') + parser.add_argument( '--build-name', default='dev', help='Release name for the package.') + parser.add_argument( + '--dry-run', action='store_true', default=False, + help='Skip running commands; just print.') + parser.add_argument( + '-v', '--verbose', action='store_true', default=False, + help='Print debug output.') multi_stage_group = parser.add_mutually_exclusive_group() multi_stage_group.add_argument( @@ -431,20 +639,38 @@ def parse_args(): '--no-multi-stage', action='store_false', dest='multi_stage', help='Do not perform multi-stage build.') - parser.add_argument( + build_all_llvm_tools_group = parser.add_mutually_exclusive_group() + build_all_llvm_tools_group.add_argument( '--build-all-llvm-tools', action='store_true', default=True, help='Build all the LLVM tools/utilities.') - - parser.add_argument( + build_all_llvm_tools_group.add_argument( '--no-build-all-llvm-tools', action='store_false', dest='build_all_llvm_tools', - help='Build all the LLVM tools/utilities.') + help='Do not build all the LLVM tools/utilities.') + + build_debug_clang_group = parser.add_mutually_exclusive_group() + build_debug_clang_group.add_argument( + '--debug-clang', action='store_true', default=True, + help='Also generate a debug version of clang (enabled by default).') + build_debug_clang_group.add_argument( + '--no-debug-clang', action='store_false', + dest='debug_clang', + help='Skip generating a debug version of clang.') return parser.parse_args() def main(): args = parse_args() + log_level = logging.INFO + if args.verbose: + log_level = logging.DEBUG + logging.basicConfig(level=log_level) + + logger().info('chdir %s', android_path()) + os.chdir(android_path()) + + Config.dry_run = args.dry_run if sys.platform.startswith('linux'): hosts = ['linux-x86', 'windows-x86'] @@ -454,7 +680,14 @@ def main(): raise RuntimeError('Unsupported host: {}'.format(sys.platform)) stage_1_out_dir = build_path('stage1') - build(out_dir=stage_1_out_dir) + + # For a multi-stage build, build a minimum clang for the first stage that is + # just enough to build the second stage. + is_stage1_final = not args.multi_stage + build(out_dir=stage_1_out_dir, + build_all_clang_tools=is_stage1_final, + build_all_llvm_tools=(is_stage1_final and args.build_all_llvm_tools), + max_jobs=args.jobs) final_out_dir = stage_1_out_dir if args.multi_stage: stage_1_install_dir = build_path('stage1-install') @@ -466,16 +699,41 @@ def main(): # Remove any previously installed toolchain so it doesn't pollute # the build. if os.path.exists(install_host_dir): - shutil.rmtree(install_host_dir) + rmtree(install_host_dir) - install_toolchain(stage_1_out_dir, install_dir, host) + install_minimal_toolchain(stage_1_out_dir, install_dir, host, True) stage_2_out_dir = build_path('stage2') build(out_dir=stage_2_out_dir, prebuilts_path=stage_1_install_dir, prebuilts_version=package_name, - build_all_llvm_tools=args.build_all_llvm_tools) + build_all_clang_tools=True, + build_all_llvm_tools=args.build_all_llvm_tools, + max_jobs=args.jobs) final_out_dir = stage_2_out_dir + if args.debug_clang: + debug_clang_out_dir = build_path('debug') + build(out_dir=debug_clang_out_dir, + prebuilts_path=stage_1_install_dir, + prebuilts_version=package_name, + build_all_clang_tools=True, + build_all_llvm_tools=args.build_all_llvm_tools, + debug_clang=args.debug_clang, + max_jobs=args.jobs) + # Install the actual debug toolchain somewhere, so it is easier to + # use. + debug_package_name = 'clang-debug' + base_debug_install_dir = build_path('debug-install') + for host in hosts: + debug_install_host_dir = os.path.join( + base_debug_install_dir, host) + debug_install_dir = os.path.join( + debug_install_host_dir, debug_package_name) + if os.path.exists(debug_install_host_dir): + rmtree(debug_install_host_dir) + install_toolchain( + debug_clang_out_dir, debug_install_dir, host, False) + dist_dir = ORIG_ENV.get('DIST_DIR', final_out_dir) for host in hosts: package_toolchain(final_out_dir, args.build_name, host, dist_dir) |