From 7528fef948b58069a5f81f1b1cb5b362af93f7ed Mon Sep 17 00:00:00 2001 From: Dan Albert Date: Tue, 12 Jan 2016 14:50:36 -0800 Subject: Differentiate OUT_DIR and DIST_DIR. DIST_DIR is archived on the build server, so we don't want to be putting everything in there. Also, this lays the groundwork for us to remove the build -> pack -> unpack -> repack -> unpack -> test workflow that we currently have and replace it with build -> test -> pack, since we can install things to their proper locations, test them where they are, and then package it directly. Change-Id: I37529b5f564d82ba0e8b126d9fd44b4b148c1478 --- build/lib/build_support.py | 39 +++-- build/tools/build-gcc.py | 2 +- build/tools/build-llvm.py | 2 +- build/tools/package.py | 10 +- checkbuild.py | 228 ++++++++++++++---------------- sources/host-tools/make-3.81/build.py | 4 +- sources/host-tools/nawk-20071023/build.py | 4 +- sources/host-tools/ndk-depends/build.py | 4 +- sources/host-tools/ndk-stack/build.py | 4 +- sources/host-tools/toolbox/build.py | 4 +- 10 files changed, 156 insertions(+), 145 deletions(-) diff --git a/build/lib/build_support.py b/build/lib/build_support.py index 1b1022026..45f10c01b 100644 --- a/build/lib/build_support.py +++ b/build/lib/build_support.py @@ -135,10 +135,11 @@ def jobs_arg(): return '-j{}'.format(multiprocessing.cpu_count() * 2) -def build(cmd, args): +def build(cmd, args, intermediate_package=False): + package_dir = args.out_dir if intermediate_package else args.dist_dir common_args = [ '--verbose', - '--package-dir={}'.format(args.package_dir), + '--package-dir={}'.format(package_dir), ] build_env = dict(os.environ) @@ -147,12 +148,19 @@ def build(cmd, args): subprocess.check_call(cmd + common_args, env=build_env) -def get_default_package_dir(): - DEFAULT_OUT_DIR = android_path('out/ndk') - out_dir = os.path.realpath(os.getenv('DIST_DIR', DEFAULT_OUT_DIR)) - if not os.path.isdir(out_dir): - os.makedirs(out_dir) - return out_dir +def _get_dir_from_env(default, env_var): + path = os.path.realpath(os.getenv(env_var, default)) + if not os.path.isdir(path): + os.makedirs(path) + return path + + +def get_out_dir(): + return _get_dir_from_env(android_path('out'), 'OUT_DIR') + + +def get_dist_dir(out_dir): + return _get_dir_from_env(os.path.join(out_dir, 'dist'), 'DIST_DIR') def get_default_host(): @@ -239,9 +247,17 @@ class ArgParser(argparse.ArgumentParser): '--host', choices=('darwin', 'linux', 'windows', 'windows64'), default=get_default_host(), help='Build binaries for given OS (e.g. linux).') + + self.add_argument( + '--out-dir', help='Directory to place temporary build files.', + type=os.path.realpath, default=get_out_dir()) + + # The default for --dist-dir has to be handled after parsing all + # arguments because the default is derived from --out-dir. This is + # handled in run(). self.add_argument( - '--package-dir', help='Directory to place the packaged artifact.', - type=os.path.realpath, default=get_default_package_dir()) + '--dist-dir', help='Directory to place the packaged artifact.', + type=os.path.realpath) def run(main_func, arg_parser=ArgParser): @@ -251,6 +267,9 @@ def run(main_func, arg_parser=ArgParser): args = arg_parser().parse_args() + if args.dist_dir is None: + args.dist_dir = get_dist_dir(args.out_dir) + # We want any paths to be relative to the invoked build script. main_filename = os.path.realpath(sys.modules['__main__'].__file__) os.chdir(os.path.dirname(main_filename)) diff --git a/build/tools/build-gcc.py b/build/tools/build-gcc.py index a97951dd3..21ff39d74 100755 --- a/build/tools/build-gcc.py +++ b/build/tools/build-gcc.py @@ -65,7 +65,7 @@ def main(args): host_tag = build_support.host_to_tag(args.host) for arch in arches: - package_gcc(args.package_dir, host_tag, arch, GCC_VERSION) + package_gcc(args.dist_dir, host_tag, arch, GCC_VERSION) if __name__ == '__main__': diff --git a/build/tools/build-llvm.py b/build/tools/build-llvm.py index 02e973d22..7081c91d6 100755 --- a/build/tools/build-llvm.py +++ b/build/tools/build-llvm.py @@ -38,7 +38,7 @@ def main(args): LLVM_VERSION = 'clang-2481030' host = args.host - package_dir = args.package_dir + package_dir = args.dist_dir # TODO(danalbert): Fix 64-bit Windows LLVM. # This wrongly packages 32-bit Windows LLVM for 64-bit as well, but the diff --git a/build/tools/package.py b/build/tools/package.py index d401695a8..53490f810 100755 --- a/build/tools/package.py +++ b/build/tools/package.py @@ -305,8 +305,8 @@ class ArgParser(argparse.ArgumentParser): '-f', '--force', dest='force', action='store_true', help='Clobber out directory if it exists.') self.add_argument( - '--package-dir', type=os.path.realpath, - default=build_support.get_default_package_dir(), + '--dist-dir', type=os.path.realpath, + default=build_support.get_dist_dir(build_support.get_out_dir()), help='Directory containing NDK modules.') self.add_argument( '--unpack', action='store_true', @@ -332,15 +332,15 @@ def main(): sys.exit(args.out_dir + ' already exists. Use -f to overwrite.') packages = get_all_packages(args.host, arches) - check_packages(args.package_dir, packages) + check_packages(args.dist_dir, packages) if args.unpack: - extract_all(args.package_dir, packages, args.out_dir) + extract_all(args.dist_dir, packages, args.out_dir) make_ndk_build_shortcut(args.out_dir, args.host) else: package_dir = tempfile.mkdtemp() try: - make_package(args.release, args.package_dir, packages, args.host, + make_package(args.release, args.dist_dir, packages, args.host, args.out_dir, package_dir) finally: shutil.rmtree(package_dir) diff --git a/checkbuild.py b/checkbuild.py index c42b015d9..831a0f21f 100644 --- a/checkbuild.py +++ b/checkbuild.py @@ -122,9 +122,9 @@ def invoke_external_build(script, args=None): _invoke_build(build_support.android_path(script), args) -def package_ndk(out_dir, args): - package_args = common_build_args(out_dir, args) - package_args.append(out_dir) +def package_ndk(out_dir, dist_dir, args): + package_args = common_build_args(out_dir, dist_dir, args) + package_args.append(dist_dir) if args.release is not None: package_args.append('--release={}'.format(args.release)) @@ -180,8 +180,9 @@ def test_ndk(out_dir, args): shutil.rmtree(unpack_dir) -def common_build_args(out_dir, args): - build_args = ['--package-dir={}'.format(out_dir)] +def common_build_args(out_dir, dist_dir, args): + build_args = ['--out-dir={}'.format(out_dir)] + build_args = ['--dist-dir={}'.format(dist_dir)] build_args.append('--host={}'.format(args.system)) return build_args @@ -285,7 +286,7 @@ def get_prebuilt_gcc(host, arch): return os.path.join(system_path, toolchain_dir) -def build_binutils(out_dir, args): +def build_binutils(out_dir, dist_dir, args): print('Extracting binutils package from GCC...') arches = build_support.ALL_ARCHITECTURES @@ -299,45 +300,44 @@ def build_binutils(out_dir, args): toolchain_path = get_prebuilt_gcc(args.system, arch) triple = fixup_toolchain_triple(toolchain) - tmpdir = tempfile.mkdtemp() - try: - install_dir = os.path.join(tmpdir, 'binutils', triple) - os.makedirs(install_dir) - has_gold = True - if host_tag == 'windows': - # Note: 64-bit Windows is fine. - has_gold = False - if arch in ('mips', 'mips64'): - has_gold = False + install_dir = os.path.join(out_dir, 'binutils', triple) + if os.path.exists(install_dir): + shutil.rmtree(install_dir) + os.makedirs(install_dir) - is_windows = host_tag.startswith('windows') - for file_name in get_binutils_files(triple, has_gold, is_windows): - install_file(file_name, toolchain_path, install_dir) + has_gold = True + if host_tag == 'windows': + # Note: 64-bit Windows is fine. + has_gold = False + if arch in ('mips', 'mips64'): + has_gold = False - license_path = build_support.android_path( - 'toolchain/binutils/binutils-2.25/COPYING') - shutil.copy2(license_path, os.path.join(install_dir, 'NOTICE')) + is_windows = host_tag.startswith('windows') + for file_name in get_binutils_files(triple, has_gold, is_windows): + install_file(file_name, toolchain_path, install_dir) - pack_binutils(arch, host_tag, out_dir, install_dir) - finally: - shutil.rmtree(tmpdir) + license_path = build_support.android_path( + 'toolchain/binutils/binutils-2.25/COPYING') + shutil.copy2(license_path, os.path.join(install_dir, 'NOTICE')) + pack_binutils(arch, host_tag, dist_dir, install_dir) -def build_clang(out_dir, args): + +def build_clang(out_dir, dist_dir, args): print('Building Clang...') - invoke_build('build-llvm.py', common_build_args(out_dir, args)) + invoke_build('build-llvm.py', common_build_args(out_dir, dist_dir, args)) -def build_gcc(out_dir, args): +def build_gcc(out_dir, dist_dir, args): print('Building GCC...') - build_args = common_build_args(out_dir, args) + build_args = common_build_args(out_dir, dist_dir, args) if args.arch is not None: build_args.append('--arch={}'.format(args.arch)) invoke_build('build-gcc.py', build_args) -def build_gcc_libs(out_dir, args): +def build_gcc_libs(out_dir, dist_dir, args): print('Packaging GCC libs...') arches = build_support.ALL_ARCHITECTURES @@ -371,35 +371,33 @@ def build_gcc_libs(out_dir, args): for subdir, lib in lib_names: libs.append((subdir, os.path.join(lib_dir, lib))) - tmpdir = tempfile.mkdtemp() - try: - install_dir = os.path.join(tmpdir, 'gcclibs', triple) - os.makedirs(install_dir) + install_dir = os.path.join(out_dir, 'gcclibs', triple) + if os.path.exists(install_dir): + shutil.rmtree(install_dir) + os.makedirs(install_dir) - # These are target libraries, so the OS we use here is not - # important. We explicitly use Linux because for whatever reason - # the Windows aarch64 toolchain doesn't include libatomic. - gcc_path = get_prebuilt_gcc('linux', arch) - for gcc_subdir, lib in libs: - src = os.path.join(gcc_path, gcc_subdir, lib) - dst = os.path.join(install_dir, lib) - dst_dir = os.path.dirname(dst) - if not os.path.exists(dst_dir): - os.makedirs(dst_dir) - shutil.copy2(src, dst) + # These are target libraries, so the OS we use here is not + # important. We explicitly use Linux because for whatever reason + # the Windows aarch64 toolchain doesn't include libatomic. + gcc_path = get_prebuilt_gcc('linux', arch) + for gcc_subdir, lib in libs: + src = os.path.join(gcc_path, gcc_subdir, lib) + dst = os.path.join(install_dir, lib) + dst_dir = os.path.dirname(dst) + if not os.path.exists(dst_dir): + os.makedirs(dst_dir) + shutil.copy2(src, dst) - shutil.copy2( - os.path.join(gcc_path, 'NOTICE'), - os.path.join(install_dir, 'NOTICE')) + shutil.copy2( + os.path.join(gcc_path, 'NOTICE'), + os.path.join(install_dir, 'NOTICE')) - archive_name = os.path.join(out_dir, 'gcclibs-' + arch) - build_support.make_package(archive_name, install_dir, out_dir) - finally: - shutil.rmtree(tmpdir) + archive_name = os.path.join('gcclibs-' + arch) + build_support.make_package(archive_name, install_dir, dist_dir) -def build_host_tools(out_dir, args): - build_args = common_build_args(out_dir, args) +def build_host_tools(out_dir, dist_dir, args): + build_args = common_build_args(out_dir, dist_dir, args) print('Building ndk-stack...') invoke_external_build( @@ -431,7 +429,7 @@ def build_host_tools(out_dir, args): print('Building YASM...') invoke_external_build('toolchain/yasm/build.py', build_args) - package_host_tools(out_dir, args.system) + package_host_tools(out_dir, dist_dir, args.system) def merge_license_files(output_path, files): @@ -444,7 +442,7 @@ def merge_license_files(output_path, files): output_file.write('\n'.join(licenses)) -def package_host_tools(out_dir, host): +def package_host_tools(out_dir, dist_dir, host): packages = [ 'gdb-multiarch-7.10', 'ndk-awk', @@ -467,109 +465,106 @@ def package_host_tools(out_dir, host): host_tag = build_support.host_to_tag(host) package_names = [p + '-' + host_tag + '.tar.bz2' for p in packages] - temp_dir = tempfile.mkdtemp() - try: - for package_name in package_names: - package_path = os.path.join(out_dir, package_name) - subprocess.check_call(['tar', 'xf', package_path, '-C', temp_dir]) - - for f in files: - shutil.copy2(f, os.path.join(temp_dir, 'host-tools/bin')) - - merge_license_files(os.path.join(temp_dir, 'host-tools/NOTICE'), [ - build_support.android_path('toolchain/gdb/gdb-7.10/COPYING'), - build_support.ndk_path('sources/host-tools/nawk-20071023/NOTICE'), - build_support.ndk_path('sources/host-tools/ndk-depends/NOTICE'), - build_support.ndk_path('sources/host-tools/make-3.81/COPYING'), - build_support.android_path( - 'toolchain/python/Python-2.7.5/LICENSE'), - build_support.ndk_path('sources/host-tools/ndk-stack/NOTICE'), - build_support.ndk_path('sources/host-tools/toolbox/NOTICE'), - build_support.android_path('toolchain/yasm/COPYING'), - build_support.android_path('toolchain/yasm/BSD.txt'), - build_support.android_path('toolchain/yasm/Artistic.txt'), - build_support.android_path('toolchain/yasm/GNU_GPL-2.0'), - build_support.android_path('toolchain/yasm/GNU_LGPL-2.0'), - ]) - - package_name = 'host-tools-' + host_tag - path = os.path.join(temp_dir, 'host-tools') - build_support.make_package(package_name, path, out_dir) - finally: - shutil.rmtree(temp_dir) + for package_name in package_names: + package_path = os.path.join(out_dir, package_name) + subprocess.check_call(['tar', 'xf', package_path, '-C', out_dir]) + + for f in files: + shutil.copy2(f, os.path.join(out_dir, 'host-tools/bin')) + + merge_license_files(os.path.join(out_dir, 'host-tools/NOTICE'), [ + build_support.android_path('toolchain/gdb/gdb-7.10/COPYING'), + build_support.ndk_path('sources/host-tools/nawk-20071023/NOTICE'), + build_support.ndk_path('sources/host-tools/ndk-depends/NOTICE'), + build_support.ndk_path('sources/host-tools/make-3.81/COPYING'), + build_support.android_path( + 'toolchain/python/Python-2.7.5/LICENSE'), + build_support.ndk_path('sources/host-tools/ndk-stack/NOTICE'), + build_support.ndk_path('sources/host-tools/toolbox/NOTICE'), + build_support.android_path('toolchain/yasm/COPYING'), + build_support.android_path('toolchain/yasm/BSD.txt'), + build_support.android_path('toolchain/yasm/Artistic.txt'), + build_support.android_path('toolchain/yasm/GNU_GPL-2.0'), + build_support.android_path('toolchain/yasm/GNU_LGPL-2.0'), + ]) + + package_name = 'host-tools-' + host_tag + path = os.path.join(out_dir, 'host-tools') + build_support.make_package(package_name, path, dist_dir) -def build_gdbserver(out_dir, args): +def build_gdbserver(out_dir, dist_dir, args): print('Building gdbserver...') - build_args = common_build_args(out_dir, args) + build_args = common_build_args(out_dir, dist_dir, args) if args.arch is not None: build_args.append('--arch={}'.format(args.arch)) invoke_build('build-gdbserver.py', build_args) -def _build_stl(out_dir, args, stl): - build_args = common_build_args(out_dir, args) +def _build_stl(out_dir, dist_dir, args, stl): + build_args = common_build_args(out_dir, dist_dir, args) if args.arch is not None: build_args.append('--arch={}'.format(args.arch)) script = 'ndk/sources/cxx-stl/{}/build.py'.format(stl) invoke_external_build(script, build_args) -def build_gnustl(out_dir, args): +def build_gnustl(out_dir, dist_dir, args): print('Building gnustl...') - _build_stl(out_dir, args, 'gnu-libstdc++') + _build_stl(out_dir, dist_dir, args, 'gnu-libstdc++') -def build_libcxx(out_dir, args): +def build_libcxx(out_dir, dist_dir, args): print('Building libc++...') - _build_stl(out_dir, args, 'llvm-libc++') + _build_stl(out_dir, dist_dir, args, 'llvm-libc++') -def build_stlport(out_dir, args): +def build_stlport(out_dir, dist_dir, args): print('Building stlport...') - _build_stl(out_dir, args, 'stlport') + _build_stl(out_dir, dist_dir, args, 'stlport') -def build_platforms(out_dir, args): +def build_platforms(out_dir, dist_dir, args): print('Building platforms...') - invoke_build('build-platforms.py', common_build_args(out_dir, args)) + build_args = common_build_args(out_dir, dist_dir, args) + invoke_build('build-platforms.py', build_args) -def build_cpufeatures(out_dir, _): +def build_cpufeatures(_, dist_dir, __): path = build_support.ndk_path('sources/android/cpufeatures') - build_support.make_package('cpufeatures', path, out_dir) + build_support.make_package('cpufeatures', path, dist_dir) -def build_native_app_glue(out_dir, _): +def build_native_app_glue(_, dist_dir, __): path = build_support.android_path( 'development/ndk/sources/android/native_app_glue') - build_support.make_package('native_app_glue', path, out_dir) + build_support.make_package('native_app_glue', path, dist_dir) -def build_ndk_helper(out_dir, _): +def build_ndk_helper(_, dist_dir, __): path = build_support.android_path( 'development/ndk/sources/android/ndk_helper') - build_support.make_package('ndk_helper', path, out_dir) + build_support.make_package('ndk_helper', path, dist_dir) -def build_gtest(out_dir, _): +def build_gtest(_, dist_dir, __): path = build_support.ndk_path('sources/third_party/googletest') - build_support.make_package('gtest', path, out_dir) + build_support.make_package('gtest', path, dist_dir) -def build_build(out_dir, _): +def build_build(_, dist_dir, __): path = build_support.ndk_path('build') - build_support.make_package('build', path, out_dir) + build_support.make_package('build', path, dist_dir) -def build_python_packages(out_dir, _): +def build_python_packages(_, dist_dir, __): # Stage the files in a temporary directory to make things easier. temp_dir = tempfile.mkdtemp() try: path = os.path.join(temp_dir, 'python-packages') shutil.copytree( build_support.android_path('development/python-packages'), path) - build_support.make_package('python-packages', path, out_dir) + build_support.make_package('python-packages', path, dist_dir) finally: shutil.rmtree(temp_dir) @@ -611,12 +606,9 @@ def main(): # Set ANDROID_BUILD_TOP. if 'ANDROID_BUILD_TOP' not in os.environ: os.environ['ANDROID_BUILD_TOP'] = os.path.realpath('..') - build_top = os.getenv('ANDROID_BUILD_TOP') - DEFAULT_OUT_DIR = os.path.join(build_top, 'out/ndk') - out_dir = os.path.realpath(os.getenv('DIST_DIR', DEFAULT_OUT_DIR)) - if not os.path.isdir(out_dir): - os.makedirs(out_dir) + out_dir = build_support.get_out_dir() + dist_dir = build_support.get_dist_dir(out_dir) print('Cleaning up...') invoke_build('dev-cleanup.sh') @@ -642,13 +634,13 @@ def main(): print('Building modules: {}'.format(' '.join(modules))) for module in modules: - module_builds[module](out_dir, args) + module_builds[module](out_dir, dist_dir, args) if do_package: - package_ndk(out_dir, args) + package_ndk(out_dir, dist_dir, args) if args.test: - result = test_ndk(out_dir, args) + result = test_ndk(dist_dir, args) sys.exit(0 if result else 1) diff --git a/sources/host-tools/make-3.81/build.py b/sources/host-tools/make-3.81/build.py index cbdc8e190..5f9d670ee 100755 --- a/sources/host-tools/make-3.81/build.py +++ b/sources/host-tools/make-3.81/build.py @@ -22,7 +22,7 @@ import site site.addsitedir(os.path.join(os.path.dirname(__file__), '../../../build/lib')) -import build_support +import build_support # pylint: disable=import-error def main(args): @@ -36,7 +36,7 @@ def main(args): if args.host != 'windows': build_cmd.append('--try-64') - build_support.build(build_cmd, args) + build_support.build(build_cmd, args, intermediate_package=True) if __name__ == '__main__': build_support.run(main) diff --git a/sources/host-tools/nawk-20071023/build.py b/sources/host-tools/nawk-20071023/build.py index aae38ed05..621e41106 100755 --- a/sources/host-tools/nawk-20071023/build.py +++ b/sources/host-tools/nawk-20071023/build.py @@ -22,7 +22,7 @@ import site site.addsitedir(os.path.join(os.path.dirname(__file__), '../../../build/lib')) -import build_support +import build_support # pylint: disable=import-error def main(args): @@ -36,7 +36,7 @@ def main(args): if args.host != 'windows': build_cmd.append('--try-64') - build_support.build(build_cmd, args) + build_support.build(build_cmd, args, intermediate_package=True) if __name__ == '__main__': build_support.run(main) diff --git a/sources/host-tools/ndk-depends/build.py b/sources/host-tools/ndk-depends/build.py index cd4d2b6d7..59c5a6fe0 100755 --- a/sources/host-tools/ndk-depends/build.py +++ b/sources/host-tools/ndk-depends/build.py @@ -22,7 +22,7 @@ import site site.addsitedir(os.path.join(os.path.dirname(__file__), '../../../build/lib')) -import build_support +import build_support # pylint: disable=import-error def main(args): @@ -38,7 +38,7 @@ def main(args): if args.host != 'windows': build_cmd.append('--try-64') - build_support.build(build_cmd, args) + build_support.build(build_cmd, args, intermediate_package=True) if __name__ == '__main__': build_support.run(main) diff --git a/sources/host-tools/ndk-stack/build.py b/sources/host-tools/ndk-stack/build.py index af667a208..a930ff437 100755 --- a/sources/host-tools/ndk-stack/build.py +++ b/sources/host-tools/ndk-stack/build.py @@ -22,7 +22,7 @@ import site site.addsitedir(os.path.join(os.path.dirname(__file__), '../../../build/lib')) -import build_support +import build_support # pylint: disable=import-error def main(args): @@ -38,7 +38,7 @@ def main(args): if args.host != 'windows': build_cmd.append('--try-64') - build_support.build(build_cmd, args) + build_support.build(build_cmd, args, intermediate_package=True) if __name__ == '__main__': build_support.run(main) diff --git a/sources/host-tools/toolbox/build.py b/sources/host-tools/toolbox/build.py index 618281433..58dd40afc 100755 --- a/sources/host-tools/toolbox/build.py +++ b/sources/host-tools/toolbox/build.py @@ -26,7 +26,7 @@ import sys site.addsitedir(os.path.join(os.path.dirname(__file__), '../../../build/lib')) -import build_support +import build_support # pylint: disable=import-error def main(args): @@ -40,7 +40,7 @@ def main(args): if args.host != 'windows': build_cmd.append('--try-64') - build_support.build(build_cmd, args) + build_support.build(build_cmd, args, intermediate_package=True) if __name__ == '__main__': build_support.run(main) -- cgit v1.2.3