diff options
author | Dan Albert <danalbert@google.com> | 2022-04-20 19:19:43 +0000 |
---|---|---|
committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2022-04-20 19:19:43 +0000 |
commit | c788e517259bad0e34edb5ea47d39dd5b8531ba6 (patch) | |
tree | 6e39ada1ddc93ab1f5f67a5358bdd6d952c2aff8 | |
parent | 2778556ebb574c1da3e31f3dd858ec3621909bae (diff) | |
parent | b414fe679a70a1823de0596cd7fbcbf4c40abbc6 (diff) | |
download | ndk-c788e517259bad0e34edb5ea47d39dd5b8531ba6.tar.gz |
Merge changes I2ae26513,Iddf3993c,I94bc5994
* changes:
Relocate libc++ libraries to keep Soong happy.
Updates for update.py.
Update script to work on off-corp machines.
-rw-r--r-- | .gitignore | 138 | ||||
-rw-r--r-- | pylintrc | 17 | ||||
-rwxr-xr-x | update.py | 211 |
3 files changed, 287 insertions, 79 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..a81c8ee12 --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ @@ -1,11 +1,22 @@ +[MASTER] +jobs = 0 + [MESSAGES CONTROL] -disable=missing-docstring,fixme,locally-disabled +disable=missing-docstring +ignore=ndk-gdb.py [BASIC] -# Good variable names which should always be accepted, separated by a comma good-names=i,j,k,ex,Run,_ +[VARIABLES] +dummy-variables-rgx=_ + +[SIMILARITIES] +ignore-imports=yes [FORMAT] -max-line-length=79 +max-line-length=88 +ignore-long-lines=(?x)( + ^\s*(\#\ )?<?https?://\S+>?$| + ^\s*\#\ \S+$) @@ -15,16 +15,16 @@ # limitations under the License. # import argparse -import glob import logging import os +from pathlib import Path import shutil import subprocess from tempfile import TemporaryDirectory import textwrap -THIS_DIR = os.path.realpath(os.path.dirname(__file__)) +THIS_DIR = Path(__file__).resolve().parent def logger(): @@ -32,153 +32,212 @@ def logger(): def check_call(cmd): - logger().debug('Running `%s`', ' '.join(cmd)) + logger().debug("Running `%s`", " ".join(cmd)) subprocess.check_call(cmd) def remove(path): - logger().debug('remove `%s`', path) + logger().debug("remove `%s`", path) os.remove(path) -def fetch_artifact(branch, build, pattern): - fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact' - cmd = [fetch_artifact_path, '--branch', branch, '--target=linux', - '--bid', build, pattern] +def fetch_artifact(branch: str, build: str, pattern: str) -> None: + """Fetches an artifact from the build server. + + Use OAuth2 authentication and the gLinux android-fetch-artifact package, + which work with both on-corp and off-corp workstations.""" + fetch_artifact_path = shutil.which("fetch_artifact") + if fetch_artifact_path is None: + raise RuntimeError( + "error: cannot find fetch_artifact in PATH. Install it using:\n" + " sudo glinux-add-repo android\n" + " sudo apt update\n" + " sudo apt install android-fetch-artifact\n" + ) + cmd = [ + fetch_artifact_path, + "--use_oauth2", + "--branch", + branch, + "--target=linux", + "--bid", + build, + pattern, + ] check_call(cmd) def api_str(api_level): - return 'android-{}'.format(api_level) + return f"android-{api_level}" def start_branch(build): - branch_name = 'update-' + (build or 'latest') - logger().info('Creating branch %s', branch_name) - check_call(['repo', 'start', branch_name, '.']) + branch_name = "update-" + (build or "latest") + logger().info("Creating branch %s", branch_name) + check_call(["repo", "start", branch_name, "."]) -def remove_old_release(install_dir): - if os.path.exists(os.path.join(install_dir, '.git')): +def remove_old_release(install_dir: Path) -> None: + if (install_dir / ".git").exists(): logger().info('Removing old install directory "%s"', install_dir) - check_call(['git', 'rm', '-rf', install_dir]) + check_call(["git", "rm", "-rf", install_dir]) # Need to check again because git won't remove directories if they have # non-git files in them. - if os.path.exists(install_dir): + if install_dir.exists(): shutil.rmtree(install_dir) -LIBUNWIND_GLOB = ( - 'toolchains/llvm/prebuilt/*/lib64/clang/*/lib/linux/*/libunwind.a' +LIBUNWIND_GLOB = "toolchains/llvm/prebuilt/*/lib64/clang/*/lib/linux/*/libunwind.a" +LIBCXX_SHARED_GLOB = ( + "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++_shared.so" ) +LIBCXX_STATIC_GLOB = ( + "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++_static.a" +) +LIBCXXABI_GLOB = "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++abi.a" -def unzip_single_directory(artifact, destination): +def unzip_single_directory(artifact: Path, destination: Path) -> None: # Use cwd so that we can use rename without having to worry about crossing # file systems. with TemporaryDirectory(dir=os.getcwd()) as temp_dir: cmd = [ - 'unzip', - artifact, - '-d', + "unzip", + str(artifact), + "-d", temp_dir, - '*/sources/android/cpufeatures/*', - '*/sources/android/native_app_glue/*', - '*/sources/android/support/*', - '*/sources/cxx-stl/*', - '*/source.properties', - os.path.join('*', LIBUNWIND_GLOB), + "*/sources/android/cpufeatures/*", + "*/sources/android/native_app_glue/*", + "*/sources/android/support/*", + "*/sources/cxx-stl/*", + "*/source.properties", + os.path.join("*", LIBUNWIND_GLOB), + os.path.join("*", LIBCXX_SHARED_GLOB), + os.path.join("*", LIBCXX_STATIC_GLOB), + os.path.join("*", LIBCXXABI_GLOB), ] check_call(cmd) dirs = os.listdir(temp_dir) assert len(dirs) == 1 - ndk_dir = os.path.join(temp_dir, dirs[0]) - for child in os.listdir(ndk_dir): - os.rename(os.path.join(ndk_dir, child), - os.path.join(destination, child)) - - -def relocate_libunwind(install_dir): - unwinds = glob.glob(os.path.join(install_dir, LIBUNWIND_GLOB)) - dest_base = os.path.join(install_dir, 'sources/cxx-stl/llvm-libc++/libs') - for libunwind in unwinds: - arch = os.path.basename(os.path.dirname(libunwind)) + ndk_dir = Path(temp_dir) / dirs[0] + for child in ndk_dir.iterdir(): + child.rename(destination / child.name) + + +def relocate_libcxx(install_dir: Path) -> None: + """Copies the libc++ libraries from the toolchain to sources. + + New versions of the NDK have removed the libraries in the sources directory because + they are duplicates and they aren't needed in typical builds. Soong still expects to + find them in that directory though. We could fix Soong, but since this whole + directory should be dead soon we'll just fix-up the install for now. + """ + dest_base = install_dir / "sources/cxx-stl/llvm-libc++/libs" + for glob in {LIBCXX_SHARED_GLOB, LIBCXX_STATIC_GLOB, LIBCXXABI_GLOB}: + file_name = Path(glob).name + for file_path in install_dir.glob(glob): + triple = file_path.parent.name + abi = { + "arm-linux-androideabi": "armeabi-v7a", + "aarch64-linux-android": "arm64-v8a", + "i686-linux-android": "x86", + "x86_64-linux-android": "x86_64", + }[triple] + dest_dir = dest_base / abi + dest_dir.mkdir(parents=True, exist_ok=True) + dest = dest_dir / file_name + logger().info("Relocating %s to %s", file_path, dest) + file_path.rename(dest) + + +def relocate_libunwind(install_dir: Path) -> None: + dest_base = install_dir / "sources/cxx-stl/llvm-libc++/libs" + for libunwind in install_dir.glob(LIBUNWIND_GLOB): + arch = libunwind.parent.name abi = { - 'arm': 'armeabi-v7a', - 'aarch64': 'arm64-v8a', - 'i386': 'x86', - 'x86_64': 'x86_64', + "arm": "armeabi-v7a", + "aarch64": "arm64-v8a", + "i386": "x86", + "x86_64": "x86_64", }[arch] - dest_dir = os.path.join(dest_base, abi) - dest = os.path.join(dest_dir, 'libunwind.a') - logger().info('Relocating %s to %s', libunwind, dest) - os.rename(libunwind, dest) + dest_dir = dest_base / abi + dest = dest_dir / "libunwind.a" + logger().info("Relocating %s to %s", libunwind, dest) + libunwind.rename(dest) -def install_new_release(branch, build, install_dir): - os.makedirs(install_dir) +def delete_android_mks(install_dir: Path) -> None: + for android_mk in install_dir.glob("**/Android.mk"): + android_mk.unlink() - artifact_pattern = 'android-ndk-*.zip' - logger().info('Fetching %s from %s (artifacts matching %s)', build, branch, - artifact_pattern) + +def install_new_release(branch: str, build: str, install_dir: Path) -> None: + install_dir.mkdir() + + artifact_pattern = "android-ndk-*.zip" + logger().info( + "Fetching %s from %s (artifacts matching %s)", build, branch, artifact_pattern + ) fetch_artifact(branch, build, artifact_pattern) - artifacts = glob.glob('android-ndk-*.zip') + artifacts = list(Path().glob("android-ndk-*.zip")) try: assert len(artifacts) == 1 artifact = artifacts[0] - logger().info('Extracting release') + logger().info("Extracting release") unzip_single_directory(artifact, install_dir) + relocate_libcxx(install_dir) relocate_libunwind(install_dir) + delete_android_mks(install_dir) finally: for artifact in artifacts: - os.unlink(artifact) + artifact.unlink() -def commit(branch, build, install_dir): - logger().info('Making commit') - check_call(['git', 'add', install_dir]) - message = textwrap.dedent("""\ +def commit(branch: str, build: str, install_dir: Path) -> None: + logger().info("Making commit") + check_call(["git", "add", str(install_dir)]) + message = textwrap.dedent( + f"""\ Update NDK prebuilts to build {build}. Taken from branch {branch}. Bug: None Test: treehugger - """).format(branch=branch, build=build) - check_call(['git', 'commit', '-m', message]) + """ + ) + check_call(["git", "commit", "-m", message]) def get_args(): parser = argparse.ArgumentParser() parser.add_argument( - '-b', '--branch', default='master-ndk', - help='Branch to pull build from.') - parser.add_argument( - 'major_release', help='Major release being installed, e.g. "r11".') - parser.add_argument('--build', required=True, help='Build number to pull.') + "-b", "--branch", default="master-ndk", help="Branch to pull build from." + ) + parser.add_argument("--build", required=True, help="Build number to pull.") parser.add_argument( - '--use-current-branch', action='store_true', - help='Perform the update in the current branch. Do not repo start.') + "--use-current-branch", + action="store_true", + help="Perform the update in the current branch. Do not repo start.", + ) parser.add_argument( - '-v', '--verbose', action='count', default=0, - help='Increase output verbosity.') + "-v", "--verbose", action="count", default=0, help="Increase output verbosity." + ) return parser.parse_args() -def main(): +def main() -> None: os.chdir(THIS_DIR) args = get_args() verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) - verbosity = args.verbose - if verbosity > 2: - verbosity = 2 + verbosity = min(args.verbose, 2) logging.basicConfig(level=verbose_map[verbosity]) - install_dir = os.path.realpath(args.major_release) + install_dir = THIS_DIR / "current" if not args.use_current_branch: start_branch(args.build) @@ -187,5 +246,5 @@ def main(): commit(args.branch, args.build, install_dir) -if __name__ == '__main__': +if __name__ == "__main__": main() |