Merge changes I2ae26513,Iddf3993c,I94bc5994 am: c788e51725 am: cd2360673f am: b24b5850aa

Original change: https://android-review.googlesource.com/c/platform/prebuilts/ndk/+/2067533

Change-Id: I4ad7e32e06fe18c5db92537f25564dce57a1808d
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

3 files changed, 287 insertions, 79 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000000000..a81c8ee12
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
diff --git a/pylintrc b/pylintrc
index 0ce266927..9e03c7dc8 100644
--- a/pylintrc
+++ b/pylintrc
@@ -1,11 +1,22 @@
+[MASTER]
+jobs = 0
+
 [MESSAGES CONTROL]
-disable=missing-docstring,fixme,locally-disabled
+disable=missing-docstring
 
+ignore=ndk-gdb.py
 
 [BASIC]
-# Good variable names which should always be accepted, separated by a comma
 good-names=i,j,k,ex,Run,_
 
+[VARIABLES]
+dummy-variables-rgx=_
+
+[SIMILARITIES]
+ignore-imports=yes
 
 [FORMAT]
-max-line-length=79
+max-line-length=88
+ignore-long-lines=(?x)(
+  ^\s*(\#\ )?<?https?://\S+>?$|
+  ^\s*\#\ \S+$)
diff --git a/update.py b/update.py
index c8c821c3e..398a68b27 100755
--- a/update.py
+++ b/update.py
@@ -15,16 +15,16 @@
 # limitations under the License.
 #
 import argparse
-import glob
 import logging
 import os
+from pathlib import Path
 import shutil
 import subprocess
 from tempfile import TemporaryDirectory
 import textwrap
 
 
-THIS_DIR = os.path.realpath(os.path.dirname(__file__))
+THIS_DIR = Path(__file__).resolve().parent
 
 
 def logger():
@@ -32,153 +32,212 @@ def logger():
 
 
 def check_call(cmd):
-    logger().debug('Running `%s`', ' '.join(cmd))
+    logger().debug("Running `%s`", " ".join(cmd))
     subprocess.check_call(cmd)
 
 
 def remove(path):
-    logger().debug('remove `%s`', path)
+    logger().debug("remove `%s`", path)
     os.remove(path)
 
 
-def fetch_artifact(branch, build, pattern):
-    fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact'
-    cmd = [fetch_artifact_path, '--branch', branch, '--target=linux',
-           '--bid', build, pattern]
+def fetch_artifact(branch: str, build: str, pattern: str) -> None:
+    """Fetches an artifact from the build server.
+
+    Use OAuth2 authentication and the gLinux android-fetch-artifact package,
+    which work with both on-corp and off-corp workstations."""
+    fetch_artifact_path = shutil.which("fetch_artifact")
+    if fetch_artifact_path is None:
+        raise RuntimeError(
+            "error: cannot find fetch_artifact in PATH. Install it using:\n"
+            "  sudo glinux-add-repo android\n"
+            "  sudo apt update\n"
+            "  sudo apt install android-fetch-artifact\n"
+        )
+    cmd = [
+        fetch_artifact_path,
+        "--use_oauth2",
+        "--branch",
+        branch,
+        "--target=linux",
+        "--bid",
+        build,
+        pattern,
+    ]
     check_call(cmd)
 
 
 def api_str(api_level):
-    return 'android-{}'.format(api_level)
+    return f"android-{api_level}"
 
 
 def start_branch(build):
-    branch_name = 'update-' + (build or 'latest')
-    logger().info('Creating branch %s', branch_name)
-    check_call(['repo', 'start', branch_name, '.'])
+    branch_name = "update-" + (build or "latest")
+    logger().info("Creating branch %s", branch_name)
+    check_call(["repo", "start", branch_name, "."])
 
 
-def remove_old_release(install_dir):
-    if os.path.exists(os.path.join(install_dir, '.git')):
+def remove_old_release(install_dir: Path) -> None:
+    if (install_dir / ".git").exists():
         logger().info('Removing old install directory "%s"', install_dir)
-        check_call(['git', 'rm', '-rf', install_dir])
+        check_call(["git", "rm", "-rf", install_dir])
 
     # Need to check again because git won't remove directories if they have
     # non-git files in them.
-    if os.path.exists(install_dir):
+    if install_dir.exists():
         shutil.rmtree(install_dir)
 
 
-LIBUNWIND_GLOB = (
-    'toolchains/llvm/prebuilt/*/lib64/clang/*/lib/linux/*/libunwind.a'
+LIBUNWIND_GLOB = "toolchains/llvm/prebuilt/*/lib64/clang/*/lib/linux/*/libunwind.a"
+LIBCXX_SHARED_GLOB = (
+    "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++_shared.so"
 )
+LIBCXX_STATIC_GLOB = (
+    "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++_static.a"
+)
+LIBCXXABI_GLOB = "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++abi.a"
 
 
-def unzip_single_directory(artifact, destination):
+def unzip_single_directory(artifact: Path, destination: Path) -> None:
     # Use cwd so that we can use rename without having to worry about crossing
     # file systems.
     with TemporaryDirectory(dir=os.getcwd()) as temp_dir:
         cmd = [
-            'unzip',
-            artifact,
-            '-d',
+            "unzip",
+            str(artifact),
+            "-d",
             temp_dir,
-            '*/sources/android/cpufeatures/*',
-            '*/sources/android/native_app_glue/*',
-            '*/sources/android/support/*',
-            '*/sources/cxx-stl/*',
-            '*/source.properties',
-            os.path.join('*', LIBUNWIND_GLOB),
+            "*/sources/android/cpufeatures/*",
+            "*/sources/android/native_app_glue/*",
+            "*/sources/android/support/*",
+            "*/sources/cxx-stl/*",
+            "*/source.properties",
+            os.path.join("*", LIBUNWIND_GLOB),
+            os.path.join("*", LIBCXX_SHARED_GLOB),
+            os.path.join("*", LIBCXX_STATIC_GLOB),
+            os.path.join("*", LIBCXXABI_GLOB),
         ]
         check_call(cmd)
 
         dirs = os.listdir(temp_dir)
         assert len(dirs) == 1
-        ndk_dir = os.path.join(temp_dir, dirs[0])
-        for child in os.listdir(ndk_dir):
-            os.rename(os.path.join(ndk_dir, child),
-                      os.path.join(destination, child))
-
-
-def relocate_libunwind(install_dir):
-    unwinds = glob.glob(os.path.join(install_dir, LIBUNWIND_GLOB))
-    dest_base = os.path.join(install_dir, 'sources/cxx-stl/llvm-libc++/libs')
-    for libunwind in unwinds:
-        arch = os.path.basename(os.path.dirname(libunwind))
+        ndk_dir = Path(temp_dir) / dirs[0]
+        for child in ndk_dir.iterdir():
+            child.rename(destination / child.name)
+
+
+def relocate_libcxx(install_dir: Path) -> None:
+    """Copies the libc++ libraries from the toolchain to sources.
+
+    New versions of the NDK have removed the libraries in the sources directory because
+    they are duplicates and they aren't needed in typical builds. Soong still expects to
+    find them in that directory though. We could fix Soong, but since this whole
+    directory should be dead soon we'll just fix-up the install for now.
+    """
+    dest_base = install_dir / "sources/cxx-stl/llvm-libc++/libs"
+    for glob in {LIBCXX_SHARED_GLOB, LIBCXX_STATIC_GLOB, LIBCXXABI_GLOB}:
+        file_name = Path(glob).name
+        for file_path in install_dir.glob(glob):
+            triple = file_path.parent.name
+            abi = {
+                "arm-linux-androideabi": "armeabi-v7a",
+                "aarch64-linux-android": "arm64-v8a",
+                "i686-linux-android": "x86",
+                "x86_64-linux-android": "x86_64",
+            }[triple]
+            dest_dir = dest_base / abi
+            dest_dir.mkdir(parents=True, exist_ok=True)
+            dest = dest_dir / file_name
+            logger().info("Relocating %s to %s", file_path, dest)
+            file_path.rename(dest)
+
+
+def relocate_libunwind(install_dir: Path) -> None:
+    dest_base = install_dir / "sources/cxx-stl/llvm-libc++/libs"
+    for libunwind in install_dir.glob(LIBUNWIND_GLOB):
+        arch = libunwind.parent.name
         abi = {
-            'arm': 'armeabi-v7a',
-            'aarch64': 'arm64-v8a',
-            'i386': 'x86',
-            'x86_64': 'x86_64',
+            "arm": "armeabi-v7a",
+            "aarch64": "arm64-v8a",
+            "i386": "x86",
+            "x86_64": "x86_64",
         }[arch]
-        dest_dir = os.path.join(dest_base, abi)
-        dest = os.path.join(dest_dir, 'libunwind.a')
-        logger().info('Relocating %s to %s', libunwind, dest)
-        os.rename(libunwind, dest)
+        dest_dir = dest_base / abi
+        dest = dest_dir / "libunwind.a"
+        logger().info("Relocating %s to %s", libunwind, dest)
+        libunwind.rename(dest)
 
 
-def install_new_release(branch, build, install_dir):
-    os.makedirs(install_dir)
+def delete_android_mks(install_dir: Path) -> None:
+    for android_mk in install_dir.glob("**/Android.mk"):
+        android_mk.unlink()
 
-    artifact_pattern = 'android-ndk-*.zip'
-    logger().info('Fetching %s from %s (artifacts matching %s)', build, branch,
-                  artifact_pattern)
+
+def install_new_release(branch: str, build: str, install_dir: Path) -> None:
+    install_dir.mkdir()
+
+    artifact_pattern = "android-ndk-*.zip"
+    logger().info(
+        "Fetching %s from %s (artifacts matching %s)", build, branch, artifact_pattern
+    )
     fetch_artifact(branch, build, artifact_pattern)
-    artifacts = glob.glob('android-ndk-*.zip')
+    artifacts = list(Path().glob("android-ndk-*.zip"))
     try:
         assert len(artifacts) == 1
         artifact = artifacts[0]
 
-        logger().info('Extracting release')
+        logger().info("Extracting release")
         unzip_single_directory(artifact, install_dir)
+        relocate_libcxx(install_dir)
         relocate_libunwind(install_dir)
+        delete_android_mks(install_dir)
     finally:
         for artifact in artifacts:
-            os.unlink(artifact)
+            artifact.unlink()
 
 
-def commit(branch, build, install_dir):
-    logger().info('Making commit')
-    check_call(['git', 'add', install_dir])
-    message = textwrap.dedent("""\
+def commit(branch: str, build: str, install_dir: Path) -> None:
+    logger().info("Making commit")
+    check_call(["git", "add", str(install_dir)])
+    message = textwrap.dedent(
+        f"""\
         Update NDK prebuilts to build {build}.
 
         Taken from branch {branch}.
 
         Bug: None
         Test: treehugger
-        """).format(branch=branch, build=build)
-    check_call(['git', 'commit', '-m', message])
+        """
+    )
+    check_call(["git", "commit", "-m", message])
 
 
 def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        '-b', '--branch', default='master-ndk',
-        help='Branch to pull build from.')
-    parser.add_argument(
-        'major_release', help='Major release being installed, e.g. "r11".')
-    parser.add_argument('--build', required=True, help='Build number to pull.')
+        "-b", "--branch", default="master-ndk", help="Branch to pull build from."
+    )
+    parser.add_argument("--build", required=True, help="Build number to pull.")
     parser.add_argument(
-        '--use-current-branch', action='store_true',
-        help='Perform the update in the current branch. Do not repo start.')
+        "--use-current-branch",
+        action="store_true",
+        help="Perform the update in the current branch. Do not repo start.",
+    )
     parser.add_argument(
-        '-v', '--verbose', action='count', default=0,
-        help='Increase output verbosity.')
+        "-v", "--verbose", action="count", default=0, help="Increase output verbosity."
+    )
     return parser.parse_args()
 
 
-def main():
+def main() -> None:
     os.chdir(THIS_DIR)
 
     args = get_args()
     verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG)
-    verbosity = args.verbose
-    if verbosity > 2:
-        verbosity = 2
+    verbosity = min(args.verbose, 2)
     logging.basicConfig(level=verbose_map[verbosity])
 
-    install_dir = os.path.realpath(args.major_release)
+    install_dir = THIS_DIR / "current"
 
     if not args.use_current_branch:
         start_branch(args.build)
@@ -187,5 +246,5 @@ def main():
     commit(args.branch, args.build, install_dir)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()