Add NDK build/test configuration.

Change-Id: I6bbf706c50be40a22466f75eb3419cc21e8b8db0

5 files changed, 484 insertions, 0 deletions

diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/__init__.py
diff --git a/builders/__init__.py b/builders/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/builders/__init__.py
diff --git a/builders/ndk_builders.py b/builders/ndk_builders.py
new file mode 100644
index 0000000..1cb6a13
--- /dev/null
+++ b/builders/ndk_builders.py
@@ -0,0 +1,393 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Module containing builders for the NDK."""
+from __future__ import print_function
+
+import contextlib
+import os
+import shutil
+import subprocess
+import time
+
+# pylint: disable=import-error
+from chromite.lib import cros_logging as logging
+
+from chromite.cbuildbot import manifest_version
+from chromite.cbuildbot.builders import generic_builders
+from chromite.cbuildbot.stages import generic_stages
+from chromite.cbuildbot.stages import sync_stages
+from chromite.lib import cros_build_lib
+from chromite.lib import timeout_util
+# pylint: enable=import-error
+
+# These classes aren't actually misssing an __init__ method, pylint just can't
+# find the imports above and so can't figure out there's an __init__ available.
+# pylint: disable=no-init
+
+# We're not really calling super on an old style class. Same pylint issue as
+# above.
+# pylint: disable=super-on-old-class
+
+
+ARCHITECTURES = ['arm', 'arm64', 'mips', 'mips64', 'x86', 'x86_64']
+OUT_DIR = 'out/ndk'
+TEST_DIR = '.test_dir'
+
+
+def wait_for_emulator(serial):
+    """Waits for the given emulator to be completely booted.
+
+    This might be valid for Nexus devices as well as emulators, but I don't
+    know if init.svc.bootanim is always guaranteed to exist and have this
+    behavior.
+    """
+    adb_cmd = os.path.expanduser('~/android-sdk-linux/platform-tools/adb')
+    cmd = [adb_cmd, '-s', serial, 'wait-for-device']
+    logging.info('Waiting for device to come online: {}'.format(' '.join(cmd)))
+    subprocess.check_call([adb_cmd, '-s', serial, 'wait-for-device'])
+    while True:
+        cmd = [adb_cmd, '-s', serial, 'shell', 'getprop', 'init.svc.bootanim']
+        logging.info('Checking for boot complete: {}'.format(' '.join(cmd)))
+        out = subprocess.check_output(cmd)
+        if 'stopped' in out:
+            return
+        time.sleep(1)
+
+
+@contextlib.contextmanager
+def emulator(api, abi):
+    """Launches an emulator on enter and kills it on exit.
+
+    Yields: Emulator's serial number.
+    """
+    name = '-'.join([abi, api])
+    emulator_cmd = os.path.expanduser('~/android-sdk-linux/tools/emulator')
+    cmd = [
+        emulator_cmd, '-avd', name, '-no-skin', '-no-audio', '-no-window',
+    ]
+    logging.info('Launching emulator: {}'.format(' '.join(cmd)))
+    proc = subprocess.Popen(cmd)
+    try:
+        # TODO: Determine what the emulator actually launched as.
+        serial = 'emulator-5554'
+        wait_for_emulator(serial)
+        yield serial
+    finally:
+        proc.terminate()
+        proc.wait()
+
+
+class CleanStage(generic_stages.BuilderStage):
+    """Cleans the old out directory, if any."""
+
+    def PerformStage(self):  # pylint: disable=no-self-use
+        if os.path.exists(OUT_DIR):
+            shutil.rmtree(OUT_DIR)
+
+
+class BuildStage(generic_stages.BuilderStage):
+    """Builds a specific NDK module for a given architecture."""
+
+    TEST_TIMEOUT_SEC = 2400
+
+    def __init__(self, builder_run, module, suffix=None, arch=None, **kwargs):
+        self.arch = arch
+        self.module = module
+        super(BuildStage, self).__init__(builder_run, suffix=suffix, **kwargs)
+
+    def PerformStage(self):
+        module_arg = '--module=' + self.module
+        system_arg = '--system=' + self._run.config.host
+
+        cmd = [
+            'python', 'ndk/checkbuild.py', module_arg, system_arg,
+            '--no-package',
+        ]
+
+        if self.arch:
+            cmd.append('--arch=' + self.arch)
+
+        with timeout_util.Timeout(self.TEST_TIMEOUT_SEC):
+            cros_build_lib.RunCommand(cmd)
+
+
+class BuildClangStage(BuildStage):
+    def __init__(self, builder_run, suffix=None, **kwargs):
+        super(BuildClangStage, self).__init__(
+            builder_run, 'clang', suffix=suffix, **kwargs)
+
+
+class BuildGccStage(BuildStage):
+    def __init__(self, builder_run, arch, suffix=None, **kwargs):
+        suffix = self.UpdateSuffix(arch, suffix)
+        super(BuildGccStage, self).__init__(
+            builder_run, 'gcc', arch=arch, suffix=suffix, **kwargs)
+
+
+class BuildGdbserverStage(BuildStage):
+    def __init__(self, builder_run, arch, suffix=None, **kwargs):
+        suffix = self.UpdateSuffix(arch, suffix)
+        super(BuildGdbserverStage, self).__init__(
+            builder_run, 'gdbserver', arch=arch, suffix=suffix, **kwargs)
+
+
+class BuildGnustlStage(BuildStage):
+    def __init__(self, builder_run, arch, suffix=None, **kwargs):
+        suffix = self.UpdateSuffix(arch, suffix)
+        super(BuildGnustlStage, self).__init__(
+            builder_run, 'gnustl', arch=arch, suffix=suffix, **kwargs)
+
+
+class BuildHostToolsStage(BuildStage):
+    def __init__(self, builder_run, suffix=None, **kwargs):
+        super(BuildHostToolsStage, self).__init__(
+            builder_run, 'host-tools', suffix=suffix, **kwargs)
+
+
+class BuildLibcxxStage(BuildStage):
+    def __init__(self, builder_run, arch, suffix=None, **kwargs):
+        suffix = self.UpdateSuffix(arch, suffix)
+        super(BuildLibcxxStage, self).__init__(
+            builder_run, 'libc++', arch=arch, suffix=suffix, **kwargs)
+
+
+class BuildPlatformsStage(BuildStage):
+    def __init__(self, builder_run, arch, suffix=None, **kwargs):
+        suffix = self.UpdateSuffix(arch, suffix)
+        super(BuildPlatformsStage, self).__init__(
+            builder_run, 'platforms', arch=arch, suffix=suffix, **kwargs)
+
+
+class BuildStlportStage(BuildStage):
+    def __init__(self, builder_run, arch, suffix=None, **kwargs):
+        suffix = self.UpdateSuffix(arch, suffix)
+        super(BuildStlportStage, self).__init__(
+            builder_run, 'stlport', arch=arch, suffix=suffix, **kwargs)
+
+
+class PackageStage(generic_stages.BuilderStage):
+    """Packages the just built NDK for testing and distribution."""
+
+    def PerformStage(self):
+        host = self._run.config.host
+        try_64 = True
+        if host in ('darwin', 'linux'):
+            host += '-x86'
+        elif host == 'windows':
+            try_64 = False
+        elif host == 'windows64':
+            host = 'windows'
+
+        # TODO: Figure out how to get the build number.
+        name = 'release'
+
+        arches_arg = '--arch=' + ','.join(ARCHITECTURES)
+        system_arg = '--systems=' + host
+        out_dir_arg = '--out-dir=' + os.path.realpath(OUT_DIR)
+        prebuilt_dir_arg = '--prebuilt-dir=' + OUT_DIR
+        release_arg = '--release=' + name
+
+        cmd = [
+            'bash', 'ndk/build/tools/package-release.sh', arches_arg,
+            out_dir_arg, prebuilt_dir_arg, release_arg, system_arg,
+        ]
+
+        if try_64:
+            cmd.append('--try-64')
+
+        cros_build_lib.RunCommand(cmd)
+
+
+class PrepareTestDirectoryStage(generic_stages.BuilderStage):
+    """Unpacks the NDK for testing."""
+
+    def PerformStage(self):  # pylint: disable=no-self-use
+        if os.path.isdir(TEST_DIR):
+            shutil.rmtree(TEST_DIR)
+        os.makedirs(TEST_DIR)
+
+        host = {
+            'darwin': 'darwin-x86_64',
+            'linux': 'linux-x86_64',
+            'windows': 'windows',
+            'windows64': 'windows-x86_64',
+        }[self._run.config.host]
+
+        # TODO: Figure out the build number.
+        name = 'release'
+
+        ext = 'tar.bz2'
+        if host.startswith('windows'):
+            ext = 'zip'
+
+        package_name = 'android-ndk-{}-{}.{}'.format(name, host, ext)
+        package_path = os.path.join(OUT_DIR, package_name)
+        cmd = [
+            'tar', 'xf', package_path, '-C', TEST_DIR,
+            '--strip-components=1',
+        ]
+        cros_build_lib.RunCommand(cmd)
+
+
+class TestStage(generic_stages.NonHaltingBuilderStage):
+    """Executes NDK tests for a given ABI on an emulator.
+
+    It is required that the host machine has been configured accordingly (the
+    NDK contains a python script tests/prepare-buildbot-emulators.py that will
+    download the SDK, install the necessary components, and create the AVDs for
+    testing.
+    """
+
+    def __init__(self, builder_run, api_level, abi, suffix=None, **kwargs):
+        self.api_level = api_level
+        self.abi = abi
+        stage_config = 'android-{}-{}'.format(api_level, abi)
+        suffix = self.UpdateSuffix(stage_config, suffix)
+        super(TestStage, self).__init__(builder_run, suffix=suffix, **kwargs)
+
+    def _RunTests(self, serial):
+        """Executes the tests for the given ABI on the emulator."""
+        abi = self.abi
+        abi_arg = '--abi=' + abi
+
+        test_path = os.path.join(TEST_DIR, 'tests/run-all.py')
+        cmd = ['python', test_path, abi_arg]
+
+        sdk_host = {
+            'darwin': 'macosx',
+            'linux': 'linux',
+            'windows': 'windows',
+            'windows64': 'windows',
+        }[self._run.config.host]
+
+        platform_tools_path = os.path.expanduser(
+            '~/android-sdk-{}/platform-tools'.format(sdk_host))
+        path = os.pathsep.join([platform_tools_path, os.environ['PATH']])
+
+        test_env = {
+            'ANDROID_SERIAL': serial,
+            'NDK': test_path,
+            'PATH': path,
+        }
+
+        cros_build_lib.RunCommand(cmd, extra_env=test_env)
+
+    def _EmulatorAbiForNdkAbi(self, abi):  # pylint: disable=no-self-use
+        """Returns the emulator ABI for testing a given NDK ABI.
+
+        There aren't actually any emulators for ARMv5 devices, and there is no
+        hard float platform, so these are just tested on ARMv7.
+        """
+        return {
+            'armeabi': 'armeabi-v7a',
+            'armeabi-v7a': 'armeabi-v7a',
+            'armeabi-v7a-hard': 'armeabi-v7a',
+            'mips': 'mips',
+            'x86': 'x86',
+            'x86_64': 'x86_64',
+        }[abi]
+
+    def PerformStage(self):
+        abi = self._EmulatorAbiForNdkAbi(self.abi)
+        with emulator(self.api_level, abi) as serial:
+            self._RunTests(serial)
+
+
+class Builder(generic_builders.Builder):
+    """Builder that performs sync, then exits."""
+
+    def GetVersionInfo(self):  # pylint: disable=no-self-use
+        """This version isn't very meaningful, just pin it."""
+        return manifest_version.VersionInfo('1.0.0')
+
+    def GetSyncInstance(self):
+        """Returns an instance of a SyncStage that should be run."""
+        return self._GetStageInstance(sync_stages.SyncStage)
+
+    def _BuildSharedStages(self):
+        """Runs stages that do not vary by architecture."""
+        self._RunStage(BuildClangStage)
+        self._RunStage(BuildHostToolsStage)
+
+    def _BuildArchSpecificStages(self):
+        """Runs stages that must be built once per architecture."""
+        arch_specific_stages = [
+            BuildGdbserverStage,
+            BuildGnustlStage,
+            BuildLibcxxStage,
+            BuildPlatformsStage,
+            BuildStlportStage,
+        ]
+
+        # Make stages the outer group so we build *all* the GCCs then *all* the
+        # gdbservers etc.
+        for stage in arch_specific_stages:
+            for arch in ARCHITECTURES:
+                self._RunStage(stage, arch)
+
+    def _RunTests(self):
+        """Run the tests against all of our configured emulators."""
+        # TODO: Get missing emulators.
+        # For whatever reason the SDK doesn't actually have system images
+        # available for everything we want to test. GingerBread only has x86,
+        # ICS only has ARM, Lollipop and Marshmallow is missing arm64, mips,
+        # and mips64.
+        test_targets = {
+            '10': [
+                #'armeabi',
+                #'armeabi-v7a',
+                #'armeabi-v7a-hard',
+                'x86',
+            ],
+            '16': [
+                'armeabi',
+                'armeabi-v7a',
+                'armeabi-v7a-hard',
+                'mips',
+                'x86',
+            ],
+            '23': [
+                'armeabi',
+                'armeabi-v7a',
+                'armeabi-v7a-hard',
+                #'arm64-v8a',
+                #'mips',
+                #'mips64',
+                'x86',
+                'x86_64',
+            ],
+        }
+
+        for api, abis in test_targets.items():
+            for abi in abis:
+                self._RunStage(TestStage, api, abi)
+
+    def RunStages(self):
+        """Runs the NDK build and tests."""
+        os.environ['ANDROID_BUILD_TOP'] = os.getcwd()
+        self._RunStage(CleanStage)
+        self._BuildSharedStages()
+        self._BuildArchSpecificStages()
+        self._RunStage(PackageStage)
+
+        # TODO: Find a solution for Windows testing.
+        # Our Windows builds are cross compiles, so we're not actually running
+        # on a Windows system and thus can't test the Windows package. We'll
+        # probably need to make the testing be a separate builder that takes a
+        # built NDK from this build for testing.
+        if self._run.config.host not in ('windows', 'windows64'):
+            self._RunStage(PrepareTestDirectoryStage)
+            self._RunTests()
diff --git a/config_dump.json b/config_dump.json
new file mode 100644
index 0000000..b267fea
--- /dev/null
+++ b/config_dump.json
@@ -0,0 +1,28 @@
+{
+  "_default": {
+    "manifest_branch": "master-ndk",
+    "manifest_repo_url": "file:///work/src/buildbot-ndk/manifest"
+  },
+  "_site_params": {},
+  "_templates": {},
+  "darwin": {
+    "boards": [],
+    "builder_class_name": "config.builders.ndk_builders.Builder",
+    "host": "darwin"
+  },
+  "linux": {
+    "boards": [],
+    "builder_class_name": "config.builders.ndk_builders.Builder",
+    "host": "linux"
+  },
+  "win32": {
+    "boards": [],
+    "builder_class_name": "config.builders.ndk_builders.Builder",
+    "host": "windows"
+  },
+  "win64": {
+    "boards": [],
+    "builder_class_name": "config.builders.ndk_builders.Builder",
+    "host": "windows64"
+  }
+}
diff --git a/gen-config.py b/gen-config.py
new file mode 100644
index 0000000..b7903d8
--- /dev/null
+++ b/gen-config.py
@@ -0,0 +1,63 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Generates config_dump.json."""
+import json
+import os
+
+
+def make_build(host):
+    """Returns a configuration dict for the given host."""
+    builder_class_name = 'config.builders.ndk_builders.Builder'
+
+    if host == 'win32':
+        host = 'windows'
+    elif host == 'win64':
+        host = 'windows64'
+
+    return {
+        'boards': [],
+        'builder_class_name':  builder_class_name,
+        'host': host,
+    }
+
+
+def main():
+    manifest_repo_url = 'file:///work/src/buildbot-ndk/manifest'
+    manifest_branch = 'master-ndk'
+
+    config = {
+        '_default': {
+            'manifest_repo_url': manifest_repo_url,
+            'manifest_branch': manifest_branch,
+        },
+        '_site_params': {},
+        '_templates': {},
+    }
+
+    hosts = ['darwin', 'linux', 'win32', 'win64']
+
+    for host in hosts:
+        config[host] = make_build(host)
+
+    this_dir = os.path.realpath(os.path.dirname(__file__))
+    config_file_path = os.path.join(this_dir, 'config_dump.json')
+    with open(config_file_path, 'w') as config_file:
+        json.dump(config, config_file, sort_keys=True, indent=2,
+                  separators=(',', ': '))
+
+
+if __name__ == '__main__':
+    main()