path: root/automation/clients/helper/android.py

# Copyright 2011 Google Inc. All Rights Reserved.
"""Helper modules for Android toolchain test infrastructure.

Provides following Android toolchain test jobs and commands.
. Checkout Android toolchain source code
. Build Android toolchain
. Checkout and build Android tree
. Checkout/build/run Android benchmarks, generate size dashboard
. Transform size dashboard to report, send perflab jobid to
  perflab dashboard server.(TODO)
"""

__author__ = 'jingyu@google.com (Jing Yu)'

import os.path

from automation.clients.helper import jobs
from automation.clients.helper import perforce
from automation.common import command as cmd
from automation.common import job


class JobsFactory(object):

  def __init__(self, gcc_version='4.4.3', build_type='DEVELOPMENT'):
    assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk']
    assert build_type in ['DEVELOPMENT', 'RELEASE']

    self.gcc_version = gcc_version
    self.commands = CommandsFactory(gcc_version, build_type)
    self.tc_tag = 'gcc-%s-%s' % (gcc_version, build_type)

  def CheckoutAndroidToolchain(self):
    """Check out Android toolchain sources by release and gcc version."""
    command = self.commands.CheckoutAndroidToolchain()
    new_job = jobs.CreateLinuxJob('AndroidCheckoutToolchain(%s)' % self.tc_tag,
                                  command)
    checkout_dir_dep = job.FolderDependency(new_job, self.commands.CHECKOUT_DIR)
    return new_job, checkout_dir_dep

  def BuildAndroidToolchain(self, checkout_dir_dep):
    """Build Android Toolchain."""
    command = self.commands.BuildAndroidToolchain()
    new_job = jobs.CreateLinuxJob('AndroidBuildToolchain(%s)' % self.tc_tag,
                                  command)
    new_job.DependsOnFolder(checkout_dir_dep)
    tc_prefix_dep = job.FolderDependency(new_job,
                                         self.commands.toolchain_prefix_dir)
    return new_job, tc_prefix_dep

  def BuildAndroidImage(self,
                        tc_prefix_dep,
                        product='stingray',
                        branch='ics-release'):
    assert product in ['stingray', 'passion', 'trygon', 'soju']
    assert branch in ['honeycomb-release', 'ics-release']
    command = self.commands.BuildAndroidImage(product, branch)
    new_job = jobs.CreateLinuxJob('AndroidGetBuildTree(%s)' % self.tc_tag,
                                  command)
    new_job.DependsOnFolder(tc_prefix_dep)
    return new_job

  def Benchmark(self, tc_prefix_dep, arch='soju'):
    assert arch in ['soju', 'stingray']
    script_cmd = self.commands.CheckoutScripts()
    experiment_tag = 'android/nightly/%s/%s/$JOB_ID' % (self.tc_tag, arch)
    build_run_benchmark_cmd = self.commands.BuildRunBenchmark(arch,
                                                              experiment_tag)
    command = cmd.Chain(script_cmd, build_run_benchmark_cmd)
    new_job = jobs.CreateLinuxJob('AndroidBenchmarking(%s)' % self.tc_tag,
                                  command)
    new_job.DependsOnFolder(tc_prefix_dep)
    return new_job


class CommandsFactory(object):
  CHECKOUT_DIR = 'androidtc-checkout-dir'
  TOOLCHAIN_SRC_DIR = os.path.join(CHECKOUT_DIR, 'src')
  TOOLCHAIN_BUILD_DIR = 'obj'
  ANDROID_TREES_DIR = 'android_trees'
  TOOLS_DIR = 'android-tools'
  BENCHMARK_OUT_DIR = 'results'

  def __init__(self, gcc_version, build_type):
    assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk']
    assert build_type in ['DEVELOPMENT', 'RELEASE']

    self.build_type = build_type
    self.gcc_version = gcc_version
    self.binutils_version = '2.21'
    self.gold_version = '2.21'
    self.toolchain_prefix_dir = 'install-gcc-%s-%s' % (gcc_version, build_type)
    self.p4client = self._CreatePerforceClient()
    self.scripts = ScriptsFactory(self.gcc_version, self.binutils_version,
                                  self.gold_version)

  def _CreatePerforceClient(self):
    p4_dev_path = 'gcctools/google_vendor_src_branch'
    mobile_rel_branch = ('branches/'
                         'mobile_toolchain_v15_release_branch/gcctools/'
                         'google_vendor_src_branch')
    gcc_443_rel_branch = ('branches/'
                          'android_compiler_v14_release_branch/gcctools/'
                          'google_vendor_src_branch')

    # Common views for tools
    p4view = perforce.View('depot2', perforce.PathMapping.ListFromPathTuples([(
        'gcctools/android/build/...', 'src/build/...'), (
            'gcctools/android/Tarballs/...', 'src/tarballs/...')]))
    for mapping in perforce.PathMapping.ListFromPathDict(
        {'gcctools/android': ['tools/scripts/...', 'master/...']}):
      p4view.add(mapping)

    # Add views for gdb
    p4view.add(perforce.PathMapping(p4_dev_path, 'src',
                                    'gdb/gdb-7.1.x-android/...'))

    # Add view for binutils for ld and gold
    if self.build_type is 'RELEASE':
      binutils_branch = mobile_rel_branch
    else:
      binutils_branch = p4_dev_path
    p4view.add(perforce.PathMapping(binutils_branch, 'src', (
        'binutils/binutils-%s/...' % self.binutils_version)))
    if self.binutils_version != self.gold_version:
      p4view.add(perforce.PathMapping(binutils_branch, 'src', (
          'binutils/binutils-%s/...' % self.gold_version)))

    # Add view for gcc if gcc_version is '4.4.3'.
    if self.gcc_version == '4.4.3':
      gcc443_path = 'gcc/gcc-4.4.3/...'
      if self.build_type is 'RELEASE':
        p4view.add(perforce.PathMapping(gcc_443_rel_branch, 'src', gcc443_path))
      else:
        p4view.add(perforce.PathMapping(p4_dev_path, 'src', gcc443_path))

    return perforce.CommandsFactory(self.CHECKOUT_DIR, p4view)

  def _CheckoutGCCFromSVN(self):
    """Check out gcc from fsf svn.

       Return the command that check out gcc from svn
       to gcc_required_dir (=TOOLCHAIN_SRC_DIR/src/gcc/gcc-xxx).

       TODO:
         Create a svn class that does these jobs.
         Parallelize p4 checkout and svn checkout.
    """
    if self.gcc_version == '4.4.3':
      return ''
    assert self.gcc_version in ['4.6', 'google_main', 'fsf_trunk']

    gcc_branches_dir = {'4.6': 'branches/google/gcc-4_6',
                        'google_main': 'branches/google/main',
                        'fsf_trunk': 'trunk'}

    # Find GCC revision number, output it to TOOLCHAIN_SRC_DIR/CLNUM_GCC
    svn_get_revision = cmd.Pipe(
        cmd.Shell('svn', 'info'),
        cmd.Shell('grep', '"Revision:"'),
        cmd.Shell('sed', '-E', '"s,Revision: ([0-9]+).*,\\1,"'),
        output='../../../CLNUM_GCC')

    svn_co_command = 'svn co svn://gcc.gnu.org/svn/gcc/%s .' % (
        gcc_branches_dir[self.gcc_version])

    gcc_required_dir = os.path.join(self.TOOLCHAIN_SRC_DIR, 'gcc',
                                    'gcc-%s' % self.gcc_version)

    return cmd.Chain(
        cmd.MakeDir(gcc_required_dir),
        cmd.Wrapper(
            cmd.Chain(svn_co_command, svn_get_revision),
            cwd=gcc_required_dir))

  def CheckoutAndroidToolchain(self):
    p4client = self.p4client
    command = p4client.SetupAndDo(p4client.Sync(),
                                  p4client.SaveCurrentCLNumber('CLNUM'),
                                  p4client.Remove())
    if self.gcc_version != '4.4.3':
      command.append(self._CheckoutGCCFromSVN())

    return command

  def BuildAndroidToolchain(self):
    script_cmd = self.scripts.BuildAndroidToolchain(
        self.toolchain_prefix_dir, self.CHECKOUT_DIR, self.TOOLCHAIN_BUILD_DIR,
        self.TOOLCHAIN_SRC_DIR)

    # Record toolchain and gcc CL number
    record_cl_cmd = cmd.Copy(
        os.path.join(self.CHECKOUT_DIR, 'CLNUM*'),
        to_dir=self.toolchain_prefix_dir)
    save_cmd = cmd.Tar(
        os.path.join('$JOB_TMP', 'results', '%s.tar.bz2' %
                     self.toolchain_prefix_dir), self.toolchain_prefix_dir)
    return cmd.Chain(script_cmd, record_cl_cmd, save_cmd)

  def _BuildAndroidTree(self, local_android_branch_dir, product):
    target_tools_prefix = os.path.join('$JOB_TMP', self.toolchain_prefix_dir,
                                       'bin', 'arm-linux-androideabi-')
    java_path = '/usr/lib/jvm/java-6-sun/bin'
    build_cmd = cmd.Shell('make', '-j8', 'PRODUCT-%s-userdebug' % product,
                          'TARGET_TOOLS_PREFIX=%s' % target_tools_prefix,
                          'PATH=%s:$PATH' % java_path)
    return cmd.Wrapper(build_cmd, cwd=local_android_branch_dir)

  def BuildAndroidImage(self, product, branch):
    assert product in ['stingray', 'passion', 'trygon', 'soju']

    # Copy the tree from atree.mtv.corp to ANDROID_TREES_DIR/branch
    androidtrees_host = 'atree.mtv.corp.google.com'
    androidtrees_path = ('/usr/local/google2/home/mobiletc-prebuild/'
                         'android_trees')
    remote_android_branch_path = os.path.join(androidtrees_path, branch)
    local_android_branch_dir = os.path.join(self.ANDROID_TREES_DIR, branch)
    gettree_cmd = cmd.RemoteCopyFrom(
        androidtrees_host, remote_android_branch_path, local_android_branch_dir)

    # Configure and build the tree
    buildtree_cmd = self._BuildAndroidTree(local_android_branch_dir, product)

    # Compress and copy system.img to result
    result_system_img = os.path.join(local_android_branch_dir, 'out', 'target',
                                     'product', product, 'system.img')
    copy_img = cmd.Copy(result_system_img, to_dir='results')
    compress_img = cmd.Shell('bzip2', os.path.join('results', 'system.img'))

    return cmd.Chain(gettree_cmd, buildtree_cmd, copy_img, compress_img)

  def CheckoutScripts(self):
    p4view = perforce.View('depot2',
                           [perforce.PathMapping('gcctools/android/tools/...',
                                                 'tools/...')])
    p4client = perforce.CommandsFactory(self.TOOLS_DIR, p4view)
    return p4client.SetupAndDo(p4client.Sync(), p4client.Remove())

  def BuildRunBenchmark(self, arch, run_experiment):
    # Copy base benchmark binaries from atree.mtv.corp
    base_benchbin_host = 'atree.mtv.corp.google.com'
    base_benchbin_path = ('/usr/local/google2/home/mobiletc-prebuild/'
                          'archive/v3binaries/2011-10-18')
    local_basebenchbin_dir = 'base_benchmark_bin'
    getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host, base_benchbin_path,
                                     local_basebenchbin_dir)

    # Build and run benchmark.
    android_arch = 'android_%s' % arch
    run_label = 'normal'
    benchmark_cmd = self.scripts.RunBenchmark(
        self.toolchain_prefix_dir, self.TOOLS_DIR, self.BENCHMARK_OUT_DIR,
        run_label, run_experiment, android_arch, local_basebenchbin_dir)

    # Extract jobid from BENCHMARK_OUT_DIR/log/jobid_normal.log file.
    # Copy jobid to www server to generate performance dashboard.
    # TODO(jingyu)

    return cmd.Chain(getbase_cmd, benchmark_cmd)


class ScriptsFactory(object):

  def __init__(self, gcc_version, binutils_version, gold_version):
    self._gcc_version = gcc_version
    self._binutils_version = binutils_version
    self._gold_version = gold_version

  def BuildAndroidToolchain(self, toolchain_prefix_dir, checkout_dir,
                            toolchain_build_dir, androidtc_src_dir):
    if self._gcc_version == '4.4.3':
      gold_option = 'both/gold'
    else:
      gold_option = 'default'

    return cmd.Shell(
        'build_androidtoolchain.sh',
        '--toolchain-src=%s' % os.path.join('$JOB_TMP', androidtc_src_dir),
        '--build-path=%s' % os.path.join('$JOB_TMP', toolchain_build_dir),
        '--install-prefix=%s' % os.path.join('$JOB_TMP', toolchain_prefix_dir),
        '--target=arm-linux-androideabi',
        '--enable-gold=%s' % gold_option,
        '--with-gcc-version=%s' % self._gcc_version,
        '--with-binutils-version=%s' % self._binutils_version,
        '--with-gold-version=%s' % self._gold_version,
        '--with-gdb-version=7.1.x-android',
        '--log-path=%s/logs' % '$JOB_HOME',
        '--android-sysroot=%s' % os.path.join('$JOB_TMP', checkout_dir,
                                              'gcctools', 'android', 'master',
                                              'honeycomb_generic_sysroot'),
        path=os.path.join(checkout_dir, 'gcctools', 'android', 'tools',
                          'scripts'))

  def RunBenchmark(self,
                   toolchain_prefix_dir,
                   checkout_dir,
                   output_dir,
                   run_label,
                   run_experiment,
                   arch,
                   base_bench_bin=None):
    if base_bench_bin:
      base_bench_opt = '--base_benchmark_bin=%s' % base_bench_bin
    else:
      base_bench_opt = ''

    return cmd.Shell(
        'benchmark.sh',
        '--android_toolchain=%s' % os.path.join('$JOB_TMP',
                                                toolchain_prefix_dir),
        '--bench_space=%s' % os.path.join('$JOB_TMP', 'bench'),
        '--benchmark_bin=%s' % os.path.join('$JOB_TMP', output_dir,
                                            'bench_bin'),
        base_bench_opt,
        '--log_path=%s' % os.path.join('$JOB_TMP', output_dir, 'log'),
        '--arch=%s' % arch,
        '--run_label=%s' % run_label,
        '--run_experiment=%s' % run_experiment,
        path=os.path.join(checkout_dir, 'tools', 'scripts'))