diff options
Diffstat (limited to 'deprecated/automation/clients/helper/android.py')
| -rw-r--r-- | deprecated/automation/clients/helper/android.py | 319 |
1 files changed, 319 insertions, 0 deletions
diff --git a/deprecated/automation/clients/helper/android.py b/deprecated/automation/clients/helper/android.py new file mode 100644 index 00000000..7ff2ac1c --- /dev/null +++ b/deprecated/automation/clients/helper/android.py @@ -0,0 +1,319 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +"""Helper modules for Android toolchain test infrastructure. + +Provides following Android toolchain test jobs and commands. +. Checkout Android toolchain source code +. Build Android toolchain +. Checkout and build Android tree +. Checkout/build/run Android benchmarks, generate size dashboard +. Transform size dashboard to report, send perflab jobid to + perflab dashboard server.(TODO) +""" + +__author__ = 'jingyu@google.com (Jing Yu)' + +import os.path + +from automation.clients.helper import jobs +from automation.clients.helper import perforce +from automation.common import command as cmd +from automation.common import job + + +class JobsFactory(object): + + def __init__(self, gcc_version='4.4.3', build_type='DEVELOPMENT'): + assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] + assert build_type in ['DEVELOPMENT', 'RELEASE'] + + self.gcc_version = gcc_version + self.commands = CommandsFactory(gcc_version, build_type) + self.tc_tag = 'gcc-%s-%s' % (gcc_version, build_type) + + def CheckoutAndroidToolchain(self): + """Check out Android toolchain sources by release and gcc version.""" + command = self.commands.CheckoutAndroidToolchain() + new_job = jobs.CreateLinuxJob('AndroidCheckoutToolchain(%s)' % self.tc_tag, + command) + checkout_dir_dep = job.FolderDependency(new_job, self.commands.CHECKOUT_DIR) + return new_job, checkout_dir_dep + + def BuildAndroidToolchain(self, checkout_dir_dep): + """Build Android Toolchain.""" + command = self.commands.BuildAndroidToolchain() + new_job = jobs.CreateLinuxJob('AndroidBuildToolchain(%s)' % self.tc_tag, + command) + new_job.DependsOnFolder(checkout_dir_dep) + tc_prefix_dep = job.FolderDependency(new_job, + self.commands.toolchain_prefix_dir) + return new_job, tc_prefix_dep + + def BuildAndroidImage(self, + tc_prefix_dep, + product='stingray', + branch='ics-release'): + assert product in ['stingray', 'passion', 'trygon', 'soju'] + assert branch in ['honeycomb-release', 'ics-release'] + command = self.commands.BuildAndroidImage(product, branch) + new_job = jobs.CreateLinuxJob('AndroidGetBuildTree(%s)' % self.tc_tag, + command) + new_job.DependsOnFolder(tc_prefix_dep) + return new_job + + def Benchmark(self, tc_prefix_dep, arch='soju'): + assert arch in ['soju', 'stingray'] + script_cmd = self.commands.CheckoutScripts() + experiment_tag = 'android/nightly/%s/%s/$JOB_ID' % (self.tc_tag, arch) + build_run_benchmark_cmd = self.commands.BuildRunBenchmark(arch, + experiment_tag) + command = cmd.Chain(script_cmd, build_run_benchmark_cmd) + new_job = jobs.CreateLinuxJob('AndroidBenchmarking(%s)' % self.tc_tag, + command) + new_job.DependsOnFolder(tc_prefix_dep) + return new_job + + +class CommandsFactory(object): + CHECKOUT_DIR = 'androidtc-checkout-dir' + TOOLCHAIN_SRC_DIR = os.path.join(CHECKOUT_DIR, 'src') + TOOLCHAIN_BUILD_DIR = 'obj' + ANDROID_TREES_DIR = 'android_trees' + TOOLS_DIR = 'android-tools' + BENCHMARK_OUT_DIR = 'results' + + def __init__(self, gcc_version, build_type): + assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] + assert build_type in ['DEVELOPMENT', 'RELEASE'] + + self.build_type = build_type + self.gcc_version = gcc_version + self.binutils_version = '2.21' + self.gold_version = '2.21' + self.toolchain_prefix_dir = 'install-gcc-%s-%s' % (gcc_version, build_type) + self.p4client = self._CreatePerforceClient() + self.scripts = ScriptsFactory(self.gcc_version, self.binutils_version, + self.gold_version) + + def _CreatePerforceClient(self): + p4_dev_path = 'gcctools/google_vendor_src_branch' + mobile_rel_branch = ('branches/' + 'mobile_toolchain_v15_release_branch/gcctools/' + 'google_vendor_src_branch') + gcc_443_rel_branch = ('branches/' + 'android_compiler_v14_release_branch/gcctools/' + 'google_vendor_src_branch') + + # Common views for tools + p4view = perforce.View('depot2', perforce.PathMapping.ListFromPathTuples([( + 'gcctools/android/build/...', 'src/build/...'), ( + 'gcctools/android/Tarballs/...', 'src/tarballs/...')])) + for mapping in perforce.PathMapping.ListFromPathDict( + {'gcctools/android': ['tools/scripts/...', 'master/...']}): + p4view.add(mapping) + + # Add views for gdb + p4view.add(perforce.PathMapping(p4_dev_path, 'src', + 'gdb/gdb-7.1.x-android/...')) + + # Add view for binutils for ld and gold + if self.build_type is 'RELEASE': + binutils_branch = mobile_rel_branch + else: + binutils_branch = p4_dev_path + p4view.add(perforce.PathMapping(binutils_branch, 'src', ( + 'binutils/binutils-%s/...' % self.binutils_version))) + if self.binutils_version != self.gold_version: + p4view.add(perforce.PathMapping(binutils_branch, 'src', ( + 'binutils/binutils-%s/...' % self.gold_version))) + + # Add view for gcc if gcc_version is '4.4.3'. + if self.gcc_version == '4.4.3': + gcc443_path = 'gcc/gcc-4.4.3/...' + if self.build_type is 'RELEASE': + p4view.add(perforce.PathMapping(gcc_443_rel_branch, 'src', gcc443_path)) + else: + p4view.add(perforce.PathMapping(p4_dev_path, 'src', gcc443_path)) + + return perforce.CommandsFactory(self.CHECKOUT_DIR, p4view) + + def _CheckoutGCCFromSVN(self): + """Check out gcc from fsf svn. + + Return the command that check out gcc from svn + to gcc_required_dir (=TOOLCHAIN_SRC_DIR/src/gcc/gcc-xxx). + + TODO: + Create a svn class that does these jobs. + Parallelize p4 checkout and svn checkout. + """ + if self.gcc_version == '4.4.3': + return '' + assert self.gcc_version in ['4.6', 'google_main', 'fsf_trunk'] + + gcc_branches_dir = {'4.6': 'branches/google/gcc-4_6', + 'google_main': 'branches/google/main', + 'fsf_trunk': 'trunk'} + + # Find GCC revision number, output it to TOOLCHAIN_SRC_DIR/CLNUM_GCC + svn_get_revision = cmd.Pipe( + cmd.Shell('svn', 'info'), + cmd.Shell('grep', '"Revision:"'), + cmd.Shell('sed', '-E', '"s,Revision: ([0-9]+).*,\\1,"'), + output='../../../CLNUM_GCC') + + svn_co_command = 'svn co svn://gcc.gnu.org/svn/gcc/%s .' % ( + gcc_branches_dir[self.gcc_version]) + + gcc_required_dir = os.path.join(self.TOOLCHAIN_SRC_DIR, 'gcc', + 'gcc-%s' % self.gcc_version) + + return cmd.Chain( + cmd.MakeDir(gcc_required_dir), + cmd.Wrapper( + cmd.Chain(svn_co_command, svn_get_revision), + cwd=gcc_required_dir)) + + def CheckoutAndroidToolchain(self): + p4client = self.p4client + command = p4client.SetupAndDo(p4client.Sync(), + p4client.SaveCurrentCLNumber('CLNUM'), + p4client.Remove()) + if self.gcc_version != '4.4.3': + command.append(self._CheckoutGCCFromSVN()) + + return command + + def BuildAndroidToolchain(self): + script_cmd = self.scripts.BuildAndroidToolchain( + self.toolchain_prefix_dir, self.CHECKOUT_DIR, self.TOOLCHAIN_BUILD_DIR, + self.TOOLCHAIN_SRC_DIR) + + # Record toolchain and gcc CL number + record_cl_cmd = cmd.Copy( + os.path.join(self.CHECKOUT_DIR, 'CLNUM*'), + to_dir=self.toolchain_prefix_dir) + save_cmd = cmd.Tar( + os.path.join('$JOB_TMP', 'results', '%s.tar.bz2' % + self.toolchain_prefix_dir), self.toolchain_prefix_dir) + return cmd.Chain(script_cmd, record_cl_cmd, save_cmd) + + def _BuildAndroidTree(self, local_android_branch_dir, product): + target_tools_prefix = os.path.join('$JOB_TMP', self.toolchain_prefix_dir, + 'bin', 'arm-linux-androideabi-') + java_path = '/usr/lib/jvm/java-6-sun/bin' + build_cmd = cmd.Shell('make', '-j8', 'PRODUCT-%s-userdebug' % product, + 'TARGET_TOOLS_PREFIX=%s' % target_tools_prefix, + 'PATH=%s:$PATH' % java_path) + return cmd.Wrapper(build_cmd, cwd=local_android_branch_dir) + + def BuildAndroidImage(self, product, branch): + assert product in ['stingray', 'passion', 'trygon', 'soju'] + + # Copy the tree from atree.mtv.corp to ANDROID_TREES_DIR/branch + androidtrees_host = 'atree.mtv.corp.google.com' + androidtrees_path = ('/usr/local/google2/home/mobiletc-prebuild/' + 'android_trees') + remote_android_branch_path = os.path.join(androidtrees_path, branch) + local_android_branch_dir = os.path.join(self.ANDROID_TREES_DIR, branch) + gettree_cmd = cmd.RemoteCopyFrom( + androidtrees_host, remote_android_branch_path, local_android_branch_dir) + + # Configure and build the tree + buildtree_cmd = self._BuildAndroidTree(local_android_branch_dir, product) + + # Compress and copy system.img to result + result_system_img = os.path.join(local_android_branch_dir, 'out', 'target', + 'product', product, 'system.img') + copy_img = cmd.Copy(result_system_img, to_dir='results') + compress_img = cmd.Shell('bzip2', os.path.join('results', 'system.img')) + + return cmd.Chain(gettree_cmd, buildtree_cmd, copy_img, compress_img) + + def CheckoutScripts(self): + p4view = perforce.View('depot2', + [perforce.PathMapping('gcctools/android/tools/...', + 'tools/...')]) + p4client = perforce.CommandsFactory(self.TOOLS_DIR, p4view) + return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) + + def BuildRunBenchmark(self, arch, run_experiment): + # Copy base benchmark binaries from atree.mtv.corp + base_benchbin_host = 'atree.mtv.corp.google.com' + base_benchbin_path = ('/usr/local/google2/home/mobiletc-prebuild/' + 'archive/v3binaries/2011-10-18') + local_basebenchbin_dir = 'base_benchmark_bin' + getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host, base_benchbin_path, + local_basebenchbin_dir) + + # Build and run benchmark. + android_arch = 'android_%s' % arch + run_label = 'normal' + benchmark_cmd = self.scripts.RunBenchmark( + self.toolchain_prefix_dir, self.TOOLS_DIR, self.BENCHMARK_OUT_DIR, + run_label, run_experiment, android_arch, local_basebenchbin_dir) + + # Extract jobid from BENCHMARK_OUT_DIR/log/jobid_normal.log file. + # Copy jobid to www server to generate performance dashboard. + # TODO(jingyu) + + return cmd.Chain(getbase_cmd, benchmark_cmd) + + +class ScriptsFactory(object): + + def __init__(self, gcc_version, binutils_version, gold_version): + self._gcc_version = gcc_version + self._binutils_version = binutils_version + self._gold_version = gold_version + + def BuildAndroidToolchain(self, toolchain_prefix_dir, checkout_dir, + toolchain_build_dir, androidtc_src_dir): + if self._gcc_version == '4.4.3': + gold_option = 'both/gold' + else: + gold_option = 'default' + + return cmd.Shell( + 'build_androidtoolchain.sh', + '--toolchain-src=%s' % os.path.join('$JOB_TMP', androidtc_src_dir), + '--build-path=%s' % os.path.join('$JOB_TMP', toolchain_build_dir), + '--install-prefix=%s' % os.path.join('$JOB_TMP', toolchain_prefix_dir), + '--target=arm-linux-androideabi', + '--enable-gold=%s' % gold_option, + '--with-gcc-version=%s' % self._gcc_version, + '--with-binutils-version=%s' % self._binutils_version, + '--with-gold-version=%s' % self._gold_version, + '--with-gdb-version=7.1.x-android', + '--log-path=%s/logs' % '$JOB_HOME', + '--android-sysroot=%s' % os.path.join('$JOB_TMP', checkout_dir, + 'gcctools', 'android', 'master', + 'honeycomb_generic_sysroot'), + path=os.path.join(checkout_dir, 'gcctools', 'android', 'tools', + 'scripts')) + + def RunBenchmark(self, + toolchain_prefix_dir, + checkout_dir, + output_dir, + run_label, + run_experiment, + arch, + base_bench_bin=None): + if base_bench_bin: + base_bench_opt = '--base_benchmark_bin=%s' % base_bench_bin + else: + base_bench_opt = '' + + return cmd.Shell( + 'benchmark.sh', + '--android_toolchain=%s' % os.path.join('$JOB_TMP', + toolchain_prefix_dir), + '--bench_space=%s' % os.path.join('$JOB_TMP', 'bench'), + '--benchmark_bin=%s' % os.path.join('$JOB_TMP', output_dir, + 'bench_bin'), + base_bench_opt, + '--log_path=%s' % os.path.join('$JOB_TMP', output_dir, 'log'), + '--arch=%s' % arch, + '--run_label=%s' % run_label, + '--run_experiment=%s' % run_experiment, + path=os.path.join(checkout_dir, 'tools', 'scripts')) |