diff options
Diffstat (limited to 'deprecated/automation/clients/helper')
| -rw-r--r-- | deprecated/automation/clients/helper/__init__.py | 1 | ||||
| -rw-r--r-- | deprecated/automation/clients/helper/android.py | 319 | ||||
| -rw-r--r-- | deprecated/automation/clients/helper/chromeos.py | 180 | ||||
| -rw-r--r-- | deprecated/automation/clients/helper/crosstool.py | 168 | ||||
| -rw-r--r-- | deprecated/automation/clients/helper/jobs.py | 11 | ||||
| -rw-r--r-- | deprecated/automation/clients/helper/perforce.py | 215 |
6 files changed, 894 insertions, 0 deletions
diff --git a/deprecated/automation/clients/helper/__init__.py b/deprecated/automation/clients/helper/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/deprecated/automation/clients/helper/__init__.py @@ -0,0 +1 @@ + diff --git a/deprecated/automation/clients/helper/android.py b/deprecated/automation/clients/helper/android.py new file mode 100644 index 00000000..7ff2ac1c --- /dev/null +++ b/deprecated/automation/clients/helper/android.py @@ -0,0 +1,319 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +"""Helper modules for Android toolchain test infrastructure. + +Provides following Android toolchain test jobs and commands. +. Checkout Android toolchain source code +. Build Android toolchain +. Checkout and build Android tree +. Checkout/build/run Android benchmarks, generate size dashboard +. Transform size dashboard to report, send perflab jobid to + perflab dashboard server.(TODO) +""" + +__author__ = 'jingyu@google.com (Jing Yu)' + +import os.path + +from automation.clients.helper import jobs +from automation.clients.helper import perforce +from automation.common import command as cmd +from automation.common import job + + +class JobsFactory(object): + + def __init__(self, gcc_version='4.4.3', build_type='DEVELOPMENT'): + assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] + assert build_type in ['DEVELOPMENT', 'RELEASE'] + + self.gcc_version = gcc_version + self.commands = CommandsFactory(gcc_version, build_type) + self.tc_tag = 'gcc-%s-%s' % (gcc_version, build_type) + + def CheckoutAndroidToolchain(self): + """Check out Android toolchain sources by release and gcc version.""" + command = self.commands.CheckoutAndroidToolchain() + new_job = jobs.CreateLinuxJob('AndroidCheckoutToolchain(%s)' % self.tc_tag, + command) + checkout_dir_dep = job.FolderDependency(new_job, self.commands.CHECKOUT_DIR) + return new_job, checkout_dir_dep + + def BuildAndroidToolchain(self, checkout_dir_dep): + """Build Android Toolchain.""" + command = self.commands.BuildAndroidToolchain() + new_job = jobs.CreateLinuxJob('AndroidBuildToolchain(%s)' % self.tc_tag, + command) + new_job.DependsOnFolder(checkout_dir_dep) + tc_prefix_dep = job.FolderDependency(new_job, + self.commands.toolchain_prefix_dir) + return new_job, tc_prefix_dep + + def BuildAndroidImage(self, + tc_prefix_dep, + product='stingray', + branch='ics-release'): + assert product in ['stingray', 'passion', 'trygon', 'soju'] + assert branch in ['honeycomb-release', 'ics-release'] + command = self.commands.BuildAndroidImage(product, branch) + new_job = jobs.CreateLinuxJob('AndroidGetBuildTree(%s)' % self.tc_tag, + command) + new_job.DependsOnFolder(tc_prefix_dep) + return new_job + + def Benchmark(self, tc_prefix_dep, arch='soju'): + assert arch in ['soju', 'stingray'] + script_cmd = self.commands.CheckoutScripts() + experiment_tag = 'android/nightly/%s/%s/$JOB_ID' % (self.tc_tag, arch) + build_run_benchmark_cmd = self.commands.BuildRunBenchmark(arch, + experiment_tag) + command = cmd.Chain(script_cmd, build_run_benchmark_cmd) + new_job = jobs.CreateLinuxJob('AndroidBenchmarking(%s)' % self.tc_tag, + command) + new_job.DependsOnFolder(tc_prefix_dep) + return new_job + + +class CommandsFactory(object): + CHECKOUT_DIR = 'androidtc-checkout-dir' + TOOLCHAIN_SRC_DIR = os.path.join(CHECKOUT_DIR, 'src') + TOOLCHAIN_BUILD_DIR = 'obj' + ANDROID_TREES_DIR = 'android_trees' + TOOLS_DIR = 'android-tools' + BENCHMARK_OUT_DIR = 'results' + + def __init__(self, gcc_version, build_type): + assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] + assert build_type in ['DEVELOPMENT', 'RELEASE'] + + self.build_type = build_type + self.gcc_version = gcc_version + self.binutils_version = '2.21' + self.gold_version = '2.21' + self.toolchain_prefix_dir = 'install-gcc-%s-%s' % (gcc_version, build_type) + self.p4client = self._CreatePerforceClient() + self.scripts = ScriptsFactory(self.gcc_version, self.binutils_version, + self.gold_version) + + def _CreatePerforceClient(self): + p4_dev_path = 'gcctools/google_vendor_src_branch' + mobile_rel_branch = ('branches/' + 'mobile_toolchain_v15_release_branch/gcctools/' + 'google_vendor_src_branch') + gcc_443_rel_branch = ('branches/' + 'android_compiler_v14_release_branch/gcctools/' + 'google_vendor_src_branch') + + # Common views for tools + p4view = perforce.View('depot2', perforce.PathMapping.ListFromPathTuples([( + 'gcctools/android/build/...', 'src/build/...'), ( + 'gcctools/android/Tarballs/...', 'src/tarballs/...')])) + for mapping in perforce.PathMapping.ListFromPathDict( + {'gcctools/android': ['tools/scripts/...', 'master/...']}): + p4view.add(mapping) + + # Add views for gdb + p4view.add(perforce.PathMapping(p4_dev_path, 'src', + 'gdb/gdb-7.1.x-android/...')) + + # Add view for binutils for ld and gold + if self.build_type is 'RELEASE': + binutils_branch = mobile_rel_branch + else: + binutils_branch = p4_dev_path + p4view.add(perforce.PathMapping(binutils_branch, 'src', ( + 'binutils/binutils-%s/...' % self.binutils_version))) + if self.binutils_version != self.gold_version: + p4view.add(perforce.PathMapping(binutils_branch, 'src', ( + 'binutils/binutils-%s/...' % self.gold_version))) + + # Add view for gcc if gcc_version is '4.4.3'. + if self.gcc_version == '4.4.3': + gcc443_path = 'gcc/gcc-4.4.3/...' + if self.build_type is 'RELEASE': + p4view.add(perforce.PathMapping(gcc_443_rel_branch, 'src', gcc443_path)) + else: + p4view.add(perforce.PathMapping(p4_dev_path, 'src', gcc443_path)) + + return perforce.CommandsFactory(self.CHECKOUT_DIR, p4view) + + def _CheckoutGCCFromSVN(self): + """Check out gcc from fsf svn. + + Return the command that check out gcc from svn + to gcc_required_dir (=TOOLCHAIN_SRC_DIR/src/gcc/gcc-xxx). + + TODO: + Create a svn class that does these jobs. + Parallelize p4 checkout and svn checkout. + """ + if self.gcc_version == '4.4.3': + return '' + assert self.gcc_version in ['4.6', 'google_main', 'fsf_trunk'] + + gcc_branches_dir = {'4.6': 'branches/google/gcc-4_6', + 'google_main': 'branches/google/main', + 'fsf_trunk': 'trunk'} + + # Find GCC revision number, output it to TOOLCHAIN_SRC_DIR/CLNUM_GCC + svn_get_revision = cmd.Pipe( + cmd.Shell('svn', 'info'), + cmd.Shell('grep', '"Revision:"'), + cmd.Shell('sed', '-E', '"s,Revision: ([0-9]+).*,\\1,"'), + output='../../../CLNUM_GCC') + + svn_co_command = 'svn co svn://gcc.gnu.org/svn/gcc/%s .' % ( + gcc_branches_dir[self.gcc_version]) + + gcc_required_dir = os.path.join(self.TOOLCHAIN_SRC_DIR, 'gcc', + 'gcc-%s' % self.gcc_version) + + return cmd.Chain( + cmd.MakeDir(gcc_required_dir), + cmd.Wrapper( + cmd.Chain(svn_co_command, svn_get_revision), + cwd=gcc_required_dir)) + + def CheckoutAndroidToolchain(self): + p4client = self.p4client + command = p4client.SetupAndDo(p4client.Sync(), + p4client.SaveCurrentCLNumber('CLNUM'), + p4client.Remove()) + if self.gcc_version != '4.4.3': + command.append(self._CheckoutGCCFromSVN()) + + return command + + def BuildAndroidToolchain(self): + script_cmd = self.scripts.BuildAndroidToolchain( + self.toolchain_prefix_dir, self.CHECKOUT_DIR, self.TOOLCHAIN_BUILD_DIR, + self.TOOLCHAIN_SRC_DIR) + + # Record toolchain and gcc CL number + record_cl_cmd = cmd.Copy( + os.path.join(self.CHECKOUT_DIR, 'CLNUM*'), + to_dir=self.toolchain_prefix_dir) + save_cmd = cmd.Tar( + os.path.join('$JOB_TMP', 'results', '%s.tar.bz2' % + self.toolchain_prefix_dir), self.toolchain_prefix_dir) + return cmd.Chain(script_cmd, record_cl_cmd, save_cmd) + + def _BuildAndroidTree(self, local_android_branch_dir, product): + target_tools_prefix = os.path.join('$JOB_TMP', self.toolchain_prefix_dir, + 'bin', 'arm-linux-androideabi-') + java_path = '/usr/lib/jvm/java-6-sun/bin' + build_cmd = cmd.Shell('make', '-j8', 'PRODUCT-%s-userdebug' % product, + 'TARGET_TOOLS_PREFIX=%s' % target_tools_prefix, + 'PATH=%s:$PATH' % java_path) + return cmd.Wrapper(build_cmd, cwd=local_android_branch_dir) + + def BuildAndroidImage(self, product, branch): + assert product in ['stingray', 'passion', 'trygon', 'soju'] + + # Copy the tree from atree.mtv.corp to ANDROID_TREES_DIR/branch + androidtrees_host = 'atree.mtv.corp.google.com' + androidtrees_path = ('/usr/local/google2/home/mobiletc-prebuild/' + 'android_trees') + remote_android_branch_path = os.path.join(androidtrees_path, branch) + local_android_branch_dir = os.path.join(self.ANDROID_TREES_DIR, branch) + gettree_cmd = cmd.RemoteCopyFrom( + androidtrees_host, remote_android_branch_path, local_android_branch_dir) + + # Configure and build the tree + buildtree_cmd = self._BuildAndroidTree(local_android_branch_dir, product) + + # Compress and copy system.img to result + result_system_img = os.path.join(local_android_branch_dir, 'out', 'target', + 'product', product, 'system.img') + copy_img = cmd.Copy(result_system_img, to_dir='results') + compress_img = cmd.Shell('bzip2', os.path.join('results', 'system.img')) + + return cmd.Chain(gettree_cmd, buildtree_cmd, copy_img, compress_img) + + def CheckoutScripts(self): + p4view = perforce.View('depot2', + [perforce.PathMapping('gcctools/android/tools/...', + 'tools/...')]) + p4client = perforce.CommandsFactory(self.TOOLS_DIR, p4view) + return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) + + def BuildRunBenchmark(self, arch, run_experiment): + # Copy base benchmark binaries from atree.mtv.corp + base_benchbin_host = 'atree.mtv.corp.google.com' + base_benchbin_path = ('/usr/local/google2/home/mobiletc-prebuild/' + 'archive/v3binaries/2011-10-18') + local_basebenchbin_dir = 'base_benchmark_bin' + getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host, base_benchbin_path, + local_basebenchbin_dir) + + # Build and run benchmark. + android_arch = 'android_%s' % arch + run_label = 'normal' + benchmark_cmd = self.scripts.RunBenchmark( + self.toolchain_prefix_dir, self.TOOLS_DIR, self.BENCHMARK_OUT_DIR, + run_label, run_experiment, android_arch, local_basebenchbin_dir) + + # Extract jobid from BENCHMARK_OUT_DIR/log/jobid_normal.log file. + # Copy jobid to www server to generate performance dashboard. + # TODO(jingyu) + + return cmd.Chain(getbase_cmd, benchmark_cmd) + + +class ScriptsFactory(object): + + def __init__(self, gcc_version, binutils_version, gold_version): + self._gcc_version = gcc_version + self._binutils_version = binutils_version + self._gold_version = gold_version + + def BuildAndroidToolchain(self, toolchain_prefix_dir, checkout_dir, + toolchain_build_dir, androidtc_src_dir): + if self._gcc_version == '4.4.3': + gold_option = 'both/gold' + else: + gold_option = 'default' + + return cmd.Shell( + 'build_androidtoolchain.sh', + '--toolchain-src=%s' % os.path.join('$JOB_TMP', androidtc_src_dir), + '--build-path=%s' % os.path.join('$JOB_TMP', toolchain_build_dir), + '--install-prefix=%s' % os.path.join('$JOB_TMP', toolchain_prefix_dir), + '--target=arm-linux-androideabi', + '--enable-gold=%s' % gold_option, + '--with-gcc-version=%s' % self._gcc_version, + '--with-binutils-version=%s' % self._binutils_version, + '--with-gold-version=%s' % self._gold_version, + '--with-gdb-version=7.1.x-android', + '--log-path=%s/logs' % '$JOB_HOME', + '--android-sysroot=%s' % os.path.join('$JOB_TMP', checkout_dir, + 'gcctools', 'android', 'master', + 'honeycomb_generic_sysroot'), + path=os.path.join(checkout_dir, 'gcctools', 'android', 'tools', + 'scripts')) + + def RunBenchmark(self, + toolchain_prefix_dir, + checkout_dir, + output_dir, + run_label, + run_experiment, + arch, + base_bench_bin=None): + if base_bench_bin: + base_bench_opt = '--base_benchmark_bin=%s' % base_bench_bin + else: + base_bench_opt = '' + + return cmd.Shell( + 'benchmark.sh', + '--android_toolchain=%s' % os.path.join('$JOB_TMP', + toolchain_prefix_dir), + '--bench_space=%s' % os.path.join('$JOB_TMP', 'bench'), + '--benchmark_bin=%s' % os.path.join('$JOB_TMP', output_dir, + 'bench_bin'), + base_bench_opt, + '--log_path=%s' % os.path.join('$JOB_TMP', output_dir, 'log'), + '--arch=%s' % arch, + '--run_label=%s' % run_label, + '--run_experiment=%s' % run_experiment, + path=os.path.join(checkout_dir, 'tools', 'scripts')) diff --git a/deprecated/automation/clients/helper/chromeos.py b/deprecated/automation/clients/helper/chromeos.py new file mode 100644 index 00000000..e7157451 --- /dev/null +++ b/deprecated/automation/clients/helper/chromeos.py @@ -0,0 +1,180 @@ +# Copyright 2011 Google Inc. All Rights Reserved. + +__author__ = 'asharif@google.com (Ahmad Sharif)' + +import os.path +import re + +from automation.clients.helper import jobs +from automation.clients.helper import perforce +from automation.common import command as cmd +from automation.common import machine + + +class ScriptsFactory(object): + + def __init__(self, chromeos_root, scripts_path): + self._chromeos_root = chromeos_root + self._scripts_path = scripts_path + + def SummarizeResults(self, logs_path): + return cmd.Shell('summarize_results.py', logs_path, path=self._scripts_path) + + def Buildbot(self, config_name): + buildbot = os.path.join(self._chromeos_root, + 'chromite/cbuildbot/cbuildbot.py') + + return cmd.Shell(buildbot, '--buildroot=%s' % self._chromeos_root, + '--resume', '--noarchive', '--noprebuilts', '--nosync', + '--nouprev', '--notests', '--noclean', config_name) + + def RunBenchmarks(self, board, tests): + image_path = os.path.join(self._chromeos_root, 'src/build/images', board, + 'latest/chromiumos_image.bin') + + return cmd.Shell('cros_run_benchmarks.py', + '--remote=$SECONDARY_MACHINES[0]', + '--board=%s' % board, + '--tests=%s' % tests, + '--full_table', + image_path, + path='/home/mobiletc-prebuild') + + def SetupChromeOS(self, version='latest', use_minilayout=False): + setup_chromeos = cmd.Shell('setup_chromeos.py', + '--public', + '--dir=%s' % self._chromeos_root, + '--version=%s' % version, + path=self._scripts_path) + + if use_minilayout: + setup_chromeos.AddOption('--minilayout') + return setup_chromeos + + +class CommandsFactory(object): + DEPOT2_DIR = '//depot2/' + P4_CHECKOUT_DIR = 'perforce2/' + P4_VERSION_DIR = os.path.join(P4_CHECKOUT_DIR, 'gcctools/chromeos/v14') + + CHROMEOS_ROOT = 'chromeos' + CHROMEOS_SCRIPTS_DIR = os.path.join(CHROMEOS_ROOT, 'src/scripts') + CHROMEOS_BUILDS_DIR = '/home/mobiletc-prebuild/www/chromeos_builds' + + def __init__(self, chromeos_version, board, toolchain, p4_snapshot): + self.chromeos_version = chromeos_version + self.board = board + self.toolchain = toolchain + self.p4_snapshot = p4_snapshot + + self.scripts = ScriptsFactory(self.CHROMEOS_ROOT, self.P4_VERSION_DIR) + + def AddBuildbotConfig(self, config_name, config_list): + config_header = 'add_config(%r, [%s])' % (config_name, + ', '.join(config_list)) + config_file = os.path.join(self.CHROMEOS_ROOT, + 'chromite/cbuildbot/cbuildbot_config.py') + quoted_config_header = '%r' % config_header + quoted_config_header = re.sub("'", "\\\"", quoted_config_header) + + return cmd.Pipe( + cmd.Shell('echo', quoted_config_header), + cmd.Shell('tee', '--append', config_file)) + + def RunBuildbot(self): + config_dict = {'board': self.board, + 'build_tests': True, + 'chrome_tests': True, + 'unittests': False, + 'vm_tests': False, + 'prebuilts': False, + 'latest_toolchain': True, + 'useflags': ['chrome_internal'], + 'usepkg_chroot': True, + self.toolchain: True} + config_name = '%s-toolchain-test' % self.board + if 'arm' in self.board: + config_list = ['arm'] + else: + config_list = [] + config_list.extend(['internal', 'full', 'official', str(config_dict)]) + + add_config_shell = self.AddBuildbotConfig(config_name, config_list) + return cmd.Chain(add_config_shell, self.scripts.Buildbot(config_name)) + + def BuildAndBenchmark(self): + return cmd.Chain( + self.CheckoutV14Dir(), + self.SetupChromeOSCheckout(self.chromeos_version, True), + self.RunBuildbot(), + self.scripts.RunBenchmarks(self.board, 'BootPerfServer,10:Page,3')) + + def GetP4Snapshot(self, p4view): + p4client = perforce.CommandsFactory(self.P4_CHECKOUT_DIR, p4view) + + if self.p4_snapshot: + return p4client.CheckoutFromSnapshot(self.p4_snapshot) + else: + return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) + + def CheckoutV14Dir(self): + p4view = perforce.View(self.DEPOT2_DIR, [ + perforce.PathMapping('gcctools/chromeos/v14/...') + ]) + return self.GetP4Snapshot(p4view) + + def SetupChromeOSCheckout(self, version, use_minilayout=False): + version_re = '^\d+\.\d+\.\d+\.[a-zA-Z0-9]+$' + + location = os.path.join(self.CHROMEOS_BUILDS_DIR, version) + + if version in ['weekly', 'quarterly']: + assert os.path.islink(location), 'Symlink %s does not exist.' % location + + location_expanded = os.path.abspath(os.path.realpath(location)) + version = os.path.basename(location_expanded) + + if version in ['top', 'latest'] or re.match(version_re, version): + return self.scripts.SetupChromeOS(version, use_minilayout) + + elif version.endswith('bz2') or version.endswith('gz'): + return cmd.UnTar(location_expanded, self.CHROMEOS_ROOT) + + else: + signature_file_location = os.path.join(location, + 'src/scripts/enter_chroot.sh') + assert os.path.exists(signature_file_location), ( + 'Signature file %s does not exist.' % signature_file_location) + + return cmd.Copy(location, to_dir=self.CHROMEOS_ROOT, recursive=True) + + +class JobsFactory(object): + + def __init__(self, + chromeos_version='top', + board='x86-mario', + toolchain='trunk', + p4_snapshot=''): + self.chromeos_version = chromeos_version + self.board = board + self.toolchain = toolchain + + self.commands = CommandsFactory(chromeos_version, board, toolchain, + p4_snapshot) + + def BuildAndBenchmark(self): + command = self.commands.BuildAndBenchmark() + + label = 'BuildAndBenchmark(%s,%s,%s)' % (self.toolchain, self.board, + self.chromeos_version) + + machine_label = 'chromeos-%s' % self.board + + job = jobs.CreateLinuxJob(label, command) + job.DependsOnMachine( + machine.MachineSpecification(label=machine_label, + lock_required=True), + False) + + return job diff --git a/deprecated/automation/clients/helper/crosstool.py b/deprecated/automation/clients/helper/crosstool.py new file mode 100644 index 00000000..80154b25 --- /dev/null +++ b/deprecated/automation/clients/helper/crosstool.py @@ -0,0 +1,168 @@ +# Copyright 2011 Google Inc. All Rights Reserved. + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +import os.path +import time + +from automation.clients.helper import jobs +from automation.clients.helper import perforce +from automation.common import command as cmd +from automation.common import job + + +class JobsFactory(object): + + def __init__(self): + self.commands = CommandsFactory() + + def CheckoutCrosstool(self, target): + command = self.commands.CheckoutCrosstool() + new_job = jobs.CreateLinuxJob('CheckoutCrosstool(%s)' % target, command) + checkout_dir_dep = job.FolderDependency(new_job, + CommandsFactory.CHECKOUT_DIR) + manifests_dir_dep = job.FolderDependency( + new_job, os.path.join(self.commands.buildit_path, target), 'manifests') + return new_job, checkout_dir_dep, manifests_dir_dep + + def BuildRelease(self, checkout_dir, target): + command = self.commands.BuildRelease(target) + new_job = jobs.CreateLinuxJob('BuildRelease(%s)' % target, command) + new_job.DependsOnFolder(checkout_dir) + build_tree_dep = job.FolderDependency(new_job, + self.commands.buildit_work_dir_path) + return new_job, build_tree_dep + + def RunTests(self, checkout_dir, build_tree_dir, target, board, component): + command = self.commands.RunTests(target, board, component) + new_job = jobs.CreateLinuxJob('RunTests(%s, %s, %s)' % + (target, component, board), command) + new_job.DependsOnFolder(checkout_dir) + new_job.DependsOnFolder(build_tree_dir) + testrun_dir_dep = job.FolderDependency( + new_job, self.commands.dejagnu_output_path, board) + return new_job, testrun_dir_dep + + def GenerateReport(self, testrun_dirs, manifests_dir, target, boards): + command = self.commands.GenerateReport(boards) + new_job = jobs.CreateLinuxJob('GenerateReport(%s)' % target, command) + new_job.DependsOnFolder(manifests_dir) + for testrun_dir in testrun_dirs: + new_job.DependsOnFolder(testrun_dir) + return new_job + + +class CommandsFactory(object): + CHECKOUT_DIR = 'crosstool-checkout-dir' + + def __init__(self): + self.buildit_path = os.path.join(self.CHECKOUT_DIR, 'gcctools', 'crosstool', + 'v15') + + self.buildit_work_dir = 'buildit-tmp' + self.buildit_work_dir_path = os.path.join('$JOB_TMP', self.buildit_work_dir) + self.dejagnu_output_path = os.path.join(self.buildit_work_dir_path, + 'dejagnu-output') + + paths = { + 'gcctools': [ + 'crosstool/v15/...', 'scripts/...' + ], + 'gcctools/google_vendor_src_branch': [ + 'binutils/binutils-2.21/...', 'gdb/gdb-7.2.x/...', + 'zlib/zlib-1.2.3/...' + ], + 'gcctools/vendor_src': [ + 'gcc/google/gcc-4_6/...' + ] + } + + p4view = perforce.View('depot2', + perforce.PathMapping.ListFromPathDict(paths)) + + self.p4client = perforce.CommandsFactory(self.CHECKOUT_DIR, p4view) + + def CheckoutCrosstool(self): + p4client = self.p4client + + return p4client.SetupAndDo(p4client.Sync(), + p4client.SaveCurrentCLNumber('CLNUM'), + p4client.Remove()) + + def BuildRelease(self, target): + clnum_path = os.path.join('$JOB_TMP', self.CHECKOUT_DIR, 'CLNUM') + + toolchain_root = os.path.join('/google/data/rw/projects/toolchains', target, + 'unstable') + toolchain_path = os.path.join(toolchain_root, '${CLNUM}') + + build_toolchain = cmd.Wrapper( + cmd.Chain( + cmd.MakeDir(toolchain_path), + cmd.Shell('buildit', + '--keep-work-dir', + '--build-type=release', + '--work-dir=%s' % self.buildit_work_dir_path, + '--results-dir=%s' % toolchain_path, + '--force-release=%s' % '${CLNUM}', + target, + path='.')), + cwd=self.buildit_path, + umask='0022', + env={'CLNUM': '$(< %s)' % clnum_path}) + + # remove all but 10 most recent directories + remove_old_toolchains_from_x20 = cmd.Wrapper( + cmd.Pipe( + cmd.Shell('ls', '-1', '-r'), cmd.Shell('sed', '-e', '1,10d'), + cmd.Shell('xargs', 'rm', '-r', '-f')), + cwd=toolchain_root) + + return cmd.Chain(build_toolchain, remove_old_toolchains_from_x20) + + def RunTests(self, target, board, component='gcc'): + dejagnu_flags = ['--outdir=%s' % self.dejagnu_output_path, + '--target_board=%s' % board] + + # Look for {pandaboard,qemu}.exp files in + # //depot/google3/experimental/users/kbaclawski/dejagnu/boards + + site_exp_file = os.path.join('/google/src/head/depot/google3', + 'experimental/users/kbaclawski', + 'dejagnu/site.exp') + + build_dir_path = os.path.join(target, 'rpmbuild/BUILD/crosstool*-0.0', + 'build-%s' % component) + + run_dejagnu = cmd.Wrapper( + cmd.Chain( + cmd.MakeDir(self.dejagnu_output_path), + cmd.Shell('make', + 'check', + '-k', + '-j $(grep -c processor /proc/cpuinfo)', + 'RUNTESTFLAGS="%s"' % ' '.join(dejagnu_flags), + 'DEJAGNU="%s"' % site_exp_file, + ignore_error=True)), + cwd=os.path.join(self.buildit_work_dir_path, build_dir_path), + env={'REMOTE_TMPDIR': 'job-$JOB_ID'}) + + save_results = cmd.Copy(self.dejagnu_output_path, + to_dir='$JOB_TMP/results', + recursive=True) + + return cmd.Chain(run_dejagnu, save_results) + + def GenerateReport(self, boards): + sumfiles = [os.path.join('$JOB_TMP', board, '*.sum') for board in boards] + + return cmd.Wrapper( + cmd.Shell('dejagnu.sh', + 'report', + '-m', + '$JOB_TMP/manifests/*.xfail', + '-o', + '$JOB_TMP/results/report.html', + *sumfiles, + path='.'), + cwd='$HOME/automation/clients/report') diff --git a/deprecated/automation/clients/helper/jobs.py b/deprecated/automation/clients/helper/jobs.py new file mode 100644 index 00000000..96a1c408 --- /dev/null +++ b/deprecated/automation/clients/helper/jobs.py @@ -0,0 +1,11 @@ +# Copyright 2010 Google Inc. All Rights Reserved. + +from automation.common import job +from automation.common import machine + + +def CreateLinuxJob(label, command, lock=False, timeout=4 * 60 * 60): + to_return = job.Job(label, command, timeout) + to_return.DependsOnMachine(machine.MachineSpecification(os='linux', + lock_required=lock)) + return to_return diff --git a/deprecated/automation/clients/helper/perforce.py b/deprecated/automation/clients/helper/perforce.py new file mode 100644 index 00000000..1f2dfe79 --- /dev/null +++ b/deprecated/automation/clients/helper/perforce.py @@ -0,0 +1,215 @@ +# Copyright 2011 Google Inc. All Rights Reserved. + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +import collections +import os.path + +from automation.common import command as cmd + + +class PathMapping(object): + """Stores information about relative path mapping (remote to local).""" + + @classmethod + def ListFromPathDict(cls, prefix_path_dict): + """Takes {'prefix1': ['path1',...], ...} and returns a list of mappings.""" + + mappings = [] + + for prefix, paths in sorted(prefix_path_dict.items()): + for path in sorted(paths): + mappings.append(cls(os.path.join(prefix, path))) + + return mappings + + @classmethod + def ListFromPathTuples(cls, tuple_list): + """Takes a list of tuples and returns a list of mappings. + + Args: + tuple_list: [('remote_path1', 'local_path1'), ...] + + Returns: + a list of mapping objects + """ + mappings = [] + for remote_path, local_path in tuple_list: + mappings.append(cls(remote_path, local_path)) + + return mappings + + def __init__(self, remote, local=None, common_suffix=None): + suffix = self._FixPath(common_suffix or '') + + self.remote = os.path.join(remote, suffix) + self.local = os.path.join(local or remote, suffix) + + @staticmethod + def _FixPath(path_s): + parts = [part for part in path_s.strip('/').split('/') if part] + + if not parts: + return '' + + return os.path.join(*parts) + + def _GetRemote(self): + return self._remote + + def _SetRemote(self, path_s): + self._remote = self._FixPath(path_s) + + remote = property(_GetRemote, _SetRemote) + + def _GetLocal(self): + return self._local + + def _SetLocal(self, path_s): + self._local = self._FixPath(path_s) + + local = property(_GetLocal, _SetLocal) + + def GetAbsolute(self, depot, client): + return (os.path.join('//', depot, self.remote), + os.path.join('//', client, self.local)) + + def __str__(self): + return '%s(%s => %s)' % (self.__class__.__name__, self.remote, self.local) + + +class View(collections.MutableSet): + """Keeps all information about local client required to work with perforce.""" + + def __init__(self, depot, mappings=None, client=None): + self.depot = depot + + if client: + self.client = client + + self._mappings = set(mappings or []) + + @staticmethod + def _FixRoot(root_s): + parts = root_s.strip('/').split('/', 1) + + if len(parts) != 1: + return None + + return parts[0] + + def _GetDepot(self): + return self._depot + + def _SetDepot(self, depot_s): + depot = self._FixRoot(depot_s) + assert depot, 'Not a valid depot name: "%s".' % depot_s + self._depot = depot + + depot = property(_GetDepot, _SetDepot) + + def _GetClient(self): + return self._client + + def _SetClient(self, client_s): + client = self._FixRoot(client_s) + assert client, 'Not a valid client name: "%s".' % client_s + self._client = client + + client = property(_GetClient, _SetClient) + + def add(self, mapping): + assert type(mapping) is PathMapping + self._mappings.add(mapping) + + def discard(self, mapping): + assert type(mapping) is PathMapping + self._mappings.discard(mapping) + + def __contains__(self, value): + return value in self._mappings + + def __len__(self): + return len(self._mappings) + + def __iter__(self): + return iter(mapping for mapping in self._mappings) + + def AbsoluteMappings(self): + return iter(mapping.GetAbsolute(self.depot, self.client) + for mapping in self._mappings) + + +class CommandsFactory(object): + """Creates shell commands used for interaction with Perforce.""" + + def __init__(self, checkout_dir, p4view, name=None, port=None): + self.port = port or 'perforce2:2666' + self.view = p4view + self.view.client = name or 'p4-automation-$HOSTNAME-$JOB_ID' + self.checkout_dir = checkout_dir + self.p4config_path = os.path.join(self.checkout_dir, '.p4config') + + def Initialize(self): + return cmd.Chain('mkdir -p %s' % self.checkout_dir, 'cp ~/.p4config %s' % + self.checkout_dir, 'chmod u+w %s' % self.p4config_path, + 'echo "P4PORT=%s" >> %s' % (self.port, self.p4config_path), + 'echo "P4CLIENT=%s" >> %s' % + (self.view.client, self.p4config_path)) + + def Create(self): + # TODO(kbaclawski): Could we support value list for options consistently? + mappings = ['-a \"%s %s\"' % mapping + for mapping in self.view.AbsoluteMappings()] + + # First command will create client with default mappings. Second one will + # replace default mapping with desired. Unfortunately, it seems that it + # cannot be done in one step. P4EDITOR is defined to /bin/true because we + # don't want "g4 client" to enter real editor and wait for user actions. + return cmd.Wrapper( + cmd.Chain( + cmd.Shell('g4', 'client'), + cmd.Shell('g4', 'client', '--replace', *mappings)), + env={'P4EDITOR': '/bin/true'}) + + def SaveSpecification(self, filename=None): + return cmd.Pipe(cmd.Shell('g4', 'client', '-o'), output=filename) + + def Sync(self, revision=None): + sync_arg = '...' + if revision: + sync_arg = '%s@%s' % (sync_arg, revision) + return cmd.Shell('g4', 'sync', sync_arg) + + def SaveCurrentCLNumber(self, filename=None): + return cmd.Pipe( + cmd.Shell('g4', 'changes', '-m1', '...#have'), + cmd.Shell('sed', '-E', '"s,Change ([0-9]+) .*,\\1,"'), + output=filename) + + def Remove(self): + return cmd.Shell('g4', 'client', '-d', self.view.client) + + def SetupAndDo(self, *commands): + return cmd.Chain(self.Initialize(), + self.InCheckoutDir(self.Create(), *commands)) + + def InCheckoutDir(self, *commands): + return cmd.Wrapper(cmd.Chain(*commands), cwd=self.checkout_dir) + + def CheckoutFromSnapshot(self, snapshot): + cmds = cmd.Chain() + + for mapping in self.view: + local_path, file_part = mapping.local.rsplit('/', 1) + + if file_part == '...': + remote_dir = os.path.join(snapshot, local_path) + local_dir = os.path.join(self.checkout_dir, os.path.dirname(local_path)) + + cmds.extend([ + cmd.Shell('mkdir', '-p', local_dir), cmd.Shell( + 'rsync', '-lr', remote_dir, local_dir) + ]) + + return cmds |