diff options
Diffstat (limited to 'deprecated/automation/clients/helper')
| -rw-r--r-- | deprecated/automation/clients/helper/__init__.py | 1 | ||||
| -rw-r--r-- | deprecated/automation/clients/helper/android.py | 319 | ||||
| -rw-r--r-- | deprecated/automation/clients/helper/chromeos.py | 180 | ||||
| -rw-r--r-- | deprecated/automation/clients/helper/crosstool.py | 168 | ||||
| -rw-r--r-- | deprecated/automation/clients/helper/jobs.py | 11 | ||||
| -rw-r--r-- | deprecated/automation/clients/helper/perforce.py | 215 |
6 files changed, 0 insertions, 894 deletions
diff --git a/deprecated/automation/clients/helper/__init__.py b/deprecated/automation/clients/helper/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/deprecated/automation/clients/helper/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/deprecated/automation/clients/helper/android.py b/deprecated/automation/clients/helper/android.py deleted file mode 100644 index 7ff2ac1c..00000000 --- a/deprecated/automation/clients/helper/android.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright 2011 Google Inc. All Rights Reserved. -"""Helper modules for Android toolchain test infrastructure. - -Provides following Android toolchain test jobs and commands. -. Checkout Android toolchain source code -. Build Android toolchain -. Checkout and build Android tree -. Checkout/build/run Android benchmarks, generate size dashboard -. Transform size dashboard to report, send perflab jobid to - perflab dashboard server.(TODO) -""" - -__author__ = 'jingyu@google.com (Jing Yu)' - -import os.path - -from automation.clients.helper import jobs -from automation.clients.helper import perforce -from automation.common import command as cmd -from automation.common import job - - -class JobsFactory(object): - - def __init__(self, gcc_version='4.4.3', build_type='DEVELOPMENT'): - assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] - assert build_type in ['DEVELOPMENT', 'RELEASE'] - - self.gcc_version = gcc_version - self.commands = CommandsFactory(gcc_version, build_type) - self.tc_tag = 'gcc-%s-%s' % (gcc_version, build_type) - - def CheckoutAndroidToolchain(self): - """Check out Android toolchain sources by release and gcc version.""" - command = self.commands.CheckoutAndroidToolchain() - new_job = jobs.CreateLinuxJob('AndroidCheckoutToolchain(%s)' % self.tc_tag, - command) - checkout_dir_dep = job.FolderDependency(new_job, self.commands.CHECKOUT_DIR) - return new_job, checkout_dir_dep - - def BuildAndroidToolchain(self, checkout_dir_dep): - """Build Android Toolchain.""" - command = self.commands.BuildAndroidToolchain() - new_job = jobs.CreateLinuxJob('AndroidBuildToolchain(%s)' % self.tc_tag, - command) - new_job.DependsOnFolder(checkout_dir_dep) - tc_prefix_dep = job.FolderDependency(new_job, - self.commands.toolchain_prefix_dir) - return new_job, tc_prefix_dep - - def BuildAndroidImage(self, - tc_prefix_dep, - product='stingray', - branch='ics-release'): - assert product in ['stingray', 'passion', 'trygon', 'soju'] - assert branch in ['honeycomb-release', 'ics-release'] - command = self.commands.BuildAndroidImage(product, branch) - new_job = jobs.CreateLinuxJob('AndroidGetBuildTree(%s)' % self.tc_tag, - command) - new_job.DependsOnFolder(tc_prefix_dep) - return new_job - - def Benchmark(self, tc_prefix_dep, arch='soju'): - assert arch in ['soju', 'stingray'] - script_cmd = self.commands.CheckoutScripts() - experiment_tag = 'android/nightly/%s/%s/$JOB_ID' % (self.tc_tag, arch) - build_run_benchmark_cmd = self.commands.BuildRunBenchmark(arch, - experiment_tag) - command = cmd.Chain(script_cmd, build_run_benchmark_cmd) - new_job = jobs.CreateLinuxJob('AndroidBenchmarking(%s)' % self.tc_tag, - command) - new_job.DependsOnFolder(tc_prefix_dep) - return new_job - - -class CommandsFactory(object): - CHECKOUT_DIR = 'androidtc-checkout-dir' - TOOLCHAIN_SRC_DIR = os.path.join(CHECKOUT_DIR, 'src') - TOOLCHAIN_BUILD_DIR = 'obj' - ANDROID_TREES_DIR = 'android_trees' - TOOLS_DIR = 'android-tools' - BENCHMARK_OUT_DIR = 'results' - - def __init__(self, gcc_version, build_type): - assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] - assert build_type in ['DEVELOPMENT', 'RELEASE'] - - self.build_type = build_type - self.gcc_version = gcc_version - self.binutils_version = '2.21' - self.gold_version = '2.21' - self.toolchain_prefix_dir = 'install-gcc-%s-%s' % (gcc_version, build_type) - self.p4client = self._CreatePerforceClient() - self.scripts = ScriptsFactory(self.gcc_version, self.binutils_version, - self.gold_version) - - def _CreatePerforceClient(self): - p4_dev_path = 'gcctools/google_vendor_src_branch' - mobile_rel_branch = ('branches/' - 'mobile_toolchain_v15_release_branch/gcctools/' - 'google_vendor_src_branch') - gcc_443_rel_branch = ('branches/' - 'android_compiler_v14_release_branch/gcctools/' - 'google_vendor_src_branch') - - # Common views for tools - p4view = perforce.View('depot2', perforce.PathMapping.ListFromPathTuples([( - 'gcctools/android/build/...', 'src/build/...'), ( - 'gcctools/android/Tarballs/...', 'src/tarballs/...')])) - for mapping in perforce.PathMapping.ListFromPathDict( - {'gcctools/android': ['tools/scripts/...', 'master/...']}): - p4view.add(mapping) - - # Add views for gdb - p4view.add(perforce.PathMapping(p4_dev_path, 'src', - 'gdb/gdb-7.1.x-android/...')) - - # Add view for binutils for ld and gold - if self.build_type is 'RELEASE': - binutils_branch = mobile_rel_branch - else: - binutils_branch = p4_dev_path - p4view.add(perforce.PathMapping(binutils_branch, 'src', ( - 'binutils/binutils-%s/...' % self.binutils_version))) - if self.binutils_version != self.gold_version: - p4view.add(perforce.PathMapping(binutils_branch, 'src', ( - 'binutils/binutils-%s/...' % self.gold_version))) - - # Add view for gcc if gcc_version is '4.4.3'. - if self.gcc_version == '4.4.3': - gcc443_path = 'gcc/gcc-4.4.3/...' - if self.build_type is 'RELEASE': - p4view.add(perforce.PathMapping(gcc_443_rel_branch, 'src', gcc443_path)) - else: - p4view.add(perforce.PathMapping(p4_dev_path, 'src', gcc443_path)) - - return perforce.CommandsFactory(self.CHECKOUT_DIR, p4view) - - def _CheckoutGCCFromSVN(self): - """Check out gcc from fsf svn. - - Return the command that check out gcc from svn - to gcc_required_dir (=TOOLCHAIN_SRC_DIR/src/gcc/gcc-xxx). - - TODO: - Create a svn class that does these jobs. - Parallelize p4 checkout and svn checkout. - """ - if self.gcc_version == '4.4.3': - return '' - assert self.gcc_version in ['4.6', 'google_main', 'fsf_trunk'] - - gcc_branches_dir = {'4.6': 'branches/google/gcc-4_6', - 'google_main': 'branches/google/main', - 'fsf_trunk': 'trunk'} - - # Find GCC revision number, output it to TOOLCHAIN_SRC_DIR/CLNUM_GCC - svn_get_revision = cmd.Pipe( - cmd.Shell('svn', 'info'), - cmd.Shell('grep', '"Revision:"'), - cmd.Shell('sed', '-E', '"s,Revision: ([0-9]+).*,\\1,"'), - output='../../../CLNUM_GCC') - - svn_co_command = 'svn co svn://gcc.gnu.org/svn/gcc/%s .' % ( - gcc_branches_dir[self.gcc_version]) - - gcc_required_dir = os.path.join(self.TOOLCHAIN_SRC_DIR, 'gcc', - 'gcc-%s' % self.gcc_version) - - return cmd.Chain( - cmd.MakeDir(gcc_required_dir), - cmd.Wrapper( - cmd.Chain(svn_co_command, svn_get_revision), - cwd=gcc_required_dir)) - - def CheckoutAndroidToolchain(self): - p4client = self.p4client - command = p4client.SetupAndDo(p4client.Sync(), - p4client.SaveCurrentCLNumber('CLNUM'), - p4client.Remove()) - if self.gcc_version != '4.4.3': - command.append(self._CheckoutGCCFromSVN()) - - return command - - def BuildAndroidToolchain(self): - script_cmd = self.scripts.BuildAndroidToolchain( - self.toolchain_prefix_dir, self.CHECKOUT_DIR, self.TOOLCHAIN_BUILD_DIR, - self.TOOLCHAIN_SRC_DIR) - - # Record toolchain and gcc CL number - record_cl_cmd = cmd.Copy( - os.path.join(self.CHECKOUT_DIR, 'CLNUM*'), - to_dir=self.toolchain_prefix_dir) - save_cmd = cmd.Tar( - os.path.join('$JOB_TMP', 'results', '%s.tar.bz2' % - self.toolchain_prefix_dir), self.toolchain_prefix_dir) - return cmd.Chain(script_cmd, record_cl_cmd, save_cmd) - - def _BuildAndroidTree(self, local_android_branch_dir, product): - target_tools_prefix = os.path.join('$JOB_TMP', self.toolchain_prefix_dir, - 'bin', 'arm-linux-androideabi-') - java_path = '/usr/lib/jvm/java-6-sun/bin' - build_cmd = cmd.Shell('make', '-j8', 'PRODUCT-%s-userdebug' % product, - 'TARGET_TOOLS_PREFIX=%s' % target_tools_prefix, - 'PATH=%s:$PATH' % java_path) - return cmd.Wrapper(build_cmd, cwd=local_android_branch_dir) - - def BuildAndroidImage(self, product, branch): - assert product in ['stingray', 'passion', 'trygon', 'soju'] - - # Copy the tree from atree.mtv.corp to ANDROID_TREES_DIR/branch - androidtrees_host = 'atree.mtv.corp.google.com' - androidtrees_path = ('/usr/local/google2/home/mobiletc-prebuild/' - 'android_trees') - remote_android_branch_path = os.path.join(androidtrees_path, branch) - local_android_branch_dir = os.path.join(self.ANDROID_TREES_DIR, branch) - gettree_cmd = cmd.RemoteCopyFrom( - androidtrees_host, remote_android_branch_path, local_android_branch_dir) - - # Configure and build the tree - buildtree_cmd = self._BuildAndroidTree(local_android_branch_dir, product) - - # Compress and copy system.img to result - result_system_img = os.path.join(local_android_branch_dir, 'out', 'target', - 'product', product, 'system.img') - copy_img = cmd.Copy(result_system_img, to_dir='results') - compress_img = cmd.Shell('bzip2', os.path.join('results', 'system.img')) - - return cmd.Chain(gettree_cmd, buildtree_cmd, copy_img, compress_img) - - def CheckoutScripts(self): - p4view = perforce.View('depot2', - [perforce.PathMapping('gcctools/android/tools/...', - 'tools/...')]) - p4client = perforce.CommandsFactory(self.TOOLS_DIR, p4view) - return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) - - def BuildRunBenchmark(self, arch, run_experiment): - # Copy base benchmark binaries from atree.mtv.corp - base_benchbin_host = 'atree.mtv.corp.google.com' - base_benchbin_path = ('/usr/local/google2/home/mobiletc-prebuild/' - 'archive/v3binaries/2011-10-18') - local_basebenchbin_dir = 'base_benchmark_bin' - getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host, base_benchbin_path, - local_basebenchbin_dir) - - # Build and run benchmark. - android_arch = 'android_%s' % arch - run_label = 'normal' - benchmark_cmd = self.scripts.RunBenchmark( - self.toolchain_prefix_dir, self.TOOLS_DIR, self.BENCHMARK_OUT_DIR, - run_label, run_experiment, android_arch, local_basebenchbin_dir) - - # Extract jobid from BENCHMARK_OUT_DIR/log/jobid_normal.log file. - # Copy jobid to www server to generate performance dashboard. - # TODO(jingyu) - - return cmd.Chain(getbase_cmd, benchmark_cmd) - - -class ScriptsFactory(object): - - def __init__(self, gcc_version, binutils_version, gold_version): - self._gcc_version = gcc_version - self._binutils_version = binutils_version - self._gold_version = gold_version - - def BuildAndroidToolchain(self, toolchain_prefix_dir, checkout_dir, - toolchain_build_dir, androidtc_src_dir): - if self._gcc_version == '4.4.3': - gold_option = 'both/gold' - else: - gold_option = 'default' - - return cmd.Shell( - 'build_androidtoolchain.sh', - '--toolchain-src=%s' % os.path.join('$JOB_TMP', androidtc_src_dir), - '--build-path=%s' % os.path.join('$JOB_TMP', toolchain_build_dir), - '--install-prefix=%s' % os.path.join('$JOB_TMP', toolchain_prefix_dir), - '--target=arm-linux-androideabi', - '--enable-gold=%s' % gold_option, - '--with-gcc-version=%s' % self._gcc_version, - '--with-binutils-version=%s' % self._binutils_version, - '--with-gold-version=%s' % self._gold_version, - '--with-gdb-version=7.1.x-android', - '--log-path=%s/logs' % '$JOB_HOME', - '--android-sysroot=%s' % os.path.join('$JOB_TMP', checkout_dir, - 'gcctools', 'android', 'master', - 'honeycomb_generic_sysroot'), - path=os.path.join(checkout_dir, 'gcctools', 'android', 'tools', - 'scripts')) - - def RunBenchmark(self, - toolchain_prefix_dir, - checkout_dir, - output_dir, - run_label, - run_experiment, - arch, - base_bench_bin=None): - if base_bench_bin: - base_bench_opt = '--base_benchmark_bin=%s' % base_bench_bin - else: - base_bench_opt = '' - - return cmd.Shell( - 'benchmark.sh', - '--android_toolchain=%s' % os.path.join('$JOB_TMP', - toolchain_prefix_dir), - '--bench_space=%s' % os.path.join('$JOB_TMP', 'bench'), - '--benchmark_bin=%s' % os.path.join('$JOB_TMP', output_dir, - 'bench_bin'), - base_bench_opt, - '--log_path=%s' % os.path.join('$JOB_TMP', output_dir, 'log'), - '--arch=%s' % arch, - '--run_label=%s' % run_label, - '--run_experiment=%s' % run_experiment, - path=os.path.join(checkout_dir, 'tools', 'scripts')) diff --git a/deprecated/automation/clients/helper/chromeos.py b/deprecated/automation/clients/helper/chromeos.py deleted file mode 100644 index e7157451..00000000 --- a/deprecated/automation/clients/helper/chromeos.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2011 Google Inc. All Rights Reserved. - -__author__ = 'asharif@google.com (Ahmad Sharif)' - -import os.path -import re - -from automation.clients.helper import jobs -from automation.clients.helper import perforce -from automation.common import command as cmd -from automation.common import machine - - -class ScriptsFactory(object): - - def __init__(self, chromeos_root, scripts_path): - self._chromeos_root = chromeos_root - self._scripts_path = scripts_path - - def SummarizeResults(self, logs_path): - return cmd.Shell('summarize_results.py', logs_path, path=self._scripts_path) - - def Buildbot(self, config_name): - buildbot = os.path.join(self._chromeos_root, - 'chromite/cbuildbot/cbuildbot.py') - - return cmd.Shell(buildbot, '--buildroot=%s' % self._chromeos_root, - '--resume', '--noarchive', '--noprebuilts', '--nosync', - '--nouprev', '--notests', '--noclean', config_name) - - def RunBenchmarks(self, board, tests): - image_path = os.path.join(self._chromeos_root, 'src/build/images', board, - 'latest/chromiumos_image.bin') - - return cmd.Shell('cros_run_benchmarks.py', - '--remote=$SECONDARY_MACHINES[0]', - '--board=%s' % board, - '--tests=%s' % tests, - '--full_table', - image_path, - path='/home/mobiletc-prebuild') - - def SetupChromeOS(self, version='latest', use_minilayout=False): - setup_chromeos = cmd.Shell('setup_chromeos.py', - '--public', - '--dir=%s' % self._chromeos_root, - '--version=%s' % version, - path=self._scripts_path) - - if use_minilayout: - setup_chromeos.AddOption('--minilayout') - return setup_chromeos - - -class CommandsFactory(object): - DEPOT2_DIR = '//depot2/' - P4_CHECKOUT_DIR = 'perforce2/' - P4_VERSION_DIR = os.path.join(P4_CHECKOUT_DIR, 'gcctools/chromeos/v14') - - CHROMEOS_ROOT = 'chromeos' - CHROMEOS_SCRIPTS_DIR = os.path.join(CHROMEOS_ROOT, 'src/scripts') - CHROMEOS_BUILDS_DIR = '/home/mobiletc-prebuild/www/chromeos_builds' - - def __init__(self, chromeos_version, board, toolchain, p4_snapshot): - self.chromeos_version = chromeos_version - self.board = board - self.toolchain = toolchain - self.p4_snapshot = p4_snapshot - - self.scripts = ScriptsFactory(self.CHROMEOS_ROOT, self.P4_VERSION_DIR) - - def AddBuildbotConfig(self, config_name, config_list): - config_header = 'add_config(%r, [%s])' % (config_name, - ', '.join(config_list)) - config_file = os.path.join(self.CHROMEOS_ROOT, - 'chromite/cbuildbot/cbuildbot_config.py') - quoted_config_header = '%r' % config_header - quoted_config_header = re.sub("'", "\\\"", quoted_config_header) - - return cmd.Pipe( - cmd.Shell('echo', quoted_config_header), - cmd.Shell('tee', '--append', config_file)) - - def RunBuildbot(self): - config_dict = {'board': self.board, - 'build_tests': True, - 'chrome_tests': True, - 'unittests': False, - 'vm_tests': False, - 'prebuilts': False, - 'latest_toolchain': True, - 'useflags': ['chrome_internal'], - 'usepkg_chroot': True, - self.toolchain: True} - config_name = '%s-toolchain-test' % self.board - if 'arm' in self.board: - config_list = ['arm'] - else: - config_list = [] - config_list.extend(['internal', 'full', 'official', str(config_dict)]) - - add_config_shell = self.AddBuildbotConfig(config_name, config_list) - return cmd.Chain(add_config_shell, self.scripts.Buildbot(config_name)) - - def BuildAndBenchmark(self): - return cmd.Chain( - self.CheckoutV14Dir(), - self.SetupChromeOSCheckout(self.chromeos_version, True), - self.RunBuildbot(), - self.scripts.RunBenchmarks(self.board, 'BootPerfServer,10:Page,3')) - - def GetP4Snapshot(self, p4view): - p4client = perforce.CommandsFactory(self.P4_CHECKOUT_DIR, p4view) - - if self.p4_snapshot: - return p4client.CheckoutFromSnapshot(self.p4_snapshot) - else: - return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) - - def CheckoutV14Dir(self): - p4view = perforce.View(self.DEPOT2_DIR, [ - perforce.PathMapping('gcctools/chromeos/v14/...') - ]) - return self.GetP4Snapshot(p4view) - - def SetupChromeOSCheckout(self, version, use_minilayout=False): - version_re = '^\d+\.\d+\.\d+\.[a-zA-Z0-9]+$' - - location = os.path.join(self.CHROMEOS_BUILDS_DIR, version) - - if version in ['weekly', 'quarterly']: - assert os.path.islink(location), 'Symlink %s does not exist.' % location - - location_expanded = os.path.abspath(os.path.realpath(location)) - version = os.path.basename(location_expanded) - - if version in ['top', 'latest'] or re.match(version_re, version): - return self.scripts.SetupChromeOS(version, use_minilayout) - - elif version.endswith('bz2') or version.endswith('gz'): - return cmd.UnTar(location_expanded, self.CHROMEOS_ROOT) - - else: - signature_file_location = os.path.join(location, - 'src/scripts/enter_chroot.sh') - assert os.path.exists(signature_file_location), ( - 'Signature file %s does not exist.' % signature_file_location) - - return cmd.Copy(location, to_dir=self.CHROMEOS_ROOT, recursive=True) - - -class JobsFactory(object): - - def __init__(self, - chromeos_version='top', - board='x86-mario', - toolchain='trunk', - p4_snapshot=''): - self.chromeos_version = chromeos_version - self.board = board - self.toolchain = toolchain - - self.commands = CommandsFactory(chromeos_version, board, toolchain, - p4_snapshot) - - def BuildAndBenchmark(self): - command = self.commands.BuildAndBenchmark() - - label = 'BuildAndBenchmark(%s,%s,%s)' % (self.toolchain, self.board, - self.chromeos_version) - - machine_label = 'chromeos-%s' % self.board - - job = jobs.CreateLinuxJob(label, command) - job.DependsOnMachine( - machine.MachineSpecification(label=machine_label, - lock_required=True), - False) - - return job diff --git a/deprecated/automation/clients/helper/crosstool.py b/deprecated/automation/clients/helper/crosstool.py deleted file mode 100644 index 80154b25..00000000 --- a/deprecated/automation/clients/helper/crosstool.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright 2011 Google Inc. All Rights Reserved. - -__author__ = 'kbaclawski@google.com (Krystian Baclawski)' - -import os.path -import time - -from automation.clients.helper import jobs -from automation.clients.helper import perforce -from automation.common import command as cmd -from automation.common import job - - -class JobsFactory(object): - - def __init__(self): - self.commands = CommandsFactory() - - def CheckoutCrosstool(self, target): - command = self.commands.CheckoutCrosstool() - new_job = jobs.CreateLinuxJob('CheckoutCrosstool(%s)' % target, command) - checkout_dir_dep = job.FolderDependency(new_job, - CommandsFactory.CHECKOUT_DIR) - manifests_dir_dep = job.FolderDependency( - new_job, os.path.join(self.commands.buildit_path, target), 'manifests') - return new_job, checkout_dir_dep, manifests_dir_dep - - def BuildRelease(self, checkout_dir, target): - command = self.commands.BuildRelease(target) - new_job = jobs.CreateLinuxJob('BuildRelease(%s)' % target, command) - new_job.DependsOnFolder(checkout_dir) - build_tree_dep = job.FolderDependency(new_job, - self.commands.buildit_work_dir_path) - return new_job, build_tree_dep - - def RunTests(self, checkout_dir, build_tree_dir, target, board, component): - command = self.commands.RunTests(target, board, component) - new_job = jobs.CreateLinuxJob('RunTests(%s, %s, %s)' % - (target, component, board), command) - new_job.DependsOnFolder(checkout_dir) - new_job.DependsOnFolder(build_tree_dir) - testrun_dir_dep = job.FolderDependency( - new_job, self.commands.dejagnu_output_path, board) - return new_job, testrun_dir_dep - - def GenerateReport(self, testrun_dirs, manifests_dir, target, boards): - command = self.commands.GenerateReport(boards) - new_job = jobs.CreateLinuxJob('GenerateReport(%s)' % target, command) - new_job.DependsOnFolder(manifests_dir) - for testrun_dir in testrun_dirs: - new_job.DependsOnFolder(testrun_dir) - return new_job - - -class CommandsFactory(object): - CHECKOUT_DIR = 'crosstool-checkout-dir' - - def __init__(self): - self.buildit_path = os.path.join(self.CHECKOUT_DIR, 'gcctools', 'crosstool', - 'v15') - - self.buildit_work_dir = 'buildit-tmp' - self.buildit_work_dir_path = os.path.join('$JOB_TMP', self.buildit_work_dir) - self.dejagnu_output_path = os.path.join(self.buildit_work_dir_path, - 'dejagnu-output') - - paths = { - 'gcctools': [ - 'crosstool/v15/...', 'scripts/...' - ], - 'gcctools/google_vendor_src_branch': [ - 'binutils/binutils-2.21/...', 'gdb/gdb-7.2.x/...', - 'zlib/zlib-1.2.3/...' - ], - 'gcctools/vendor_src': [ - 'gcc/google/gcc-4_6/...' - ] - } - - p4view = perforce.View('depot2', - perforce.PathMapping.ListFromPathDict(paths)) - - self.p4client = perforce.CommandsFactory(self.CHECKOUT_DIR, p4view) - - def CheckoutCrosstool(self): - p4client = self.p4client - - return p4client.SetupAndDo(p4client.Sync(), - p4client.SaveCurrentCLNumber('CLNUM'), - p4client.Remove()) - - def BuildRelease(self, target): - clnum_path = os.path.join('$JOB_TMP', self.CHECKOUT_DIR, 'CLNUM') - - toolchain_root = os.path.join('/google/data/rw/projects/toolchains', target, - 'unstable') - toolchain_path = os.path.join(toolchain_root, '${CLNUM}') - - build_toolchain = cmd.Wrapper( - cmd.Chain( - cmd.MakeDir(toolchain_path), - cmd.Shell('buildit', - '--keep-work-dir', - '--build-type=release', - '--work-dir=%s' % self.buildit_work_dir_path, - '--results-dir=%s' % toolchain_path, - '--force-release=%s' % '${CLNUM}', - target, - path='.')), - cwd=self.buildit_path, - umask='0022', - env={'CLNUM': '$(< %s)' % clnum_path}) - - # remove all but 10 most recent directories - remove_old_toolchains_from_x20 = cmd.Wrapper( - cmd.Pipe( - cmd.Shell('ls', '-1', '-r'), cmd.Shell('sed', '-e', '1,10d'), - cmd.Shell('xargs', 'rm', '-r', '-f')), - cwd=toolchain_root) - - return cmd.Chain(build_toolchain, remove_old_toolchains_from_x20) - - def RunTests(self, target, board, component='gcc'): - dejagnu_flags = ['--outdir=%s' % self.dejagnu_output_path, - '--target_board=%s' % board] - - # Look for {pandaboard,qemu}.exp files in - # //depot/google3/experimental/users/kbaclawski/dejagnu/boards - - site_exp_file = os.path.join('/google/src/head/depot/google3', - 'experimental/users/kbaclawski', - 'dejagnu/site.exp') - - build_dir_path = os.path.join(target, 'rpmbuild/BUILD/crosstool*-0.0', - 'build-%s' % component) - - run_dejagnu = cmd.Wrapper( - cmd.Chain( - cmd.MakeDir(self.dejagnu_output_path), - cmd.Shell('make', - 'check', - '-k', - '-j $(grep -c processor /proc/cpuinfo)', - 'RUNTESTFLAGS="%s"' % ' '.join(dejagnu_flags), - 'DEJAGNU="%s"' % site_exp_file, - ignore_error=True)), - cwd=os.path.join(self.buildit_work_dir_path, build_dir_path), - env={'REMOTE_TMPDIR': 'job-$JOB_ID'}) - - save_results = cmd.Copy(self.dejagnu_output_path, - to_dir='$JOB_TMP/results', - recursive=True) - - return cmd.Chain(run_dejagnu, save_results) - - def GenerateReport(self, boards): - sumfiles = [os.path.join('$JOB_TMP', board, '*.sum') for board in boards] - - return cmd.Wrapper( - cmd.Shell('dejagnu.sh', - 'report', - '-m', - '$JOB_TMP/manifests/*.xfail', - '-o', - '$JOB_TMP/results/report.html', - *sumfiles, - path='.'), - cwd='$HOME/automation/clients/report') diff --git a/deprecated/automation/clients/helper/jobs.py b/deprecated/automation/clients/helper/jobs.py deleted file mode 100644 index 96a1c408..00000000 --- a/deprecated/automation/clients/helper/jobs.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2010 Google Inc. All Rights Reserved. - -from automation.common import job -from automation.common import machine - - -def CreateLinuxJob(label, command, lock=False, timeout=4 * 60 * 60): - to_return = job.Job(label, command, timeout) - to_return.DependsOnMachine(machine.MachineSpecification(os='linux', - lock_required=lock)) - return to_return diff --git a/deprecated/automation/clients/helper/perforce.py b/deprecated/automation/clients/helper/perforce.py deleted file mode 100644 index 1f2dfe79..00000000 --- a/deprecated/automation/clients/helper/perforce.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2011 Google Inc. All Rights Reserved. - -__author__ = 'kbaclawski@google.com (Krystian Baclawski)' - -import collections -import os.path - -from automation.common import command as cmd - - -class PathMapping(object): - """Stores information about relative path mapping (remote to local).""" - - @classmethod - def ListFromPathDict(cls, prefix_path_dict): - """Takes {'prefix1': ['path1',...], ...} and returns a list of mappings.""" - - mappings = [] - - for prefix, paths in sorted(prefix_path_dict.items()): - for path in sorted(paths): - mappings.append(cls(os.path.join(prefix, path))) - - return mappings - - @classmethod - def ListFromPathTuples(cls, tuple_list): - """Takes a list of tuples and returns a list of mappings. - - Args: - tuple_list: [('remote_path1', 'local_path1'), ...] - - Returns: - a list of mapping objects - """ - mappings = [] - for remote_path, local_path in tuple_list: - mappings.append(cls(remote_path, local_path)) - - return mappings - - def __init__(self, remote, local=None, common_suffix=None): - suffix = self._FixPath(common_suffix or '') - - self.remote = os.path.join(remote, suffix) - self.local = os.path.join(local or remote, suffix) - - @staticmethod - def _FixPath(path_s): - parts = [part for part in path_s.strip('/').split('/') if part] - - if not parts: - return '' - - return os.path.join(*parts) - - def _GetRemote(self): - return self._remote - - def _SetRemote(self, path_s): - self._remote = self._FixPath(path_s) - - remote = property(_GetRemote, _SetRemote) - - def _GetLocal(self): - return self._local - - def _SetLocal(self, path_s): - self._local = self._FixPath(path_s) - - local = property(_GetLocal, _SetLocal) - - def GetAbsolute(self, depot, client): - return (os.path.join('//', depot, self.remote), - os.path.join('//', client, self.local)) - - def __str__(self): - return '%s(%s => %s)' % (self.__class__.__name__, self.remote, self.local) - - -class View(collections.MutableSet): - """Keeps all information about local client required to work with perforce.""" - - def __init__(self, depot, mappings=None, client=None): - self.depot = depot - - if client: - self.client = client - - self._mappings = set(mappings or []) - - @staticmethod - def _FixRoot(root_s): - parts = root_s.strip('/').split('/', 1) - - if len(parts) != 1: - return None - - return parts[0] - - def _GetDepot(self): - return self._depot - - def _SetDepot(self, depot_s): - depot = self._FixRoot(depot_s) - assert depot, 'Not a valid depot name: "%s".' % depot_s - self._depot = depot - - depot = property(_GetDepot, _SetDepot) - - def _GetClient(self): - return self._client - - def _SetClient(self, client_s): - client = self._FixRoot(client_s) - assert client, 'Not a valid client name: "%s".' % client_s - self._client = client - - client = property(_GetClient, _SetClient) - - def add(self, mapping): - assert type(mapping) is PathMapping - self._mappings.add(mapping) - - def discard(self, mapping): - assert type(mapping) is PathMapping - self._mappings.discard(mapping) - - def __contains__(self, value): - return value in self._mappings - - def __len__(self): - return len(self._mappings) - - def __iter__(self): - return iter(mapping for mapping in self._mappings) - - def AbsoluteMappings(self): - return iter(mapping.GetAbsolute(self.depot, self.client) - for mapping in self._mappings) - - -class CommandsFactory(object): - """Creates shell commands used for interaction with Perforce.""" - - def __init__(self, checkout_dir, p4view, name=None, port=None): - self.port = port or 'perforce2:2666' - self.view = p4view - self.view.client = name or 'p4-automation-$HOSTNAME-$JOB_ID' - self.checkout_dir = checkout_dir - self.p4config_path = os.path.join(self.checkout_dir, '.p4config') - - def Initialize(self): - return cmd.Chain('mkdir -p %s' % self.checkout_dir, 'cp ~/.p4config %s' % - self.checkout_dir, 'chmod u+w %s' % self.p4config_path, - 'echo "P4PORT=%s" >> %s' % (self.port, self.p4config_path), - 'echo "P4CLIENT=%s" >> %s' % - (self.view.client, self.p4config_path)) - - def Create(self): - # TODO(kbaclawski): Could we support value list for options consistently? - mappings = ['-a \"%s %s\"' % mapping - for mapping in self.view.AbsoluteMappings()] - - # First command will create client with default mappings. Second one will - # replace default mapping with desired. Unfortunately, it seems that it - # cannot be done in one step. P4EDITOR is defined to /bin/true because we - # don't want "g4 client" to enter real editor and wait for user actions. - return cmd.Wrapper( - cmd.Chain( - cmd.Shell('g4', 'client'), - cmd.Shell('g4', 'client', '--replace', *mappings)), - env={'P4EDITOR': '/bin/true'}) - - def SaveSpecification(self, filename=None): - return cmd.Pipe(cmd.Shell('g4', 'client', '-o'), output=filename) - - def Sync(self, revision=None): - sync_arg = '...' - if revision: - sync_arg = '%s@%s' % (sync_arg, revision) - return cmd.Shell('g4', 'sync', sync_arg) - - def SaveCurrentCLNumber(self, filename=None): - return cmd.Pipe( - cmd.Shell('g4', 'changes', '-m1', '...#have'), - cmd.Shell('sed', '-E', '"s,Change ([0-9]+) .*,\\1,"'), - output=filename) - - def Remove(self): - return cmd.Shell('g4', 'client', '-d', self.view.client) - - def SetupAndDo(self, *commands): - return cmd.Chain(self.Initialize(), - self.InCheckoutDir(self.Create(), *commands)) - - def InCheckoutDir(self, *commands): - return cmd.Wrapper(cmd.Chain(*commands), cwd=self.checkout_dir) - - def CheckoutFromSnapshot(self, snapshot): - cmds = cmd.Chain() - - for mapping in self.view: - local_path, file_part = mapping.local.rsplit('/', 1) - - if file_part == '...': - remote_dir = os.path.join(snapshot, local_path) - local_dir = os.path.join(self.checkout_dir, os.path.dirname(local_path)) - - cmds.extend([ - cmd.Shell('mkdir', '-p', local_dir), cmd.Shell( - 'rsync', '-lr', remote_dir, local_dir) - ]) - - return cmds |