diff options
Diffstat (limited to 'deprecated/automation/clients')
22 files changed, 2353 insertions, 0 deletions
diff --git a/deprecated/automation/clients/__init__.py b/deprecated/automation/clients/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/deprecated/automation/clients/__init__.py @@ -0,0 +1 @@ + diff --git a/deprecated/automation/clients/android.py b/deprecated/automation/clients/android.py new file mode 100755 index 00000000..06e76d29 --- /dev/null +++ b/deprecated/automation/clients/android.py @@ -0,0 +1,87 @@ +#!/usr/bin/python2 +# +# Copyright 2011 Google Inc. All Rights Reserved. +"""Client for Android nightly jobs. + +Does the following jobs: + 1. Checkout android toolchain sources + 2. Build Android toolchain + 3. Build Android tree + 4. Checkout/build/run Android benchmarks (TODO) + 5. Generate size/performance dashboard ? (TODO) +""" + +__author__ = 'jingyu@google.com (Jing Yu)' + +import optparse +import pickle +import sys +import xmlrpclib + +from automation.clients.helper import android +from automation.common import job_group +from automation.common import logger + + +class AndroidToolchainNightlyClient(object): + VALID_GCC_VERSIONS = ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] + + def __init__(self, gcc_version, is_release): + assert gcc_version in self.VALID_GCC_VERSIONS + self.gcc_version = gcc_version + if is_release: + self.build_type = 'RELEASE' + else: + self.build_type = 'DEVELOPMENT' + + def Run(self): + server = xmlrpclib.Server('http://localhost:8000') + server.ExecuteJobGroup(pickle.dumps(self.CreateJobGroup())) + + def CreateJobGroup(self): + factory = android.JobsFactory(self.gcc_version, self.build_type) + + p4_androidtc_job, checkout_dir_dep = factory.CheckoutAndroidToolchain() + + tc_build_job, tc_prefix_dep = factory.BuildAndroidToolchain( + checkout_dir_dep) + + tree_build_job = factory.BuildAndroidImage(tc_prefix_dep) + + benchmark_job = factory.Benchmark(tc_prefix_dep) + + all_jobs = [p4_androidtc_job, tc_build_job, tree_build_job, benchmark_job] + + return job_group.JobGroup('androidtoolchain_nightly', all_jobs, True, False) + + +@logger.HandleUncaughtExceptions +def Main(argv): + valid_gcc_versions_string = ', '.join( + AndroidToolchainNightlyClient.VALID_GCC_VERSIONS) + + parser = optparse.OptionParser() + parser.add_option('--with-gcc-version', + dest='gcc_version', + default='4.6', + action='store', + choices=AndroidToolchainNightlyClient.VALID_GCC_VERSIONS, + help='gcc version: %s.' % valid_gcc_versions_string) + parser.add_option('-r', + '--release', + dest='is_release', + default=False, + action='store_true', + help='Build a release toolchain?') + options, _ = parser.parse_args(argv) + + option_list = [opt.dest for opt in parser.option_list if opt.dest] + + kwargs = dict((option, getattr(options, option)) for option in option_list) + + client = AndroidToolchainNightlyClient(**kwargs) + client.Run() + + +if __name__ == '__main__': + Main(sys.argv) diff --git a/deprecated/automation/clients/chromeos.py b/deprecated/automation/clients/chromeos.py new file mode 100755 index 00000000..572320fd --- /dev/null +++ b/deprecated/automation/clients/chromeos.py @@ -0,0 +1,104 @@ +#!/usr/bin/python2 +# +# Copyright 2011 Google Inc. All Rights Reserved. +"""chromeos.py: Build & Test ChromeOS using custom compilers.""" + +__author__ = 'asharif@google.com (Ahmad Sharif)' + +import logging +import optparse +import os +import pickle +import sys +import xmlrpclib + +from automation.clients.helper import jobs +from automation.clients.helper import perforce +from automation.common import command as cmd +from automation.common import job_group +from automation.common import logger + + +class ChromeOSNightlyClient(object): + DEPOT2_DIR = '//depot2/' + P4_CHECKOUT_DIR = 'perforce2/' + P4_VERSION_DIR = os.path.join(P4_CHECKOUT_DIR, 'gcctools/chromeos/v14') + + def __init__(self, board, remote, gcc_githash, p4_snapshot=''): + self._board = board + self._remote = remote + self._gcc_githash = gcc_githash + self._p4_snapshot = p4_snapshot + + def Run(self): + server = xmlrpclib.Server('http://localhost:8000') + server.ExecuteJobGroup(pickle.dumps(self.CreateJobGroup())) + + def CheckoutV14Dir(self): + p4view = perforce.View(self.DEPOT2_DIR, [ + perforce.PathMapping('gcctools/chromeos/v14/...') + ]) + return self.GetP4Snapshot(p4view) + + def GetP4Snapshot(self, p4view): + p4client = perforce.CommandsFactory(self.P4_CHECKOUT_DIR, p4view) + + if self._p4_snapshot: + return p4client.CheckoutFromSnapshot(self._p4_snapshot) + else: + return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) + + def CreateJobGroup(self): + chain = cmd.Chain( + self.CheckoutV14Dir(), + cmd.Shell('python', + os.path.join(self.P4_VERSION_DIR, 'test_toolchains.py'), + '--force-mismatch', + '--clean', + '--public', # crbug.com/145822 + '--board=%s' % self._board, + '--remote=%s' % self._remote, + '--githashes=%s' % self._gcc_githash)) + label = 'testlabel' + job = jobs.CreateLinuxJob(label, chain, timeout=24 * 60 * 60) + + return job_group.JobGroup(label, [job], True, False) + + +@logger.HandleUncaughtExceptions +def Main(argv): + parser = optparse.OptionParser() + parser.add_option('-b', + '--board', + dest='board', + help='Run performance tests on these boards') + parser.add_option('-r', + '--remote', + dest='remote', + help='Run performance tests on these remotes') + parser.add_option('-g', + '--gcc_githash', + dest='gcc_githash', + help='Use this gcc_githash.') + parser.add_option('-p', + '--p4_snapshot', + dest='p4_snapshot', + default='', + help='Use this p4_snapshot.') + options, _ = parser.parse_args(argv) + + if not all([options.board, options.remote, options.gcc_githash]): + logging.error('Specify a board, remote and gcc_githash') + return 1 + + client = ChromeOSNightlyClient(options.board, + options.remote, + options.gcc_githash, + p4_snapshot=options.p4_snapshot) + client.Run() + return 0 + + +if __name__ == '__main__': + logger.SetUpRootLogger(level=logging.DEBUG, display_flags={'name': False}) + sys.exit(Main(sys.argv)) diff --git a/deprecated/automation/clients/crosstool.py b/deprecated/automation/clients/crosstool.py new file mode 100755 index 00000000..9ba83807 --- /dev/null +++ b/deprecated/automation/clients/crosstool.py @@ -0,0 +1,102 @@ +#!/usr/bin/python2 +# +# Copyright 2011 Google Inc. All Rights Reserved. + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +import logging +import optparse +import pickle +import sys +import xmlrpclib + +from automation.clients.helper import crosstool +from automation.common import job_group +from automation.common import logger + + +class CrosstoolNightlyClient(object): + VALID_TARGETS = ['gcc-4.6.x-ubuntu_lucid-arm', + 'gcc-4.6.x-ubuntu_lucid-x86_64', + 'gcc-4.6.x-grtev2-armv7a-vfpv3.d16-hard', + 'gcc-4.6.x-glibc-2.11.1-grte', + 'gcc-4.6.x-glibc-2.11.1-powerpc'] + VALID_BOARDS = ['qemu', 'pandaboard', 'unix'] + + def __init__(self, target, boards): + assert target in self.VALID_TARGETS + assert all(board in self.VALID_BOARDS for board in boards) + + self._target = target + self._boards = boards + + def Run(self): + server = xmlrpclib.Server('http://localhost:8000') + server.ExecuteJobGroup(pickle.dumps(self.CreateJobGroup())) + + def CreateJobGroup(self): + factory = crosstool.JobsFactory() + + checkout_crosstool_job, checkout_dir, manifests_dir = \ + factory.CheckoutCrosstool(self._target) + + all_jobs = [checkout_crosstool_job] + + # Build crosstool target + build_release_job, build_tree_dir = factory.BuildRelease(checkout_dir, + self._target) + all_jobs.append(build_release_job) + + testruns = [] + + # Perform crosstool tests + for board in self._boards: + for component in ('gcc', 'binutils'): + test_job, testrun_dir = factory.RunTests(checkout_dir, build_tree_dir, + self._target, board, component) + all_jobs.append(test_job) + testruns.append(testrun_dir) + + if testruns: + all_jobs.append(factory.GenerateReport(testruns, manifests_dir, + self._target, self._boards)) + + return job_group.JobGroup('Crosstool Nightly Build (%s)' % self._target, + all_jobs, True, False) + + +@logger.HandleUncaughtExceptions +def Main(argv): + valid_boards_string = ', '.join(CrosstoolNightlyClient.VALID_BOARDS) + + parser = optparse.OptionParser() + parser.add_option( + '-b', + '--board', + dest='boards', + action='append', + choices=CrosstoolNightlyClient.VALID_BOARDS, + default=[], + help=('Run DejaGNU tests on selected boards: %s.' % valid_boards_string)) + options, args = parser.parse_args(argv) + + if len(args) == 2: + target = args[1] + else: + logging.error('Exactly one target required as a command line argument!') + logging.info('List of valid targets:') + for pair in enumerate(CrosstoolNightlyClient.VALID_TARGETS, start=1): + logging.info('%d) %s', pair) + sys.exit(1) + + option_list = [opt.dest for opt in parser.option_list if opt.dest] + + kwargs = dict((option, getattr(options, option)) for option in option_list) + + client = CrosstoolNightlyClient(target, **kwargs) + client.Run() + + +if __name__ == '__main__': + logger.SetUpRootLogger(level=logging.DEBUG, display_flags={'name': False}) + Main(sys.argv) diff --git a/deprecated/automation/clients/dejagnu_compiler.py b/deprecated/automation/clients/dejagnu_compiler.py new file mode 100755 index 00000000..7448b87e --- /dev/null +++ b/deprecated/automation/clients/dejagnu_compiler.py @@ -0,0 +1,98 @@ +#!/usr/bin/python2 +# +# Copyright 2012 Google Inc. All Rights Reserved. +"""dejagnu_compiler.py: Run dejagnu test.""" + +__author__ = 'shenhan@google.com (Han Shen)' + +import logging +import optparse +import os +import pickle +import sys +import xmlrpclib + +from automation.clients.helper import jobs +from automation.clients.helper import perforce +from automation.common import command as cmd +from automation.common import job_group +from automation.common import logger + + +class DejagnuCompilerNightlyClient: + DEPOT2_DIR = '//depot2/' + P4_CHECKOUT_DIR = 'perforce2/' + P4_VERSION_DIR = os.path.join(P4_CHECKOUT_DIR, 'gcctools/chromeos/v14') + + def __init__(self, board, remote, p4_snapshot, cleanup): + self._board = board + self._remote = remote + self._p4_snapshot = p4_snapshot + self._cleanup = cleanup + + def Run(self): + server = xmlrpclib.Server('http://localhost:8000') + server.ExecuteJobGroup(pickle.dumps(self.CreateJobGroup())) + + def CheckoutV14Dir(self): + p4view = perforce.View(self.DEPOT2_DIR, [ + perforce.PathMapping('gcctools/chromeos/v14/...') + ]) + return self.GetP4Snapshot(p4view) + + def GetP4Snapshot(self, p4view): + p4client = perforce.CommandsFactory(self.P4_CHECKOUT_DIR, p4view) + + if self._p4_snapshot: + return p4client.CheckoutFromSnapshot(self._p4_snapshot) + else: + return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) + + def CreateJobGroup(self): + chain = cmd.Chain(self.CheckoutV14Dir(), cmd.Shell( + 'python', os.path.join(self.P4_VERSION_DIR, 'test_gcc_dejagnu.py'), + '--board=%s' % self._board, '--remote=%s' % self._remote, + '--cleanup=%s' % self._cleanup)) + label = 'dejagnu' + job = jobs.CreateLinuxJob(label, chain, timeout=8 * 60 * 60) + return job_group.JobGroup(label, + [job], + cleanup_on_failure=True, + cleanup_on_completion=True) + + +@logger.HandleUncaughtExceptions +def Main(argv): + parser = optparse.OptionParser() + parser.add_option('-b', + '--board', + dest='board', + help='Run performance tests on these boards') + parser.add_option('-r', + '--remote', + dest='remote', + help='Run performance tests on these remotes') + parser.add_option('-p', + '--p4_snapshot', + dest='p4_snapshot', + help=('For development only. ' + 'Use snapshot instead of checking out.')) + parser.add_option('--cleanup', + dest='cleanup', + default='mount', + help=('Cleanup test directory, values could be one of ' + '"mount", "chroot" or "chromeos"')) + options, _ = parser.parse_args(argv) + + if not all([options.board, options.remote]): + logging.error('Specify a board and remote.') + return 1 + + client = DejagnuCompilerNightlyClient(options.board, options.remote, + options.p4_snapshot, options.cleanup) + client.Run() + return 0 + + +if __name__ == '__main__': + sys.exit(Main(sys.argv)) diff --git a/deprecated/automation/clients/helper/__init__.py b/deprecated/automation/clients/helper/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/deprecated/automation/clients/helper/__init__.py @@ -0,0 +1 @@ + diff --git a/deprecated/automation/clients/helper/android.py b/deprecated/automation/clients/helper/android.py new file mode 100644 index 00000000..7ff2ac1c --- /dev/null +++ b/deprecated/automation/clients/helper/android.py @@ -0,0 +1,319 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +"""Helper modules for Android toolchain test infrastructure. + +Provides following Android toolchain test jobs and commands. +. Checkout Android toolchain source code +. Build Android toolchain +. Checkout and build Android tree +. Checkout/build/run Android benchmarks, generate size dashboard +. Transform size dashboard to report, send perflab jobid to + perflab dashboard server.(TODO) +""" + +__author__ = 'jingyu@google.com (Jing Yu)' + +import os.path + +from automation.clients.helper import jobs +from automation.clients.helper import perforce +from automation.common import command as cmd +from automation.common import job + + +class JobsFactory(object): + + def __init__(self, gcc_version='4.4.3', build_type='DEVELOPMENT'): + assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] + assert build_type in ['DEVELOPMENT', 'RELEASE'] + + self.gcc_version = gcc_version + self.commands = CommandsFactory(gcc_version, build_type) + self.tc_tag = 'gcc-%s-%s' % (gcc_version, build_type) + + def CheckoutAndroidToolchain(self): + """Check out Android toolchain sources by release and gcc version.""" + command = self.commands.CheckoutAndroidToolchain() + new_job = jobs.CreateLinuxJob('AndroidCheckoutToolchain(%s)' % self.tc_tag, + command) + checkout_dir_dep = job.FolderDependency(new_job, self.commands.CHECKOUT_DIR) + return new_job, checkout_dir_dep + + def BuildAndroidToolchain(self, checkout_dir_dep): + """Build Android Toolchain.""" + command = self.commands.BuildAndroidToolchain() + new_job = jobs.CreateLinuxJob('AndroidBuildToolchain(%s)' % self.tc_tag, + command) + new_job.DependsOnFolder(checkout_dir_dep) + tc_prefix_dep = job.FolderDependency(new_job, + self.commands.toolchain_prefix_dir) + return new_job, tc_prefix_dep + + def BuildAndroidImage(self, + tc_prefix_dep, + product='stingray', + branch='ics-release'): + assert product in ['stingray', 'passion', 'trygon', 'soju'] + assert branch in ['honeycomb-release', 'ics-release'] + command = self.commands.BuildAndroidImage(product, branch) + new_job = jobs.CreateLinuxJob('AndroidGetBuildTree(%s)' % self.tc_tag, + command) + new_job.DependsOnFolder(tc_prefix_dep) + return new_job + + def Benchmark(self, tc_prefix_dep, arch='soju'): + assert arch in ['soju', 'stingray'] + script_cmd = self.commands.CheckoutScripts() + experiment_tag = 'android/nightly/%s/%s/$JOB_ID' % (self.tc_tag, arch) + build_run_benchmark_cmd = self.commands.BuildRunBenchmark(arch, + experiment_tag) + command = cmd.Chain(script_cmd, build_run_benchmark_cmd) + new_job = jobs.CreateLinuxJob('AndroidBenchmarking(%s)' % self.tc_tag, + command) + new_job.DependsOnFolder(tc_prefix_dep) + return new_job + + +class CommandsFactory(object): + CHECKOUT_DIR = 'androidtc-checkout-dir' + TOOLCHAIN_SRC_DIR = os.path.join(CHECKOUT_DIR, 'src') + TOOLCHAIN_BUILD_DIR = 'obj' + ANDROID_TREES_DIR = 'android_trees' + TOOLS_DIR = 'android-tools' + BENCHMARK_OUT_DIR = 'results' + + def __init__(self, gcc_version, build_type): + assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] + assert build_type in ['DEVELOPMENT', 'RELEASE'] + + self.build_type = build_type + self.gcc_version = gcc_version + self.binutils_version = '2.21' + self.gold_version = '2.21' + self.toolchain_prefix_dir = 'install-gcc-%s-%s' % (gcc_version, build_type) + self.p4client = self._CreatePerforceClient() + self.scripts = ScriptsFactory(self.gcc_version, self.binutils_version, + self.gold_version) + + def _CreatePerforceClient(self): + p4_dev_path = 'gcctools/google_vendor_src_branch' + mobile_rel_branch = ('branches/' + 'mobile_toolchain_v15_release_branch/gcctools/' + 'google_vendor_src_branch') + gcc_443_rel_branch = ('branches/' + 'android_compiler_v14_release_branch/gcctools/' + 'google_vendor_src_branch') + + # Common views for tools + p4view = perforce.View('depot2', perforce.PathMapping.ListFromPathTuples([( + 'gcctools/android/build/...', 'src/build/...'), ( + 'gcctools/android/Tarballs/...', 'src/tarballs/...')])) + for mapping in perforce.PathMapping.ListFromPathDict( + {'gcctools/android': ['tools/scripts/...', 'master/...']}): + p4view.add(mapping) + + # Add views for gdb + p4view.add(perforce.PathMapping(p4_dev_path, 'src', + 'gdb/gdb-7.1.x-android/...')) + + # Add view for binutils for ld and gold + if self.build_type is 'RELEASE': + binutils_branch = mobile_rel_branch + else: + binutils_branch = p4_dev_path + p4view.add(perforce.PathMapping(binutils_branch, 'src', ( + 'binutils/binutils-%s/...' % self.binutils_version))) + if self.binutils_version != self.gold_version: + p4view.add(perforce.PathMapping(binutils_branch, 'src', ( + 'binutils/binutils-%s/...' % self.gold_version))) + + # Add view for gcc if gcc_version is '4.4.3'. + if self.gcc_version == '4.4.3': + gcc443_path = 'gcc/gcc-4.4.3/...' + if self.build_type is 'RELEASE': + p4view.add(perforce.PathMapping(gcc_443_rel_branch, 'src', gcc443_path)) + else: + p4view.add(perforce.PathMapping(p4_dev_path, 'src', gcc443_path)) + + return perforce.CommandsFactory(self.CHECKOUT_DIR, p4view) + + def _CheckoutGCCFromSVN(self): + """Check out gcc from fsf svn. + + Return the command that check out gcc from svn + to gcc_required_dir (=TOOLCHAIN_SRC_DIR/src/gcc/gcc-xxx). + + TODO: + Create a svn class that does these jobs. + Parallelize p4 checkout and svn checkout. + """ + if self.gcc_version == '4.4.3': + return '' + assert self.gcc_version in ['4.6', 'google_main', 'fsf_trunk'] + + gcc_branches_dir = {'4.6': 'branches/google/gcc-4_6', + 'google_main': 'branches/google/main', + 'fsf_trunk': 'trunk'} + + # Find GCC revision number, output it to TOOLCHAIN_SRC_DIR/CLNUM_GCC + svn_get_revision = cmd.Pipe( + cmd.Shell('svn', 'info'), + cmd.Shell('grep', '"Revision:"'), + cmd.Shell('sed', '-E', '"s,Revision: ([0-9]+).*,\\1,"'), + output='../../../CLNUM_GCC') + + svn_co_command = 'svn co svn://gcc.gnu.org/svn/gcc/%s .' % ( + gcc_branches_dir[self.gcc_version]) + + gcc_required_dir = os.path.join(self.TOOLCHAIN_SRC_DIR, 'gcc', + 'gcc-%s' % self.gcc_version) + + return cmd.Chain( + cmd.MakeDir(gcc_required_dir), + cmd.Wrapper( + cmd.Chain(svn_co_command, svn_get_revision), + cwd=gcc_required_dir)) + + def CheckoutAndroidToolchain(self): + p4client = self.p4client + command = p4client.SetupAndDo(p4client.Sync(), + p4client.SaveCurrentCLNumber('CLNUM'), + p4client.Remove()) + if self.gcc_version != '4.4.3': + command.append(self._CheckoutGCCFromSVN()) + + return command + + def BuildAndroidToolchain(self): + script_cmd = self.scripts.BuildAndroidToolchain( + self.toolchain_prefix_dir, self.CHECKOUT_DIR, self.TOOLCHAIN_BUILD_DIR, + self.TOOLCHAIN_SRC_DIR) + + # Record toolchain and gcc CL number + record_cl_cmd = cmd.Copy( + os.path.join(self.CHECKOUT_DIR, 'CLNUM*'), + to_dir=self.toolchain_prefix_dir) + save_cmd = cmd.Tar( + os.path.join('$JOB_TMP', 'results', '%s.tar.bz2' % + self.toolchain_prefix_dir), self.toolchain_prefix_dir) + return cmd.Chain(script_cmd, record_cl_cmd, save_cmd) + + def _BuildAndroidTree(self, local_android_branch_dir, product): + target_tools_prefix = os.path.join('$JOB_TMP', self.toolchain_prefix_dir, + 'bin', 'arm-linux-androideabi-') + java_path = '/usr/lib/jvm/java-6-sun/bin' + build_cmd = cmd.Shell('make', '-j8', 'PRODUCT-%s-userdebug' % product, + 'TARGET_TOOLS_PREFIX=%s' % target_tools_prefix, + 'PATH=%s:$PATH' % java_path) + return cmd.Wrapper(build_cmd, cwd=local_android_branch_dir) + + def BuildAndroidImage(self, product, branch): + assert product in ['stingray', 'passion', 'trygon', 'soju'] + + # Copy the tree from atree.mtv.corp to ANDROID_TREES_DIR/branch + androidtrees_host = 'atree.mtv.corp.google.com' + androidtrees_path = ('/usr/local/google2/home/mobiletc-prebuild/' + 'android_trees') + remote_android_branch_path = os.path.join(androidtrees_path, branch) + local_android_branch_dir = os.path.join(self.ANDROID_TREES_DIR, branch) + gettree_cmd = cmd.RemoteCopyFrom( + androidtrees_host, remote_android_branch_path, local_android_branch_dir) + + # Configure and build the tree + buildtree_cmd = self._BuildAndroidTree(local_android_branch_dir, product) + + # Compress and copy system.img to result + result_system_img = os.path.join(local_android_branch_dir, 'out', 'target', + 'product', product, 'system.img') + copy_img = cmd.Copy(result_system_img, to_dir='results') + compress_img = cmd.Shell('bzip2', os.path.join('results', 'system.img')) + + return cmd.Chain(gettree_cmd, buildtree_cmd, copy_img, compress_img) + + def CheckoutScripts(self): + p4view = perforce.View('depot2', + [perforce.PathMapping('gcctools/android/tools/...', + 'tools/...')]) + p4client = perforce.CommandsFactory(self.TOOLS_DIR, p4view) + return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) + + def BuildRunBenchmark(self, arch, run_experiment): + # Copy base benchmark binaries from atree.mtv.corp + base_benchbin_host = 'atree.mtv.corp.google.com' + base_benchbin_path = ('/usr/local/google2/home/mobiletc-prebuild/' + 'archive/v3binaries/2011-10-18') + local_basebenchbin_dir = 'base_benchmark_bin' + getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host, base_benchbin_path, + local_basebenchbin_dir) + + # Build and run benchmark. + android_arch = 'android_%s' % arch + run_label = 'normal' + benchmark_cmd = self.scripts.RunBenchmark( + self.toolchain_prefix_dir, self.TOOLS_DIR, self.BENCHMARK_OUT_DIR, + run_label, run_experiment, android_arch, local_basebenchbin_dir) + + # Extract jobid from BENCHMARK_OUT_DIR/log/jobid_normal.log file. + # Copy jobid to www server to generate performance dashboard. + # TODO(jingyu) + + return cmd.Chain(getbase_cmd, benchmark_cmd) + + +class ScriptsFactory(object): + + def __init__(self, gcc_version, binutils_version, gold_version): + self._gcc_version = gcc_version + self._binutils_version = binutils_version + self._gold_version = gold_version + + def BuildAndroidToolchain(self, toolchain_prefix_dir, checkout_dir, + toolchain_build_dir, androidtc_src_dir): + if self._gcc_version == '4.4.3': + gold_option = 'both/gold' + else: + gold_option = 'default' + + return cmd.Shell( + 'build_androidtoolchain.sh', + '--toolchain-src=%s' % os.path.join('$JOB_TMP', androidtc_src_dir), + '--build-path=%s' % os.path.join('$JOB_TMP', toolchain_build_dir), + '--install-prefix=%s' % os.path.join('$JOB_TMP', toolchain_prefix_dir), + '--target=arm-linux-androideabi', + '--enable-gold=%s' % gold_option, + '--with-gcc-version=%s' % self._gcc_version, + '--with-binutils-version=%s' % self._binutils_version, + '--with-gold-version=%s' % self._gold_version, + '--with-gdb-version=7.1.x-android', + '--log-path=%s/logs' % '$JOB_HOME', + '--android-sysroot=%s' % os.path.join('$JOB_TMP', checkout_dir, + 'gcctools', 'android', 'master', + 'honeycomb_generic_sysroot'), + path=os.path.join(checkout_dir, 'gcctools', 'android', 'tools', + 'scripts')) + + def RunBenchmark(self, + toolchain_prefix_dir, + checkout_dir, + output_dir, + run_label, + run_experiment, + arch, + base_bench_bin=None): + if base_bench_bin: + base_bench_opt = '--base_benchmark_bin=%s' % base_bench_bin + else: + base_bench_opt = '' + + return cmd.Shell( + 'benchmark.sh', + '--android_toolchain=%s' % os.path.join('$JOB_TMP', + toolchain_prefix_dir), + '--bench_space=%s' % os.path.join('$JOB_TMP', 'bench'), + '--benchmark_bin=%s' % os.path.join('$JOB_TMP', output_dir, + 'bench_bin'), + base_bench_opt, + '--log_path=%s' % os.path.join('$JOB_TMP', output_dir, 'log'), + '--arch=%s' % arch, + '--run_label=%s' % run_label, + '--run_experiment=%s' % run_experiment, + path=os.path.join(checkout_dir, 'tools', 'scripts')) diff --git a/deprecated/automation/clients/helper/chromeos.py b/deprecated/automation/clients/helper/chromeos.py new file mode 100644 index 00000000..e7157451 --- /dev/null +++ b/deprecated/automation/clients/helper/chromeos.py @@ -0,0 +1,180 @@ +# Copyright 2011 Google Inc. All Rights Reserved. + +__author__ = 'asharif@google.com (Ahmad Sharif)' + +import os.path +import re + +from automation.clients.helper import jobs +from automation.clients.helper import perforce +from automation.common import command as cmd +from automation.common import machine + + +class ScriptsFactory(object): + + def __init__(self, chromeos_root, scripts_path): + self._chromeos_root = chromeos_root + self._scripts_path = scripts_path + + def SummarizeResults(self, logs_path): + return cmd.Shell('summarize_results.py', logs_path, path=self._scripts_path) + + def Buildbot(self, config_name): + buildbot = os.path.join(self._chromeos_root, + 'chromite/cbuildbot/cbuildbot.py') + + return cmd.Shell(buildbot, '--buildroot=%s' % self._chromeos_root, + '--resume', '--noarchive', '--noprebuilts', '--nosync', + '--nouprev', '--notests', '--noclean', config_name) + + def RunBenchmarks(self, board, tests): + image_path = os.path.join(self._chromeos_root, 'src/build/images', board, + 'latest/chromiumos_image.bin') + + return cmd.Shell('cros_run_benchmarks.py', + '--remote=$SECONDARY_MACHINES[0]', + '--board=%s' % board, + '--tests=%s' % tests, + '--full_table', + image_path, + path='/home/mobiletc-prebuild') + + def SetupChromeOS(self, version='latest', use_minilayout=False): + setup_chromeos = cmd.Shell('setup_chromeos.py', + '--public', + '--dir=%s' % self._chromeos_root, + '--version=%s' % version, + path=self._scripts_path) + + if use_minilayout: + setup_chromeos.AddOption('--minilayout') + return setup_chromeos + + +class CommandsFactory(object): + DEPOT2_DIR = '//depot2/' + P4_CHECKOUT_DIR = 'perforce2/' + P4_VERSION_DIR = os.path.join(P4_CHECKOUT_DIR, 'gcctools/chromeos/v14') + + CHROMEOS_ROOT = 'chromeos' + CHROMEOS_SCRIPTS_DIR = os.path.join(CHROMEOS_ROOT, 'src/scripts') + CHROMEOS_BUILDS_DIR = '/home/mobiletc-prebuild/www/chromeos_builds' + + def __init__(self, chromeos_version, board, toolchain, p4_snapshot): + self.chromeos_version = chromeos_version + self.board = board + self.toolchain = toolchain + self.p4_snapshot = p4_snapshot + + self.scripts = ScriptsFactory(self.CHROMEOS_ROOT, self.P4_VERSION_DIR) + + def AddBuildbotConfig(self, config_name, config_list): + config_header = 'add_config(%r, [%s])' % (config_name, + ', '.join(config_list)) + config_file = os.path.join(self.CHROMEOS_ROOT, + 'chromite/cbuildbot/cbuildbot_config.py') + quoted_config_header = '%r' % config_header + quoted_config_header = re.sub("'", "\\\"", quoted_config_header) + + return cmd.Pipe( + cmd.Shell('echo', quoted_config_header), + cmd.Shell('tee', '--append', config_file)) + + def RunBuildbot(self): + config_dict = {'board': self.board, + 'build_tests': True, + 'chrome_tests': True, + 'unittests': False, + 'vm_tests': False, + 'prebuilts': False, + 'latest_toolchain': True, + 'useflags': ['chrome_internal'], + 'usepkg_chroot': True, + self.toolchain: True} + config_name = '%s-toolchain-test' % self.board + if 'arm' in self.board: + config_list = ['arm'] + else: + config_list = [] + config_list.extend(['internal', 'full', 'official', str(config_dict)]) + + add_config_shell = self.AddBuildbotConfig(config_name, config_list) + return cmd.Chain(add_config_shell, self.scripts.Buildbot(config_name)) + + def BuildAndBenchmark(self): + return cmd.Chain( + self.CheckoutV14Dir(), + self.SetupChromeOSCheckout(self.chromeos_version, True), + self.RunBuildbot(), + self.scripts.RunBenchmarks(self.board, 'BootPerfServer,10:Page,3')) + + def GetP4Snapshot(self, p4view): + p4client = perforce.CommandsFactory(self.P4_CHECKOUT_DIR, p4view) + + if self.p4_snapshot: + return p4client.CheckoutFromSnapshot(self.p4_snapshot) + else: + return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) + + def CheckoutV14Dir(self): + p4view = perforce.View(self.DEPOT2_DIR, [ + perforce.PathMapping('gcctools/chromeos/v14/...') + ]) + return self.GetP4Snapshot(p4view) + + def SetupChromeOSCheckout(self, version, use_minilayout=False): + version_re = '^\d+\.\d+\.\d+\.[a-zA-Z0-9]+$' + + location = os.path.join(self.CHROMEOS_BUILDS_DIR, version) + + if version in ['weekly', 'quarterly']: + assert os.path.islink(location), 'Symlink %s does not exist.' % location + + location_expanded = os.path.abspath(os.path.realpath(location)) + version = os.path.basename(location_expanded) + + if version in ['top', 'latest'] or re.match(version_re, version): + return self.scripts.SetupChromeOS(version, use_minilayout) + + elif version.endswith('bz2') or version.endswith('gz'): + return cmd.UnTar(location_expanded, self.CHROMEOS_ROOT) + + else: + signature_file_location = os.path.join(location, + 'src/scripts/enter_chroot.sh') + assert os.path.exists(signature_file_location), ( + 'Signature file %s does not exist.' % signature_file_location) + + return cmd.Copy(location, to_dir=self.CHROMEOS_ROOT, recursive=True) + + +class JobsFactory(object): + + def __init__(self, + chromeos_version='top', + board='x86-mario', + toolchain='trunk', + p4_snapshot=''): + self.chromeos_version = chromeos_version + self.board = board + self.toolchain = toolchain + + self.commands = CommandsFactory(chromeos_version, board, toolchain, + p4_snapshot) + + def BuildAndBenchmark(self): + command = self.commands.BuildAndBenchmark() + + label = 'BuildAndBenchmark(%s,%s,%s)' % (self.toolchain, self.board, + self.chromeos_version) + + machine_label = 'chromeos-%s' % self.board + + job = jobs.CreateLinuxJob(label, command) + job.DependsOnMachine( + machine.MachineSpecification(label=machine_label, + lock_required=True), + False) + + return job diff --git a/deprecated/automation/clients/helper/crosstool.py b/deprecated/automation/clients/helper/crosstool.py new file mode 100644 index 00000000..80154b25 --- /dev/null +++ b/deprecated/automation/clients/helper/crosstool.py @@ -0,0 +1,168 @@ +# Copyright 2011 Google Inc. All Rights Reserved. + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +import os.path +import time + +from automation.clients.helper import jobs +from automation.clients.helper import perforce +from automation.common import command as cmd +from automation.common import job + + +class JobsFactory(object): + + def __init__(self): + self.commands = CommandsFactory() + + def CheckoutCrosstool(self, target): + command = self.commands.CheckoutCrosstool() + new_job = jobs.CreateLinuxJob('CheckoutCrosstool(%s)' % target, command) + checkout_dir_dep = job.FolderDependency(new_job, + CommandsFactory.CHECKOUT_DIR) + manifests_dir_dep = job.FolderDependency( + new_job, os.path.join(self.commands.buildit_path, target), 'manifests') + return new_job, checkout_dir_dep, manifests_dir_dep + + def BuildRelease(self, checkout_dir, target): + command = self.commands.BuildRelease(target) + new_job = jobs.CreateLinuxJob('BuildRelease(%s)' % target, command) + new_job.DependsOnFolder(checkout_dir) + build_tree_dep = job.FolderDependency(new_job, + self.commands.buildit_work_dir_path) + return new_job, build_tree_dep + + def RunTests(self, checkout_dir, build_tree_dir, target, board, component): + command = self.commands.RunTests(target, board, component) + new_job = jobs.CreateLinuxJob('RunTests(%s, %s, %s)' % + (target, component, board), command) + new_job.DependsOnFolder(checkout_dir) + new_job.DependsOnFolder(build_tree_dir) + testrun_dir_dep = job.FolderDependency( + new_job, self.commands.dejagnu_output_path, board) + return new_job, testrun_dir_dep + + def GenerateReport(self, testrun_dirs, manifests_dir, target, boards): + command = self.commands.GenerateReport(boards) + new_job = jobs.CreateLinuxJob('GenerateReport(%s)' % target, command) + new_job.DependsOnFolder(manifests_dir) + for testrun_dir in testrun_dirs: + new_job.DependsOnFolder(testrun_dir) + return new_job + + +class CommandsFactory(object): + CHECKOUT_DIR = 'crosstool-checkout-dir' + + def __init__(self): + self.buildit_path = os.path.join(self.CHECKOUT_DIR, 'gcctools', 'crosstool', + 'v15') + + self.buildit_work_dir = 'buildit-tmp' + self.buildit_work_dir_path = os.path.join('$JOB_TMP', self.buildit_work_dir) + self.dejagnu_output_path = os.path.join(self.buildit_work_dir_path, + 'dejagnu-output') + + paths = { + 'gcctools': [ + 'crosstool/v15/...', 'scripts/...' + ], + 'gcctools/google_vendor_src_branch': [ + 'binutils/binutils-2.21/...', 'gdb/gdb-7.2.x/...', + 'zlib/zlib-1.2.3/...' + ], + 'gcctools/vendor_src': [ + 'gcc/google/gcc-4_6/...' + ] + } + + p4view = perforce.View('depot2', + perforce.PathMapping.ListFromPathDict(paths)) + + self.p4client = perforce.CommandsFactory(self.CHECKOUT_DIR, p4view) + + def CheckoutCrosstool(self): + p4client = self.p4client + + return p4client.SetupAndDo(p4client.Sync(), + p4client.SaveCurrentCLNumber('CLNUM'), + p4client.Remove()) + + def BuildRelease(self, target): + clnum_path = os.path.join('$JOB_TMP', self.CHECKOUT_DIR, 'CLNUM') + + toolchain_root = os.path.join('/google/data/rw/projects/toolchains', target, + 'unstable') + toolchain_path = os.path.join(toolchain_root, '${CLNUM}') + + build_toolchain = cmd.Wrapper( + cmd.Chain( + cmd.MakeDir(toolchain_path), + cmd.Shell('buildit', + '--keep-work-dir', + '--build-type=release', + '--work-dir=%s' % self.buildit_work_dir_path, + '--results-dir=%s' % toolchain_path, + '--force-release=%s' % '${CLNUM}', + target, + path='.')), + cwd=self.buildit_path, + umask='0022', + env={'CLNUM': '$(< %s)' % clnum_path}) + + # remove all but 10 most recent directories + remove_old_toolchains_from_x20 = cmd.Wrapper( + cmd.Pipe( + cmd.Shell('ls', '-1', '-r'), cmd.Shell('sed', '-e', '1,10d'), + cmd.Shell('xargs', 'rm', '-r', '-f')), + cwd=toolchain_root) + + return cmd.Chain(build_toolchain, remove_old_toolchains_from_x20) + + def RunTests(self, target, board, component='gcc'): + dejagnu_flags = ['--outdir=%s' % self.dejagnu_output_path, + '--target_board=%s' % board] + + # Look for {pandaboard,qemu}.exp files in + # //depot/google3/experimental/users/kbaclawski/dejagnu/boards + + site_exp_file = os.path.join('/google/src/head/depot/google3', + 'experimental/users/kbaclawski', + 'dejagnu/site.exp') + + build_dir_path = os.path.join(target, 'rpmbuild/BUILD/crosstool*-0.0', + 'build-%s' % component) + + run_dejagnu = cmd.Wrapper( + cmd.Chain( + cmd.MakeDir(self.dejagnu_output_path), + cmd.Shell('make', + 'check', + '-k', + '-j $(grep -c processor /proc/cpuinfo)', + 'RUNTESTFLAGS="%s"' % ' '.join(dejagnu_flags), + 'DEJAGNU="%s"' % site_exp_file, + ignore_error=True)), + cwd=os.path.join(self.buildit_work_dir_path, build_dir_path), + env={'REMOTE_TMPDIR': 'job-$JOB_ID'}) + + save_results = cmd.Copy(self.dejagnu_output_path, + to_dir='$JOB_TMP/results', + recursive=True) + + return cmd.Chain(run_dejagnu, save_results) + + def GenerateReport(self, boards): + sumfiles = [os.path.join('$JOB_TMP', board, '*.sum') for board in boards] + + return cmd.Wrapper( + cmd.Shell('dejagnu.sh', + 'report', + '-m', + '$JOB_TMP/manifests/*.xfail', + '-o', + '$JOB_TMP/results/report.html', + *sumfiles, + path='.'), + cwd='$HOME/automation/clients/report') diff --git a/deprecated/automation/clients/helper/jobs.py b/deprecated/automation/clients/helper/jobs.py new file mode 100644 index 00000000..96a1c408 --- /dev/null +++ b/deprecated/automation/clients/helper/jobs.py @@ -0,0 +1,11 @@ +# Copyright 2010 Google Inc. All Rights Reserved. + +from automation.common import job +from automation.common import machine + + +def CreateLinuxJob(label, command, lock=False, timeout=4 * 60 * 60): + to_return = job.Job(label, command, timeout) + to_return.DependsOnMachine(machine.MachineSpecification(os='linux', + lock_required=lock)) + return to_return diff --git a/deprecated/automation/clients/helper/perforce.py b/deprecated/automation/clients/helper/perforce.py new file mode 100644 index 00000000..1f2dfe79 --- /dev/null +++ b/deprecated/automation/clients/helper/perforce.py @@ -0,0 +1,215 @@ +# Copyright 2011 Google Inc. All Rights Reserved. + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +import collections +import os.path + +from automation.common import command as cmd + + +class PathMapping(object): + """Stores information about relative path mapping (remote to local).""" + + @classmethod + def ListFromPathDict(cls, prefix_path_dict): + """Takes {'prefix1': ['path1',...], ...} and returns a list of mappings.""" + + mappings = [] + + for prefix, paths in sorted(prefix_path_dict.items()): + for path in sorted(paths): + mappings.append(cls(os.path.join(prefix, path))) + + return mappings + + @classmethod + def ListFromPathTuples(cls, tuple_list): + """Takes a list of tuples and returns a list of mappings. + + Args: + tuple_list: [('remote_path1', 'local_path1'), ...] + + Returns: + a list of mapping objects + """ + mappings = [] + for remote_path, local_path in tuple_list: + mappings.append(cls(remote_path, local_path)) + + return mappings + + def __init__(self, remote, local=None, common_suffix=None): + suffix = self._FixPath(common_suffix or '') + + self.remote = os.path.join(remote, suffix) + self.local = os.path.join(local or remote, suffix) + + @staticmethod + def _FixPath(path_s): + parts = [part for part in path_s.strip('/').split('/') if part] + + if not parts: + return '' + + return os.path.join(*parts) + + def _GetRemote(self): + return self._remote + + def _SetRemote(self, path_s): + self._remote = self._FixPath(path_s) + + remote = property(_GetRemote, _SetRemote) + + def _GetLocal(self): + return self._local + + def _SetLocal(self, path_s): + self._local = self._FixPath(path_s) + + local = property(_GetLocal, _SetLocal) + + def GetAbsolute(self, depot, client): + return (os.path.join('//', depot, self.remote), + os.path.join('//', client, self.local)) + + def __str__(self): + return '%s(%s => %s)' % (self.__class__.__name__, self.remote, self.local) + + +class View(collections.MutableSet): + """Keeps all information about local client required to work with perforce.""" + + def __init__(self, depot, mappings=None, client=None): + self.depot = depot + + if client: + self.client = client + + self._mappings = set(mappings or []) + + @staticmethod + def _FixRoot(root_s): + parts = root_s.strip('/').split('/', 1) + + if len(parts) != 1: + return None + + return parts[0] + + def _GetDepot(self): + return self._depot + + def _SetDepot(self, depot_s): + depot = self._FixRoot(depot_s) + assert depot, 'Not a valid depot name: "%s".' % depot_s + self._depot = depot + + depot = property(_GetDepot, _SetDepot) + + def _GetClient(self): + return self._client + + def _SetClient(self, client_s): + client = self._FixRoot(client_s) + assert client, 'Not a valid client name: "%s".' % client_s + self._client = client + + client = property(_GetClient, _SetClient) + + def add(self, mapping): + assert type(mapping) is PathMapping + self._mappings.add(mapping) + + def discard(self, mapping): + assert type(mapping) is PathMapping + self._mappings.discard(mapping) + + def __contains__(self, value): + return value in self._mappings + + def __len__(self): + return len(self._mappings) + + def __iter__(self): + return iter(mapping for mapping in self._mappings) + + def AbsoluteMappings(self): + return iter(mapping.GetAbsolute(self.depot, self.client) + for mapping in self._mappings) + + +class CommandsFactory(object): + """Creates shell commands used for interaction with Perforce.""" + + def __init__(self, checkout_dir, p4view, name=None, port=None): + self.port = port or 'perforce2:2666' + self.view = p4view + self.view.client = name or 'p4-automation-$HOSTNAME-$JOB_ID' + self.checkout_dir = checkout_dir + self.p4config_path = os.path.join(self.checkout_dir, '.p4config') + + def Initialize(self): + return cmd.Chain('mkdir -p %s' % self.checkout_dir, 'cp ~/.p4config %s' % + self.checkout_dir, 'chmod u+w %s' % self.p4config_path, + 'echo "P4PORT=%s" >> %s' % (self.port, self.p4config_path), + 'echo "P4CLIENT=%s" >> %s' % + (self.view.client, self.p4config_path)) + + def Create(self): + # TODO(kbaclawski): Could we support value list for options consistently? + mappings = ['-a \"%s %s\"' % mapping + for mapping in self.view.AbsoluteMappings()] + + # First command will create client with default mappings. Second one will + # replace default mapping with desired. Unfortunately, it seems that it + # cannot be done in one step. P4EDITOR is defined to /bin/true because we + # don't want "g4 client" to enter real editor and wait for user actions. + return cmd.Wrapper( + cmd.Chain( + cmd.Shell('g4', 'client'), + cmd.Shell('g4', 'client', '--replace', *mappings)), + env={'P4EDITOR': '/bin/true'}) + + def SaveSpecification(self, filename=None): + return cmd.Pipe(cmd.Shell('g4', 'client', '-o'), output=filename) + + def Sync(self, revision=None): + sync_arg = '...' + if revision: + sync_arg = '%s@%s' % (sync_arg, revision) + return cmd.Shell('g4', 'sync', sync_arg) + + def SaveCurrentCLNumber(self, filename=None): + return cmd.Pipe( + cmd.Shell('g4', 'changes', '-m1', '...#have'), + cmd.Shell('sed', '-E', '"s,Change ([0-9]+) .*,\\1,"'), + output=filename) + + def Remove(self): + return cmd.Shell('g4', 'client', '-d', self.view.client) + + def SetupAndDo(self, *commands): + return cmd.Chain(self.Initialize(), + self.InCheckoutDir(self.Create(), *commands)) + + def InCheckoutDir(self, *commands): + return cmd.Wrapper(cmd.Chain(*commands), cwd=self.checkout_dir) + + def CheckoutFromSnapshot(self, snapshot): + cmds = cmd.Chain() + + for mapping in self.view: + local_path, file_part = mapping.local.rsplit('/', 1) + + if file_part == '...': + remote_dir = os.path.join(snapshot, local_path) + local_dir = os.path.join(self.checkout_dir, os.path.dirname(local_path)) + + cmds.extend([ + cmd.Shell('mkdir', '-p', local_dir), cmd.Shell( + 'rsync', '-lr', remote_dir, local_dir) + ]) + + return cmds diff --git a/deprecated/automation/clients/nightly.py b/deprecated/automation/clients/nightly.py new file mode 100755 index 00000000..d35c4eca --- /dev/null +++ b/deprecated/automation/clients/nightly.py @@ -0,0 +1,51 @@ +#!/usr/bin/python2 +# +# Copyright 2010 Google Inc. All Rights Reserved. + +import optparse +import pickle +import sys +import xmlrpclib + +from automation.clients.helper import chromeos +from automation.common import job_group + + +def Main(argv): + parser = optparse.OptionParser() + parser.add_option('-c', + '--chromeos_version', + dest='chromeos_version', + default='quarterly', + help='ChromeOS version to use.') + parser.add_option('-t', + '--toolchain', + dest='toolchain', + default='latest-toolchain', + help='Toolchain to use {latest-toolchain,gcc_46}.') + parser.add_option('-b', + '--board', + dest='board', + default='x86-generic', + help='Board to use for the nightly job.') + options = parser.parse_args(argv)[0] + + toolchain = options.toolchain + board = options.board + chromeos_version = options.chromeos_version + + # Build toolchain + jobs_factory = chromeos.JobsFactory(chromeos_version=chromeos_version, + board=board, + toolchain=toolchain) + benchmark_job = jobs_factory.BuildAndBenchmark() + + group_label = 'nightly_client_%s' % board + group = job_group.JobGroup(group_label, [benchmark_job], True, False) + + server = xmlrpclib.Server('http://localhost:8000') + server.ExecuteJobGroup(pickle.dumps(group)) + + +if __name__ == '__main__': + Main(sys.argv) diff --git a/deprecated/automation/clients/output_test.py b/deprecated/automation/clients/output_test.py new file mode 100755 index 00000000..73c26eed --- /dev/null +++ b/deprecated/automation/clients/output_test.py @@ -0,0 +1,29 @@ +#!/usr/bin/python2 +# +# Copyright 2010 Google Inc. All Rights Reserved. + +import os.path +import pickle +import sys +import xmlrpclib + +from automation.common import job +from automation.common import job_group +from automation.common import machine + + +def Main(): + server = xmlrpclib.Server('http://localhost:8000') + + command = os.path.join( + os.path.dirname(sys.argv[0]), '../../produce_output.py') + + pwd_job = job.Job('pwd_job', command) + pwd_job.DependsOnMachine(machine.MachineSpecification(os='linux')) + + group = job_group.JobGroup('pwd_client', [pwd_job]) + server.ExecuteJobGroup(pickle.dumps(group)) + + +if __name__ == '__main__': + Main() diff --git a/deprecated/automation/clients/pwd_test.py b/deprecated/automation/clients/pwd_test.py new file mode 100755 index 00000000..493444d5 --- /dev/null +++ b/deprecated/automation/clients/pwd_test.py @@ -0,0 +1,27 @@ +#!/usr/bin/python2 +# +# Copyright 2010 Google Inc. All Rights Reserved. + +import pickle +import xmlrpclib + +from automation.common import job +from automation.common import job_group +from automation.common import machine + + +def Main(): + server = xmlrpclib.Server('http://localhost:8000') + + command = ['echo These following 3 lines should be the same', 'pwd', '$(pwd)', + 'echo ${PWD}'] + + pwd_job = job.Job('pwd_job', ' && '.join(command)) + pwd_job.DependsOnMachine(machine.MachineSpecification(os='linux')) + + group = job_group.JobGroup('pwd_client', [pwd_job]) + server.ExecuteJobGroup(pickle.dumps(group)) + + +if __name__ == '__main__': + Main() diff --git a/deprecated/automation/clients/report/dejagnu.sh b/deprecated/automation/clients/report/dejagnu.sh new file mode 100755 index 00000000..fadd8a0c --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +export PYTHONPATH="$(pwd)" + +python dejagnu/main.py $@ diff --git a/deprecated/automation/clients/report/dejagnu/__init__.py b/deprecated/automation/clients/report/dejagnu/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/__init__.py @@ -0,0 +1 @@ + diff --git a/deprecated/automation/clients/report/dejagnu/main.py b/deprecated/automation/clients/report/dejagnu/main.py new file mode 100644 index 00000000..62f095e1 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/main.py @@ -0,0 +1,137 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +from contextlib import contextmanager +import glob +from itertools import chain +import logging +import optparse +import os.path +import sys + +from manifest import Manifest +import report +from summary import DejaGnuTestRun + + +def ExpandGlobExprList(paths): + """Returns an iterator that goes over expanded glob paths.""" + return chain.from_iterable(map(glob.glob, paths)) + + +@contextmanager +def OptionChecker(parser): + """Provides scoped environment for command line option checking.""" + try: + yield + except SystemExit as ex: + parser.print_help() + print '' + sys.exit('ERROR: %s' % str(ex)) + + +def ManifestCommand(argv): + parser = optparse.OptionParser( + description= + ('Read in one or more DejaGNU summary files (.sum), parse their ' + 'content and generate manifest files. Manifest files store a list ' + 'of failed tests that should be ignored. Generated files are ' + 'stored in current directory under following name: ' + '${tool}-${board}.xfail (e.g. "gcc-unix.xfail").'), + usage='Usage: %prog manifest [file.sum] (file2.sum ...)') + + _, args = parser.parse_args(argv[2:]) + + with OptionChecker(parser): + if not args: + sys.exit('At least one *.sum file required.') + + for filename in chain.from_iterable(map(glob.glob, args)): + test_run = DejaGnuTestRun.FromFile(filename) + + manifest = Manifest.FromDejaGnuTestRun(test_run) + manifest_filename = '%s-%s.xfail' % (test_run.tool, test_run.board) + + with open(manifest_filename, 'w') as manifest_file: + manifest_file.write(manifest.Generate()) + + logging.info('Wrote manifest to "%s" file.', manifest_filename) + + +def ReportCommand(argv): + parser = optparse.OptionParser( + description= + ('Read in one or more DejaGNU summary files (.sum), parse their ' + 'content and generate a single report file in selected format ' + '(currently only HTML).'), + usage=('Usage: %prog report (-m manifest.xfail) [-o report.html] ' + '[file.sum (file2.sum ...)')) + parser.add_option( + '-o', + dest='output', + type='string', + default=None, + help=('Suppress failures for test listed in provided manifest files. ' + '(use -m for each manifest file you want to read)')) + parser.add_option( + '-m', + dest='manifests', + type='string', + action='append', + default=None, + help=('Suppress failures for test listed in provided manifest files. ' + '(use -m for each manifest file you want to read)')) + + opts, args = parser.parse_args(argv[2:]) + + with OptionChecker(parser): + if not args: + sys.exit('At least one *.sum file required.') + + if not opts.output: + sys.exit('Please provide name for report file.') + + manifests = [] + + for filename in ExpandGlobExprList(opts.manifests or []): + logging.info('Using "%s" manifest.', filename) + manifests.append(Manifest.FromFile(filename)) + + test_runs = [DejaGnuTestRun.FromFile(filename) + for filename in chain.from_iterable(map(glob.glob, args))] + + html = report.Generate(test_runs, manifests) + + if html: + with open(opts.output, 'w') as html_file: + html_file.write(html) + logging.info('Wrote report to "%s" file.', opts.output) + else: + sys.exit(1) + + +def HelpCommand(argv): + sys.exit('\n'.join([ + 'Usage: %s command [options]' % os.path.basename(argv[ + 0]), '', 'Commands:', + ' manifest - manage files containing a list of suppressed test failures', + ' report - generate report file for selected test runs' + ])) + + +def Main(argv): + try: + cmd_name = argv[1] + except IndexError: + cmd_name = None + + cmd_map = {'manifest': ManifestCommand, 'report': ReportCommand} + cmd_map.get(cmd_name, HelpCommand)(argv) + + +if __name__ == '__main__': + FORMAT = '%(asctime)-15s %(levelname)s %(message)s' + logging.basicConfig(format=FORMAT, level=logging.INFO) + + Main(sys.argv) diff --git a/deprecated/automation/clients/report/dejagnu/manifest.py b/deprecated/automation/clients/report/dejagnu/manifest.py new file mode 100644 index 00000000..5831d1b0 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/manifest.py @@ -0,0 +1,103 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' + +from collections import namedtuple +from cStringIO import StringIO +import logging + +from summary import DejaGnuTestResult + + +class Manifest(namedtuple('Manifest', 'tool board results')): + """Stores a list of unsuccessful tests. + + Any line that starts with '#@' marker carries auxiliary data in form of a + key-value pair, for example: + + #@ tool: * + #@ board: unix + + So far tool and board parameters are recognized. Their value can contain + arbitrary glob expression. Based on aforementioned parameters given manifest + will be applied for all test results, but only in selected test runs. Note + that all parameters are optional. Their default value is '*' (i.e. for all + tools/boards). + + The meaning of lines above is as follows: corresponding test results to follow + should only be suppressed if test run was performed on "unix" board. + + The summary line used to build the test result should have this format: + + attrlist | UNRESOLVED: gcc.dg/unroll_1.c (test for excess errors) + ^^^^^^^^ ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^ + optional result name variant + attributes + """ + SUPPRESSIBLE_RESULTS = ['FAIL', 'UNRESOLVED', 'XPASS', 'ERROR'] + + @classmethod + def FromDejaGnuTestRun(cls, test_run): + results = [result + for result in test_run.results + if result.result in cls.SUPPRESSIBLE_RESULTS] + + return cls(test_run.tool, test_run.board, results) + + @classmethod + def FromFile(cls, filename): + """Creates manifest instance from a file in format described above.""" + params = {} + results = [] + + with open(filename, 'r') as manifest_file: + for line in manifest_file: + if line.startswith('#@'): + # parse a line with a parameter + try: + key, value = line[2:].split(':', 1) + except ValueError: + logging.warning('Malformed parameter line: "%s".', line) + else: + params[key.strip()] = value.strip() + else: + # remove comment + try: + line, _ = line.split('#', 1) + except ValueError: + pass + + line = line.strip() + + if line: + # parse a line with a test result + result = DejaGnuTestResult.FromLine(line) + + if result: + results.append(result) + else: + logging.warning('Malformed test result line: "%s".', line) + + tool = params.get('tool', '*') + board = params.get('board', '*') + + return cls(tool, board, results) + + def Generate(self): + """Dumps manifest to string.""" + text = StringIO() + + for name in ['tool', 'board']: + text.write('#@ {0}: {1}\n'.format(name, getattr(self, name))) + + text.write('\n') + + for result in sorted(self.results, key=lambda r: r.result): + text.write('{0}\n'.format(result)) + + return text.getvalue() + + def __iter__(self): + return iter(self.results) diff --git a/deprecated/automation/clients/report/dejagnu/report.html b/deprecated/automation/clients/report/dejagnu/report.html new file mode 100644 index 00000000..39b39e09 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/report.html @@ -0,0 +1,94 @@ +<link type="text/css" rel="Stylesheet" +href="http://ajax.googleapis.com/ajax/libs/jqueryui/1.8.16/themes/ui-lightness/jquery-ui.css"/> + +<script type="text/javascript" src="https://www.google.com/jsapi"></script> +<script type="text/javascript"> + google.load("visualization", "1.1", {packages: ["corechart", "table"]}); + google.load("jquery", "1.6.2"); + google.load("jqueryui", "1.8.16"); + + function drawChart(name, label, table) { + var data = google.visualization.arrayToDataTable(table); + var chart = new google.visualization.PieChart( + document.getElementById(name)); + + chart.draw(data, + {title: label, pieSliceText: "value", width: 800, height: 400}); + } + + function drawTable(name, table) { + var data = google.visualization.arrayToDataTable(table); + var table = new google.visualization.Table( + document.getElementById(name)); + + table.draw(data, { + showRowNumber: false, allowHtml: true, sortColumn: 0}); + } + + google.setOnLoadCallback(function () { + $( "#testruns" ).tabs(); + + {% for test_run in test_runs %} + $( "#testrun{{ test_run.id }}" ).tabs(); + + {% for result_type, group in test_run.groups.items %} + $( "#testrun{{ test_run.id }}-{{ result_type }}-tables" ).accordion({ + autoHeight: false, collapsible: true, active: false }); + + drawChart( + "testrun{{ test_run.id }}-{{ result_type }}-chart", + "DejaGNU test {{ result_type }} summary for {{ test_run.name }}", + [ + ["Result", "Count"], + {% for result, count in group.summary %} + ["{{ result }}", {{ count }}],{% endfor %} + ]); + + {% for description, test_list in group.tests %} + {% if test_list %} + drawTable( + "testrun{{ test_run.id }}-{{ result_type }}-table-{{ forloop.counter }}", + [ + ["Test", "Variant"], + {% for test, variant in test_list %} + ["{{ test }}", "{{ variant }}"],{% endfor %} + ]); + {% endif %} + {% endfor %} + {% endfor %} + {% endfor %} + }); +</script> + +<div id="testruns"> + <ul> + {% for test_run in test_runs %} + <li><a href="#testrun{{ test_run.id }}">{{ test_run.name }}</a></li> + {% endfor %} + </ul> + + {% for test_run in test_runs %} + <div id="testrun{{ test_run.id }}" style="padding: 0px"> + <ul> + {% for result_type, group in test_run.groups.items %} + <li> + <a href="#testrun{{ test_run.id }}-{{ forloop.counter }}">{{ result_type }}</a> + </li> + {% endfor %} + </ul> + {% for result_type, group in test_run.groups.items %} + <div id="testrun{{ test_run.id }}-{{ forloop.counter }}"> + <div id="testrun{{ test_run.id }}-{{ result_type }}-chart" style="text-align: center"></div> + <div id="testrun{{ test_run.id }}-{{ result_type }}-tables"> + {% for description, test_list in group.tests %} + {% if test_list %} + <h3><a href="#">{{ description }}</a></h3> + <div id="testrun{{ test_run.id }}-{{ result_type }}-table-{{ forloop.counter }}"></div> + {% endif %} + {% endfor %} + </div> + </div> + {% endfor %} + </div> +{% endfor %} +</div> diff --git a/deprecated/automation/clients/report/dejagnu/report.py b/deprecated/automation/clients/report/dejagnu/report.py new file mode 100644 index 00000000..191a5389 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/report.py @@ -0,0 +1,115 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +import logging +import os.path + +RESULT_DESCRIPTION = { + 'ERROR': 'DejaGNU errors', + 'FAIL': 'Failed tests', + 'NOTE': 'DejaGNU notices', + 'PASS': 'Passed tests', + 'UNRESOLVED': 'Unresolved tests', + 'UNSUPPORTED': 'Unsupported tests', + 'UNTESTED': 'Not executed tests', + 'WARNING': 'DejaGNU warnings', + 'XFAIL': 'Expected test failures', + 'XPASS': 'Unexpectedly passed tests' +} + +RESULT_GROUPS = { + 'Successes': ['PASS', 'XFAIL'], + 'Failures': ['FAIL', 'XPASS', 'UNRESOLVED'], + 'Suppressed': ['!FAIL', '!XPASS', '!UNRESOLVED', '!ERROR'], + 'Framework': ['UNTESTED', 'UNSUPPORTED', 'ERROR', 'WARNING', 'NOTE'] +} + +ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) + + +def _GetResultDescription(name): + if name.startswith('!'): + name = name[1:] + + try: + return RESULT_DESCRIPTION[name] + except KeyError: + raise ValueError('Unknown result: "%s"' % name) + + +def _PrepareSummary(res_types, summary): + + def GetResultCount(res_type): + return summary.get(res_type, 0) + + return [(_GetResultDescription(rt), GetResultCount(rt)) for rt in res_types] + + +def _PrepareTestList(res_types, tests): + + def GetTestsByResult(res_type): + return [(test.name, test.variant or '') + for test in sorted(tests) if test.result == res_type] + + return [(_GetResultDescription(rt), GetTestsByResult(rt)) + for rt in res_types if rt != 'PASS'] + + +def Generate(test_runs, manifests): + """Generate HTML report from provided test runs. + + Args: + test_runs: DejaGnuTestRun objects list. + manifests: Manifest object list that will drive test result suppression. + + Returns: + String to which the HTML report was rendered. + """ + tmpl_args = [] + + for test_run_id, test_run in enumerate(test_runs): + logging.info('Generating report for: %s.', test_run) + + test_run.CleanUpTestResults() + test_run.SuppressTestResults(manifests) + + # Generate summary and test list for each result group + groups = {} + + for res_group, res_types in RESULT_GROUPS.items(): + summary_all = _PrepareSummary(res_types, test_run.summary) + tests_all = _PrepareTestList(res_types, test_run.results) + + has_2nd = lambda tuple2: bool(tuple2[1]) + summary = filter(has_2nd, summary_all) + tests = filter(has_2nd, tests_all) + + if summary or tests: + groups[res_group] = {'summary': summary, 'tests': tests} + + tmpl_args.append({ + 'id': test_run_id, + 'name': '%s @%s' % (test_run.tool, test_run.board), + 'groups': groups + }) + + logging.info('Rendering report in HTML format.') + + try: + from django import template + from django.template import loader + from django.conf import settings + except ImportError: + logging.error('Django framework not installed!') + logging.error('Failed to generate report in HTML format!') + return '' + + settings.configure(DEBUG=True, + TEMPLATE_DEBUG=True, + TEMPLATE_DIRS=(ROOT_PATH,)) + + tmpl = loader.get_template('report.html') + ctx = template.Context({'test_runs': tmpl_args}) + + return tmpl.render(ctx) diff --git a/deprecated/automation/clients/report/dejagnu/summary.py b/deprecated/automation/clients/report/dejagnu/summary.py new file mode 100644 index 00000000..d573c691 --- /dev/null +++ b/deprecated/automation/clients/report/dejagnu/summary.py @@ -0,0 +1,262 @@ +# Copyright 2011 Google Inc. All Rights Reserved. +# Author: kbaclawski@google.com (Krystian Baclawski) +# + +from collections import defaultdict +from collections import namedtuple +from datetime import datetime +from fnmatch import fnmatch +from itertools import groupby +import logging +import os.path +import re + + +class DejaGnuTestResult(namedtuple('Result', 'name variant result flaky')): + """Stores the result of a single test case.""" + + # avoid adding __dict__ to the class + __slots__ = () + + LINE_RE = re.compile(r'([A-Z]+):\s+([\w/+.-]+)(.*)') + + @classmethod + def FromLine(cls, line): + """Alternate constructor which takes a string and parses it.""" + try: + attrs, line = line.split('|', 1) + + if attrs.strip() != 'flaky': + return None + + line = line.strip() + flaky = True + except ValueError: + flaky = False + + fields = cls.LINE_RE.match(line.strip()) + + if fields: + result, path, variant = fields.groups() + + # some of the tests are generated in build dir and are issued from there, + # because every test run is performed in randomly named tmp directory we + # need to remove random part + try: + # assume that 2nd field is a test path + path_parts = path.split('/') + + index = path_parts.index('testsuite') + path = '/'.join(path_parts[index + 1:]) + except ValueError: + path = '/'.join(path_parts) + + # Remove junk from test description. + variant = variant.strip(', ') + + substitutions = [ + # remove include paths - they contain name of tmp directory + ('-I\S+', ''), + # compress white spaces + ('\s+', ' ') + ] + + for pattern, replacement in substitutions: + variant = re.sub(pattern, replacement, variant) + + # Some tests separate last component of path by space, so actual filename + # ends up in description instead of path part. Correct that. + try: + first, rest = variant.split(' ', 1) + except ValueError: + pass + else: + if first.endswith('.o'): + path = os.path.join(path, first) + variant = rest + + # DejaGNU framework errors don't contain path part at all, so description + # part has to be reconstructed. + if not any(os.path.basename(path).endswith('.%s' % suffix) + for suffix in ['h', 'c', 'C', 'S', 'H', 'cc', 'i', 'o']): + variant = '%s %s' % (path, variant) + path = '' + + # Some tests are picked up from current directory (presumably DejaGNU + # generates some test files). Remove the prefix for these files. + if path.startswith('./'): + path = path[2:] + + return cls(path, variant or '', result, flaky=flaky) + + def __str__(self): + """Returns string representation of a test result.""" + if self.flaky: + fmt = 'flaky | ' + else: + fmt = '' + fmt += '{2}: {0}' + if self.variant: + fmt += ' {1}' + return fmt.format(*self) + + +class DejaGnuTestRun(object): + """Container for test results that were a part of single test run. + + The class stores also metadata related to the test run. + + Attributes: + board: Name of DejaGNU board, which was used to run the tests. + date: The date when the test run was started. + target: Target triple. + host: Host triple. + tool: The tool that was tested (e.g. gcc, binutils, g++, etc.) + results: a list of DejaGnuTestResult objects. + """ + + __slots__ = ('board', 'date', 'target', 'host', 'tool', 'results') + + def __init__(self, **kwargs): + assert all(name in self.__slots__ for name in kwargs) + + self.results = set() + self.date = kwargs.get('date', datetime.now()) + + for name in ('board', 'target', 'tool', 'host'): + setattr(self, name, kwargs.get(name, 'unknown')) + + @classmethod + def FromFile(cls, filename): + """Alternate constructor - reads a DejaGNU output file.""" + test_run = cls() + test_run.FromDejaGnuOutput(filename) + test_run.CleanUpTestResults() + return test_run + + @property + def summary(self): + """Returns a summary as {ResultType -> Count} dictionary.""" + summary = defaultdict(int) + + for r in self.results: + summary[r.result] += 1 + + return summary + + def _ParseBoard(self, fields): + self.board = fields.group(1).strip() + + def _ParseDate(self, fields): + self.date = datetime.strptime(fields.group(2).strip(), '%a %b %d %X %Y') + + def _ParseTarget(self, fields): + self.target = fields.group(2).strip() + + def _ParseHost(self, fields): + self.host = fields.group(2).strip() + + def _ParseTool(self, fields): + self.tool = fields.group(1).strip() + + def FromDejaGnuOutput(self, filename): + """Read in and parse DejaGNU output file.""" + + logging.info('Reading "%s" DejaGNU output file.', filename) + + with open(filename, 'r') as report: + lines = [line.strip() for line in report.readlines() if line.strip()] + + parsers = ((re.compile(r'Running target (.*)'), self._ParseBoard), + (re.compile(r'Test Run By (.*) on (.*)'), self._ParseDate), + (re.compile(r'=== (.*) tests ==='), self._ParseTool), + (re.compile(r'Target(\s+)is (.*)'), self._ParseTarget), + (re.compile(r'Host(\s+)is (.*)'), self._ParseHost)) + + for line in lines: + result = DejaGnuTestResult.FromLine(line) + + if result: + self.results.add(result) + else: + for regexp, parser in parsers: + fields = regexp.match(line) + if fields: + parser(fields) + break + + logging.debug('DejaGNU output file parsed successfully.') + logging.debug(self) + + def CleanUpTestResults(self): + """Remove certain test results considered to be spurious. + + 1) Large number of test reported as UNSUPPORTED are also marked as + UNRESOLVED. If that's the case remove latter result. + 2) If a test is performed on compiler output and for some reason compiler + fails, we don't want to report all failures that depend on the former. + """ + name_key = lambda v: v.name + results_by_name = sorted(self.results, key=name_key) + + for name, res_iter in groupby(results_by_name, key=name_key): + results = set(res_iter) + + # If DejaGnu was unable to compile a test it will create following result: + failed = DejaGnuTestResult(name, '(test for excess errors)', 'FAIL', + False) + + # If a test compilation failed, remove all results that are dependent. + if failed in results: + dependants = set(filter(lambda r: r.result != 'FAIL', results)) + + self.results -= dependants + + for res in dependants: + logging.info('Removed {%s} dependance.', res) + + # Remove all UNRESOLVED results that were also marked as UNSUPPORTED. + unresolved = [res._replace(result='UNRESOLVED') + for res in results if res.result == 'UNSUPPORTED'] + + for res in unresolved: + if res in self.results: + self.results.remove(res) + logging.info('Removed {%s} duplicate.', res) + + def _IsApplicable(self, manifest): + """Checks if test results need to be reconsidered based on the manifest.""" + check_list = [(self.tool, manifest.tool), (self.board, manifest.board)] + + return all(fnmatch(text, pattern) for text, pattern in check_list) + + def SuppressTestResults(self, manifests): + """Suppresses all test results listed in manifests.""" + + # Get a set of tests results that are going to be suppressed if they fail. + manifest_results = set() + + for manifest in filter(self._IsApplicable, manifests): + manifest_results |= set(manifest.results) + + suppressed_results = self.results & manifest_results + + for result in sorted(suppressed_results): + logging.debug('Result suppressed for {%s}.', result) + + new_result = '!' + result.result + + # Mark result suppression as applied. + manifest_results.remove(result) + + # Rewrite test result. + self.results.remove(result) + self.results.add(result._replace(result=new_result)) + + for result in sorted(manifest_results): + logging.warning('Result {%s} listed in manifest but not suppressed.', + result) + + def __str__(self): + return '{0}, {1} @{2} on {3}'.format(self.target, self.tool, self.board, + self.date) diff --git a/deprecated/automation/clients/report/validate_failures.py b/deprecated/automation/clients/report/validate_failures.py new file mode 100755 index 00000000..d8776ba5 --- /dev/null +++ b/deprecated/automation/clients/report/validate_failures.py @@ -0,0 +1,239 @@ +#!/usr/bin/python2 + +# Script to compare testsuite failures against a list of known-to-fail +# tests. + +# Contributed by Diego Novillo <dnovillo@google.com> +# Overhaul by Krystian Baclawski <kbaclawski@google.com> +# +# Copyright (C) 2011 Free Software Foundation, Inc. +# +# This file is part of GCC. +# +# GCC is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# GCC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GCC; see the file COPYING. If not, write to +# the Free Software Foundation, 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. +"""This script provides a coarser XFAILing mechanism that requires no +detailed DejaGNU markings. This is useful in a variety of scenarios: + +- Development branches with many known failures waiting to be fixed. +- Release branches with known failures that are not considered + important for the particular release criteria used in that branch. + +The script must be executed from the toplevel build directory. When +executed it will: + +1) Determine the target built: TARGET +2) Determine the source directory: SRCDIR +3) Look for a failure manifest file in + <SRCDIR>/contrib/testsuite-management/<TARGET>.xfail +4) Collect all the <tool>.sum files from the build tree. +5) Produce a report stating: + a) Failures expected in the manifest but not present in the build. + b) Failures in the build not expected in the manifest. +6) If all the build failures are expected in the manifest, it exits + with exit code 0. Otherwise, it exits with error code 1. +""" + +import optparse +import logging +import os +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from dejagnu.manifest import Manifest +from dejagnu.summary import DejaGnuTestResult +from dejagnu.summary import DejaGnuTestRun + +# Pattern for naming manifest files. The first argument should be +# the toplevel GCC source directory. The second argument is the +# target triple used during the build. +_MANIFEST_PATH_PATTERN = '%s/contrib/testsuite-management/%s.xfail' + + +def GetMakefileVars(makefile_path): + assert os.path.exists(makefile_path) + + with open(makefile_path) as lines: + kvs = [line.split('=', 1) for line in lines if '=' in line] + + return dict((k.strip(), v.strip()) for k, v in kvs) + + +def GetSumFiles(build_dir): + summaries = [] + + for root, _, filenames in os.walk(build_dir): + summaries.extend([os.path.join(root, filename) + for filename in filenames if filename.endswith('.sum')]) + + return map(os.path.normpath, summaries) + + +def ValidBuildDirectory(build_dir, target): + mandatory_paths = [build_dir, os.path.join(build_dir, 'Makefile')] + + extra_paths = [os.path.join(build_dir, target), + os.path.join(build_dir, 'build-%s' % target)] + + return (all(map(os.path.exists, mandatory_paths)) and + any(map(os.path.exists, extra_paths))) + + +def GetManifestPath(build_dir): + makefile = GetMakefileVars(os.path.join(build_dir, 'Makefile')) + srcdir = makefile['srcdir'] + target = makefile['target'] + + if not ValidBuildDirectory(build_dir, target): + target = makefile['target_alias'] + + if not ValidBuildDirectory(build_dir, target): + logging.error('%s is not a valid GCC top level build directory.', build_dir) + sys.exit(1) + + logging.info('Discovered source directory: "%s"', srcdir) + logging.info('Discovered build target: "%s"', target) + + return _MANIFEST_PATH_PATTERN % (srcdir, target) + + +def CompareResults(manifest, actual): + """Compare sets of results and return two lists: + - List of results present in MANIFEST but missing from ACTUAL. + - List of results present in ACTUAL but missing from MANIFEST. + """ + # Report all the actual results not present in the manifest. + actual_vs_manifest = actual - manifest + + # Filter out tests marked flaky. + manifest_without_flaky_tests = set(filter(lambda result: not result.flaky, + manifest)) + + # Simlarly for all the tests in the manifest. + manifest_vs_actual = manifest_without_flaky_tests - actual + + return actual_vs_manifest, manifest_vs_actual + + +def LogResults(level, results): + log_fun = getattr(logging, level) + + for num, result in enumerate(sorted(results), start=1): + log_fun(' %d) %s', num, result) + + +def CheckExpectedResults(manifest_path, build_dir): + logging.info('Reading manifest file: "%s"', manifest_path) + + manifest = set(Manifest.FromFile(manifest_path)) + + logging.info('Getting actual results from build directory: "%s"', + os.path.realpath(build_dir)) + + summaries = GetSumFiles(build_dir) + + actual = set() + + for summary in summaries: + test_run = DejaGnuTestRun.FromFile(summary) + failures = set(Manifest.FromDejaGnuTestRun(test_run)) + actual.update(failures) + + if manifest: + logging.debug('Tests expected to fail:') + LogResults('debug', manifest) + + if actual: + logging.debug('Actual test failures:') + LogResults('debug', actual) + + actual_vs_manifest, manifest_vs_actual = CompareResults(manifest, actual) + + if actual_vs_manifest: + logging.info('Build results not in the manifest:') + LogResults('info', actual_vs_manifest) + + if manifest_vs_actual: + logging.info('Manifest results not present in the build:') + LogResults('info', manifest_vs_actual) + logging.info('NOTE: This is not a failure! ', + 'It just means that the manifest expected these tests to ' + 'fail, but they worked in this configuration.') + + if actual_vs_manifest or manifest_vs_actual: + sys.exit(1) + + logging.info('No unexpected failures.') + + +def ProduceManifest(manifest_path, build_dir, overwrite): + if os.path.exists(manifest_path) and not overwrite: + logging.error('Manifest file "%s" already exists.', manifest_path) + logging.error('Use --force to overwrite.') + sys.exit(1) + + testruns = map(DejaGnuTestRun.FromFile, GetSumFiles(build_dir)) + manifests = map(Manifest.FromDejaGnuTestRun, testruns) + + with open(manifest_path, 'w') as manifest_file: + manifest_strings = [manifest.Generate() for manifest in manifests] + logging.info('Writing manifest to "%s".', manifest_path) + manifest_file.write('\n'.join(manifest_strings)) + + +def Main(argv): + parser = optparse.OptionParser(usage=__doc__) + parser.add_option( + '-b', + '--build_dir', + dest='build_dir', + action='store', + metavar='PATH', + default=os.getcwd(), + help='Build directory to check. (default: current directory)') + parser.add_option('-m', + '--manifest', + dest='manifest', + action='store_true', + help='Produce the manifest for the current build.') + parser.add_option( + '-f', + '--force', + dest='force', + action='store_true', + help=('Overwrite an existing manifest file, if user requested creating ' + 'new one. (default: False)')) + parser.add_option('-v', + '--verbose', + dest='verbose', + action='store_true', + help='Increase verbosity.') + options, _ = parser.parse_args(argv[1:]) + + if options.verbose: + logging.root.setLevel(logging.DEBUG) + + manifest_path = GetManifestPath(options.build_dir) + + if options.manifest: + ProduceManifest(manifest_path, options.build_dir, options.force) + else: + CheckExpectedResults(manifest_path, options.build_dir) + + +if __name__ == '__main__': + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + Main(sys.argv) |