diff options
author | Luis Lozano <llozano@chromium.org> | 2015-12-15 13:49:30 -0800 |
---|---|---|
committer | Luis Lozano <llozano@chromium.org> | 2015-12-16 17:36:06 +0000 |
commit | f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe (patch) | |
tree | 185d243c7eed7c7a0db6f0e640746cadc1479ea9 /automation/clients | |
parent | 2a66f70fef907c1cb15229cb58e5129cb620ac98 (diff) | |
download | toolchain-utils-f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe.tar.gz |
Run pyformat on all the toolchain-utils files.
This gets rid of a lot of lint issues.
Ran by doing this:
for f in *.py; do echo -n "$f " ; if [ -x $f ]; then pyformat -i
--remove_trailing_comma --yapf --force_quote_type=double $f ; else
pyformat -i --remove_shebang --remove_trailing_comma --yapf
--force_quote_type=double $f ; fi ; done
BUG=chromium:567921
TEST=Ran simple crosperf run.
Change-Id: I59778835fdaa5f706d2e1765924389f9e97433d1
Reviewed-on: https://chrome-internal-review.googlesource.com/242031
Reviewed-by: Luis Lozano <llozano@chromium.org>
Commit-Queue: Luis Lozano <llozano@chromium.org>
Tested-by: Luis Lozano <llozano@chromium.org>
Reviewed-by: Yunlian Jiang <yunlian@google.com>
Diffstat (limited to 'automation/clients')
-rw-r--r-- | automation/clients/__init__.py | 1 | ||||
-rwxr-xr-x | automation/clients/android.py | 1 | ||||
-rwxr-xr-x | automation/clients/chromeos.py | 12 | ||||
-rwxr-xr-x | automation/clients/crosstool.py | 30 | ||||
-rwxr-xr-x | automation/clients/dejagnu_compiler.py | 42 | ||||
-rw-r--r-- | automation/clients/helper/android.py | 101 | ||||
-rw-r--r-- | automation/clients/helper/chromeos.py | 145 | ||||
-rw-r--r-- | automation/clients/helper/crosstool.py | 85 | ||||
-rw-r--r-- | automation/clients/helper/jobs.py | 8 | ||||
-rw-r--r-- | automation/clients/helper/perforce.py | 37 | ||||
-rwxr-xr-x | automation/clients/nightly.py | 39 | ||||
-rwxr-xr-x | automation/clients/output_test.py | 14 | ||||
-rwxr-xr-x | automation/clients/pwd_test.py | 16 | ||||
-rw-r--r-- | automation/clients/report/dejagnu/__init__.py | 1 | ||||
-rw-r--r-- | automation/clients/report/dejagnu/main.py | 49 | ||||
-rw-r--r-- | automation/clients/report/dejagnu/manifest.py | 5 | ||||
-rw-r--r-- | automation/clients/report/dejagnu/report.py | 23 | ||||
-rw-r--r-- | automation/clients/report/dejagnu/summary.py | 23 | ||||
-rwxr-xr-x | automation/clients/report/validate_failures.py | 48 |
19 files changed, 342 insertions, 338 deletions
diff --git a/automation/clients/__init__.py b/automation/clients/__init__.py index e69de29b..8b137891 100644 --- a/automation/clients/__init__.py +++ b/automation/clients/__init__.py @@ -0,0 +1 @@ + diff --git a/automation/clients/android.py b/automation/clients/android.py index a65cba2e..603744b1 100755 --- a/automation/clients/android.py +++ b/automation/clients/android.py @@ -1,7 +1,6 @@ #!/usr/bin/python # # Copyright 2011 Google Inc. All Rights Reserved. - """Client for Android nightly jobs. Does the following jobs: diff --git a/automation/clients/chromeos.py b/automation/clients/chromeos.py index 18b29b7c..084f7840 100755 --- a/automation/clients/chromeos.py +++ b/automation/clients/chromeos.py @@ -1,12 +1,10 @@ #!/usr/bin/python # # Copyright 2011 Google Inc. All Rights Reserved. - """chromeos.py: Build & Test ChromeOS using custom compilers.""" __author__ = 'asharif@google.com (Ahmad Sharif)' - import logging import optparse import os @@ -38,7 +36,8 @@ class ChromeOSNightlyClient(object): def CheckoutV14Dir(self): p4view = perforce.View(self.DEPOT2_DIR, [ - perforce.PathMapping('gcctools/chromeos/v14/...')]) + perforce.PathMapping('gcctools/chromeos/v14/...') + ]) return self.GetP4Snapshot(p4view) def GetP4Snapshot(self, p4view): @@ -56,12 +55,12 @@ class ChromeOSNightlyClient(object): os.path.join(self.P4_VERSION_DIR, 'test_toolchains.py'), '--force-mismatch', '--clean', - '--public', # crbug.com/145822 + '--public', # crbug.com/145822 '--board=%s' % self._board, '--remote=%s' % self._remote, '--githashes=%s' % self._gcc_githash)) label = 'testlabel' - job = jobs.CreateLinuxJob(label, chain, timeout=24*60*60) + job = jobs.CreateLinuxJob(label, chain, timeout=24 * 60 * 60) return job_group.JobGroup(label, [job], True, False) @@ -92,7 +91,8 @@ def Main(argv): logging.error('Specify a board, remote and gcc_githash') return 1 - client = ChromeOSNightlyClient(options.board, options.remote, + client = ChromeOSNightlyClient(options.board, + options.remote, options.gcc_githash, p4_snapshot=options.p4_snapshot) client.Run() diff --git a/automation/clients/crosstool.py b/automation/clients/crosstool.py index 450d6030..65720343 100755 --- a/automation/clients/crosstool.py +++ b/automation/clients/crosstool.py @@ -43,8 +43,8 @@ class CrosstoolNightlyClient(object): all_jobs = [checkout_crosstool_job] # Build crosstool target - build_release_job, build_tree_dir = factory.BuildRelease( - checkout_dir, self._target) + build_release_job, build_tree_dir = factory.BuildRelease(checkout_dir, + self._target) all_jobs.append(build_release_job) testruns = [] @@ -52,8 +52,8 @@ class CrosstoolNightlyClient(object): # Perform crosstool tests for board in self._boards: for component in ('gcc', 'binutils'): - test_job, testrun_dir = factory.RunTests( - checkout_dir, build_tree_dir, self._target, board, component) + test_job, testrun_dir = factory.RunTests(checkout_dir, build_tree_dir, + self._target, board, component) all_jobs.append(test_job) testruns.append(testrun_dir) @@ -61,8 +61,8 @@ class CrosstoolNightlyClient(object): all_jobs.append(factory.GenerateReport(testruns, manifests_dir, self._target, self._boards)) - return job_group.JobGroup( - 'Crosstool Nightly Build (%s)' % self._target, all_jobs, True, False) + return job_group.JobGroup('Crosstool Nightly Build (%s)' % self._target, + all_jobs, True, False) @logger.HandleUncaughtExceptions @@ -70,14 +70,14 @@ def Main(argv): valid_boards_string = ', '.join(CrosstoolNightlyClient.VALID_BOARDS) parser = optparse.OptionParser() - parser.add_option('-b', - '--board', - dest='boards', - action='append', - choices=CrosstoolNightlyClient.VALID_BOARDS, - default=[], - help=('Run DejaGNU tests on selected boards: %s.' % - valid_boards_string)) + parser.add_option( + '-b', + '--board', + dest='boards', + action='append', + choices=CrosstoolNightlyClient.VALID_BOARDS, + default=[], + help=('Run DejaGNU tests on selected boards: %s.' % valid_boards_string)) options, args = parser.parse_args(argv) if len(args) == 2: @@ -86,7 +86,7 @@ def Main(argv): logging.error('Exactly one target required as a command line argument!') logging.info('List of valid targets:') for pair in enumerate(CrosstoolNightlyClient.VALID_TARGETS, start=1): - logging.info('%d) %s' % pair) + logging.info('%d) %s', pair) sys.exit(1) option_list = [opt.dest for opt in parser.option_list if opt.dest] diff --git a/automation/clients/dejagnu_compiler.py b/automation/clients/dejagnu_compiler.py index e4342160..eb923d5d 100755 --- a/automation/clients/dejagnu_compiler.py +++ b/automation/clients/dejagnu_compiler.py @@ -1,12 +1,10 @@ #!/usr/bin/python # # Copyright 2012 Google Inc. All Rights Reserved. - """dejagnu_compiler.py: Run dejagnu test.""" __author__ = 'shenhan@google.com (Han Shen)' - import logging import optparse import os @@ -38,7 +36,8 @@ class DejagnuCompilerNightlyClient: def CheckoutV14Dir(self): p4view = perforce.View(self.DEPOT2_DIR, [ - perforce.PathMapping('gcctools/chromeos/v14/...')]) + perforce.PathMapping('gcctools/chromeos/v14/...') + ]) return self.GetP4Snapshot(p4view) def GetP4Snapshot(self, p4view): @@ -50,30 +49,37 @@ class DejagnuCompilerNightlyClient: return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) def CreateJobGroup(self): - chain = cmd.Chain( - self.CheckoutV14Dir(), - cmd.Shell('python', - os.path.join(self.P4_VERSION_DIR, 'test_gcc_dejagnu.py'), - '--board=%s' % self._board, - '--remote=%s' % self._remote, - '--cleanup=%s' % self._cleanup)) + chain = cmd.Chain(self.CheckoutV14Dir(), cmd.Shell( + 'python', os.path.join(self.P4_VERSION_DIR, 'test_gcc_dejagnu.py'), + '--board=%s' % self._board, '--remote=%s' % self._remote, + '--cleanup=%s' % self._cleanup)) label = 'dejagnu' - job = jobs.CreateLinuxJob(label, chain, timeout=8*60*60) - return job_group.JobGroup(label, [job], cleanup_on_failure=True, + job = jobs.CreateLinuxJob(label, chain, timeout=8 * 60 * 60) + return job_group.JobGroup(label, + [job], + cleanup_on_failure=True, cleanup_on_completion=True) @logger.HandleUncaughtExceptions def Main(argv): parser = optparse.OptionParser() - parser.add_option('-b', '--board', dest='board', + parser.add_option('-b', + '--board', + dest='board', help='Run performance tests on these boards') - parser.add_option('-r', '--remote', dest='remote', + parser.add_option('-r', + '--remote', + dest='remote', help='Run performance tests on these remotes') - parser.add_option('-p', '--p4_snapshot', dest='p4_snapshot', + parser.add_option('-p', + '--p4_snapshot', + dest='p4_snapshot', help=('For development only. ' 'Use snapshot instead of checking out.')) - parser.add_option('--cleanup', dest='cleanup', default='mount', + parser.add_option('--cleanup', + dest='cleanup', + default='mount', help=('Cleanup test directory, values could be one of ' '"mount", "chroot" or "chromeos"')) options, _ = parser.parse_args(argv) @@ -82,8 +88,8 @@ def Main(argv): logging.error('Specify a board and remote.') return 1 - client = DejagnuCompilerNightlyClient( - options.board, options.remote, options.p4_snapshot, options.cleanup) + client = DejagnuCompilerNightlyClient(options.board, options.remote, + options.p4_snapshot, options.cleanup) client.Run() return 0 diff --git a/automation/clients/helper/android.py b/automation/clients/helper/android.py index fc068aca..7ff2ac1c 100644 --- a/automation/clients/helper/android.py +++ b/automation/clients/helper/android.py @@ -1,7 +1,4 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. - """Helper modules for Android toolchain test infrastructure. Provides following Android toolchain test jobs and commands. @@ -24,6 +21,7 @@ from automation.common import job class JobsFactory(object): + def __init__(self, gcc_version='4.4.3', build_type='DEVELOPMENT'): assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] assert build_type in ['DEVELOPMENT', 'RELEASE'] @@ -46,11 +44,13 @@ class JobsFactory(object): new_job = jobs.CreateLinuxJob('AndroidBuildToolchain(%s)' % self.tc_tag, command) new_job.DependsOnFolder(checkout_dir_dep) - tc_prefix_dep = job.FolderDependency( - new_job, self.commands.toolchain_prefix_dir) + tc_prefix_dep = job.FolderDependency(new_job, + self.commands.toolchain_prefix_dir) return new_job, tc_prefix_dep - def BuildAndroidImage(self, tc_prefix_dep, product='stingray', + def BuildAndroidImage(self, + tc_prefix_dep, + product='stingray', branch='ics-release'): assert product in ['stingray', 'passion', 'trygon', 'soju'] assert branch in ['honeycomb-release', 'ics-release'] @@ -89,8 +89,7 @@ class CommandsFactory(object): self.gcc_version = gcc_version self.binutils_version = '2.21' self.gold_version = '2.21' - self.toolchain_prefix_dir = 'install-gcc-%s-%s' % ( - gcc_version, build_type) + self.toolchain_prefix_dir = 'install-gcc-%s-%s' % (gcc_version, build_type) self.p4client = self._CreatePerforceClient() self.scripts = ScriptsFactory(self.gcc_version, self.binutils_version, self.gold_version) @@ -105,11 +104,9 @@ class CommandsFactory(object): 'google_vendor_src_branch') # Common views for tools - p4view = perforce.View('depot2', - perforce.PathMapping.ListFromPathTuples( - [('gcctools/android/build/...', 'src/build/...'), - ('gcctools/android/Tarballs/...', - 'src/tarballs/...')])) + p4view = perforce.View('depot2', perforce.PathMapping.ListFromPathTuples([( + 'gcctools/android/build/...', 'src/build/...'), ( + 'gcctools/android/Tarballs/...', 'src/tarballs/...')])) for mapping in perforce.PathMapping.ListFromPathDict( {'gcctools/android': ['tools/scripts/...', 'master/...']}): p4view.add(mapping) @@ -123,13 +120,11 @@ class CommandsFactory(object): binutils_branch = mobile_rel_branch else: binutils_branch = p4_dev_path - p4view.add(perforce.PathMapping(binutils_branch, 'src', - ('binutils/binutils-%s/...' % - self.binutils_version))) + p4view.add(perforce.PathMapping(binutils_branch, 'src', ( + 'binutils/binutils-%s/...' % self.binutils_version))) if self.binutils_version != self.gold_version: - p4view.add(perforce.PathMapping(binutils_branch, 'src', - ('binutils/binutils-%s/...' % - self.gold_version))) + p4view.add(perforce.PathMapping(binutils_branch, 'src', ( + 'binutils/binutils-%s/...' % self.gold_version))) # Add view for gcc if gcc_version is '4.4.3'. if self.gcc_version == '4.4.3': @@ -172,9 +167,11 @@ class CommandsFactory(object): gcc_required_dir = os.path.join(self.TOOLCHAIN_SRC_DIR, 'gcc', 'gcc-%s' % self.gcc_version) - return cmd.Chain(cmd.MakeDir(gcc_required_dir), - cmd.Wrapper(cmd.Chain(svn_co_command, svn_get_revision), - cwd=gcc_required_dir)) + return cmd.Chain( + cmd.MakeDir(gcc_required_dir), + cmd.Wrapper( + cmd.Chain(svn_co_command, svn_get_revision), + cwd=gcc_required_dir)) def CheckoutAndroidToolchain(self): p4client = self.p4client @@ -187,25 +184,24 @@ class CommandsFactory(object): return command def BuildAndroidToolchain(self): - script_cmd = self.scripts.BuildAndroidToolchain(self.toolchain_prefix_dir, - self.CHECKOUT_DIR, - self.TOOLCHAIN_BUILD_DIR, - self.TOOLCHAIN_SRC_DIR) + script_cmd = self.scripts.BuildAndroidToolchain( + self.toolchain_prefix_dir, self.CHECKOUT_DIR, self.TOOLCHAIN_BUILD_DIR, + self.TOOLCHAIN_SRC_DIR) # Record toolchain and gcc CL number - record_cl_cmd = cmd.Copy(os.path.join(self.CHECKOUT_DIR, 'CLNUM*'), - to_dir=self.toolchain_prefix_dir) - save_cmd = cmd.Tar(os.path.join('$JOB_TMP', 'results', - '%s.tar.bz2' % self.toolchain_prefix_dir), - self.toolchain_prefix_dir) + record_cl_cmd = cmd.Copy( + os.path.join(self.CHECKOUT_DIR, 'CLNUM*'), + to_dir=self.toolchain_prefix_dir) + save_cmd = cmd.Tar( + os.path.join('$JOB_TMP', 'results', '%s.tar.bz2' % + self.toolchain_prefix_dir), self.toolchain_prefix_dir) return cmd.Chain(script_cmd, record_cl_cmd, save_cmd) def _BuildAndroidTree(self, local_android_branch_dir, product): target_tools_prefix = os.path.join('$JOB_TMP', self.toolchain_prefix_dir, 'bin', 'arm-linux-androideabi-') java_path = '/usr/lib/jvm/java-6-sun/bin' - build_cmd = cmd.Shell('make', '-j8', - 'PRODUCT-%s-userdebug' % product, + build_cmd = cmd.Shell('make', '-j8', 'PRODUCT-%s-userdebug' % product, 'TARGET_TOOLS_PREFIX=%s' % target_tools_prefix, 'PATH=%s:$PATH' % java_path) return cmd.Wrapper(build_cmd, cwd=local_android_branch_dir) @@ -219,9 +215,8 @@ class CommandsFactory(object): 'android_trees') remote_android_branch_path = os.path.join(androidtrees_path, branch) local_android_branch_dir = os.path.join(self.ANDROID_TREES_DIR, branch) - gettree_cmd = cmd.RemoteCopyFrom(androidtrees_host, - remote_android_branch_path, - local_android_branch_dir) + gettree_cmd = cmd.RemoteCopyFrom( + androidtrees_host, remote_android_branch_path, local_android_branch_dir) # Configure and build the tree buildtree_cmd = self._BuildAndroidTree(local_android_branch_dir, product) @@ -235,8 +230,9 @@ class CommandsFactory(object): return cmd.Chain(gettree_cmd, buildtree_cmd, copy_img, compress_img) def CheckoutScripts(self): - p4view = perforce.View('depot2', [perforce.PathMapping( - 'gcctools/android/tools/...', 'tools/...')]) + p4view = perforce.View('depot2', + [perforce.PathMapping('gcctools/android/tools/...', + 'tools/...')]) p4client = perforce.CommandsFactory(self.TOOLS_DIR, p4view) return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) @@ -246,19 +242,15 @@ class CommandsFactory(object): base_benchbin_path = ('/usr/local/google2/home/mobiletc-prebuild/' 'archive/v3binaries/2011-10-18') local_basebenchbin_dir = 'base_benchmark_bin' - getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host, - base_benchbin_path, + getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host, base_benchbin_path, local_basebenchbin_dir) # Build and run benchmark. android_arch = 'android_%s' % arch run_label = 'normal' - benchmark_cmd = self.scripts.RunBenchmark(self.toolchain_prefix_dir, - self.TOOLS_DIR, - self.BENCHMARK_OUT_DIR, - run_label, run_experiment, - android_arch, - local_basebenchbin_dir) + benchmark_cmd = self.scripts.RunBenchmark( + self.toolchain_prefix_dir, self.TOOLS_DIR, self.BENCHMARK_OUT_DIR, + run_label, run_experiment, android_arch, local_basebenchbin_dir) # Extract jobid from BENCHMARK_OUT_DIR/log/jobid_normal.log file. # Copy jobid to www server to generate performance dashboard. @@ -268,6 +260,7 @@ class CommandsFactory(object): class ScriptsFactory(object): + def __init__(self, gcc_version, binutils_version, gold_version): self._gcc_version = gcc_version self._binutils_version = binutils_version @@ -292,14 +285,20 @@ class ScriptsFactory(object): '--with-gold-version=%s' % self._gold_version, '--with-gdb-version=7.1.x-android', '--log-path=%s/logs' % '$JOB_HOME', - '--android-sysroot=%s' % - os.path.join('$JOB_TMP', checkout_dir, 'gcctools', 'android', - 'master', 'honeycomb_generic_sysroot'), + '--android-sysroot=%s' % os.path.join('$JOB_TMP', checkout_dir, + 'gcctools', 'android', 'master', + 'honeycomb_generic_sysroot'), path=os.path.join(checkout_dir, 'gcctools', 'android', 'tools', 'scripts')) - def RunBenchmark(self, toolchain_prefix_dir, checkout_dir, output_dir, - run_label, run_experiment, arch, base_bench_bin=None): + def RunBenchmark(self, + toolchain_prefix_dir, + checkout_dir, + output_dir, + run_label, + run_experiment, + arch, + base_bench_bin=None): if base_bench_bin: base_bench_opt = '--base_benchmark_bin=%s' % base_bench_bin else: diff --git a/automation/clients/helper/chromeos.py b/automation/clients/helper/chromeos.py index ddfd59fe..e7157451 100644 --- a/automation/clients/helper/chromeos.py +++ b/automation/clients/helper/chromeos.py @@ -1,8 +1,6 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. -__author__ = "asharif@google.com (Ahmad Sharif)" +__author__ = 'asharif@google.com (Ahmad Sharif)' import os.path import re @@ -14,64 +12,54 @@ from automation.common import machine class ScriptsFactory(object): + def __init__(self, chromeos_root, scripts_path): self._chromeos_root = chromeos_root self._scripts_path = scripts_path def SummarizeResults(self, logs_path): - return cmd.Shell("summarize_results.py", - logs_path, - path=self._scripts_path) + return cmd.Shell('summarize_results.py', logs_path, path=self._scripts_path) def Buildbot(self, config_name): buildbot = os.path.join(self._chromeos_root, - "chromite/cbuildbot/cbuildbot.py") - - return cmd.Shell(buildbot, - "--buildroot=%s" % self._chromeos_root, - "--resume", - "--noarchive", - "--noprebuilts", - "--nosync", - "--nouprev", - "--notests", - "--noclean", - config_name) + 'chromite/cbuildbot/cbuildbot.py') + + return cmd.Shell(buildbot, '--buildroot=%s' % self._chromeos_root, + '--resume', '--noarchive', '--noprebuilts', '--nosync', + '--nouprev', '--notests', '--noclean', config_name) def RunBenchmarks(self, board, tests): - image_path = os.path.join(self._chromeos_root, - "src/build/images", - board, - "latest/chromiumos_image.bin") - - return cmd.Shell("cros_run_benchmarks.py", - "--remote=$SECONDARY_MACHINES[0]", - "--board=%s" % board, - "--tests=%s" % tests, - "--full_table", + image_path = os.path.join(self._chromeos_root, 'src/build/images', board, + 'latest/chromiumos_image.bin') + + return cmd.Shell('cros_run_benchmarks.py', + '--remote=$SECONDARY_MACHINES[0]', + '--board=%s' % board, + '--tests=%s' % tests, + '--full_table', image_path, - path="/home/mobiletc-prebuild") + path='/home/mobiletc-prebuild') - def SetupChromeOS(self, version="latest", use_minilayout=False): - setup_chromeos = cmd.Shell("setup_chromeos.py", - "--public", - "--dir=%s" % self._chromeos_root, - "--version=%s" % version, + def SetupChromeOS(self, version='latest', use_minilayout=False): + setup_chromeos = cmd.Shell('setup_chromeos.py', + '--public', + '--dir=%s' % self._chromeos_root, + '--version=%s' % version, path=self._scripts_path) if use_minilayout: - setup_chromeos.AddOption("--minilayout") + setup_chromeos.AddOption('--minilayout') return setup_chromeos class CommandsFactory(object): - DEPOT2_DIR = "//depot2/" - P4_CHECKOUT_DIR = "perforce2/" - P4_VERSION_DIR = os.path.join(P4_CHECKOUT_DIR, "gcctools/chromeos/v14") + DEPOT2_DIR = '//depot2/' + P4_CHECKOUT_DIR = 'perforce2/' + P4_VERSION_DIR = os.path.join(P4_CHECKOUT_DIR, 'gcctools/chromeos/v14') - CHROMEOS_ROOT = "chromeos" - CHROMEOS_SCRIPTS_DIR = os.path.join(CHROMEOS_ROOT, "src/scripts") - CHROMEOS_BUILDS_DIR = "/home/mobiletc-prebuild/www/chromeos_builds" + CHROMEOS_ROOT = 'chromeos' + CHROMEOS_SCRIPTS_DIR = os.path.join(CHROMEOS_ROOT, 'src/scripts') + CHROMEOS_BUILDS_DIR = '/home/mobiletc-prebuild/www/chromeos_builds' def __init__(self, chromeos_version, board, toolchain, p4_snapshot): self.chromeos_version = chromeos_version @@ -82,33 +70,34 @@ class CommandsFactory(object): self.scripts = ScriptsFactory(self.CHROMEOS_ROOT, self.P4_VERSION_DIR) def AddBuildbotConfig(self, config_name, config_list): - config_header = "add_config(%r, [%s])" % (config_name, - ", ".join(config_list)) + config_header = 'add_config(%r, [%s])' % (config_name, + ', '.join(config_list)) config_file = os.path.join(self.CHROMEOS_ROOT, - "chromite/cbuildbot/cbuildbot_config.py") - quoted_config_header = "%r" % config_header + 'chromite/cbuildbot/cbuildbot_config.py') + quoted_config_header = '%r' % config_header quoted_config_header = re.sub("'", "\\\"", quoted_config_header) - return cmd.Pipe(cmd.Shell("echo", quoted_config_header), - cmd.Shell("tee", "--append", config_file)) + return cmd.Pipe( + cmd.Shell('echo', quoted_config_header), + cmd.Shell('tee', '--append', config_file)) def RunBuildbot(self): - config_dict = {"board": self.board, - "build_tests": True, - "chrome_tests": True, - "unittests": False, - "vm_tests": False, - "prebuilts": False, - "latest_toolchain": True, - "useflags": ["chrome_internal"], - "usepkg_chroot": True, + config_dict = {'board': self.board, + 'build_tests': True, + 'chrome_tests': True, + 'unittests': False, + 'vm_tests': False, + 'prebuilts': False, + 'latest_toolchain': True, + 'useflags': ['chrome_internal'], + 'usepkg_chroot': True, self.toolchain: True} - config_name = "%s-toolchain-test" % self.board - if "arm" in self.board: - config_list = ["arm"] + config_name = '%s-toolchain-test' % self.board + if 'arm' in self.board: + config_list = ['arm'] else: config_list = [] - config_list.extend(["internal", "full", "official", str(config_dict)]) + config_list.extend(['internal', 'full', 'official', str(config_dict)]) add_config_shell = self.AddBuildbotConfig(config_name, config_list) return cmd.Chain(add_config_shell, self.scripts.Buildbot(config_name)) @@ -118,7 +107,7 @@ class CommandsFactory(object): self.CheckoutV14Dir(), self.SetupChromeOSCheckout(self.chromeos_version, True), self.RunBuildbot(), - self.scripts.RunBenchmarks(self.board, "BootPerfServer,10:Page,3")) + self.scripts.RunBenchmarks(self.board, 'BootPerfServer,10:Page,3')) def GetP4Snapshot(self, p4view): p4client = perforce.CommandsFactory(self.P4_CHECKOUT_DIR, p4view) @@ -130,38 +119,43 @@ class CommandsFactory(object): def CheckoutV14Dir(self): p4view = perforce.View(self.DEPOT2_DIR, [ - perforce.PathMapping("gcctools/chromeos/v14/...")]) + perforce.PathMapping('gcctools/chromeos/v14/...') + ]) return self.GetP4Snapshot(p4view) def SetupChromeOSCheckout(self, version, use_minilayout=False): - version_re = "^\d+\.\d+\.\d+\.[a-zA-Z0-9]+$" + version_re = '^\d+\.\d+\.\d+\.[a-zA-Z0-9]+$' location = os.path.join(self.CHROMEOS_BUILDS_DIR, version) - if version in ["weekly", "quarterly"]: - assert os.path.islink(location), "Symlink %s does not exist." % location + if version in ['weekly', 'quarterly']: + assert os.path.islink(location), 'Symlink %s does not exist.' % location location_expanded = os.path.abspath(os.path.realpath(location)) version = os.path.basename(location_expanded) - if version in ["top", "latest"] or re.match(version_re, version): + if version in ['top', 'latest'] or re.match(version_re, version): return self.scripts.SetupChromeOS(version, use_minilayout) - elif version.endswith("bz2") or version.endswith("gz"): + elif version.endswith('bz2') or version.endswith('gz'): return cmd.UnTar(location_expanded, self.CHROMEOS_ROOT) else: signature_file_location = os.path.join(location, - "src/scripts/enter_chroot.sh") + 'src/scripts/enter_chroot.sh') assert os.path.exists(signature_file_location), ( - "Signature file %s does not exist." % signature_file_location) + 'Signature file %s does not exist.' % signature_file_location) return cmd.Copy(location, to_dir=self.CHROMEOS_ROOT, recursive=True) class JobsFactory(object): - def __init__(self, chromeos_version="top", board="x86-mario", - toolchain="trunk", p4_snapshot=""): + + def __init__(self, + chromeos_version='top', + board='x86-mario', + toolchain='trunk', + p4_snapshot=''): self.chromeos_version = chromeos_version self.board = board self.toolchain = toolchain @@ -172,14 +166,15 @@ class JobsFactory(object): def BuildAndBenchmark(self): command = self.commands.BuildAndBenchmark() - label = "BuildAndBenchmark(%s,%s,%s)" % ( - self.toolchain, self.board, self.chromeos_version) + label = 'BuildAndBenchmark(%s,%s,%s)' % (self.toolchain, self.board, + self.chromeos_version) - machine_label = "chromeos-%s" % self.board + machine_label = 'chromeos-%s' % self.board job = jobs.CreateLinuxJob(label, command) job.DependsOnMachine( - machine.MachineSpecification(label=machine_label, lock_required=True), + machine.MachineSpecification(label=machine_label, + lock_required=True), False) return job diff --git a/automation/clients/helper/crosstool.py b/automation/clients/helper/crosstool.py index f3b24005..80154b25 100644 --- a/automation/clients/helper/crosstool.py +++ b/automation/clients/helper/crosstool.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. __author__ = 'kbaclawski@google.com (Krystian Baclawski)' @@ -14,14 +12,15 @@ from automation.common import job class JobsFactory(object): + def __init__(self): self.commands = CommandsFactory() def CheckoutCrosstool(self, target): command = self.commands.CheckoutCrosstool() new_job = jobs.CreateLinuxJob('CheckoutCrosstool(%s)' % target, command) - checkout_dir_dep = job.FolderDependency( - new_job, CommandsFactory.CHECKOUT_DIR) + checkout_dir_dep = job.FolderDependency(new_job, + CommandsFactory.CHECKOUT_DIR) manifests_dir_dep = job.FolderDependency( new_job, os.path.join(self.commands.buildit_path, target), 'manifests') return new_job, checkout_dir_dep, manifests_dir_dep @@ -30,14 +29,14 @@ class JobsFactory(object): command = self.commands.BuildRelease(target) new_job = jobs.CreateLinuxJob('BuildRelease(%s)' % target, command) new_job.DependsOnFolder(checkout_dir) - build_tree_dep = job.FolderDependency( - new_job, self.commands.buildit_work_dir_path) + build_tree_dep = job.FolderDependency(new_job, + self.commands.buildit_work_dir_path) return new_job, build_tree_dep def RunTests(self, checkout_dir, build_tree_dir, target, board, component): command = self.commands.RunTests(target, board, component) - new_job = jobs.CreateLinuxJob( - 'RunTests(%s, %s, %s)' % (target, component, board), command) + new_job = jobs.CreateLinuxJob('RunTests(%s, %s, %s)' % + (target, component, board), command) new_job.DependsOnFolder(checkout_dir) new_job.DependsOnFolder(build_tree_dir) testrun_dir_dep = job.FolderDependency( @@ -57,24 +56,26 @@ class CommandsFactory(object): CHECKOUT_DIR = 'crosstool-checkout-dir' def __init__(self): - self.buildit_path = os.path.join( - self.CHECKOUT_DIR, 'gcctools', 'crosstool', 'v15') + self.buildit_path = os.path.join(self.CHECKOUT_DIR, 'gcctools', 'crosstool', + 'v15') self.buildit_work_dir = 'buildit-tmp' self.buildit_work_dir_path = os.path.join('$JOB_TMP', self.buildit_work_dir) - self.dejagnu_output_path = os.path.join( - self.buildit_work_dir_path, 'dejagnu-output') + self.dejagnu_output_path = os.path.join(self.buildit_work_dir_path, + 'dejagnu-output') paths = { 'gcctools': [ - 'crosstool/v15/...', - 'scripts/...'], + 'crosstool/v15/...', 'scripts/...' + ], 'gcctools/google_vendor_src_branch': [ - 'binutils/binutils-2.21/...', - 'gdb/gdb-7.2.x/...', - 'zlib/zlib-1.2.3/...'], + 'binutils/binutils-2.21/...', 'gdb/gdb-7.2.x/...', + 'zlib/zlib-1.2.3/...' + ], 'gcctools/vendor_src': [ - 'gcc/google/gcc-4_6/...']} + 'gcc/google/gcc-4_6/...' + ] + } p4view = perforce.View('depot2', perforce.PathMapping.ListFromPathDict(paths)) @@ -91,22 +92,21 @@ class CommandsFactory(object): def BuildRelease(self, target): clnum_path = os.path.join('$JOB_TMP', self.CHECKOUT_DIR, 'CLNUM') - toolchain_root = os.path.join( - '/google/data/rw/projects/toolchains', target, 'unstable') + toolchain_root = os.path.join('/google/data/rw/projects/toolchains', target, + 'unstable') toolchain_path = os.path.join(toolchain_root, '${CLNUM}') build_toolchain = cmd.Wrapper( cmd.Chain( - cmd.MakeDir(toolchain_path), - cmd.Shell( - 'buildit', - '--keep-work-dir', - '--build-type=release', - '--work-dir=%s' % self.buildit_work_dir_path, - '--results-dir=%s' % toolchain_path, - '--force-release=%s' % '${CLNUM}', - target, - path='.')), + cmd.MakeDir(toolchain_path), + cmd.Shell('buildit', + '--keep-work-dir', + '--build-type=release', + '--work-dir=%s' % self.buildit_work_dir_path, + '--results-dir=%s' % toolchain_path, + '--force-release=%s' % '${CLNUM}', + target, + path='.')), cwd=self.buildit_path, umask='0022', env={'CLNUM': '$(< %s)' % clnum_path}) @@ -114,8 +114,7 @@ class CommandsFactory(object): # remove all but 10 most recent directories remove_old_toolchains_from_x20 = cmd.Wrapper( cmd.Pipe( - cmd.Shell('ls', '-1', '-r'), - cmd.Shell('sed', '-e', '1,10d'), + cmd.Shell('ls', '-1', '-r'), cmd.Shell('sed', '-e', '1,10d'), cmd.Shell('xargs', 'rm', '-r', '-f')), cwd=toolchain_root) @@ -132,13 +131,15 @@ class CommandsFactory(object): 'experimental/users/kbaclawski', 'dejagnu/site.exp') - build_dir_path = os.path.join( - target, 'rpmbuild/BUILD/crosstool*-0.0', 'build-%s' % component) + build_dir_path = os.path.join(target, 'rpmbuild/BUILD/crosstool*-0.0', + 'build-%s' % component) run_dejagnu = cmd.Wrapper( cmd.Chain( cmd.MakeDir(self.dejagnu_output_path), - cmd.Shell('make', 'check', '-k', + cmd.Shell('make', + 'check', + '-k', '-j $(grep -c processor /proc/cpuinfo)', 'RUNTESTFLAGS="%s"' % ' '.join(dejagnu_flags), 'DEJAGNU="%s"' % site_exp_file, @@ -146,8 +147,9 @@ class CommandsFactory(object): cwd=os.path.join(self.buildit_work_dir_path, build_dir_path), env={'REMOTE_TMPDIR': 'job-$JOB_ID'}) - save_results = cmd.Copy( - self.dejagnu_output_path, to_dir='$JOB_TMP/results', recursive=True) + save_results = cmd.Copy(self.dejagnu_output_path, + to_dir='$JOB_TMP/results', + recursive=True) return cmd.Chain(run_dejagnu, save_results) @@ -155,9 +157,12 @@ class CommandsFactory(object): sumfiles = [os.path.join('$JOB_TMP', board, '*.sum') for board in boards] return cmd.Wrapper( - cmd.Shell('dejagnu.sh', 'report', - '-m', '$JOB_TMP/manifests/*.xfail', - '-o', '$JOB_TMP/results/report.html', + cmd.Shell('dejagnu.sh', + 'report', + '-m', + '$JOB_TMP/manifests/*.xfail', + '-o', + '$JOB_TMP/results/report.html', *sumfiles, path='.'), cwd='$HOME/automation/clients/report') diff --git a/automation/clients/helper/jobs.py b/automation/clients/helper/jobs.py index ea9c9691..96a1c408 100644 --- a/automation/clients/helper/jobs.py +++ b/automation/clients/helper/jobs.py @@ -1,13 +1,11 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. from automation.common import job from automation.common import machine -def CreateLinuxJob(label, command, lock=False, timeout=4*60*60): +def CreateLinuxJob(label, command, lock=False, timeout=4 * 60 * 60): to_return = job.Job(label, command, timeout) - to_return.DependsOnMachine( - machine.MachineSpecification(os="linux", lock_required=lock)) + to_return.DependsOnMachine(machine.MachineSpecification(os='linux', + lock_required=lock)) return to_return diff --git a/automation/clients/helper/perforce.py b/automation/clients/helper/perforce.py index 1100a1ee..1f2dfe79 100644 --- a/automation/clients/helper/perforce.py +++ b/automation/clients/helper/perforce.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. __author__ = 'kbaclawski@google.com (Krystian Baclawski)' @@ -153,17 +151,16 @@ class CommandsFactory(object): self.p4config_path = os.path.join(self.checkout_dir, '.p4config') def Initialize(self): - return cmd.Chain( - 'mkdir -p %s' % self.checkout_dir, - 'cp ~/.p4config %s' % self.checkout_dir, - 'chmod u+w %s' % self.p4config_path, - 'echo "P4PORT=%s" >> %s' % (self.port, self.p4config_path), - 'echo "P4CLIENT=%s" >> %s' % (self.view.client, self.p4config_path)) + return cmd.Chain('mkdir -p %s' % self.checkout_dir, 'cp ~/.p4config %s' % + self.checkout_dir, 'chmod u+w %s' % self.p4config_path, + 'echo "P4PORT=%s" >> %s' % (self.port, self.p4config_path), + 'echo "P4CLIENT=%s" >> %s' % + (self.view.client, self.p4config_path)) def Create(self): # TODO(kbaclawski): Could we support value list for options consistently? - mappings = ['-a \"%s %s\"' % mapping for mapping in - self.view.AbsoluteMappings()] + mappings = ['-a \"%s %s\"' % mapping + for mapping in self.view.AbsoluteMappings()] # First command will create client with default mappings. Second one will # replace default mapping with desired. Unfortunately, it seems that it @@ -176,14 +173,12 @@ class CommandsFactory(object): env={'P4EDITOR': '/bin/true'}) def SaveSpecification(self, filename=None): - return cmd.Pipe( - cmd.Shell('g4', 'client', '-o'), - output=filename) + return cmd.Pipe(cmd.Shell('g4', 'client', '-o'), output=filename) def Sync(self, revision=None): sync_arg = '...' if revision: - sync_arg = "%s@%s" % (sync_arg, revision) + sync_arg = '%s@%s' % (sync_arg, revision) return cmd.Shell('g4', 'sync', sync_arg) def SaveCurrentCLNumber(self, filename=None): @@ -196,14 +191,11 @@ class CommandsFactory(object): return cmd.Shell('g4', 'client', '-d', self.view.client) def SetupAndDo(self, *commands): - return cmd.Chain( - self.Initialize(), - self.InCheckoutDir(self.Create(), *commands)) + return cmd.Chain(self.Initialize(), + self.InCheckoutDir(self.Create(), *commands)) def InCheckoutDir(self, *commands): - return cmd.Wrapper( - cmd.Chain(*commands), - cwd=self.checkout_dir) + return cmd.Wrapper(cmd.Chain(*commands), cwd=self.checkout_dir) def CheckoutFromSnapshot(self, snapshot): cmds = cmd.Chain() @@ -216,7 +208,8 @@ class CommandsFactory(object): local_dir = os.path.join(self.checkout_dir, os.path.dirname(local_path)) cmds.extend([ - cmd.Shell('mkdir', '-p', local_dir), - cmd.Shell('rsync', '-lr', remote_dir, local_dir)]) + cmd.Shell('mkdir', '-p', local_dir), cmd.Shell( + 'rsync', '-lr', remote_dir, local_dir) + ]) return cmds diff --git a/automation/clients/nightly.py b/automation/clients/nightly.py index a6157b0a..98e2b081 100755 --- a/automation/clients/nightly.py +++ b/automation/clients/nightly.py @@ -13,21 +13,21 @@ from automation.common import job_group def Main(argv): parser = optparse.OptionParser() - parser.add_option("-c", - "--chromeos_version", - dest="chromeos_version", - default="quarterly", - help="ChromeOS version to use.") - parser.add_option("-t", - "--toolchain", - dest="toolchain", - default="latest-toolchain", - help="Toolchain to use {latest-toolchain,gcc_46}.") - parser.add_option("-b", - "--board", - dest="board", - default="x86-generic", - help="Board to use for the nightly job.") + parser.add_option('-c', + '--chromeos_version', + dest='chromeos_version', + default='quarterly', + help='ChromeOS version to use.') + parser.add_option('-t', + '--toolchain', + dest='toolchain', + default='latest-toolchain', + help='Toolchain to use {latest-toolchain,gcc_46}.') + parser.add_option('-b', + '--board', + dest='board', + default='x86-generic', + help='Board to use for the nightly job.') options = parser.parse_args(argv)[0] toolchain = options.toolchain @@ -36,15 +36,16 @@ def Main(argv): # Build toolchain jobs_factory = chromeos.JobsFactory(chromeos_version=chromeos_version, - board=board, toolchain=toolchain) + board=board, + toolchain=toolchain) benchmark_job = jobs_factory.BuildAndBenchmark() - group_label = "nightly_client_%s" % board + group_label = 'nightly_client_%s' % board group = job_group.JobGroup(group_label, [benchmark_job], True, False) - server = xmlrpclib.Server("http://localhost:8000") + server = xmlrpclib.Server('http://localhost:8000') server.ExecuteJobGroup(pickle.dumps(group)) -if __name__ == "__main__": +if __name__ == '__main__': Main(sys.argv) diff --git a/automation/clients/output_test.py b/automation/clients/output_test.py index 3369e768..3126f050 100755 --- a/automation/clients/output_test.py +++ b/automation/clients/output_test.py @@ -13,17 +13,17 @@ from automation.common import machine def Main(): - server = xmlrpclib.Server("http://localhost:8000") + server = xmlrpclib.Server('http://localhost:8000') - command = os.path.join(os.path.dirname(sys.argv[0]), - "../../produce_output.py") + command = os.path.join( + os.path.dirname(sys.argv[0]), '../../produce_output.py') - pwd_job = job.Job("pwd_job", command) - pwd_job.DependsOnMachine(machine.MachineSpecification(os="linux")) + pwd_job = job.Job('pwd_job', command) + pwd_job.DependsOnMachine(machine.MachineSpecification(os='linux')) - group = job_group.JobGroup("pwd_client", [pwd_job]) + group = job_group.JobGroup('pwd_client', [pwd_job]) server.ExecuteJobGroup(pickle.dumps(group)) -if __name__ == "__main__": +if __name__ == '__main__': Main() diff --git a/automation/clients/pwd_test.py b/automation/clients/pwd_test.py index 565fe739..a4b28552 100755 --- a/automation/clients/pwd_test.py +++ b/automation/clients/pwd_test.py @@ -11,19 +11,17 @@ from automation.common import machine def Main(): - server = xmlrpclib.Server("http://localhost:8000") + server = xmlrpclib.Server('http://localhost:8000') - command = ["echo These following 3 lines should be the same", - "pwd", - "$(pwd)", - "echo ${PWD}"] + command = ['echo These following 3 lines should be the same', 'pwd', '$(pwd)', + 'echo ${PWD}'] - pwd_job = job.Job("pwd_job", " && ".join(command)) - pwd_job.DependsOnMachine(machine.MachineSpecification(os="linux")) + pwd_job = job.Job('pwd_job', ' && '.join(command)) + pwd_job.DependsOnMachine(machine.MachineSpecification(os='linux')) - group = job_group.JobGroup("pwd_client", [pwd_job]) + group = job_group.JobGroup('pwd_client', [pwd_job]) server.ExecuteJobGroup(pickle.dumps(group)) -if __name__ == "__main__": +if __name__ == '__main__': Main() diff --git a/automation/clients/report/dejagnu/__init__.py b/automation/clients/report/dejagnu/__init__.py index e69de29b..8b137891 100644 --- a/automation/clients/report/dejagnu/__init__.py +++ b/automation/clients/report/dejagnu/__init__.py @@ -0,0 +1 @@ + diff --git a/automation/clients/report/dejagnu/main.py b/automation/clients/report/dejagnu/main.py index 6e5e6872..62f095e1 100644 --- a/automation/clients/report/dejagnu/main.py +++ b/automation/clients/report/dejagnu/main.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # Author: kbaclawski@google.com (Krystian Baclawski) # @@ -35,12 +33,12 @@ def OptionChecker(parser): def ManifestCommand(argv): parser = optparse.OptionParser( - description=( - 'Read in one or more DejaGNU summary files (.sum), parse their ' - 'content and generate manifest files. Manifest files store a list ' - 'of failed tests that should be ignored. Generated files are ' - 'stored in current directory under following name: ' - '${tool}-${board}.xfail (e.g. "gcc-unix.xfail").'), + description= + ('Read in one or more DejaGNU summary files (.sum), parse their ' + 'content and generate manifest files. Manifest files store a list ' + 'of failed tests that should be ignored. Generated files are ' + 'stored in current directory under following name: ' + '${tool}-${board}.xfail (e.g. "gcc-unix.xfail").'), usage='Usage: %prog manifest [file.sum] (file2.sum ...)') _, args = parser.parse_args(argv[2:]) @@ -53,8 +51,7 @@ def ManifestCommand(argv): test_run = DejaGnuTestRun.FromFile(filename) manifest = Manifest.FromDejaGnuTestRun(test_run) - manifest_filename = '%s-%s.xfail' % ( - test_run.tool, test_run.board) + manifest_filename = '%s-%s.xfail' % (test_run.tool, test_run.board) with open(manifest_filename, 'w') as manifest_file: manifest_file.write(manifest.Generate()) @@ -64,18 +61,25 @@ def ManifestCommand(argv): def ReportCommand(argv): parser = optparse.OptionParser( - description=( - 'Read in one or more DejaGNU summary files (.sum), parse their ' - 'content and generate a single report file in selected format ' - '(currently only HTML).'), + description= + ('Read in one or more DejaGNU summary files (.sum), parse their ' + 'content and generate a single report file in selected format ' + '(currently only HTML).'), usage=('Usage: %prog report (-m manifest.xfail) [-o report.html] ' '[file.sum (file2.sum ...)')) parser.add_option( - '-o', dest='output', type='string', default=None, + '-o', + dest='output', + type='string', + default=None, help=('Suppress failures for test listed in provided manifest files. ' '(use -m for each manifest file you want to read)')) parser.add_option( - '-m', dest='manifests', type='string', action='append', default=None, + '-m', + dest='manifests', + type='string', + action='append', + default=None, help=('Suppress failures for test listed in provided manifest files. ' '(use -m for each manifest file you want to read)')) @@ -109,11 +113,11 @@ def ReportCommand(argv): def HelpCommand(argv): sys.exit('\n'.join([ - 'Usage: %s command [options]' % os.path.basename(argv[0]), - '', - 'Commands:', + 'Usage: %s command [options]' % os.path.basename(argv[ + 0]), '', 'Commands:', ' manifest - manage files containing a list of suppressed test failures', - ' report - generate report file for selected test runs'])) + ' report - generate report file for selected test runs' + ])) def Main(argv): @@ -122,11 +126,10 @@ def Main(argv): except IndexError: cmd_name = None - cmd_map = { - 'manifest': ManifestCommand, - 'report': ReportCommand} + cmd_map = {'manifest': ManifestCommand, 'report': ReportCommand} cmd_map.get(cmd_name, HelpCommand)(argv) + if __name__ == '__main__': FORMAT = '%(asctime)-15s %(levelname)s %(message)s' logging.basicConfig(format=FORMAT, level=logging.INFO) diff --git a/automation/clients/report/dejagnu/manifest.py b/automation/clients/report/dejagnu/manifest.py index 2124d577..5831d1b0 100644 --- a/automation/clients/report/dejagnu/manifest.py +++ b/automation/clients/report/dejagnu/manifest.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # Author: kbaclawski@google.com (Krystian Baclawski) # @@ -42,7 +40,8 @@ class Manifest(namedtuple('Manifest', 'tool board results')): @classmethod def FromDejaGnuTestRun(cls, test_run): - results = [result for result in test_run.results + results = [result + for result in test_run.results if result.result in cls.SUPPRESSIBLE_RESULTS] return cls(test_run.tool, test_run.board, results) diff --git a/automation/clients/report/dejagnu/report.py b/automation/clients/report/dejagnu/report.py index 7b56590d..191a5389 100644 --- a/automation/clients/report/dejagnu/report.py +++ b/automation/clients/report/dejagnu/report.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # Author: kbaclawski@google.com (Krystian Baclawski) # @@ -7,7 +5,6 @@ import logging import os.path - RESULT_DESCRIPTION = { 'ERROR': 'DejaGNU errors', 'FAIL': 'Failed tests', @@ -18,13 +15,15 @@ RESULT_DESCRIPTION = { 'UNTESTED': 'Not executed tests', 'WARNING': 'DejaGNU warnings', 'XFAIL': 'Expected test failures', - 'XPASS': 'Unexpectedly passed tests'} + 'XPASS': 'Unexpectedly passed tests' +} RESULT_GROUPS = { 'Successes': ['PASS', 'XFAIL'], 'Failures': ['FAIL', 'XPASS', 'UNRESOLVED'], 'Suppressed': ['!FAIL', '!XPASS', '!UNRESOLVED', '!ERROR'], - 'Framework': ['UNTESTED', 'UNSUPPORTED', 'ERROR', 'WARNING', 'NOTE']} + 'Framework': ['UNTESTED', 'UNSUPPORTED', 'ERROR', 'WARNING', 'NOTE'] +} ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -40,18 +39,18 @@ def _GetResultDescription(name): def _PrepareSummary(res_types, summary): + def GetResultCount(res_type): return summary.get(res_type, 0) - return [(_GetResultDescription(rt), GetResultCount(rt)) - for rt in res_types] + return [(_GetResultDescription(rt), GetResultCount(rt)) for rt in res_types] def _PrepareTestList(res_types, tests): + def GetTestsByResult(res_type): return [(test.name, test.variant or '') - for test in sorted(tests) - if test.result == res_type] + for test in sorted(tests) if test.result == res_type] return [(_GetResultDescription(rt), GetTestsByResult(rt)) for rt in res_types if rt != 'PASS'] @@ -92,7 +91,8 @@ def Generate(test_runs, manifests): tmpl_args.append({ 'id': test_run_id, 'name': '%s @%s' % (test_run.tool, test_run.board), - 'groups': groups}) + 'groups': groups + }) logging.info('Rendering report in HTML format.') @@ -105,7 +105,8 @@ def Generate(test_runs, manifests): logging.error('Failed to generate report in HTML format!') return '' - settings.configure(DEBUG=True, TEMPLATE_DEBUG=True, + settings.configure(DEBUG=True, + TEMPLATE_DEBUG=True, TEMPLATE_DIRS=(ROOT_PATH,)) tmpl = loader.get_template('report.html') diff --git a/automation/clients/report/dejagnu/summary.py b/automation/clients/report/dejagnu/summary.py index 269793e6..d573c691 100644 --- a/automation/clients/report/dejagnu/summary.py +++ b/automation/clients/report/dejagnu/summary.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # Author: kbaclawski@google.com (Krystian Baclawski) # @@ -60,7 +58,8 @@ class DejaGnuTestResult(namedtuple('Result', 'name variant result flaky')): # remove include paths - they contain name of tmp directory ('-I\S+', ''), # compress white spaces - ('\s+', ' ')] + ('\s+', ' ') + ] for pattern, replacement in substitutions: variant = re.sub(pattern, replacement, variant) @@ -168,12 +167,11 @@ class DejaGnuTestRun(object): with open(filename, 'r') as report: lines = [line.strip() for line in report.readlines() if line.strip()] - parsers = ( - (re.compile(r'Running target (.*)'), self._ParseBoard), - (re.compile(r'Test Run By (.*) on (.*)'), self._ParseDate), - (re.compile(r'=== (.*) tests ==='), self._ParseTool), - (re.compile(r'Target(\s+)is (.*)'), self._ParseTarget), - (re.compile(r'Host(\s+)is (.*)'), self._ParseHost)) + parsers = ((re.compile(r'Running target (.*)'), self._ParseBoard), + (re.compile(r'Test Run By (.*) on (.*)'), self._ParseDate), + (re.compile(r'=== (.*) tests ==='), self._ParseTool), + (re.compile(r'Target(\s+)is (.*)'), self._ParseTarget), + (re.compile(r'Host(\s+)is (.*)'), self._ParseHost)) for line in lines: result = DejaGnuTestResult.FromLine(line) @@ -219,8 +217,7 @@ class DejaGnuTestRun(object): # Remove all UNRESOLVED results that were also marked as UNSUPPORTED. unresolved = [res._replace(result='UNRESOLVED') - for res in results - if res.result == 'UNSUPPORTED'] + for res in results if res.result == 'UNSUPPORTED'] for res in unresolved: if res in self.results: @@ -257,8 +254,8 @@ class DejaGnuTestRun(object): self.results.add(result._replace(result=new_result)) for result in sorted(manifest_results): - logging.warning( - 'Result {%s} listed in manifest but not suppressed.', result) + logging.warning('Result {%s} listed in manifest but not suppressed.', + result) def __str__(self): return '{0}, {1} @{2} on {3}'.format(self.target, self.tool, self.board, diff --git a/automation/clients/report/validate_failures.py b/automation/clients/report/validate_failures.py index e99c9054..5db356d1 100755 --- a/automation/clients/report/validate_failures.py +++ b/automation/clients/report/validate_failures.py @@ -24,7 +24,6 @@ # along with GCC; see the file COPYING. If not, write to # the Free Software Foundation, 51 Franklin Street, Fifth Floor, # Boston, MA 02110-1301, USA. - """This script provides a coarser XFAILing mechanism that requires no detailed DejaGNU markings. This is useful in a variety of scenarios: @@ -78,15 +77,13 @@ def GetSumFiles(build_dir): for root, _, filenames in os.walk(build_dir): summaries.extend([os.path.join(root, filename) - for filename in filenames - if filename.endswith('.sum')]) + for filename in filenames if filename.endswith('.sum')]) return map(os.path.normpath, summaries) def ValidBuildDirectory(build_dir, target): - mandatory_paths = [build_dir, - os.path.join(build_dir, 'Makefile')] + mandatory_paths = [build_dir, os.path.join(build_dir, 'Makefile')] extra_paths = [os.path.join(build_dir, target), os.path.join(build_dir, 'build-%s' % target)] @@ -104,8 +101,7 @@ def GetManifestPath(build_dir): target = makefile['target_alias'] if not ValidBuildDirectory(build_dir, target): - logging.error( - '%s is not a valid GCC top level build directory.', build_dir) + logging.error('%s is not a valid GCC top level build directory.', build_dir) sys.exit(1) logging.info('Discovered source directory: "%s"', srcdir) @@ -123,8 +119,8 @@ def CompareResults(manifest, actual): actual_vs_manifest = actual - manifest # Filter out tests marked flaky. - manifest_without_flaky_tests = set( - filter(lambda result: not result.flaky, manifest)) + manifest_without_flaky_tests = set(filter(lambda result: not result.flaky, + manifest)) # Simlarly for all the tests in the manifest. manifest_vs_actual = manifest_without_flaky_tests - actual @@ -136,7 +132,7 @@ def LogResults(level, results): log_fun = getattr(logging, level) for num, result in enumerate(sorted(results), start=1): - log_fun(" %d) %s", num, result) + log_fun(' %d) %s', num, result) def CheckExpectedResults(manifest_path, build_dir): @@ -194,26 +190,37 @@ def ProduceManifest(manifest_path, build_dir, overwrite): with open(manifest_path, 'w') as manifest_file: manifest_strings = [manifest.Generate() for manifest in manifests] - logging.info('Writing manifest to "%s".' % manifest_path) + logging.info('Writing manifest to "%s".', manifest_path) manifest_file.write('\n'.join(manifest_strings)) def Main(argv): parser = optparse.OptionParser(usage=__doc__) parser.add_option( - '-b', '--build_dir', - dest='build_dir', action='store',metavar='PATH', default=os.getcwd(), + '-b', + '--build_dir', + dest='build_dir', + action='store', + metavar='PATH', + default=os.getcwd(), help='Build directory to check. (default: current directory)') + parser.add_option('-m', + '--manifest', + dest='manifest', + action='store_true', + help='Produce the manifest for the current build.') parser.add_option( - '-m', '--manifest', dest='manifest', action='store_true', - help='Produce the manifest for the current build.') - parser.add_option( - '-f', '--force', dest='force', action='store_true', + '-f', + '--force', + dest='force', + action='store_true', help=('Overwrite an existing manifest file, if user requested creating ' 'new one. (default: False)')) - parser.add_option( - '-v', '--verbose', dest='verbose', action='store_true', - help='Increase verbosity.') + parser.add_option('-v', + '--verbose', + dest='verbose', + action='store_true', + help='Increase verbosity.') options, _ = parser.parse_args(argv[1:]) if options.verbose: @@ -226,6 +233,7 @@ def Main(argv): else: CheckExpectedResults(manifest_path, options.build_dir) + if __name__ == '__main__': logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) Main(sys.argv) |