diff options
| author | Luis Lozano <llozano@chromium.org> | 2015-12-15 13:49:30 -0800 |
|---|---|---|
| committer | Luis Lozano <llozano@chromium.org> | 2015-12-16 17:36:06 +0000 |
| commit | f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe (patch) | |
| tree | 185d243c7eed7c7a0db6f0e640746cadc1479ea9 /automation | |
| parent | 2a66f70fef907c1cb15229cb58e5129cb620ac98 (diff) | |
| download | toolchain-utils-f2a3ef46f75d2196a93d3ed27f4d1fcf22b54fbe.tar.gz | |
Run pyformat on all the toolchain-utils files.
This gets rid of a lot of lint issues.
Ran by doing this:
for f in *.py; do echo -n "$f " ; if [ -x $f ]; then pyformat -i
--remove_trailing_comma --yapf --force_quote_type=double $f ; else
pyformat -i --remove_shebang --remove_trailing_comma --yapf
--force_quote_type=double $f ; fi ; done
BUG=chromium:567921
TEST=Ran simple crosperf run.
Change-Id: I59778835fdaa5f706d2e1765924389f9e97433d1
Reviewed-on: https://chrome-internal-review.googlesource.com/242031
Reviewed-by: Luis Lozano <llozano@chromium.org>
Commit-Queue: Luis Lozano <llozano@chromium.org>
Tested-by: Luis Lozano <llozano@chromium.org>
Reviewed-by: Yunlian Jiang <yunlian@google.com>
Diffstat (limited to 'automation')
45 files changed, 561 insertions, 574 deletions
diff --git a/automation/__init__.py b/automation/__init__.py index e69de29b..8b137891 100644 --- a/automation/__init__.py +++ b/automation/__init__.py @@ -0,0 +1 @@ + diff --git a/automation/all_tests.py b/automation/all_tests.py index 3ca4f52a..e7b70884 100644 --- a/automation/all_tests.py +++ b/automation/all_tests.py @@ -2,15 +2,15 @@ import glob import sys import unittest -sys.path.insert(0, "server") -sys.path.insert(0, "clients") -sys.path.insert(0, "common") +sys.path.insert(0, 'server') +sys.path.insert(0, 'clients') +sys.path.insert(0, 'common') test_file_strings = glob.glob('*/*_test.py') module_strings = [str[0:len(str) - 3] for str in test_file_strings] for i in range(len(module_strings)): - module_strings[i] = module_strings[i].split("/")[-1] -suites = [unittest.defaultTestLoader.loadTestsFromName(str) for str - in module_strings] + module_strings[i] = module_strings[i].split('/')[-1] +suites = [unittest.defaultTestLoader.loadTestsFromName(str) + for str in module_strings] testSuite = unittest.TestSuite(suites) text_runner = unittest.TextTestRunner().run(testSuite) diff --git a/automation/clients/__init__.py b/automation/clients/__init__.py index e69de29b..8b137891 100644 --- a/automation/clients/__init__.py +++ b/automation/clients/__init__.py @@ -0,0 +1 @@ + diff --git a/automation/clients/android.py b/automation/clients/android.py index a65cba2e..603744b1 100755 --- a/automation/clients/android.py +++ b/automation/clients/android.py @@ -1,7 +1,6 @@ #!/usr/bin/python # # Copyright 2011 Google Inc. All Rights Reserved. - """Client for Android nightly jobs. Does the following jobs: diff --git a/automation/clients/chromeos.py b/automation/clients/chromeos.py index 18b29b7c..084f7840 100755 --- a/automation/clients/chromeos.py +++ b/automation/clients/chromeos.py @@ -1,12 +1,10 @@ #!/usr/bin/python # # Copyright 2011 Google Inc. All Rights Reserved. - """chromeos.py: Build & Test ChromeOS using custom compilers.""" __author__ = 'asharif@google.com (Ahmad Sharif)' - import logging import optparse import os @@ -38,7 +36,8 @@ class ChromeOSNightlyClient(object): def CheckoutV14Dir(self): p4view = perforce.View(self.DEPOT2_DIR, [ - perforce.PathMapping('gcctools/chromeos/v14/...')]) + perforce.PathMapping('gcctools/chromeos/v14/...') + ]) return self.GetP4Snapshot(p4view) def GetP4Snapshot(self, p4view): @@ -56,12 +55,12 @@ class ChromeOSNightlyClient(object): os.path.join(self.P4_VERSION_DIR, 'test_toolchains.py'), '--force-mismatch', '--clean', - '--public', # crbug.com/145822 + '--public', # crbug.com/145822 '--board=%s' % self._board, '--remote=%s' % self._remote, '--githashes=%s' % self._gcc_githash)) label = 'testlabel' - job = jobs.CreateLinuxJob(label, chain, timeout=24*60*60) + job = jobs.CreateLinuxJob(label, chain, timeout=24 * 60 * 60) return job_group.JobGroup(label, [job], True, False) @@ -92,7 +91,8 @@ def Main(argv): logging.error('Specify a board, remote and gcc_githash') return 1 - client = ChromeOSNightlyClient(options.board, options.remote, + client = ChromeOSNightlyClient(options.board, + options.remote, options.gcc_githash, p4_snapshot=options.p4_snapshot) client.Run() diff --git a/automation/clients/crosstool.py b/automation/clients/crosstool.py index 450d6030..65720343 100755 --- a/automation/clients/crosstool.py +++ b/automation/clients/crosstool.py @@ -43,8 +43,8 @@ class CrosstoolNightlyClient(object): all_jobs = [checkout_crosstool_job] # Build crosstool target - build_release_job, build_tree_dir = factory.BuildRelease( - checkout_dir, self._target) + build_release_job, build_tree_dir = factory.BuildRelease(checkout_dir, + self._target) all_jobs.append(build_release_job) testruns = [] @@ -52,8 +52,8 @@ class CrosstoolNightlyClient(object): # Perform crosstool tests for board in self._boards: for component in ('gcc', 'binutils'): - test_job, testrun_dir = factory.RunTests( - checkout_dir, build_tree_dir, self._target, board, component) + test_job, testrun_dir = factory.RunTests(checkout_dir, build_tree_dir, + self._target, board, component) all_jobs.append(test_job) testruns.append(testrun_dir) @@ -61,8 +61,8 @@ class CrosstoolNightlyClient(object): all_jobs.append(factory.GenerateReport(testruns, manifests_dir, self._target, self._boards)) - return job_group.JobGroup( - 'Crosstool Nightly Build (%s)' % self._target, all_jobs, True, False) + return job_group.JobGroup('Crosstool Nightly Build (%s)' % self._target, + all_jobs, True, False) @logger.HandleUncaughtExceptions @@ -70,14 +70,14 @@ def Main(argv): valid_boards_string = ', '.join(CrosstoolNightlyClient.VALID_BOARDS) parser = optparse.OptionParser() - parser.add_option('-b', - '--board', - dest='boards', - action='append', - choices=CrosstoolNightlyClient.VALID_BOARDS, - default=[], - help=('Run DejaGNU tests on selected boards: %s.' % - valid_boards_string)) + parser.add_option( + '-b', + '--board', + dest='boards', + action='append', + choices=CrosstoolNightlyClient.VALID_BOARDS, + default=[], + help=('Run DejaGNU tests on selected boards: %s.' % valid_boards_string)) options, args = parser.parse_args(argv) if len(args) == 2: @@ -86,7 +86,7 @@ def Main(argv): logging.error('Exactly one target required as a command line argument!') logging.info('List of valid targets:') for pair in enumerate(CrosstoolNightlyClient.VALID_TARGETS, start=1): - logging.info('%d) %s' % pair) + logging.info('%d) %s', pair) sys.exit(1) option_list = [opt.dest for opt in parser.option_list if opt.dest] diff --git a/automation/clients/dejagnu_compiler.py b/automation/clients/dejagnu_compiler.py index e4342160..eb923d5d 100755 --- a/automation/clients/dejagnu_compiler.py +++ b/automation/clients/dejagnu_compiler.py @@ -1,12 +1,10 @@ #!/usr/bin/python # # Copyright 2012 Google Inc. All Rights Reserved. - """dejagnu_compiler.py: Run dejagnu test.""" __author__ = 'shenhan@google.com (Han Shen)' - import logging import optparse import os @@ -38,7 +36,8 @@ class DejagnuCompilerNightlyClient: def CheckoutV14Dir(self): p4view = perforce.View(self.DEPOT2_DIR, [ - perforce.PathMapping('gcctools/chromeos/v14/...')]) + perforce.PathMapping('gcctools/chromeos/v14/...') + ]) return self.GetP4Snapshot(p4view) def GetP4Snapshot(self, p4view): @@ -50,30 +49,37 @@ class DejagnuCompilerNightlyClient: return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) def CreateJobGroup(self): - chain = cmd.Chain( - self.CheckoutV14Dir(), - cmd.Shell('python', - os.path.join(self.P4_VERSION_DIR, 'test_gcc_dejagnu.py'), - '--board=%s' % self._board, - '--remote=%s' % self._remote, - '--cleanup=%s' % self._cleanup)) + chain = cmd.Chain(self.CheckoutV14Dir(), cmd.Shell( + 'python', os.path.join(self.P4_VERSION_DIR, 'test_gcc_dejagnu.py'), + '--board=%s' % self._board, '--remote=%s' % self._remote, + '--cleanup=%s' % self._cleanup)) label = 'dejagnu' - job = jobs.CreateLinuxJob(label, chain, timeout=8*60*60) - return job_group.JobGroup(label, [job], cleanup_on_failure=True, + job = jobs.CreateLinuxJob(label, chain, timeout=8 * 60 * 60) + return job_group.JobGroup(label, + [job], + cleanup_on_failure=True, cleanup_on_completion=True) @logger.HandleUncaughtExceptions def Main(argv): parser = optparse.OptionParser() - parser.add_option('-b', '--board', dest='board', + parser.add_option('-b', + '--board', + dest='board', help='Run performance tests on these boards') - parser.add_option('-r', '--remote', dest='remote', + parser.add_option('-r', + '--remote', + dest='remote', help='Run performance tests on these remotes') - parser.add_option('-p', '--p4_snapshot', dest='p4_snapshot', + parser.add_option('-p', + '--p4_snapshot', + dest='p4_snapshot', help=('For development only. ' 'Use snapshot instead of checking out.')) - parser.add_option('--cleanup', dest='cleanup', default='mount', + parser.add_option('--cleanup', + dest='cleanup', + default='mount', help=('Cleanup test directory, values could be one of ' '"mount", "chroot" or "chromeos"')) options, _ = parser.parse_args(argv) @@ -82,8 +88,8 @@ def Main(argv): logging.error('Specify a board and remote.') return 1 - client = DejagnuCompilerNightlyClient( - options.board, options.remote, options.p4_snapshot, options.cleanup) + client = DejagnuCompilerNightlyClient(options.board, options.remote, + options.p4_snapshot, options.cleanup) client.Run() return 0 diff --git a/automation/clients/helper/android.py b/automation/clients/helper/android.py index fc068aca..7ff2ac1c 100644 --- a/automation/clients/helper/android.py +++ b/automation/clients/helper/android.py @@ -1,7 +1,4 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. - """Helper modules for Android toolchain test infrastructure. Provides following Android toolchain test jobs and commands. @@ -24,6 +21,7 @@ from automation.common import job class JobsFactory(object): + def __init__(self, gcc_version='4.4.3', build_type='DEVELOPMENT'): assert gcc_version in ['4.4.3', '4.6', 'google_main', 'fsf_trunk'] assert build_type in ['DEVELOPMENT', 'RELEASE'] @@ -46,11 +44,13 @@ class JobsFactory(object): new_job = jobs.CreateLinuxJob('AndroidBuildToolchain(%s)' % self.tc_tag, command) new_job.DependsOnFolder(checkout_dir_dep) - tc_prefix_dep = job.FolderDependency( - new_job, self.commands.toolchain_prefix_dir) + tc_prefix_dep = job.FolderDependency(new_job, + self.commands.toolchain_prefix_dir) return new_job, tc_prefix_dep - def BuildAndroidImage(self, tc_prefix_dep, product='stingray', + def BuildAndroidImage(self, + tc_prefix_dep, + product='stingray', branch='ics-release'): assert product in ['stingray', 'passion', 'trygon', 'soju'] assert branch in ['honeycomb-release', 'ics-release'] @@ -89,8 +89,7 @@ class CommandsFactory(object): self.gcc_version = gcc_version self.binutils_version = '2.21' self.gold_version = '2.21' - self.toolchain_prefix_dir = 'install-gcc-%s-%s' % ( - gcc_version, build_type) + self.toolchain_prefix_dir = 'install-gcc-%s-%s' % (gcc_version, build_type) self.p4client = self._CreatePerforceClient() self.scripts = ScriptsFactory(self.gcc_version, self.binutils_version, self.gold_version) @@ -105,11 +104,9 @@ class CommandsFactory(object): 'google_vendor_src_branch') # Common views for tools - p4view = perforce.View('depot2', - perforce.PathMapping.ListFromPathTuples( - [('gcctools/android/build/...', 'src/build/...'), - ('gcctools/android/Tarballs/...', - 'src/tarballs/...')])) + p4view = perforce.View('depot2', perforce.PathMapping.ListFromPathTuples([( + 'gcctools/android/build/...', 'src/build/...'), ( + 'gcctools/android/Tarballs/...', 'src/tarballs/...')])) for mapping in perforce.PathMapping.ListFromPathDict( {'gcctools/android': ['tools/scripts/...', 'master/...']}): p4view.add(mapping) @@ -123,13 +120,11 @@ class CommandsFactory(object): binutils_branch = mobile_rel_branch else: binutils_branch = p4_dev_path - p4view.add(perforce.PathMapping(binutils_branch, 'src', - ('binutils/binutils-%s/...' % - self.binutils_version))) + p4view.add(perforce.PathMapping(binutils_branch, 'src', ( + 'binutils/binutils-%s/...' % self.binutils_version))) if self.binutils_version != self.gold_version: - p4view.add(perforce.PathMapping(binutils_branch, 'src', - ('binutils/binutils-%s/...' % - self.gold_version))) + p4view.add(perforce.PathMapping(binutils_branch, 'src', ( + 'binutils/binutils-%s/...' % self.gold_version))) # Add view for gcc if gcc_version is '4.4.3'. if self.gcc_version == '4.4.3': @@ -172,9 +167,11 @@ class CommandsFactory(object): gcc_required_dir = os.path.join(self.TOOLCHAIN_SRC_DIR, 'gcc', 'gcc-%s' % self.gcc_version) - return cmd.Chain(cmd.MakeDir(gcc_required_dir), - cmd.Wrapper(cmd.Chain(svn_co_command, svn_get_revision), - cwd=gcc_required_dir)) + return cmd.Chain( + cmd.MakeDir(gcc_required_dir), + cmd.Wrapper( + cmd.Chain(svn_co_command, svn_get_revision), + cwd=gcc_required_dir)) def CheckoutAndroidToolchain(self): p4client = self.p4client @@ -187,25 +184,24 @@ class CommandsFactory(object): return command def BuildAndroidToolchain(self): - script_cmd = self.scripts.BuildAndroidToolchain(self.toolchain_prefix_dir, - self.CHECKOUT_DIR, - self.TOOLCHAIN_BUILD_DIR, - self.TOOLCHAIN_SRC_DIR) + script_cmd = self.scripts.BuildAndroidToolchain( + self.toolchain_prefix_dir, self.CHECKOUT_DIR, self.TOOLCHAIN_BUILD_DIR, + self.TOOLCHAIN_SRC_DIR) # Record toolchain and gcc CL number - record_cl_cmd = cmd.Copy(os.path.join(self.CHECKOUT_DIR, 'CLNUM*'), - to_dir=self.toolchain_prefix_dir) - save_cmd = cmd.Tar(os.path.join('$JOB_TMP', 'results', - '%s.tar.bz2' % self.toolchain_prefix_dir), - self.toolchain_prefix_dir) + record_cl_cmd = cmd.Copy( + os.path.join(self.CHECKOUT_DIR, 'CLNUM*'), + to_dir=self.toolchain_prefix_dir) + save_cmd = cmd.Tar( + os.path.join('$JOB_TMP', 'results', '%s.tar.bz2' % + self.toolchain_prefix_dir), self.toolchain_prefix_dir) return cmd.Chain(script_cmd, record_cl_cmd, save_cmd) def _BuildAndroidTree(self, local_android_branch_dir, product): target_tools_prefix = os.path.join('$JOB_TMP', self.toolchain_prefix_dir, 'bin', 'arm-linux-androideabi-') java_path = '/usr/lib/jvm/java-6-sun/bin' - build_cmd = cmd.Shell('make', '-j8', - 'PRODUCT-%s-userdebug' % product, + build_cmd = cmd.Shell('make', '-j8', 'PRODUCT-%s-userdebug' % product, 'TARGET_TOOLS_PREFIX=%s' % target_tools_prefix, 'PATH=%s:$PATH' % java_path) return cmd.Wrapper(build_cmd, cwd=local_android_branch_dir) @@ -219,9 +215,8 @@ class CommandsFactory(object): 'android_trees') remote_android_branch_path = os.path.join(androidtrees_path, branch) local_android_branch_dir = os.path.join(self.ANDROID_TREES_DIR, branch) - gettree_cmd = cmd.RemoteCopyFrom(androidtrees_host, - remote_android_branch_path, - local_android_branch_dir) + gettree_cmd = cmd.RemoteCopyFrom( + androidtrees_host, remote_android_branch_path, local_android_branch_dir) # Configure and build the tree buildtree_cmd = self._BuildAndroidTree(local_android_branch_dir, product) @@ -235,8 +230,9 @@ class CommandsFactory(object): return cmd.Chain(gettree_cmd, buildtree_cmd, copy_img, compress_img) def CheckoutScripts(self): - p4view = perforce.View('depot2', [perforce.PathMapping( - 'gcctools/android/tools/...', 'tools/...')]) + p4view = perforce.View('depot2', + [perforce.PathMapping('gcctools/android/tools/...', + 'tools/...')]) p4client = perforce.CommandsFactory(self.TOOLS_DIR, p4view) return p4client.SetupAndDo(p4client.Sync(), p4client.Remove()) @@ -246,19 +242,15 @@ class CommandsFactory(object): base_benchbin_path = ('/usr/local/google2/home/mobiletc-prebuild/' 'archive/v3binaries/2011-10-18') local_basebenchbin_dir = 'base_benchmark_bin' - getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host, - base_benchbin_path, + getbase_cmd = cmd.RemoteCopyFrom(base_benchbin_host, base_benchbin_path, local_basebenchbin_dir) # Build and run benchmark. android_arch = 'android_%s' % arch run_label = 'normal' - benchmark_cmd = self.scripts.RunBenchmark(self.toolchain_prefix_dir, - self.TOOLS_DIR, - self.BENCHMARK_OUT_DIR, - run_label, run_experiment, - android_arch, - local_basebenchbin_dir) + benchmark_cmd = self.scripts.RunBenchmark( + self.toolchain_prefix_dir, self.TOOLS_DIR, self.BENCHMARK_OUT_DIR, + run_label, run_experiment, android_arch, local_basebenchbin_dir) # Extract jobid from BENCHMARK_OUT_DIR/log/jobid_normal.log file. # Copy jobid to www server to generate performance dashboard. @@ -268,6 +260,7 @@ class CommandsFactory(object): class ScriptsFactory(object): + def __init__(self, gcc_version, binutils_version, gold_version): self._gcc_version = gcc_version self._binutils_version = binutils_version @@ -292,14 +285,20 @@ class ScriptsFactory(object): '--with-gold-version=%s' % self._gold_version, '--with-gdb-version=7.1.x-android', '--log-path=%s/logs' % '$JOB_HOME', - '--android-sysroot=%s' % - os.path.join('$JOB_TMP', checkout_dir, 'gcctools', 'android', - 'master', 'honeycomb_generic_sysroot'), + '--android-sysroot=%s' % os.path.join('$JOB_TMP', checkout_dir, + 'gcctools', 'android', 'master', + 'honeycomb_generic_sysroot'), path=os.path.join(checkout_dir, 'gcctools', 'android', 'tools', 'scripts')) - def RunBenchmark(self, toolchain_prefix_dir, checkout_dir, output_dir, - run_label, run_experiment, arch, base_bench_bin=None): + def RunBenchmark(self, + toolchain_prefix_dir, + checkout_dir, + output_dir, + run_label, + run_experiment, + arch, + base_bench_bin=None): if base_bench_bin: base_bench_opt = '--base_benchmark_bin=%s' % base_bench_bin else: diff --git a/automation/clients/helper/chromeos.py b/automation/clients/helper/chromeos.py index ddfd59fe..e7157451 100644 --- a/automation/clients/helper/chromeos.py +++ b/automation/clients/helper/chromeos.py @@ -1,8 +1,6 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. -__author__ = "asharif@google.com (Ahmad Sharif)" +__author__ = 'asharif@google.com (Ahmad Sharif)' import os.path import re @@ -14,64 +12,54 @@ from automation.common import machine class ScriptsFactory(object): + def __init__(self, chromeos_root, scripts_path): self._chromeos_root = chromeos_root self._scripts_path = scripts_path def SummarizeResults(self, logs_path): - return cmd.Shell("summarize_results.py", - logs_path, - path=self._scripts_path) + return cmd.Shell('summarize_results.py', logs_path, path=self._scripts_path) def Buildbot(self, config_name): buildbot = os.path.join(self._chromeos_root, - "chromite/cbuildbot/cbuildbot.py") - - return cmd.Shell(buildbot, - "--buildroot=%s" % self._chromeos_root, - "--resume", - "--noarchive", - "--noprebuilts", - "--nosync", - "--nouprev", - "--notests", - "--noclean", - config_name) + 'chromite/cbuildbot/cbuildbot.py') + + return cmd.Shell(buildbot, '--buildroot=%s' % self._chromeos_root, + '--resume', '--noarchive', '--noprebuilts', '--nosync', + '--nouprev', '--notests', '--noclean', config_name) def RunBenchmarks(self, board, tests): - image_path = os.path.join(self._chromeos_root, - "src/build/images", - board, - "latest/chromiumos_image.bin") - - return cmd.Shell("cros_run_benchmarks.py", - "--remote=$SECONDARY_MACHINES[0]", - "--board=%s" % board, - "--tests=%s" % tests, - "--full_table", + image_path = os.path.join(self._chromeos_root, 'src/build/images', board, + 'latest/chromiumos_image.bin') + + return cmd.Shell('cros_run_benchmarks.py', + '--remote=$SECONDARY_MACHINES[0]', + '--board=%s' % board, + '--tests=%s' % tests, + '--full_table', image_path, - path="/home/mobiletc-prebuild") + path='/home/mobiletc-prebuild') - def SetupChromeOS(self, version="latest", use_minilayout=False): - setup_chromeos = cmd.Shell("setup_chromeos.py", - "--public", - "--dir=%s" % self._chromeos_root, - "--version=%s" % version, + def SetupChromeOS(self, version='latest', use_minilayout=False): + setup_chromeos = cmd.Shell('setup_chromeos.py', + '--public', + '--dir=%s' % self._chromeos_root, + '--version=%s' % version, path=self._scripts_path) if use_minilayout: - setup_chromeos.AddOption("--minilayout") + setup_chromeos.AddOption('--minilayout') return setup_chromeos class CommandsFactory(object): - DEPOT2_DIR = "//depot2/" - P4_CHECKOUT_DIR = "perforce2/" - P4_VERSION_DIR = os.path.join(P4_CHECKOUT_DIR, "gcctools/chromeos/v14") + DEPOT2_DIR = '//depot2/' + P4_CHECKOUT_DIR = 'perforce2/' + P4_VERSION_DIR = os.path.join(P4_CHECKOUT_DIR, 'gcctools/chromeos/v14') - CHROMEOS_ROOT = "chromeos" - CHROMEOS_SCRIPTS_DIR = os.path.join(CHROMEOS_ROOT, "src/scripts") - CHROMEOS_BUILDS_DIR = "/home/mobiletc-prebuild/www/chromeos_builds" + CHROMEOS_ROOT = 'chromeos' + CHROMEOS_SCRIPTS_DIR = os.path.join(CHROMEOS_ROOT, 'src/scripts') + CHROMEOS_BUILDS_DIR = '/home/mobiletc-prebuild/www/chromeos_builds' def __init__(self, chromeos_version, board, toolchain, p4_snapshot): self.chromeos_version = chromeos_version @@ -82,33 +70,34 @@ class CommandsFactory(object): self.scripts = ScriptsFactory(self.CHROMEOS_ROOT, self.P4_VERSION_DIR) def AddBuildbotConfig(self, config_name, config_list): - config_header = "add_config(%r, [%s])" % (config_name, - ", ".join(config_list)) + config_header = 'add_config(%r, [%s])' % (config_name, + ', '.join(config_list)) config_file = os.path.join(self.CHROMEOS_ROOT, - "chromite/cbuildbot/cbuildbot_config.py") - quoted_config_header = "%r" % config_header + 'chromite/cbuildbot/cbuildbot_config.py') + quoted_config_header = '%r' % config_header quoted_config_header = re.sub("'", "\\\"", quoted_config_header) - return cmd.Pipe(cmd.Shell("echo", quoted_config_header), - cmd.Shell("tee", "--append", config_file)) + return cmd.Pipe( + cmd.Shell('echo', quoted_config_header), + cmd.Shell('tee', '--append', config_file)) def RunBuildbot(self): - config_dict = {"board": self.board, - "build_tests": True, - "chrome_tests": True, - "unittests": False, - "vm_tests": False, - "prebuilts": False, - "latest_toolchain": True, - "useflags": ["chrome_internal"], - "usepkg_chroot": True, + config_dict = {'board': self.board, + 'build_tests': True, + 'chrome_tests': True, + 'unittests': False, + 'vm_tests': False, + 'prebuilts': False, + 'latest_toolchain': True, + 'useflags': ['chrome_internal'], + 'usepkg_chroot': True, self.toolchain: True} - config_name = "%s-toolchain-test" % self.board - if "arm" in self.board: - config_list = ["arm"] + config_name = '%s-toolchain-test' % self.board + if 'arm' in self.board: + config_list = ['arm'] else: config_list = [] - config_list.extend(["internal", "full", "official", str(config_dict)]) + config_list.extend(['internal', 'full', 'official', str(config_dict)]) add_config_shell = self.AddBuildbotConfig(config_name, config_list) return cmd.Chain(add_config_shell, self.scripts.Buildbot(config_name)) @@ -118,7 +107,7 @@ class CommandsFactory(object): self.CheckoutV14Dir(), self.SetupChromeOSCheckout(self.chromeos_version, True), self.RunBuildbot(), - self.scripts.RunBenchmarks(self.board, "BootPerfServer,10:Page,3")) + self.scripts.RunBenchmarks(self.board, 'BootPerfServer,10:Page,3')) def GetP4Snapshot(self, p4view): p4client = perforce.CommandsFactory(self.P4_CHECKOUT_DIR, p4view) @@ -130,38 +119,43 @@ class CommandsFactory(object): def CheckoutV14Dir(self): p4view = perforce.View(self.DEPOT2_DIR, [ - perforce.PathMapping("gcctools/chromeos/v14/...")]) + perforce.PathMapping('gcctools/chromeos/v14/...') + ]) return self.GetP4Snapshot(p4view) def SetupChromeOSCheckout(self, version, use_minilayout=False): - version_re = "^\d+\.\d+\.\d+\.[a-zA-Z0-9]+$" + version_re = '^\d+\.\d+\.\d+\.[a-zA-Z0-9]+$' location = os.path.join(self.CHROMEOS_BUILDS_DIR, version) - if version in ["weekly", "quarterly"]: - assert os.path.islink(location), "Symlink %s does not exist." % location + if version in ['weekly', 'quarterly']: + assert os.path.islink(location), 'Symlink %s does not exist.' % location location_expanded = os.path.abspath(os.path.realpath(location)) version = os.path.basename(location_expanded) - if version in ["top", "latest"] or re.match(version_re, version): + if version in ['top', 'latest'] or re.match(version_re, version): return self.scripts.SetupChromeOS(version, use_minilayout) - elif version.endswith("bz2") or version.endswith("gz"): + elif version.endswith('bz2') or version.endswith('gz'): return cmd.UnTar(location_expanded, self.CHROMEOS_ROOT) else: signature_file_location = os.path.join(location, - "src/scripts/enter_chroot.sh") + 'src/scripts/enter_chroot.sh') assert os.path.exists(signature_file_location), ( - "Signature file %s does not exist." % signature_file_location) + 'Signature file %s does not exist.' % signature_file_location) return cmd.Copy(location, to_dir=self.CHROMEOS_ROOT, recursive=True) class JobsFactory(object): - def __init__(self, chromeos_version="top", board="x86-mario", - toolchain="trunk", p4_snapshot=""): + + def __init__(self, + chromeos_version='top', + board='x86-mario', + toolchain='trunk', + p4_snapshot=''): self.chromeos_version = chromeos_version self.board = board self.toolchain = toolchain @@ -172,14 +166,15 @@ class JobsFactory(object): def BuildAndBenchmark(self): command = self.commands.BuildAndBenchmark() - label = "BuildAndBenchmark(%s,%s,%s)" % ( - self.toolchain, self.board, self.chromeos_version) + label = 'BuildAndBenchmark(%s,%s,%s)' % (self.toolchain, self.board, + self.chromeos_version) - machine_label = "chromeos-%s" % self.board + machine_label = 'chromeos-%s' % self.board job = jobs.CreateLinuxJob(label, command) job.DependsOnMachine( - machine.MachineSpecification(label=machine_label, lock_required=True), + machine.MachineSpecification(label=machine_label, + lock_required=True), False) return job diff --git a/automation/clients/helper/crosstool.py b/automation/clients/helper/crosstool.py index f3b24005..80154b25 100644 --- a/automation/clients/helper/crosstool.py +++ b/automation/clients/helper/crosstool.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. __author__ = 'kbaclawski@google.com (Krystian Baclawski)' @@ -14,14 +12,15 @@ from automation.common import job class JobsFactory(object): + def __init__(self): self.commands = CommandsFactory() def CheckoutCrosstool(self, target): command = self.commands.CheckoutCrosstool() new_job = jobs.CreateLinuxJob('CheckoutCrosstool(%s)' % target, command) - checkout_dir_dep = job.FolderDependency( - new_job, CommandsFactory.CHECKOUT_DIR) + checkout_dir_dep = job.FolderDependency(new_job, + CommandsFactory.CHECKOUT_DIR) manifests_dir_dep = job.FolderDependency( new_job, os.path.join(self.commands.buildit_path, target), 'manifests') return new_job, checkout_dir_dep, manifests_dir_dep @@ -30,14 +29,14 @@ class JobsFactory(object): command = self.commands.BuildRelease(target) new_job = jobs.CreateLinuxJob('BuildRelease(%s)' % target, command) new_job.DependsOnFolder(checkout_dir) - build_tree_dep = job.FolderDependency( - new_job, self.commands.buildit_work_dir_path) + build_tree_dep = job.FolderDependency(new_job, + self.commands.buildit_work_dir_path) return new_job, build_tree_dep def RunTests(self, checkout_dir, build_tree_dir, target, board, component): command = self.commands.RunTests(target, board, component) - new_job = jobs.CreateLinuxJob( - 'RunTests(%s, %s, %s)' % (target, component, board), command) + new_job = jobs.CreateLinuxJob('RunTests(%s, %s, %s)' % + (target, component, board), command) new_job.DependsOnFolder(checkout_dir) new_job.DependsOnFolder(build_tree_dir) testrun_dir_dep = job.FolderDependency( @@ -57,24 +56,26 @@ class CommandsFactory(object): CHECKOUT_DIR = 'crosstool-checkout-dir' def __init__(self): - self.buildit_path = os.path.join( - self.CHECKOUT_DIR, 'gcctools', 'crosstool', 'v15') + self.buildit_path = os.path.join(self.CHECKOUT_DIR, 'gcctools', 'crosstool', + 'v15') self.buildit_work_dir = 'buildit-tmp' self.buildit_work_dir_path = os.path.join('$JOB_TMP', self.buildit_work_dir) - self.dejagnu_output_path = os.path.join( - self.buildit_work_dir_path, 'dejagnu-output') + self.dejagnu_output_path = os.path.join(self.buildit_work_dir_path, + 'dejagnu-output') paths = { 'gcctools': [ - 'crosstool/v15/...', - 'scripts/...'], + 'crosstool/v15/...', 'scripts/...' + ], 'gcctools/google_vendor_src_branch': [ - 'binutils/binutils-2.21/...', - 'gdb/gdb-7.2.x/...', - 'zlib/zlib-1.2.3/...'], + 'binutils/binutils-2.21/...', 'gdb/gdb-7.2.x/...', + 'zlib/zlib-1.2.3/...' + ], 'gcctools/vendor_src': [ - 'gcc/google/gcc-4_6/...']} + 'gcc/google/gcc-4_6/...' + ] + } p4view = perforce.View('depot2', perforce.PathMapping.ListFromPathDict(paths)) @@ -91,22 +92,21 @@ class CommandsFactory(object): def BuildRelease(self, target): clnum_path = os.path.join('$JOB_TMP', self.CHECKOUT_DIR, 'CLNUM') - toolchain_root = os.path.join( - '/google/data/rw/projects/toolchains', target, 'unstable') + toolchain_root = os.path.join('/google/data/rw/projects/toolchains', target, + 'unstable') toolchain_path = os.path.join(toolchain_root, '${CLNUM}') build_toolchain = cmd.Wrapper( cmd.Chain( - cmd.MakeDir(toolchain_path), - cmd.Shell( - 'buildit', - '--keep-work-dir', - '--build-type=release', - '--work-dir=%s' % self.buildit_work_dir_path, - '--results-dir=%s' % toolchain_path, - '--force-release=%s' % '${CLNUM}', - target, - path='.')), + cmd.MakeDir(toolchain_path), + cmd.Shell('buildit', + '--keep-work-dir', + '--build-type=release', + '--work-dir=%s' % self.buildit_work_dir_path, + '--results-dir=%s' % toolchain_path, + '--force-release=%s' % '${CLNUM}', + target, + path='.')), cwd=self.buildit_path, umask='0022', env={'CLNUM': '$(< %s)' % clnum_path}) @@ -114,8 +114,7 @@ class CommandsFactory(object): # remove all but 10 most recent directories remove_old_toolchains_from_x20 = cmd.Wrapper( cmd.Pipe( - cmd.Shell('ls', '-1', '-r'), - cmd.Shell('sed', '-e', '1,10d'), + cmd.Shell('ls', '-1', '-r'), cmd.Shell('sed', '-e', '1,10d'), cmd.Shell('xargs', 'rm', '-r', '-f')), cwd=toolchain_root) @@ -132,13 +131,15 @@ class CommandsFactory(object): 'experimental/users/kbaclawski', 'dejagnu/site.exp') - build_dir_path = os.path.join( - target, 'rpmbuild/BUILD/crosstool*-0.0', 'build-%s' % component) + build_dir_path = os.path.join(target, 'rpmbuild/BUILD/crosstool*-0.0', + 'build-%s' % component) run_dejagnu = cmd.Wrapper( cmd.Chain( cmd.MakeDir(self.dejagnu_output_path), - cmd.Shell('make', 'check', '-k', + cmd.Shell('make', + 'check', + '-k', '-j $(grep -c processor /proc/cpuinfo)', 'RUNTESTFLAGS="%s"' % ' '.join(dejagnu_flags), 'DEJAGNU="%s"' % site_exp_file, @@ -146,8 +147,9 @@ class CommandsFactory(object): cwd=os.path.join(self.buildit_work_dir_path, build_dir_path), env={'REMOTE_TMPDIR': 'job-$JOB_ID'}) - save_results = cmd.Copy( - self.dejagnu_output_path, to_dir='$JOB_TMP/results', recursive=True) + save_results = cmd.Copy(self.dejagnu_output_path, + to_dir='$JOB_TMP/results', + recursive=True) return cmd.Chain(run_dejagnu, save_results) @@ -155,9 +157,12 @@ class CommandsFactory(object): sumfiles = [os.path.join('$JOB_TMP', board, '*.sum') for board in boards] return cmd.Wrapper( - cmd.Shell('dejagnu.sh', 'report', - '-m', '$JOB_TMP/manifests/*.xfail', - '-o', '$JOB_TMP/results/report.html', + cmd.Shell('dejagnu.sh', + 'report', + '-m', + '$JOB_TMP/manifests/*.xfail', + '-o', + '$JOB_TMP/results/report.html', *sumfiles, path='.'), cwd='$HOME/automation/clients/report') diff --git a/automation/clients/helper/jobs.py b/automation/clients/helper/jobs.py index ea9c9691..96a1c408 100644 --- a/automation/clients/helper/jobs.py +++ b/automation/clients/helper/jobs.py @@ -1,13 +1,11 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. from automation.common import job from automation.common import machine -def CreateLinuxJob(label, command, lock=False, timeout=4*60*60): +def CreateLinuxJob(label, command, lock=False, timeout=4 * 60 * 60): to_return = job.Job(label, command, timeout) - to_return.DependsOnMachine( - machine.MachineSpecification(os="linux", lock_required=lock)) + to_return.DependsOnMachine(machine.MachineSpecification(os='linux', + lock_required=lock)) return to_return diff --git a/automation/clients/helper/perforce.py b/automation/clients/helper/perforce.py index 1100a1ee..1f2dfe79 100644 --- a/automation/clients/helper/perforce.py +++ b/automation/clients/helper/perforce.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. __author__ = 'kbaclawski@google.com (Krystian Baclawski)' @@ -153,17 +151,16 @@ class CommandsFactory(object): self.p4config_path = os.path.join(self.checkout_dir, '.p4config') def Initialize(self): - return cmd.Chain( - 'mkdir -p %s' % self.checkout_dir, - 'cp ~/.p4config %s' % self.checkout_dir, - 'chmod u+w %s' % self.p4config_path, - 'echo "P4PORT=%s" >> %s' % (self.port, self.p4config_path), - 'echo "P4CLIENT=%s" >> %s' % (self.view.client, self.p4config_path)) + return cmd.Chain('mkdir -p %s' % self.checkout_dir, 'cp ~/.p4config %s' % + self.checkout_dir, 'chmod u+w %s' % self.p4config_path, + 'echo "P4PORT=%s" >> %s' % (self.port, self.p4config_path), + 'echo "P4CLIENT=%s" >> %s' % + (self.view.client, self.p4config_path)) def Create(self): # TODO(kbaclawski): Could we support value list for options consistently? - mappings = ['-a \"%s %s\"' % mapping for mapping in - self.view.AbsoluteMappings()] + mappings = ['-a \"%s %s\"' % mapping + for mapping in self.view.AbsoluteMappings()] # First command will create client with default mappings. Second one will # replace default mapping with desired. Unfortunately, it seems that it @@ -176,14 +173,12 @@ class CommandsFactory(object): env={'P4EDITOR': '/bin/true'}) def SaveSpecification(self, filename=None): - return cmd.Pipe( - cmd.Shell('g4', 'client', '-o'), - output=filename) + return cmd.Pipe(cmd.Shell('g4', 'client', '-o'), output=filename) def Sync(self, revision=None): sync_arg = '...' if revision: - sync_arg = "%s@%s" % (sync_arg, revision) + sync_arg = '%s@%s' % (sync_arg, revision) return cmd.Shell('g4', 'sync', sync_arg) def SaveCurrentCLNumber(self, filename=None): @@ -196,14 +191,11 @@ class CommandsFactory(object): return cmd.Shell('g4', 'client', '-d', self.view.client) def SetupAndDo(self, *commands): - return cmd.Chain( - self.Initialize(), - self.InCheckoutDir(self.Create(), *commands)) + return cmd.Chain(self.Initialize(), + self.InCheckoutDir(self.Create(), *commands)) def InCheckoutDir(self, *commands): - return cmd.Wrapper( - cmd.Chain(*commands), - cwd=self.checkout_dir) + return cmd.Wrapper(cmd.Chain(*commands), cwd=self.checkout_dir) def CheckoutFromSnapshot(self, snapshot): cmds = cmd.Chain() @@ -216,7 +208,8 @@ class CommandsFactory(object): local_dir = os.path.join(self.checkout_dir, os.path.dirname(local_path)) cmds.extend([ - cmd.Shell('mkdir', '-p', local_dir), - cmd.Shell('rsync', '-lr', remote_dir, local_dir)]) + cmd.Shell('mkdir', '-p', local_dir), cmd.Shell( + 'rsync', '-lr', remote_dir, local_dir) + ]) return cmds diff --git a/automation/clients/nightly.py b/automation/clients/nightly.py index a6157b0a..98e2b081 100755 --- a/automation/clients/nightly.py +++ b/automation/clients/nightly.py @@ -13,21 +13,21 @@ from automation.common import job_group def Main(argv): parser = optparse.OptionParser() - parser.add_option("-c", - "--chromeos_version", - dest="chromeos_version", - default="quarterly", - help="ChromeOS version to use.") - parser.add_option("-t", - "--toolchain", - dest="toolchain", - default="latest-toolchain", - help="Toolchain to use {latest-toolchain,gcc_46}.") - parser.add_option("-b", - "--board", - dest="board", - default="x86-generic", - help="Board to use for the nightly job.") + parser.add_option('-c', + '--chromeos_version', + dest='chromeos_version', + default='quarterly', + help='ChromeOS version to use.') + parser.add_option('-t', + '--toolchain', + dest='toolchain', + default='latest-toolchain', + help='Toolchain to use {latest-toolchain,gcc_46}.') + parser.add_option('-b', + '--board', + dest='board', + default='x86-generic', + help='Board to use for the nightly job.') options = parser.parse_args(argv)[0] toolchain = options.toolchain @@ -36,15 +36,16 @@ def Main(argv): # Build toolchain jobs_factory = chromeos.JobsFactory(chromeos_version=chromeos_version, - board=board, toolchain=toolchain) + board=board, + toolchain=toolchain) benchmark_job = jobs_factory.BuildAndBenchmark() - group_label = "nightly_client_%s" % board + group_label = 'nightly_client_%s' % board group = job_group.JobGroup(group_label, [benchmark_job], True, False) - server = xmlrpclib.Server("http://localhost:8000") + server = xmlrpclib.Server('http://localhost:8000') server.ExecuteJobGroup(pickle.dumps(group)) -if __name__ == "__main__": +if __name__ == '__main__': Main(sys.argv) diff --git a/automation/clients/output_test.py b/automation/clients/output_test.py index 3369e768..3126f050 100755 --- a/automation/clients/output_test.py +++ b/automation/clients/output_test.py @@ -13,17 +13,17 @@ from automation.common import machine def Main(): - server = xmlrpclib.Server("http://localhost:8000") + server = xmlrpclib.Server('http://localhost:8000') - command = os.path.join(os.path.dirname(sys.argv[0]), - "../../produce_output.py") + command = os.path.join( + os.path.dirname(sys.argv[0]), '../../produce_output.py') - pwd_job = job.Job("pwd_job", command) - pwd_job.DependsOnMachine(machine.MachineSpecification(os="linux")) + pwd_job = job.Job('pwd_job', command) + pwd_job.DependsOnMachine(machine.MachineSpecification(os='linux')) - group = job_group.JobGroup("pwd_client", [pwd_job]) + group = job_group.JobGroup('pwd_client', [pwd_job]) server.ExecuteJobGroup(pickle.dumps(group)) -if __name__ == "__main__": +if __name__ == '__main__': Main() diff --git a/automation/clients/pwd_test.py b/automation/clients/pwd_test.py index 565fe739..a4b28552 100755 --- a/automation/clients/pwd_test.py +++ b/automation/clients/pwd_test.py @@ -11,19 +11,17 @@ from automation.common import machine def Main(): - server = xmlrpclib.Server("http://localhost:8000") + server = xmlrpclib.Server('http://localhost:8000') - command = ["echo These following 3 lines should be the same", - "pwd", - "$(pwd)", - "echo ${PWD}"] + command = ['echo These following 3 lines should be the same', 'pwd', '$(pwd)', + 'echo ${PWD}'] - pwd_job = job.Job("pwd_job", " && ".join(command)) - pwd_job.DependsOnMachine(machine.MachineSpecification(os="linux")) + pwd_job = job.Job('pwd_job', ' && '.join(command)) + pwd_job.DependsOnMachine(machine.MachineSpecification(os='linux')) - group = job_group.JobGroup("pwd_client", [pwd_job]) + group = job_group.JobGroup('pwd_client', [pwd_job]) server.ExecuteJobGroup(pickle.dumps(group)) -if __name__ == "__main__": +if __name__ == '__main__': Main() diff --git a/automation/clients/report/dejagnu/__init__.py b/automation/clients/report/dejagnu/__init__.py index e69de29b..8b137891 100644 --- a/automation/clients/report/dejagnu/__init__.py +++ b/automation/clients/report/dejagnu/__init__.py @@ -0,0 +1 @@ + diff --git a/automation/clients/report/dejagnu/main.py b/automation/clients/report/dejagnu/main.py index 6e5e6872..62f095e1 100644 --- a/automation/clients/report/dejagnu/main.py +++ b/automation/clients/report/dejagnu/main.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # Author: kbaclawski@google.com (Krystian Baclawski) # @@ -35,12 +33,12 @@ def OptionChecker(parser): def ManifestCommand(argv): parser = optparse.OptionParser( - description=( - 'Read in one or more DejaGNU summary files (.sum), parse their ' - 'content and generate manifest files. Manifest files store a list ' - 'of failed tests that should be ignored. Generated files are ' - 'stored in current directory under following name: ' - '${tool}-${board}.xfail (e.g. "gcc-unix.xfail").'), + description= + ('Read in one or more DejaGNU summary files (.sum), parse their ' + 'content and generate manifest files. Manifest files store a list ' + 'of failed tests that should be ignored. Generated files are ' + 'stored in current directory under following name: ' + '${tool}-${board}.xfail (e.g. "gcc-unix.xfail").'), usage='Usage: %prog manifest [file.sum] (file2.sum ...)') _, args = parser.parse_args(argv[2:]) @@ -53,8 +51,7 @@ def ManifestCommand(argv): test_run = DejaGnuTestRun.FromFile(filename) manifest = Manifest.FromDejaGnuTestRun(test_run) - manifest_filename = '%s-%s.xfail' % ( - test_run.tool, test_run.board) + manifest_filename = '%s-%s.xfail' % (test_run.tool, test_run.board) with open(manifest_filename, 'w') as manifest_file: manifest_file.write(manifest.Generate()) @@ -64,18 +61,25 @@ def ManifestCommand(argv): def ReportCommand(argv): parser = optparse.OptionParser( - description=( - 'Read in one or more DejaGNU summary files (.sum), parse their ' - 'content and generate a single report file in selected format ' - '(currently only HTML).'), + description= + ('Read in one or more DejaGNU summary files (.sum), parse their ' + 'content and generate a single report file in selected format ' + '(currently only HTML).'), usage=('Usage: %prog report (-m manifest.xfail) [-o report.html] ' '[file.sum (file2.sum ...)')) parser.add_option( - '-o', dest='output', type='string', default=None, + '-o', + dest='output', + type='string', + default=None, help=('Suppress failures for test listed in provided manifest files. ' '(use -m for each manifest file you want to read)')) parser.add_option( - '-m', dest='manifests', type='string', action='append', default=None, + '-m', + dest='manifests', + type='string', + action='append', + default=None, help=('Suppress failures for test listed in provided manifest files. ' '(use -m for each manifest file you want to read)')) @@ -109,11 +113,11 @@ def ReportCommand(argv): def HelpCommand(argv): sys.exit('\n'.join([ - 'Usage: %s command [options]' % os.path.basename(argv[0]), - '', - 'Commands:', + 'Usage: %s command [options]' % os.path.basename(argv[ + 0]), '', 'Commands:', ' manifest - manage files containing a list of suppressed test failures', - ' report - generate report file for selected test runs'])) + ' report - generate report file for selected test runs' + ])) def Main(argv): @@ -122,11 +126,10 @@ def Main(argv): except IndexError: cmd_name = None - cmd_map = { - 'manifest': ManifestCommand, - 'report': ReportCommand} + cmd_map = {'manifest': ManifestCommand, 'report': ReportCommand} cmd_map.get(cmd_name, HelpCommand)(argv) + if __name__ == '__main__': FORMAT = '%(asctime)-15s %(levelname)s %(message)s' logging.basicConfig(format=FORMAT, level=logging.INFO) diff --git a/automation/clients/report/dejagnu/manifest.py b/automation/clients/report/dejagnu/manifest.py index 2124d577..5831d1b0 100644 --- a/automation/clients/report/dejagnu/manifest.py +++ b/automation/clients/report/dejagnu/manifest.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # Author: kbaclawski@google.com (Krystian Baclawski) # @@ -42,7 +40,8 @@ class Manifest(namedtuple('Manifest', 'tool board results')): @classmethod def FromDejaGnuTestRun(cls, test_run): - results = [result for result in test_run.results + results = [result + for result in test_run.results if result.result in cls.SUPPRESSIBLE_RESULTS] return cls(test_run.tool, test_run.board, results) diff --git a/automation/clients/report/dejagnu/report.py b/automation/clients/report/dejagnu/report.py index 7b56590d..191a5389 100644 --- a/automation/clients/report/dejagnu/report.py +++ b/automation/clients/report/dejagnu/report.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # Author: kbaclawski@google.com (Krystian Baclawski) # @@ -7,7 +5,6 @@ import logging import os.path - RESULT_DESCRIPTION = { 'ERROR': 'DejaGNU errors', 'FAIL': 'Failed tests', @@ -18,13 +15,15 @@ RESULT_DESCRIPTION = { 'UNTESTED': 'Not executed tests', 'WARNING': 'DejaGNU warnings', 'XFAIL': 'Expected test failures', - 'XPASS': 'Unexpectedly passed tests'} + 'XPASS': 'Unexpectedly passed tests' +} RESULT_GROUPS = { 'Successes': ['PASS', 'XFAIL'], 'Failures': ['FAIL', 'XPASS', 'UNRESOLVED'], 'Suppressed': ['!FAIL', '!XPASS', '!UNRESOLVED', '!ERROR'], - 'Framework': ['UNTESTED', 'UNSUPPORTED', 'ERROR', 'WARNING', 'NOTE']} + 'Framework': ['UNTESTED', 'UNSUPPORTED', 'ERROR', 'WARNING', 'NOTE'] +} ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) @@ -40,18 +39,18 @@ def _GetResultDescription(name): def _PrepareSummary(res_types, summary): + def GetResultCount(res_type): return summary.get(res_type, 0) - return [(_GetResultDescription(rt), GetResultCount(rt)) - for rt in res_types] + return [(_GetResultDescription(rt), GetResultCount(rt)) for rt in res_types] def _PrepareTestList(res_types, tests): + def GetTestsByResult(res_type): return [(test.name, test.variant or '') - for test in sorted(tests) - if test.result == res_type] + for test in sorted(tests) if test.result == res_type] return [(_GetResultDescription(rt), GetTestsByResult(rt)) for rt in res_types if rt != 'PASS'] @@ -92,7 +91,8 @@ def Generate(test_runs, manifests): tmpl_args.append({ 'id': test_run_id, 'name': '%s @%s' % (test_run.tool, test_run.board), - 'groups': groups}) + 'groups': groups + }) logging.info('Rendering report in HTML format.') @@ -105,7 +105,8 @@ def Generate(test_runs, manifests): logging.error('Failed to generate report in HTML format!') return '' - settings.configure(DEBUG=True, TEMPLATE_DEBUG=True, + settings.configure(DEBUG=True, + TEMPLATE_DEBUG=True, TEMPLATE_DIRS=(ROOT_PATH,)) tmpl = loader.get_template('report.html') diff --git a/automation/clients/report/dejagnu/summary.py b/automation/clients/report/dejagnu/summary.py index 269793e6..d573c691 100644 --- a/automation/clients/report/dejagnu/summary.py +++ b/automation/clients/report/dejagnu/summary.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # Author: kbaclawski@google.com (Krystian Baclawski) # @@ -60,7 +58,8 @@ class DejaGnuTestResult(namedtuple('Result', 'name variant result flaky')): # remove include paths - they contain name of tmp directory ('-I\S+', ''), # compress white spaces - ('\s+', ' ')] + ('\s+', ' ') + ] for pattern, replacement in substitutions: variant = re.sub(pattern, replacement, variant) @@ -168,12 +167,11 @@ class DejaGnuTestRun(object): with open(filename, 'r') as report: lines = [line.strip() for line in report.readlines() if line.strip()] - parsers = ( - (re.compile(r'Running target (.*)'), self._ParseBoard), - (re.compile(r'Test Run By (.*) on (.*)'), self._ParseDate), - (re.compile(r'=== (.*) tests ==='), self._ParseTool), - (re.compile(r'Target(\s+)is (.*)'), self._ParseTarget), - (re.compile(r'Host(\s+)is (.*)'), self._ParseHost)) + parsers = ((re.compile(r'Running target (.*)'), self._ParseBoard), + (re.compile(r'Test Run By (.*) on (.*)'), self._ParseDate), + (re.compile(r'=== (.*) tests ==='), self._ParseTool), + (re.compile(r'Target(\s+)is (.*)'), self._ParseTarget), + (re.compile(r'Host(\s+)is (.*)'), self._ParseHost)) for line in lines: result = DejaGnuTestResult.FromLine(line) @@ -219,8 +217,7 @@ class DejaGnuTestRun(object): # Remove all UNRESOLVED results that were also marked as UNSUPPORTED. unresolved = [res._replace(result='UNRESOLVED') - for res in results - if res.result == 'UNSUPPORTED'] + for res in results if res.result == 'UNSUPPORTED'] for res in unresolved: if res in self.results: @@ -257,8 +254,8 @@ class DejaGnuTestRun(object): self.results.add(result._replace(result=new_result)) for result in sorted(manifest_results): - logging.warning( - 'Result {%s} listed in manifest but not suppressed.', result) + logging.warning('Result {%s} listed in manifest but not suppressed.', + result) def __str__(self): return '{0}, {1} @{2} on {3}'.format(self.target, self.tool, self.board, diff --git a/automation/clients/report/validate_failures.py b/automation/clients/report/validate_failures.py index e99c9054..5db356d1 100755 --- a/automation/clients/report/validate_failures.py +++ b/automation/clients/report/validate_failures.py @@ -24,7 +24,6 @@ # along with GCC; see the file COPYING. If not, write to # the Free Software Foundation, 51 Franklin Street, Fifth Floor, # Boston, MA 02110-1301, USA. - """This script provides a coarser XFAILing mechanism that requires no detailed DejaGNU markings. This is useful in a variety of scenarios: @@ -78,15 +77,13 @@ def GetSumFiles(build_dir): for root, _, filenames in os.walk(build_dir): summaries.extend([os.path.join(root, filename) - for filename in filenames - if filename.endswith('.sum')]) + for filename in filenames if filename.endswith('.sum')]) return map(os.path.normpath, summaries) def ValidBuildDirectory(build_dir, target): - mandatory_paths = [build_dir, - os.path.join(build_dir, 'Makefile')] + mandatory_paths = [build_dir, os.path.join(build_dir, 'Makefile')] extra_paths = [os.path.join(build_dir, target), os.path.join(build_dir, 'build-%s' % target)] @@ -104,8 +101,7 @@ def GetManifestPath(build_dir): target = makefile['target_alias'] if not ValidBuildDirectory(build_dir, target): - logging.error( - '%s is not a valid GCC top level build directory.', build_dir) + logging.error('%s is not a valid GCC top level build directory.', build_dir) sys.exit(1) logging.info('Discovered source directory: "%s"', srcdir) @@ -123,8 +119,8 @@ def CompareResults(manifest, actual): actual_vs_manifest = actual - manifest # Filter out tests marked flaky. - manifest_without_flaky_tests = set( - filter(lambda result: not result.flaky, manifest)) + manifest_without_flaky_tests = set(filter(lambda result: not result.flaky, + manifest)) # Simlarly for all the tests in the manifest. manifest_vs_actual = manifest_without_flaky_tests - actual @@ -136,7 +132,7 @@ def LogResults(level, results): log_fun = getattr(logging, level) for num, result in enumerate(sorted(results), start=1): - log_fun(" %d) %s", num, result) + log_fun(' %d) %s', num, result) def CheckExpectedResults(manifest_path, build_dir): @@ -194,26 +190,37 @@ def ProduceManifest(manifest_path, build_dir, overwrite): with open(manifest_path, 'w') as manifest_file: manifest_strings = [manifest.Generate() for manifest in manifests] - logging.info('Writing manifest to "%s".' % manifest_path) + logging.info('Writing manifest to "%s".', manifest_path) manifest_file.write('\n'.join(manifest_strings)) def Main(argv): parser = optparse.OptionParser(usage=__doc__) parser.add_option( - '-b', '--build_dir', - dest='build_dir', action='store',metavar='PATH', default=os.getcwd(), + '-b', + '--build_dir', + dest='build_dir', + action='store', + metavar='PATH', + default=os.getcwd(), help='Build directory to check. (default: current directory)') + parser.add_option('-m', + '--manifest', + dest='manifest', + action='store_true', + help='Produce the manifest for the current build.') parser.add_option( - '-m', '--manifest', dest='manifest', action='store_true', - help='Produce the manifest for the current build.') - parser.add_option( - '-f', '--force', dest='force', action='store_true', + '-f', + '--force', + dest='force', + action='store_true', help=('Overwrite an existing manifest file, if user requested creating ' 'new one. (default: False)')) - parser.add_option( - '-v', '--verbose', dest='verbose', action='store_true', - help='Increase verbosity.') + parser.add_option('-v', + '--verbose', + dest='verbose', + action='store_true', + help='Increase verbosity.') options, _ = parser.parse_args(argv[1:]) if options.verbose: @@ -226,6 +233,7 @@ def Main(argv): else: CheckExpectedResults(manifest_path, options.build_dir) + if __name__ == '__main__': logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) Main(sys.argv) diff --git a/automation/common/__init__.py b/automation/common/__init__.py index e69de29b..8b137891 100644 --- a/automation/common/__init__.py +++ b/automation/common/__init__.py @@ -0,0 +1 @@ + diff --git a/automation/common/command.py b/automation/common/command.py index 192e2bd8..c56e9fad 100644 --- a/automation/common/command.py +++ b/automation/common/command.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. __author__ = 'kbaclawski@google.com (Krystian Baclawski)' @@ -197,8 +195,8 @@ def RemoteCopyFrom(from_machine, from_path, to_path, username=None): login = '%s@%s' % (username, from_machine) return Chain( - MakeDir(to_path), - Shell('rsync', '-a', '%s:%s' % (login, from_path), to_path)) + MakeDir(to_path), Shell('rsync', '-a', '%s:%s' % + (login, from_path), to_path)) def MakeSymlink(to_path, link_name): @@ -224,8 +222,7 @@ def RmTree(*dirs): def UnTar(tar_file, dest_dir): return Chain( - MakeDir(dest_dir), - Shell('tar', '-x', '-f', tar_file, '-C', dest_dir)) + MakeDir(dest_dir), Shell('tar', '-x', '-f', tar_file, '-C', dest_dir)) def Tar(tar_file, *args): @@ -241,6 +238,4 @@ def Tar(tar_file, *args): options.extend(['-f', tar_file]) options.extend(args) - return Chain( - MakeDir(os.path.dirname(tar_file)), - Shell('tar', *options)) + return Chain(MakeDir(os.path.dirname(tar_file)), Shell('tar', *options)) diff --git a/automation/common/command_executer.py b/automation/common/command_executer.py index 2db0048e..c0f314f5 100644 --- a/automation/common/command_executer.py +++ b/automation/common/command_executer.py @@ -1,15 +1,12 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # - """Classes that help running commands in a subshell. Commands can be run locally, or remotly using SSH connection. You may log the output of a command to a terminal or a file, or any other destination. """ -__author__ = "kbaclawski@google.com (Krystian Baclawski)" +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' import fcntl import logging @@ -32,8 +29,12 @@ class CommandExecuter(object): def Configure(cls, dry_run): cls.DRY_RUN = dry_run - def RunCommand(self, cmd, machine=None, username=None, - command_terminator=None, command_timeout=None): + def RunCommand(self, + cmd, + machine=None, + username=None, + command_terminator=None, + command_timeout=None): cmd = str(cmd) if self._dry_run: @@ -43,13 +44,13 @@ class CommandExecuter(object): command_terminator = CommandTerminator() if command_terminator.IsTerminated(): - self._logger.warning("Command has been already terminated!") + self._logger.warning('Command has been already terminated!') return 1 # Rewrite command for remote execution. if machine: if username: - login = "%s@%s" % (username, machine) + login = '%s@%s' % (username, machine) else: login = machine @@ -62,8 +63,8 @@ class CommandExecuter(object): child = self._SpawnProcess(cmd, command_terminator, command_timeout) - self._logger.debug( - "{PID: %d} Finished with %d code.", child.pid, child.returncode) + self._logger.debug('{PID: %d} Finished with %d code.', child.pid, + child.returncode) return child.returncode @@ -71,10 +72,10 @@ class CommandExecuter(object): """Gracefully shutdown the child by sending SIGTERM.""" if command_timeout: - self._logger.warning("{PID: %d} Timeout of %s seconds reached since " - "process started.", child.pid, command_timeout) + self._logger.warning('{PID: %d} Timeout of %s seconds reached since ' + 'process started.', child.pid, command_timeout) - self._logger.warning("{PID: %d} Terminating child.", child.pid) + self._logger.warning('{PID: %d} Terminating child.', child.pid) try: child.terminate() @@ -92,16 +93,18 @@ class CommandExecuter(object): def _Kill(self, child): """Kill the child with immediate result.""" - self._logger.warning("{PID: %d} Process still alive.", child.pid) - self._logger.warning("{PID: %d} Killing child.", child.pid) + self._logger.warning('{PID: %d} Process still alive.', child.pid) + self._logger.warning('{PID: %d} Killing child.', child.pid) child.kill() child.wait() def _SpawnProcess(self, cmd, command_terminator, command_timeout): # Create a child process executing provided command. - child = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - stdin=subprocess.PIPE, shell=True) + child = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + shell=True) # Close stdin so the child won't be able to block on read. child.stdin.close() @@ -157,10 +160,10 @@ class CommandExecuter(object): data = os.read(fd, 4096) except OSError: # terminate loop if EWOULDBLOCK (EAGAIN) is received - data = "" + data = '' if not already_terminated: - self._logger.debug("Waiting for command to finish.") + self._logger.debug('Waiting for command to finish.') child.wait() return child @@ -175,6 +178,7 @@ class CommandExecuter(object): class LoggingCommandExecuter(CommandExecuter): + def __init__(self, *args, **kwargs): super(LoggingCommandExecuter, self).__init__(*args, **kwargs) @@ -183,8 +187,8 @@ class LoggingCommandExecuter(CommandExecuter): def OpenLog(self, log_path): """The messages are going to be saved to gzip compressed file.""" - formatter = logging.Formatter( - '%(asctime)s %(prefix)s: %(message)s', '%Y-%m-%d %H:%M:%S') + formatter = logging.Formatter('%(asctime)s %(prefix)s: %(message)s', + '%Y-%m-%d %H:%M:%S') handler = logger.CompressedFileHandler(log_path, delay=True) handler.setFormatter(formatter) self._output.addHandler(handler) @@ -215,6 +219,7 @@ class LoggingCommandExecuter(CommandExecuter): class CommandTerminator(object): + def __init__(self): self.terminated = False diff --git a/automation/common/command_executer_test.py b/automation/common/command_executer_test.py index 5b6ca639..4aa245f0 100755 --- a/automation/common/command_executer_test.py +++ b/automation/common/command_executer_test.py @@ -32,6 +32,7 @@ from automation.common.command_executer import CommandExecuter class LoggerMock(object): + def LogCmd(self, cmd, machine='', user=''): if machine: logging.info('[%s] Executing: %s', machine, cmd) @@ -49,6 +50,7 @@ class LoggerMock(object): class CommandExecuterUnderTest(CommandExecuter): + def __init__(self): CommandExecuter.__init__(self, logger_to_set=LoggerMock()) @@ -83,8 +85,9 @@ class CommandExecuterLocalTests(unittest.TestCase): def RunCommand(self, method, **kwargs): program = os.path.abspath(sys.argv[0]) - return self._executer.RunCommand( - '%s runHelper %s' % (program, method), machine=self.HOSTNAME, **kwargs) + return self._executer.RunCommand('%s runHelper %s' % (program, method), + machine=self.HOSTNAME, + **kwargs) def testCommandTimeout(self): exit_code = self.RunCommand('SleepForMinute', command_timeout=3) @@ -114,12 +117,14 @@ class CommandExecuterLocalTests(unittest.TestCase): self.assertEquals(self._executer.stdout, '') def testOutputStreamNonInteractive(self): - self.assertFalse(self.RunCommand('IsOutputStreamInteractive'), - 'stdout stream is a terminal!') + self.assertFalse( + self.RunCommand('IsOutputStreamInteractive'), + 'stdout stream is a terminal!') def testErrorStreamNonInteractive(self): - self.assertFalse(self.RunCommand('IsErrorStreamInteractive'), - 'stderr stream is a terminal!') + self.assertFalse( + self.RunCommand('IsErrorStreamInteractive'), + 'stderr stream is a terminal!') def testAttemptToRead(self): self.assertFalse(self.RunCommand('WaitForInput', command_timeout=3)) @@ -149,6 +154,7 @@ class CommandExecuterRemoteTests(CommandExecuterLocalTests): class CommandExecuterTestHelpers(object): + def SleepForMinute(self): time.sleep(60) return 1 diff --git a/automation/common/events.py b/automation/common/events.py index 851a982c..ad3ec844 100644 --- a/automation/common/events.py +++ b/automation/common/events.py @@ -1,9 +1,7 @@ -#!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright 2011 Google Inc. All Rights Reserved. # - """Tools for recording and reporting timeline of abstract events. You can store any events provided that they can be stringified. @@ -56,8 +54,7 @@ class _EventRecord(object): return self._time_elapsed is not None def GetTimeStartedFormatted(self): - return time.strftime('%m/%d/%Y %H:%M:%S', - time.gmtime(self._time_started)) + return time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime(self._time_started)) def GetTimeElapsedRounded(self): return datetime.timedelta(seconds=int(self.time_elapsed.seconds)) @@ -104,8 +101,8 @@ class EventHistory(collections.Sequence): def GetTotalTime(self): if self._records: - total_time_elapsed = sum(evrec.time_elapsed.seconds for evrec in - self._records) + total_time_elapsed = sum(evrec.time_elapsed.seconds + for evrec in self._records) return datetime.timedelta(seconds=int(total_time_elapsed)) @@ -114,15 +111,16 @@ class EventHistory(collections.Sequence): if self._records: for num, next_evrec in enumerate(self._records[1:], start=1): - evrec = self._records[num-1] + evrec = self._records[num - 1] - records.append(_EventRecord(_Transition(evrec.event, next_evrec.event), - evrec.time_started, evrec.time_elapsed)) + records.append(_EventRecord( + _Transition(evrec.event, next_evrec.event), evrec.time_started, + evrec.time_elapsed)) if not self.last.has_finished: - records.append(_EventRecord(_Transition(self.last.event, 'NOW'), - self.last.time_started, - self.last.time_elapsed)) + records.append(_EventRecord( + _Transition(self.last.event, + 'NOW'), self.last.time_started, self.last.time_elapsed)) return EventHistory(records) diff --git a/automation/common/job.py b/automation/common/job.py index d32fa59a..e845ab25 100644 --- a/automation/common/job.py +++ b/automation/common/job.py @@ -1,14 +1,9 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. # - """A module for a job in the infrastructure.""" - __author__ = 'raymes@google.com (Raymes Khoury)' - import os.path from automation.common import state_machine @@ -22,6 +17,7 @@ STATUS_FAILED = 'FAILED' class FolderDependency(object): + def __init__(self, job, src, dest=None): if not dest: dest = src @@ -41,12 +37,14 @@ class JobStateMachine(state_machine.BasicStateMachine): STATUS_NOT_EXECUTED: [STATUS_SETUP], STATUS_SETUP: [STATUS_COPYING, STATUS_FAILED], STATUS_COPYING: [STATUS_RUNNING, STATUS_FAILED], - STATUS_RUNNING: [STATUS_SUCCEEDED, STATUS_FAILED]} + STATUS_RUNNING: [STATUS_SUCCEEDED, STATUS_FAILED] + } final_states = [STATUS_SUCCEEDED, STATUS_FAILED] class JobFailure(Exception): + def __init__(self, message, exit_code): Exception.__init__(self, message) self.exit_code = exit_code @@ -57,7 +55,7 @@ class Job(object): WORKDIR_PREFIX = '/usr/local/google/tmp/automation' - def __init__(self, label, command, timeout=4*60*60): + def __init__(self, label, command, timeout=4 * 60 * 60): self._state = JobStateMachine(STATUS_NOT_EXECUTED) self.predecessors = set() self.successors = set() @@ -110,15 +108,15 @@ class Job(object): def GetCommand(self): substitutions = [ - ('$JOB_ID', str(self.id)), - ('$JOB_TMP', self.work_dir), + ('$JOB_ID', str(self.id)), ('$JOB_TMP', self.work_dir), ('$JOB_HOME', self.home_dir), - ('$PRIMARY_MACHINE', self.primary_machine.hostname)] + ('$PRIMARY_MACHINE', self.primary_machine.hostname) + ] if len(self.machines) > 1: for num, machine in enumerate(self.machines[1:]): - substitutions.append( - ('$SECONDARY_MACHINES[%d]' % num, machine.hostname)) + substitutions.append(('$SECONDARY_MACHINES[%d]' % num, machine.hostname + )) return self._FormatCommand(str(self.command), substitutions) @@ -127,7 +125,8 @@ class Job(object): # non existing Command class. If one is created then PrettyFormatCommand # shall become its method. return self._FormatCommand(self.GetCommand(), [ - ('\{ ', ''), ('; \}', ''), ('\} ', '\n'), ('\s*&&\s*', '\n')]) + ('\{ ', ''), ('; \}', ''), ('\} ', '\n'), ('\s*&&\s*', '\n') + ]) def DependsOnFolder(self, dependency): self.folder_dependencies.append(dependency) diff --git a/automation/common/job_group.py b/automation/common/job_group.py index 09321e1a..96912fc1 100644 --- a/automation/common/job_group.py +++ b/automation/common/job_group.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. # @@ -8,25 +6,30 @@ import os from automation.common.state_machine import BasicStateMachine -STATUS_NOT_EXECUTED = "NOT_EXECUTED" -STATUS_EXECUTING = "EXECUTING" -STATUS_SUCCEEDED = "SUCCEEDED" -STATUS_FAILED = "FAILED" +STATUS_NOT_EXECUTED = 'NOT_EXECUTED' +STATUS_EXECUTING = 'EXECUTING' +STATUS_SUCCEEDED = 'SUCCEEDED' +STATUS_FAILED = 'FAILED' class JobGroupStateMachine(BasicStateMachine): state_machine = { STATUS_NOT_EXECUTED: [STATUS_EXECUTING], - STATUS_EXECUTING: [STATUS_SUCCEEDED, STATUS_FAILED]} + STATUS_EXECUTING: [STATUS_SUCCEEDED, STATUS_FAILED] + } final_states = [STATUS_SUCCEEDED, STATUS_FAILED] class JobGroup(object): - HOMEDIR_PREFIX = os.path.join("/home", getpass.getuser(), "www", "automation") - - def __init__(self, label, jobs=None, cleanup_on_completion=True, - cleanup_on_failure=False, description=""): + HOMEDIR_PREFIX = os.path.join('/home', getpass.getuser(), 'www', 'automation') + + def __init__(self, + label, + jobs=None, + cleanup_on_completion=True, + cleanup_on_failure=False, + description=''): self._state = JobGroupStateMachine(STATUS_NOT_EXECUTED) self.id = 0 self.label = label @@ -49,7 +52,7 @@ class JobGroup(object): @property def home_dir(self): - return os.path.join(self.HOMEDIR_PREFIX, "job-group-%d" % self.id) + return os.path.join(self.HOMEDIR_PREFIX, 'job-group-%d' % self.id) @property def time_submitted(self): @@ -62,9 +65,8 @@ class JobGroup(object): return '{%s: %s}' % (self.__class__.__name__, self.id) def __str__(self): - return "\n".join(["Job-Group:", - "ID: %s" % self.id] + - [str(job) for job in self.jobs]) + return '\n'.join(['Job-Group:', 'ID: %s' % self.id] + [str( + job) for job in self.jobs]) def AddJob(self, job): self.jobs.append(job) diff --git a/automation/common/logger.py b/automation/common/logger.py index efa1a904..4aeee052 100644 --- a/automation/common/logger.py +++ b/automation/common/logger.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. from itertools import chain @@ -17,7 +15,10 @@ def SetUpRootLogger(filename=None, level=None, display_flags={}): if filename: file_handler = logging.handlers.RotatingFileHandler( - filename, maxBytes=10*1024*1024, backupCount=9, delay=True) + filename, + maxBytes=10 * 1024 * 1024, + backupCount=9, + delay=True) file_handler.setFormatter(CustomFormatter(NullColorCoder(), display_flags)) logging.root.addHandler(file_handler) @@ -26,12 +27,13 @@ def SetUpRootLogger(filename=None, level=None, display_flags={}): class NullColorCoder(object): + def __call__(self, *args): return '' class AnsiColorCoder(object): - CODES = {'reset': (0, ), + CODES = {'reset': (0,), 'bold': (1, 22), 'italics': (3, 23), 'underline': (4, 24), @@ -82,8 +84,8 @@ class CustomFormatter(logging.Formatter): def formatTime(self, record): ct = self.converter(record.created) - t = time.strftime("%Y-%m-%d %H:%M:%S", ct) - return "%s.%02d" % (t, record.msecs / 10) + t = time.strftime('%Y-%m-%d %H:%M:%S', ct) + return '%s.%02d' % (t, record.msecs / 10) def formatLevelName(self, record): if record.levelname in ['WARNING', 'CRITICAL']: @@ -96,8 +98,8 @@ class CustomFormatter(logging.Formatter): def formatMessagePrefix(self, record): try: - return ' %s%s:%s ' % ( - self._coder('black', 'bold'), record.prefix, self._coder('reset')) + return ' %s%s:%s ' % (self._coder('black', 'bold'), record.prefix, + self._coder('reset')) except AttributeError: return '' @@ -125,6 +127,7 @@ class CustomFormatter(logging.Formatter): class CompressedFileHandler(logging.FileHandler): + def _open(self): return gzip.open(self.baseFilename + '.gz', self.mode, 9) @@ -136,6 +139,6 @@ def HandleUncaughtExceptions(fun): try: return fun(*args, **kwargs) except StandardError: - logging.exception("Uncaught exception:") + logging.exception('Uncaught exception:') return _Interceptor diff --git a/automation/common/machine.py b/automation/common/machine.py index ae9f0b49..4db0db0d 100644 --- a/automation/common/machine.py +++ b/automation/common/machine.py @@ -1,8 +1,6 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. -__author__ = "asharif@google.com (Ahmad Sharif)" +__author__ = 'asharif@google.com (Ahmad Sharif)' from fnmatch import fnmatch @@ -38,24 +36,20 @@ class Machine(object): self.locked = False def __repr__(self): - return "{%s: %s@%s}" % ( - self.__class__.__name__, self.username, self.hostname) + return '{%s: %s@%s}' % (self.__class__.__name__, self.username, + self.hostname) def __str__(self): - return "\n".join(["Machine Information:", - "Hostname: %s" % self.hostname, - "Label: %s" % self.label, - "CPU: %s" % self.cpu, - "Cores: %d" % self.cores, - "OS: %s" % self.os, - "Uses: %d" % self.uses, - "Locked: %s" % self.locked]) + return '\n'.join( + ['Machine Information:', 'Hostname: %s' % self.hostname, 'Label: %s' % + self.label, 'CPU: %s' % self.cpu, 'Cores: %d' % self.cores, 'OS: %s' % + self.os, 'Uses: %d' % self.uses, 'Locked: %s' % self.locked]) class MachineSpecification(object): """Helper class used to find a machine matching your requirements.""" - def __init__(self, hostname="*", label="*", os="*", lock_required=False): + def __init__(self, hostname='*', label='*', os='*', lock_required=False): self.hostname = hostname self.label = label self.os = os @@ -63,16 +57,13 @@ class MachineSpecification(object): self.preferred_machines = [] def __str__(self): - return "\n".join(["Machine Specification:", - "Name: %s" % self.name, - "OS: %s" % self.os, - "Lock required: %s" % self.lock_required]) + return '\n'.join(['Machine Specification:', 'Name: %s' % self.name, 'OS: %s' + % self.os, 'Lock required: %s' % self.lock_required]) def IsMatch(self, machine): - return all([not machine.locked, - fnmatch(machine.hostname, self.hostname), - fnmatch(machine.label, self.label), - fnmatch(machine.os, self.os)]) + return all([not machine.locked, fnmatch(machine.hostname, self.hostname), + fnmatch(machine.label, self.label), fnmatch(machine.os, + self.os)]) def AddPreferredMachine(self, hostname): if hostname not in self.preferred_machines: diff --git a/automation/common/machine_test.py b/automation/common/machine_test.py index 96e8f823..c9c200a9 100755 --- a/automation/common/machine_test.py +++ b/automation/common/machine_test.py @@ -1,28 +1,26 @@ #!/usr/bin/python # # Copyright 2010 Google Inc. All Rights Reserved. - """Machine manager unittest. MachineManagerTest tests MachineManager. """ -__author__ = "asharif@google.com (Ahmad Sharif)" - +__author__ = 'asharif@google.com (Ahmad Sharif)' import machine import unittest class MachineTest(unittest.TestCase): + def setUp(self): pass - def testPrintMachine(self): - mach = machine.Machine("ahmad.mtv", "core2duo", 4, "linux", "asharif") - self.assertTrue("ahmad.mtv" in str(mach)) + mach = machine.Machine('ahmad.mtv', 'core2duo', 4, 'linux', 'asharif') + self.assertTrue('ahmad.mtv' in str(mach)) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/automation/common/state_machine.py b/automation/common/state_machine.py index 4d8dc49b..d1cf42c8 100644 --- a/automation/common/state_machine.py +++ b/automation/common/state_machine.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright 2011 Google Inc. All Rights Reserved. diff --git a/automation/server/__init__.py b/automation/server/__init__.py index e69de29b..8b137891 100644 --- a/automation/server/__init__.py +++ b/automation/server/__init__.py @@ -0,0 +1 @@ + diff --git a/automation/server/job_executer.py b/automation/server/job_executer.py index 8f48ca08..30b59463 100644 --- a/automation/server/job_executer.py +++ b/automation/server/job_executer.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. # @@ -15,9 +13,10 @@ from automation.common.command_executer import CommandTerminator class JobExecuter(threading.Thread): + def __init__(self, job_to_execute, machines, listeners): threading.Thread.__init__(self) - + assert machines self.job = job_to_execute @@ -25,13 +24,13 @@ class JobExecuter(threading.Thread): self.machines = machines # Set Thread name. - self.name = "%s-%s" % (self.__class__.__name__, self.job.id) + self.name = '%s-%s' % (self.__class__.__name__, self.job.id) self._logger = logging.getLogger(self.__class__.__name__) self._executer = LoggingCommandExecuter(self.job.dry_run) self._terminator = CommandTerminator() - def _RunRemotely(self, command, fail_msg, command_timeout=1*60*60): + def _RunRemotely(self, command, fail_msg, command_timeout=1 * 60 * 60): exit_code = self._executer.RunCommand(command, self.job.primary_machine.hostname, self.job.primary_machine.username, @@ -40,7 +39,7 @@ class JobExecuter(threading.Thread): if exit_code: raise job.JobFailure(fail_msg, exit_code) - def _RunLocally(self, command, fail_msg, command_timeout=1*60*60): + def _RunLocally(self, command, fail_msg, command_timeout=1 * 60 * 60): exit_code = self._executer.RunCommand(command, command_terminator=self._terminator, command_timeout=command_timeout) @@ -52,22 +51,20 @@ class JobExecuter(threading.Thread): def CleanUpWorkDir(self): self._logger.debug('Cleaning up %r work directory.', self.job) - self._RunRemotely( - cmd.RmTree(self.job.work_dir), "Cleanup workdir failed.") + self._RunRemotely(cmd.RmTree(self.job.work_dir), 'Cleanup workdir failed.') def CleanUpHomeDir(self): self._logger.debug('Cleaning up %r home directory.', self.job) - self._RunLocally( - cmd.RmTree(self.job.home_dir), "Cleanup homedir failed.") + self._RunLocally(cmd.RmTree(self.job.home_dir), 'Cleanup homedir failed.') def _PrepareRuntimeEnvironment(self): self._RunRemotely( cmd.MakeDir(self.job.work_dir, self.job.logs_dir, self.job.results_dir), - "Creating new job directory failed.") + 'Creating new job directory failed.') # The log directory is ready, so we can prepare to log command's output. - self._executer.OpenLog( - os.path.join(self.job.logs_dir, self.job.log_filename_prefix)) + self._executer.OpenLog(os.path.join(self.job.logs_dir, + self.job.log_filename_prefix)) def _SatisfyFolderDependencies(self): for dependency in self.job.folder_dependencies: @@ -79,18 +76,21 @@ class JobExecuter(threading.Thread): # No need to make a copy, just symlink it self._RunRemotely( cmd.MakeSymlink(from_folder, to_folder), - "Failed to create symlink to required directory.") + 'Failed to create symlink to required directory.') else: self._RunRemotely( - cmd.RemoteCopyFrom(from_machine.hostname, from_folder, to_folder, + cmd.RemoteCopyFrom(from_machine.hostname, + from_folder, + to_folder, username=from_machine.username), - "Failed to copy required files.") + 'Failed to copy required files.') def _LaunchJobCommand(self): command = self.job.GetCommand() - self._RunRemotely("%s; %s" % ("PS1=. TERM=linux source ~/.bashrc", - cmd.Wrapper(command, cwd=self.job.work_dir)), + self._RunRemotely('%s; %s' % ('PS1=. TERM=linux source ~/.bashrc', + cmd.Wrapper(command, + cwd=self.job.work_dir)), "Command failed to execute: '%s'." % command, self.job.timeout) @@ -101,14 +101,13 @@ class JobExecuter(threading.Thread): self.job.results_dir, self.job.home_dir, username=self.job.primary_machine.username), - "Failed to copy results.") + 'Failed to copy results.') def run(self): self.job.status = job.STATUS_SETUP self.job.machines = self.machines - self._logger.debug( - "Executing %r on %r in directory %s.", - self.job, self.job.primary_machine.hostname, self.job.work_dir) + self._logger.debug('Executing %r on %r in directory %s.', self.job, + self.job.primary_machine.hostname, self.job.work_dir) try: self.CleanUpWorkDir() @@ -127,10 +126,9 @@ class JobExecuter(threading.Thread): # If we get here, the job succeeded. self.job.status = job.STATUS_SUCCEEDED except job.JobFailure as ex: - self._logger.error( - "Job failed. Exit code %s. %s", ex.exit_code, ex) + self._logger.error('Job failed. Exit code %s. %s', ex.exit_code, ex) if self._terminator.IsTerminated(): - self._logger.info("%r was killed", self.job) + self._logger.info('%r was killed', self.job) self.job.status = job.STATUS_FAILED diff --git a/automation/server/job_group_manager.py b/automation/server/job_group_manager.py index 1a0b1b08..d66f5e07 100644 --- a/automation/server/job_group_manager.py +++ b/automation/server/job_group_manager.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. # @@ -16,6 +14,7 @@ from automation.server.job_manager import IdProducerPolicy class JobGroupManager(object): + def __init__(self, job_manager): self.all_job_groups = [] @@ -27,7 +26,7 @@ class JobGroupManager(object): self._id_producer = IdProducerPolicy() self._id_producer.Initialize(job_group.JobGroup.HOMEDIR_PREFIX, - "job-group-(?P<id>\d+)") + 'job-group-(?P<id>\d+)') self._logger = logging.getLogger(self.__class__.__name__) @@ -49,9 +48,8 @@ class JobGroupManager(object): self._logger.debug('Creating runtime environment for %r.', group) - CommandExecuter().RunCommand( - cmd.Chain(cmd.RmTree(group.home_dir), - cmd.MakeDir(group.home_dir))) + CommandExecuter().RunCommand(cmd.Chain( + cmd.RmTree(group.home_dir), cmd.MakeDir(group.home_dir))) with self._lock: self.all_job_groups.append(group) @@ -61,7 +59,7 @@ class JobGroupManager(object): group.status = job_group.STATUS_EXECUTING - self._logger.info("Added %r to queue.", group) + self._logger.info('Added %r to queue.', group) return group.id diff --git a/automation/server/job_manager.py b/automation/server/job_manager.py index de5bc47c..7a65b918 100644 --- a/automation/server/job_manager.py +++ b/automation/server/job_manager.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. # @@ -60,6 +58,7 @@ class IdProducerPolicy(object): class JobManager(threading.Thread): + def __init__(self, machine_manager): threading.Thread.__init__(self, name=self.__class__.__name__) self.all_jobs = [] @@ -81,14 +80,14 @@ class JobManager(threading.Thread): self._logger = logging.getLogger(self.__class__.__name__) def StartJobManager(self): - self._logger.info("Starting...") + self._logger.info('Starting...') with self._lock: self.start() self._jobs_available.notifyAll() def StopJobManager(self): - self._logger.info("Shutdown request received.") + self._logger.info('Shutdown request received.') with self._lock: for job_ in self.all_jobs: @@ -117,7 +116,7 @@ class JobManager(threading.Thread): return None def _KillJob(self, job_id): - self._logger.info("Killing [Job: %d].", job_id) + self._logger.info('Killing [Job: %d].', job_id) if job_id in self.job_executer_mapping: self.job_executer_mapping[job_id].Kill() @@ -165,7 +164,7 @@ class JobManager(threading.Thread): @logger.HandleUncaughtExceptions def run(self): - self._logger.info("Started.") + self._logger.info('Started.') while not self._exit_request: with self._lock: @@ -192,4 +191,4 @@ class JobManager(threading.Thread): executer.start() self.job_executer_mapping[ready_job.id] = executer - self._logger.info("Stopped.") + self._logger.info('Stopped.') diff --git a/automation/server/machine_manager.py b/automation/server/machine_manager.py index b54f19d4..b7186077 100644 --- a/automation/server/machine_manager.py +++ b/automation/server/machine_manager.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. __author__ = 'asharif@google.com (Ahmad Sharif)' @@ -41,7 +39,8 @@ class MachineManager(object): mach = min(available_pool, key=uses) if mach_spec.preferred_machines: - preferred_pool = [m for m in available_pool + preferred_pool = [m + for m in available_pool if m.hostname in mach_spec.preferred_machines] if preferred_pool: mach = min(preferred_pool, key=uses) diff --git a/automation/server/machine_manager_test.py b/automation/server/machine_manager_test.py index ebdaea5f..67fdcc2b 100755 --- a/automation/server/machine_manager_test.py +++ b/automation/server/machine_manager_test.py @@ -2,7 +2,7 @@ # # Copyright 2010 Google Inc. All Rights Reserved. -__author__ = "asharif@google.com (Ahmad Sharif)" +__author__ = 'asharif@google.com (Ahmad Sharif)' import unittest from automation.common import machine @@ -18,15 +18,15 @@ class MachineManagerTest(unittest.TestCase): print self.machine_manager def testGetLinuxBox(self): - mach_spec_list = [machine.MachineSpecification(os="linux")] + mach_spec_list = [machine.MachineSpecification(os='linux')] machines = self.machine_manager.GetMachines(mach_spec_list) self.assertTrue(machines) def testGetChromeOSBox(self): - mach_spec_list = [machine.MachineSpecification(os="chromeos")] + mach_spec_list = [machine.MachineSpecification(os='chromeos')] machines = self.machine_manager.GetMachines(mach_spec_list) self.assertTrue(machines) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/automation/server/monitor/__init__.py b/automation/server/monitor/__init__.py index e69de29b..8b137891 100644 --- a/automation/server/monitor/__init__.py +++ b/automation/server/monitor/__init__.py @@ -0,0 +1 @@ + diff --git a/automation/server/monitor/dashboard.py b/automation/server/monitor/dashboard.py index 33d7c3d7..f6befed8 100644 --- a/automation/server/monitor/dashboard.py +++ b/automation/server/monitor/dashboard.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # @@ -19,7 +17,6 @@ from django.shortcuts import render_to_response from django.template import Context from django.views import static - Link = namedtuple('Link', 'href name') @@ -29,8 +26,8 @@ def GetServerConnection(): def MakeDefaultContext(*args): context = Context({'links': [ - Link('/job-group', 'Job Groups'), - Link('/machine', 'Machines')]}) + Link('/job-group', 'Job Groups'), Link('/machine', 'Machines') + ]}) for arg in args: context.update(arg) @@ -39,6 +36,7 @@ def MakeDefaultContext(*args): class JobInfo(object): + def __init__(self, job_id): self._job = pickle.loads(GetServerConnection().GetJob(job_id)) @@ -60,12 +58,9 @@ class JobInfo(object): commands = enumerate(job.PrettyFormatCommand().split('\n'), start=1) - return {'text': [('Label', job.label), - ('Directory', job.work_dir)], - 'link': [('Group', group), - ('Predecessors', predecessors), - ('Successors', successors), - ('Machines', machines), + return {'text': [('Label', job.label), ('Directory', job.work_dir)], + 'link': [('Group', group), ('Predecessors', predecessors), + ('Successors', successors), ('Machines', machines), ('Logs', logs)], 'code': [('Command', commands)]} @@ -77,8 +72,8 @@ class JobInfo(object): for evlog in self._job.timeline.GetTransitionEventHistory()] def GetLog(self): - log_path = os.path.join( - self._job.logs_dir, '%s.gz' % self._job.log_filename_prefix) + log_path = os.path.join(self._job.logs_dir, + '%s.gz' % self._job.log_filename_prefix) try: log = gzip.open(log_path, 'r') @@ -104,9 +99,10 @@ class JobInfo(object): class JobGroupInfo(object): + def __init__(self, job_group_id): - self._job_group = pickle.loads( - GetServerConnection().GetJobGroup(job_group_id)) + self._job_group = pickle.loads(GetServerConnection().GetJobGroup( + job_group_id)) def GetAttributes(self): group = self._job_group @@ -159,9 +155,9 @@ class JobGroupInfo(object): class JobGroupListInfo(object): + def __init__(self): - self._all_job_groups = pickle.loads( - GetServerConnection().GetAllJobGroups()) + self._all_job_groups = pickle.loads(GetServerConnection().GetAllJobGroups()) def _GetJobGroupState(self, group): return str(group.status) @@ -188,7 +184,8 @@ def JobPageHandler(request, job_id): ctx = MakeDefaultContext({ 'job_id': job_id, 'attributes': job.GetAttributes(), - 'timeline': job.GetTimeline()}) + 'timeline': job.GetTimeline() + }) return render_to_response('job.html', ctx) @@ -196,9 +193,7 @@ def JobPageHandler(request, job_id): def LogPageHandler(request, job_id): job = JobInfo(int(job_id)) - ctx = MakeDefaultContext({ - 'job_id': job_id, - 'log_lines': job.GetLog()}) + ctx = MakeDefaultContext({'job_id': job_id, 'log_lines': job.GetLog()}) return render_to_response('job_log.html', ctx) @@ -210,7 +205,8 @@ def JobGroupPageHandler(request, job_group_id): 'group_id': job_group_id, 'attributes': group.GetAttributes(), 'job_list': group.GetJobList(), - 'reports': group.GetReportList()}) + 'reports': group.GetReportList() + }) return render_to_response('job_group.html', ctx) @@ -218,8 +214,10 @@ def JobGroupPageHandler(request, job_group_id): def JobGroupFilesPageHandler(request, job_group_id, path): group = JobGroupInfo(int(job_group_id)) - return static.serve( - request, path, document_root=group.GetHomeDirectory(), show_indexes=True) + return static.serve(request, + path, + document_root=group.GetHomeDirectory(), + show_indexes=True) class FilterJobGroupsForm(forms.Form): @@ -245,9 +243,7 @@ def JobGroupListPageHandler(request): else: form = FilterJobGroupsForm({'initial': '*'}) - ctx = MakeDefaultContext({ - 'filter': form, - 'groups': group_list}) + ctx = MakeDefaultContext({'filter': form, 'groups': group_list}) return render_to_response('job_group_list.html', ctx) diff --git a/automation/server/monitor/manage.py b/automation/server/monitor/manage.py index 1733753c..57deb5c2 100755 --- a/automation/server/monitor/manage.py +++ b/automation/server/monitor/manage.py @@ -8,7 +8,7 @@ __author__ = 'kbaclawski@google.com (Krystian Baclawski)' from django.core.management import execute_manager try: - import settings # Assumed to be in the same directory. + import settings # Assumed to be in the same directory. except ImportError: import sys @@ -16,5 +16,5 @@ except ImportError: 'containing %r.' % __file__) sys.exit(1) -if __name__ == "__main__": +if __name__ == '__main__': execute_manager(settings) diff --git a/automation/server/monitor/settings.py b/automation/server/monitor/settings.py index 9048da50..8cd20e35 100644 --- a/automation/server/monitor/settings.py +++ b/automation/server/monitor/settings.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # # Django settings for monitor project. @@ -48,6 +46,4 @@ SECRET_KEY = '13p5p_4q91*8@yo+tvvt#2k&6#d_&e_zvxdpdil53k419i5sop' ROOT_URLCONF = 'monitor.urls' # List of locations of the template source files, in search order. -TEMPLATE_DIRS = ( - os.path.join(ROOT_PATH, 'templates'), -) +TEMPLATE_DIRS = (os.path.join(ROOT_PATH, 'templates'),) diff --git a/automation/server/monitor/urls.py b/automation/server/monitor/urls.py index 19a1ef9c..1a6b2485 100644 --- a/automation/server/monitor/urls.py +++ b/automation/server/monitor/urls.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # @@ -9,17 +7,15 @@ from django.conf import settings from django.conf.urls.defaults import patterns urlpatterns = patterns( - 'dashboard', - (r'^job-group$', 'JobGroupListPageHandler'), + 'dashboard', (r'^job-group$', 'JobGroupListPageHandler'), (r'^machine$', 'MachineListPageHandler'), (r'^job/(?P<job_id>\d+)/log$', 'LogPageHandler'), - (r'^job/(?P<job_id>\d+)$', 'JobPageHandler'), - (r'^job-group/(?P<job_group_id>\d+)/files/(?P<path>.*)$', - 'JobGroupFilesPageHandler'), + (r'^job/(?P<job_id>\d+)$', 'JobPageHandler'), ( + r'^job-group/(?P<job_group_id>\d+)/files/(?P<path>.*)$', + 'JobGroupFilesPageHandler'), (r'^job-group/(?P<job_group_id>\d+)$', 'JobGroupPageHandler'), (r'^$', 'DefaultPageHandler')) -urlpatterns += patterns( - '', - (r'^static/(?P<path>.*)$', 'django.views.static.serve', - {'document_root': settings.MEDIA_ROOT})) +urlpatterns += patterns('', + (r'^static/(?P<path>.*)$', 'django.views.static.serve', + {'document_root': settings.MEDIA_ROOT})) diff --git a/automation/server/server.py b/automation/server/server.py index e9243cbb..f02a1d0f 100755 --- a/automation/server/server.py +++ b/automation/server/server.py @@ -38,7 +38,7 @@ class Server(object): def ExecuteJobGroup(self, job_group, dry_run=False): job_group = pickle.loads(job_group) - self._logger.info("Received ExecuteJobGroup(%r, dry_run=%s) request.", + self._logger.info('Received ExecuteJobGroup(%r, dry_run=%s) request.', job_group, dry_run) for job in job_group.jobs: @@ -46,25 +46,25 @@ class Server(object): return self.job_group_manager.AddJobGroup(job_group) def GetAllJobGroups(self): - self._logger.info("Received GetAllJobGroups() request.") + self._logger.info('Received GetAllJobGroups() request.') return pickle.dumps(self.job_group_manager.GetAllJobGroups()) def KillJobGroup(self, job_group_id): - self._logger.info("Received KillJobGroup(%d) request.", job_group_id) + self._logger.info('Received KillJobGroup(%d) request.', job_group_id) self.job_group_manager.KillJobGroup(pickle.loads(job_group_id)) def GetJobGroup(self, job_group_id): - self._logger.info("Received GetJobGroup(%d) request.", job_group_id) + self._logger.info('Received GetJobGroup(%d) request.', job_group_id) return pickle.dumps(self.job_group_manager.GetJobGroup(job_group_id)) def GetJob(self, job_id): - self._logger.info("Received GetJob(%d) request.", job_id) + self._logger.info('Received GetJob(%d) request.', job_id) return pickle.dumps(self.job_manager.GetJob(job_id)) def GetMachineList(self): - self._logger.info("Received GetMachineList() request.") + self._logger.info('Received GetMachineList() request.') return pickle.dumps(self.job_manager.machine_manager.GetMachineList()) @@ -79,18 +79,18 @@ class Server(object): def GetServerOptions(): """Get server's settings from command line options.""" parser = optparse.OptionParser() - parser.add_option("-m", - "--machines-file", - dest="machines_file", - help="The location of the file " - "containing the machines database", + parser.add_option('-m', + '--machines-file', + dest='machines_file', + help='The location of the file ' + 'containing the machines database', default=machine_manager.DEFAULT_MACHINES_FILE) - parser.add_option("-n", - "--dry-run", - dest="dry_run", - help="Start the server in dry-run mode, where jobs will " - "not actually be executed.", - action="store_true", + parser.add_option('-n', + '--dry-run', + dest='dry_run', + help='Start the server in dry-run mode, where jobs will ' + 'not actually be executed.', + action='store_true', default=False) return parser.parse_args()[0] @@ -110,7 +110,9 @@ def Main(): try: xmlserver = SimpleXMLRPCServer( - ("localhost", 8000), allow_none=True, logRequests=False) + ('localhost', 8000), + allow_none=True, + logRequests=False) xmlserver.register_instance(server) xmlserver.serve_forever() except Exception as ex: @@ -119,5 +121,5 @@ def Main(): sys.exit(1) -if __name__ == "__main__": +if __name__ == '__main__': Main() diff --git a/automation/server/server_test.py b/automation/server/server_test.py index c2e4b0ae..bcf1b9f5 100755 --- a/automation/server/server_test.py +++ b/automation/server/server_test.py @@ -1,27 +1,26 @@ #!/usr/bin/python # # Copyright 2010 Google Inc. All Rights Reserved. - """Machine manager unittest. MachineManagerTest tests MachineManager. """ -__author__ = "asharif@google.com (Ahmad Sharif)" +__author__ = 'asharif@google.com (Ahmad Sharif)' import server import unittest class ServerTest(unittest.TestCase): + def setUp(self): pass - def testGetAllJobs(self): s = server.Server() print s.GetAllJobs() -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() |