diff options
Diffstat (limited to 'automation/common')
| -rw-r--r-- | automation/common/__init__.py | 1 | ||||
| -rw-r--r-- | automation/common/command.py | 13 | ||||
| -rw-r--r-- | automation/common/command_executer.py | 49 | ||||
| -rwxr-xr-x | automation/common/command_executer_test.py | 18 | ||||
| -rw-r--r-- | automation/common/events.py | 22 | ||||
| -rw-r--r-- | automation/common/job.py | 25 | ||||
| -rw-r--r-- | automation/common/job_group.py | 32 | ||||
| -rw-r--r-- | automation/common/logger.py | 21 | ||||
| -rw-r--r-- | automation/common/machine.py | 35 | ||||
| -rwxr-xr-x | automation/common/machine_test.py | 12 | ||||
| -rw-r--r-- | automation/common/state_machine.py | 1 |
11 files changed, 113 insertions, 116 deletions
diff --git a/automation/common/__init__.py b/automation/common/__init__.py index e69de29b..8b137891 100644 --- a/automation/common/__init__.py +++ b/automation/common/__init__.py @@ -0,0 +1 @@ + diff --git a/automation/common/command.py b/automation/common/command.py index 192e2bd8..c56e9fad 100644 --- a/automation/common/command.py +++ b/automation/common/command.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. __author__ = 'kbaclawski@google.com (Krystian Baclawski)' @@ -197,8 +195,8 @@ def RemoteCopyFrom(from_machine, from_path, to_path, username=None): login = '%s@%s' % (username, from_machine) return Chain( - MakeDir(to_path), - Shell('rsync', '-a', '%s:%s' % (login, from_path), to_path)) + MakeDir(to_path), Shell('rsync', '-a', '%s:%s' % + (login, from_path), to_path)) def MakeSymlink(to_path, link_name): @@ -224,8 +222,7 @@ def RmTree(*dirs): def UnTar(tar_file, dest_dir): return Chain( - MakeDir(dest_dir), - Shell('tar', '-x', '-f', tar_file, '-C', dest_dir)) + MakeDir(dest_dir), Shell('tar', '-x', '-f', tar_file, '-C', dest_dir)) def Tar(tar_file, *args): @@ -241,6 +238,4 @@ def Tar(tar_file, *args): options.extend(['-f', tar_file]) options.extend(args) - return Chain( - MakeDir(os.path.dirname(tar_file)), - Shell('tar', *options)) + return Chain(MakeDir(os.path.dirname(tar_file)), Shell('tar', *options)) diff --git a/automation/common/command_executer.py b/automation/common/command_executer.py index 2db0048e..c0f314f5 100644 --- a/automation/common/command_executer.py +++ b/automation/common/command_executer.py @@ -1,15 +1,12 @@ -#!/usr/bin/python -# # Copyright 2011 Google Inc. All Rights Reserved. # - """Classes that help running commands in a subshell. Commands can be run locally, or remotly using SSH connection. You may log the output of a command to a terminal or a file, or any other destination. """ -__author__ = "kbaclawski@google.com (Krystian Baclawski)" +__author__ = 'kbaclawski@google.com (Krystian Baclawski)' import fcntl import logging @@ -32,8 +29,12 @@ class CommandExecuter(object): def Configure(cls, dry_run): cls.DRY_RUN = dry_run - def RunCommand(self, cmd, machine=None, username=None, - command_terminator=None, command_timeout=None): + def RunCommand(self, + cmd, + machine=None, + username=None, + command_terminator=None, + command_timeout=None): cmd = str(cmd) if self._dry_run: @@ -43,13 +44,13 @@ class CommandExecuter(object): command_terminator = CommandTerminator() if command_terminator.IsTerminated(): - self._logger.warning("Command has been already terminated!") + self._logger.warning('Command has been already terminated!') return 1 # Rewrite command for remote execution. if machine: if username: - login = "%s@%s" % (username, machine) + login = '%s@%s' % (username, machine) else: login = machine @@ -62,8 +63,8 @@ class CommandExecuter(object): child = self._SpawnProcess(cmd, command_terminator, command_timeout) - self._logger.debug( - "{PID: %d} Finished with %d code.", child.pid, child.returncode) + self._logger.debug('{PID: %d} Finished with %d code.', child.pid, + child.returncode) return child.returncode @@ -71,10 +72,10 @@ class CommandExecuter(object): """Gracefully shutdown the child by sending SIGTERM.""" if command_timeout: - self._logger.warning("{PID: %d} Timeout of %s seconds reached since " - "process started.", child.pid, command_timeout) + self._logger.warning('{PID: %d} Timeout of %s seconds reached since ' + 'process started.', child.pid, command_timeout) - self._logger.warning("{PID: %d} Terminating child.", child.pid) + self._logger.warning('{PID: %d} Terminating child.', child.pid) try: child.terminate() @@ -92,16 +93,18 @@ class CommandExecuter(object): def _Kill(self, child): """Kill the child with immediate result.""" - self._logger.warning("{PID: %d} Process still alive.", child.pid) - self._logger.warning("{PID: %d} Killing child.", child.pid) + self._logger.warning('{PID: %d} Process still alive.', child.pid) + self._logger.warning('{PID: %d} Killing child.', child.pid) child.kill() child.wait() def _SpawnProcess(self, cmd, command_terminator, command_timeout): # Create a child process executing provided command. - child = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - stdin=subprocess.PIPE, shell=True) + child = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + shell=True) # Close stdin so the child won't be able to block on read. child.stdin.close() @@ -157,10 +160,10 @@ class CommandExecuter(object): data = os.read(fd, 4096) except OSError: # terminate loop if EWOULDBLOCK (EAGAIN) is received - data = "" + data = '' if not already_terminated: - self._logger.debug("Waiting for command to finish.") + self._logger.debug('Waiting for command to finish.') child.wait() return child @@ -175,6 +178,7 @@ class CommandExecuter(object): class LoggingCommandExecuter(CommandExecuter): + def __init__(self, *args, **kwargs): super(LoggingCommandExecuter, self).__init__(*args, **kwargs) @@ -183,8 +187,8 @@ class LoggingCommandExecuter(CommandExecuter): def OpenLog(self, log_path): """The messages are going to be saved to gzip compressed file.""" - formatter = logging.Formatter( - '%(asctime)s %(prefix)s: %(message)s', '%Y-%m-%d %H:%M:%S') + formatter = logging.Formatter('%(asctime)s %(prefix)s: %(message)s', + '%Y-%m-%d %H:%M:%S') handler = logger.CompressedFileHandler(log_path, delay=True) handler.setFormatter(formatter) self._output.addHandler(handler) @@ -215,6 +219,7 @@ class LoggingCommandExecuter(CommandExecuter): class CommandTerminator(object): + def __init__(self): self.terminated = False diff --git a/automation/common/command_executer_test.py b/automation/common/command_executer_test.py index 5b6ca639..4aa245f0 100755 --- a/automation/common/command_executer_test.py +++ b/automation/common/command_executer_test.py @@ -32,6 +32,7 @@ from automation.common.command_executer import CommandExecuter class LoggerMock(object): + def LogCmd(self, cmd, machine='', user=''): if machine: logging.info('[%s] Executing: %s', machine, cmd) @@ -49,6 +50,7 @@ class LoggerMock(object): class CommandExecuterUnderTest(CommandExecuter): + def __init__(self): CommandExecuter.__init__(self, logger_to_set=LoggerMock()) @@ -83,8 +85,9 @@ class CommandExecuterLocalTests(unittest.TestCase): def RunCommand(self, method, **kwargs): program = os.path.abspath(sys.argv[0]) - return self._executer.RunCommand( - '%s runHelper %s' % (program, method), machine=self.HOSTNAME, **kwargs) + return self._executer.RunCommand('%s runHelper %s' % (program, method), + machine=self.HOSTNAME, + **kwargs) def testCommandTimeout(self): exit_code = self.RunCommand('SleepForMinute', command_timeout=3) @@ -114,12 +117,14 @@ class CommandExecuterLocalTests(unittest.TestCase): self.assertEquals(self._executer.stdout, '') def testOutputStreamNonInteractive(self): - self.assertFalse(self.RunCommand('IsOutputStreamInteractive'), - 'stdout stream is a terminal!') + self.assertFalse( + self.RunCommand('IsOutputStreamInteractive'), + 'stdout stream is a terminal!') def testErrorStreamNonInteractive(self): - self.assertFalse(self.RunCommand('IsErrorStreamInteractive'), - 'stderr stream is a terminal!') + self.assertFalse( + self.RunCommand('IsErrorStreamInteractive'), + 'stderr stream is a terminal!') def testAttemptToRead(self): self.assertFalse(self.RunCommand('WaitForInput', command_timeout=3)) @@ -149,6 +154,7 @@ class CommandExecuterRemoteTests(CommandExecuterLocalTests): class CommandExecuterTestHelpers(object): + def SleepForMinute(self): time.sleep(60) return 1 diff --git a/automation/common/events.py b/automation/common/events.py index 851a982c..ad3ec844 100644 --- a/automation/common/events.py +++ b/automation/common/events.py @@ -1,9 +1,7 @@ -#!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright 2011 Google Inc. All Rights Reserved. # - """Tools for recording and reporting timeline of abstract events. You can store any events provided that they can be stringified. @@ -56,8 +54,7 @@ class _EventRecord(object): return self._time_elapsed is not None def GetTimeStartedFormatted(self): - return time.strftime('%m/%d/%Y %H:%M:%S', - time.gmtime(self._time_started)) + return time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime(self._time_started)) def GetTimeElapsedRounded(self): return datetime.timedelta(seconds=int(self.time_elapsed.seconds)) @@ -104,8 +101,8 @@ class EventHistory(collections.Sequence): def GetTotalTime(self): if self._records: - total_time_elapsed = sum(evrec.time_elapsed.seconds for evrec in - self._records) + total_time_elapsed = sum(evrec.time_elapsed.seconds + for evrec in self._records) return datetime.timedelta(seconds=int(total_time_elapsed)) @@ -114,15 +111,16 @@ class EventHistory(collections.Sequence): if self._records: for num, next_evrec in enumerate(self._records[1:], start=1): - evrec = self._records[num-1] + evrec = self._records[num - 1] - records.append(_EventRecord(_Transition(evrec.event, next_evrec.event), - evrec.time_started, evrec.time_elapsed)) + records.append(_EventRecord( + _Transition(evrec.event, next_evrec.event), evrec.time_started, + evrec.time_elapsed)) if not self.last.has_finished: - records.append(_EventRecord(_Transition(self.last.event, 'NOW'), - self.last.time_started, - self.last.time_elapsed)) + records.append(_EventRecord( + _Transition(self.last.event, + 'NOW'), self.last.time_started, self.last.time_elapsed)) return EventHistory(records) diff --git a/automation/common/job.py b/automation/common/job.py index d32fa59a..e845ab25 100644 --- a/automation/common/job.py +++ b/automation/common/job.py @@ -1,14 +1,9 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. # - """A module for a job in the infrastructure.""" - __author__ = 'raymes@google.com (Raymes Khoury)' - import os.path from automation.common import state_machine @@ -22,6 +17,7 @@ STATUS_FAILED = 'FAILED' class FolderDependency(object): + def __init__(self, job, src, dest=None): if not dest: dest = src @@ -41,12 +37,14 @@ class JobStateMachine(state_machine.BasicStateMachine): STATUS_NOT_EXECUTED: [STATUS_SETUP], STATUS_SETUP: [STATUS_COPYING, STATUS_FAILED], STATUS_COPYING: [STATUS_RUNNING, STATUS_FAILED], - STATUS_RUNNING: [STATUS_SUCCEEDED, STATUS_FAILED]} + STATUS_RUNNING: [STATUS_SUCCEEDED, STATUS_FAILED] + } final_states = [STATUS_SUCCEEDED, STATUS_FAILED] class JobFailure(Exception): + def __init__(self, message, exit_code): Exception.__init__(self, message) self.exit_code = exit_code @@ -57,7 +55,7 @@ class Job(object): WORKDIR_PREFIX = '/usr/local/google/tmp/automation' - def __init__(self, label, command, timeout=4*60*60): + def __init__(self, label, command, timeout=4 * 60 * 60): self._state = JobStateMachine(STATUS_NOT_EXECUTED) self.predecessors = set() self.successors = set() @@ -110,15 +108,15 @@ class Job(object): def GetCommand(self): substitutions = [ - ('$JOB_ID', str(self.id)), - ('$JOB_TMP', self.work_dir), + ('$JOB_ID', str(self.id)), ('$JOB_TMP', self.work_dir), ('$JOB_HOME', self.home_dir), - ('$PRIMARY_MACHINE', self.primary_machine.hostname)] + ('$PRIMARY_MACHINE', self.primary_machine.hostname) + ] if len(self.machines) > 1: for num, machine in enumerate(self.machines[1:]): - substitutions.append( - ('$SECONDARY_MACHINES[%d]' % num, machine.hostname)) + substitutions.append(('$SECONDARY_MACHINES[%d]' % num, machine.hostname + )) return self._FormatCommand(str(self.command), substitutions) @@ -127,7 +125,8 @@ class Job(object): # non existing Command class. If one is created then PrettyFormatCommand # shall become its method. return self._FormatCommand(self.GetCommand(), [ - ('\{ ', ''), ('; \}', ''), ('\} ', '\n'), ('\s*&&\s*', '\n')]) + ('\{ ', ''), ('; \}', ''), ('\} ', '\n'), ('\s*&&\s*', '\n') + ]) def DependsOnFolder(self, dependency): self.folder_dependencies.append(dependency) diff --git a/automation/common/job_group.py b/automation/common/job_group.py index 09321e1a..96912fc1 100644 --- a/automation/common/job_group.py +++ b/automation/common/job_group.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. # @@ -8,25 +6,30 @@ import os from automation.common.state_machine import BasicStateMachine -STATUS_NOT_EXECUTED = "NOT_EXECUTED" -STATUS_EXECUTING = "EXECUTING" -STATUS_SUCCEEDED = "SUCCEEDED" -STATUS_FAILED = "FAILED" +STATUS_NOT_EXECUTED = 'NOT_EXECUTED' +STATUS_EXECUTING = 'EXECUTING' +STATUS_SUCCEEDED = 'SUCCEEDED' +STATUS_FAILED = 'FAILED' class JobGroupStateMachine(BasicStateMachine): state_machine = { STATUS_NOT_EXECUTED: [STATUS_EXECUTING], - STATUS_EXECUTING: [STATUS_SUCCEEDED, STATUS_FAILED]} + STATUS_EXECUTING: [STATUS_SUCCEEDED, STATUS_FAILED] + } final_states = [STATUS_SUCCEEDED, STATUS_FAILED] class JobGroup(object): - HOMEDIR_PREFIX = os.path.join("/home", getpass.getuser(), "www", "automation") - - def __init__(self, label, jobs=None, cleanup_on_completion=True, - cleanup_on_failure=False, description=""): + HOMEDIR_PREFIX = os.path.join('/home', getpass.getuser(), 'www', 'automation') + + def __init__(self, + label, + jobs=None, + cleanup_on_completion=True, + cleanup_on_failure=False, + description=''): self._state = JobGroupStateMachine(STATUS_NOT_EXECUTED) self.id = 0 self.label = label @@ -49,7 +52,7 @@ class JobGroup(object): @property def home_dir(self): - return os.path.join(self.HOMEDIR_PREFIX, "job-group-%d" % self.id) + return os.path.join(self.HOMEDIR_PREFIX, 'job-group-%d' % self.id) @property def time_submitted(self): @@ -62,9 +65,8 @@ class JobGroup(object): return '{%s: %s}' % (self.__class__.__name__, self.id) def __str__(self): - return "\n".join(["Job-Group:", - "ID: %s" % self.id] + - [str(job) for job in self.jobs]) + return '\n'.join(['Job-Group:', 'ID: %s' % self.id] + [str( + job) for job in self.jobs]) def AddJob(self, job): self.jobs.append(job) diff --git a/automation/common/logger.py b/automation/common/logger.py index efa1a904..4aeee052 100644 --- a/automation/common/logger.py +++ b/automation/common/logger.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. from itertools import chain @@ -17,7 +15,10 @@ def SetUpRootLogger(filename=None, level=None, display_flags={}): if filename: file_handler = logging.handlers.RotatingFileHandler( - filename, maxBytes=10*1024*1024, backupCount=9, delay=True) + filename, + maxBytes=10 * 1024 * 1024, + backupCount=9, + delay=True) file_handler.setFormatter(CustomFormatter(NullColorCoder(), display_flags)) logging.root.addHandler(file_handler) @@ -26,12 +27,13 @@ def SetUpRootLogger(filename=None, level=None, display_flags={}): class NullColorCoder(object): + def __call__(self, *args): return '' class AnsiColorCoder(object): - CODES = {'reset': (0, ), + CODES = {'reset': (0,), 'bold': (1, 22), 'italics': (3, 23), 'underline': (4, 24), @@ -82,8 +84,8 @@ class CustomFormatter(logging.Formatter): def formatTime(self, record): ct = self.converter(record.created) - t = time.strftime("%Y-%m-%d %H:%M:%S", ct) - return "%s.%02d" % (t, record.msecs / 10) + t = time.strftime('%Y-%m-%d %H:%M:%S', ct) + return '%s.%02d' % (t, record.msecs / 10) def formatLevelName(self, record): if record.levelname in ['WARNING', 'CRITICAL']: @@ -96,8 +98,8 @@ class CustomFormatter(logging.Formatter): def formatMessagePrefix(self, record): try: - return ' %s%s:%s ' % ( - self._coder('black', 'bold'), record.prefix, self._coder('reset')) + return ' %s%s:%s ' % (self._coder('black', 'bold'), record.prefix, + self._coder('reset')) except AttributeError: return '' @@ -125,6 +127,7 @@ class CustomFormatter(logging.Formatter): class CompressedFileHandler(logging.FileHandler): + def _open(self): return gzip.open(self.baseFilename + '.gz', self.mode, 9) @@ -136,6 +139,6 @@ def HandleUncaughtExceptions(fun): try: return fun(*args, **kwargs) except StandardError: - logging.exception("Uncaught exception:") + logging.exception('Uncaught exception:') return _Interceptor diff --git a/automation/common/machine.py b/automation/common/machine.py index ae9f0b49..4db0db0d 100644 --- a/automation/common/machine.py +++ b/automation/common/machine.py @@ -1,8 +1,6 @@ -#!/usr/bin/python -# # Copyright 2010 Google Inc. All Rights Reserved. -__author__ = "asharif@google.com (Ahmad Sharif)" +__author__ = 'asharif@google.com (Ahmad Sharif)' from fnmatch import fnmatch @@ -38,24 +36,20 @@ class Machine(object): self.locked = False def __repr__(self): - return "{%s: %s@%s}" % ( - self.__class__.__name__, self.username, self.hostname) + return '{%s: %s@%s}' % (self.__class__.__name__, self.username, + self.hostname) def __str__(self): - return "\n".join(["Machine Information:", - "Hostname: %s" % self.hostname, - "Label: %s" % self.label, - "CPU: %s" % self.cpu, - "Cores: %d" % self.cores, - "OS: %s" % self.os, - "Uses: %d" % self.uses, - "Locked: %s" % self.locked]) + return '\n'.join( + ['Machine Information:', 'Hostname: %s' % self.hostname, 'Label: %s' % + self.label, 'CPU: %s' % self.cpu, 'Cores: %d' % self.cores, 'OS: %s' % + self.os, 'Uses: %d' % self.uses, 'Locked: %s' % self.locked]) class MachineSpecification(object): """Helper class used to find a machine matching your requirements.""" - def __init__(self, hostname="*", label="*", os="*", lock_required=False): + def __init__(self, hostname='*', label='*', os='*', lock_required=False): self.hostname = hostname self.label = label self.os = os @@ -63,16 +57,13 @@ class MachineSpecification(object): self.preferred_machines = [] def __str__(self): - return "\n".join(["Machine Specification:", - "Name: %s" % self.name, - "OS: %s" % self.os, - "Lock required: %s" % self.lock_required]) + return '\n'.join(['Machine Specification:', 'Name: %s' % self.name, 'OS: %s' + % self.os, 'Lock required: %s' % self.lock_required]) def IsMatch(self, machine): - return all([not machine.locked, - fnmatch(machine.hostname, self.hostname), - fnmatch(machine.label, self.label), - fnmatch(machine.os, self.os)]) + return all([not machine.locked, fnmatch(machine.hostname, self.hostname), + fnmatch(machine.label, self.label), fnmatch(machine.os, + self.os)]) def AddPreferredMachine(self, hostname): if hostname not in self.preferred_machines: diff --git a/automation/common/machine_test.py b/automation/common/machine_test.py index 96e8f823..c9c200a9 100755 --- a/automation/common/machine_test.py +++ b/automation/common/machine_test.py @@ -1,28 +1,26 @@ #!/usr/bin/python # # Copyright 2010 Google Inc. All Rights Reserved. - """Machine manager unittest. MachineManagerTest tests MachineManager. """ -__author__ = "asharif@google.com (Ahmad Sharif)" - +__author__ = 'asharif@google.com (Ahmad Sharif)' import machine import unittest class MachineTest(unittest.TestCase): + def setUp(self): pass - def testPrintMachine(self): - mach = machine.Machine("ahmad.mtv", "core2duo", 4, "linux", "asharif") - self.assertTrue("ahmad.mtv" in str(mach)) + mach = machine.Machine('ahmad.mtv', 'core2duo', 4, 'linux', 'asharif') + self.assertTrue('ahmad.mtv' in str(mach)) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/automation/common/state_machine.py b/automation/common/state_machine.py index 4d8dc49b..d1cf42c8 100644 --- a/automation/common/state_machine.py +++ b/automation/common/state_machine.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright 2011 Google Inc. All Rights Reserved. |