diff options
| -rwxr-xr-x | afe_lock_machine.py | 43 | ||||
| -rw-r--r-- | crosperf/experiment.py | 9 | ||||
| -rw-r--r-- | crosperf/experiment_runner.py | 32 | ||||
| -rw-r--r-- | crosperf/machine_manager.py | 9 |
4 files changed, 74 insertions, 19 deletions
diff --git a/afe_lock_machine.py b/afe_lock_machine.py index 29693cc7..5d6ae921 100755 --- a/afe_lock_machine.py +++ b/afe_lock_machine.py @@ -41,14 +41,6 @@ class LockingError(AFELockException): """Raised when server fails to lock/unlock machine as requested.""" -class DuplicateLock(AFELockException): - """Raised when user attempts to lock an already locked machine.""" - - -class DuplicateUnlock(AFELockException): - """Raised when user attempts to unlock an already unlocked machine.""" - - class DontOwnLock(AFELockException): """Raised when user attmepts to unlock machine locked by someone else.""" # This should not be raised if the user specified '--force' @@ -378,7 +370,11 @@ class AFELockManager(object): Args: lock_machines: Boolean indicating whether to lock the machines (True) or unlock the machines (False). + + Returns: + A list of the machines whose state was successfully updated. """ + updated_machines = [] for m in self.machines: self.UpdateLockInAFE(lock_machines, m) @@ -388,6 +384,25 @@ class AFELockManager(object): self.logger.LogOutput('Locked machine(s) %s.' % m) else: self.logger.LogOutput('Unlocked machine(s) %s.' % m) + updated_machines.append(m) + + return updated_machines + + def _InternalRemoveMachine(self, machine): + """Remove machine from internal list of machines. + + Args: + machine: Name of machine to be removed from internal list. + """ + # Check to see if machine is lab machine and if so, make sure it has + # ".cros" on the end. + cros_machine = machine + if machine.find('rack') > 0 and machine.find('row') > 0: + if machine.find('.cros') == -1: + cros_machine = cros_machine + '.cros' + + self.machines = [m for m in self.machines if m != cros_machine and + m != machine] def CheckMachineLocks(self, machine_states, cmd): """Check that every machine in requested list is in the proper state. @@ -402,22 +417,22 @@ class AFELockManager(object): cmd: 'lock' or 'unlock'. The user-requested action for the machines. Raises: - DuplicateLock: A machine requested to be locked is already locked. - DuplicateUnlock: A machine requested to be unlocked is already unlocked. DontOwnLock: The lock on a requested machine is owned by someone else. """ for k, state in machine_states.iteritems(): if cmd == 'unlock': if not state['locked']: - raise DuplicateUnlock('Attempt to unlock already unlocked machine ' - '(%s).' % k) + self.logger.LogWarning('Attempt to unlock already unlocked machine ' + '(%s).' % k) + self._InternalRemoveMachine(k) - if state['locked_by'] != self.user: + if state['locked'] and state['locked_by'] != self.user: raise DontOwnLock('Attempt to unlock machine (%s) locked by someone ' 'else (%s).' % (k, state['locked_by'])) elif cmd == 'lock': if state['locked']: - raise DuplicateLock('Attempt to lock already locked machine (%s)' % k) + self.logger.LogWarning('Attempt to lock already locked machine (%s)' % k) + self._InternalRemoveMachine(k) def HasAFEServer(self, local): """Verifies that the AFELockManager has appropriate AFE server. diff --git a/crosperf/experiment.py b/crosperf/experiment.py index 33b3f878..9f88ed01 100644 --- a/crosperf/experiment.py +++ b/crosperf/experiment.py @@ -50,6 +50,7 @@ class Experiment(object): # locking mechanism; if it is blank then we will use the AFE server # locking mechanism. self.locks_dir = locks_directory + self.locked_machines = [] # We need one chromeos_root to run the benchmarks in, but it doesn't # matter where it is, unless the ABIs are different. @@ -146,9 +147,11 @@ class Experiment(object): # to unlock everything. self.machine_manager.Cleanup() else: - all_machines = self.remote - for l in self.labels: - all_machines += l.remote + all_machines = self.locked_machines + if not all_machines: + return + + # If we locked any machines earlier, make sure we unlock them now. lock_mgr = afe_lock_machine.AFELockManager(all_machines, "", self.labels[0].chromeos_root, None) diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py index dae3c2e3..541bebe0 100644 --- a/crosperf/experiment_runner.py +++ b/crosperf/experiment_runner.py @@ -31,6 +31,7 @@ class ExperimentRunner(object): self.l = log or logger.GetLogger(experiment.log_dir) self._ce = cmd_exec or command_executer.GetCommandExecuter(self.l) self._terminated = False + self.locked_machines = [] if experiment.log_level != "verbose": self.STATUS_TIME_DELAY = 10 @@ -46,6 +47,25 @@ class ExperimentRunner(object): machines += l.remote return machines + def _UpdateMachineList(self, locked_machines): + """Update machines lists to contain only locked machines. + + Go through all the lists of requested machines, both global and + label-specific requests, and remove any machine that we were not + able to lock. + + Args: + locked_machines: A list of the machines we successfully locked. + """ + for m in self._experiment.remote: + if m not in locked_machines: + self._experiment.remote.remove(m) + + for l in self._experiment.labels: + for m in l.remote: + if m not in locked_machines: + l.remote.remove(m) + def _LockAllMachines(self, experiment): """Attempt to globally lock all of the machines requested for run. @@ -65,7 +85,12 @@ class ExperimentRunner(object): lock_mgr.AddLocalMachine(m) machine_states = lock_mgr.GetMachineStates("lock") lock_mgr.CheckMachineLocks(machine_states, "lock") - lock_mgr.UpdateMachines(True) + self.locked_machines = lock_mgr.UpdateMachines(True) + self._experiment.locked_machines = self.locked_machines + self._UpdateMachineList(self.locked_machines) + self._experiment.machine_manager.RemoveNonLockedMachines(self.locked_machines) + if len(self.locked_machines) == 0: + raise RuntimeError("Unable to lock any machines.") def _UnlockAllMachines(self, experiment): """Attempt to globally unlock all of the machines requested for run. @@ -73,8 +98,11 @@ class ExperimentRunner(object): The method will use the AFE server to globally unlock all of the machines requested for this crosperf run. """ + if not self.locked_machines: + return + lock_mgr = afe_lock_machine.AFELockManager( - self._GetMachineList(), + self.locked_machines, "", experiment.labels[0].chromeos_root, None, diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py index e36ae7d5..79423481 100644 --- a/crosperf/machine_manager.py +++ b/crosperf/machine_manager.py @@ -206,6 +206,15 @@ class MachineManager(object): self._initialized_machines = [] self.chromeos_root = chromeos_root + def RemoveNonLockedMachines(self, locked_machines): + for m in self._all_machines: + if m.name not in locked_machines: + self._all_machines.remove(m) + + for m in self._machines: + if m.name not in locked_machines: + self._machines.remove(m) + def ImageMachine(self, machine, label): if label.image_type == "local": checksum = ImageChecksummer().Checksum(label, self.log_level) |