Make duplicate locking/unlocking warning instead of error.

Change the afe_lock_manager, so that an attempt to lock an already
locked machine or unlock an already unlocked machine is a warning
rather than an exception; also return a list of machines which we
were successfully able to lock/unlock.

Also update Crosperf: Previously if multiple machines were requested
and we were not able to lock all of them, the Crosperf job failed.
Now if we are able to successfully lock any of the requested machines,
Crosperf will try to run the job.

BUG=None
TEST=Tested in my workspace with various locked/unlocked
configurations for crosperf job with two machines.

Change-Id: I65a0a9cce4b8be8d0c043913dc2dd7f9f07cda0f
Reviewed-on: https://chrome-internal-review.googlesource.com/224180
Reviewed-by: Han Shen <shenhan@google.com>
Commit-Queue: Caroline Tice <cmtice@google.com>
Tested-by: Caroline Tice <cmtice@google.com>

1 files changed, 30 insertions, 2 deletions

diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py
index dae3c2e3..541bebe0 100644
--- a/crosperf/experiment_runner.py
+++ b/crosperf/experiment_runner.py
@@ -31,6 +31,7 @@ class ExperimentRunner(object):
     self.l = log or logger.GetLogger(experiment.log_dir)
     self._ce = cmd_exec or command_executer.GetCommandExecuter(self.l)
     self._terminated = False
+    self.locked_machines = []
     if experiment.log_level != "verbose":
       self.STATUS_TIME_DELAY = 10
 
@@ -46,6 +47,25 @@ class ExperimentRunner(object):
         machines += l.remote
     return machines
 
+  def _UpdateMachineList(self, locked_machines):
+    """Update machines lists to contain only locked machines.
+
+    Go through all the lists of requested machines, both global and
+    label-specific requests, and remove any machine that we were not
+    able to lock.
+
+    Args:
+      locked_machines: A list of the machines we successfully locked.
+    """
+    for m in self._experiment.remote:
+      if m not in locked_machines:
+        self._experiment.remote.remove(m)
+
+    for l in self._experiment.labels:
+      for m in l.remote:
+        if m not in locked_machines:
+          l.remote.remove(m)
+
   def _LockAllMachines(self, experiment):
     """Attempt to globally lock all of the machines requested for run.
 
@@ -65,7 +85,12 @@ class ExperimentRunner(object):
         lock_mgr.AddLocalMachine(m)
     machine_states = lock_mgr.GetMachineStates("lock")
     lock_mgr.CheckMachineLocks(machine_states, "lock")
-    lock_mgr.UpdateMachines(True)
+    self.locked_machines = lock_mgr.UpdateMachines(True)
+    self._experiment.locked_machines = self.locked_machines
+    self._UpdateMachineList(self.locked_machines)
+    self._experiment.machine_manager.RemoveNonLockedMachines(self.locked_machines)
+    if len(self.locked_machines) == 0:
+        raise RuntimeError("Unable to lock any machines.")
 
   def _UnlockAllMachines(self, experiment):
     """Attempt to globally unlock all of the machines requested for run.
@@ -73,8 +98,11 @@ class ExperimentRunner(object):
     The method will use the AFE server to globally unlock all of the machines
     requested for this crosperf run.
     """
+    if not self.locked_machines:
+        return
+
     lock_mgr = afe_lock_machine.AFELockManager(
-        self._GetMachineList(),
+        self.locked_machines,
         "",
         experiment.labels[0].chromeos_root,
         None,