Add flag to crosperf to use file lock mechanism.

This CL adds a flag to crosperf that specifies using the old file
locking mechanism, rather than the new AFE server locking mechanism.
This is useful when there are issues with accessing the AFE server.

BUG=None
TEST=Tested crosperf with this, both using the AFE locking mechanism
and using the file locking mechanism.

Change-Id: If6aa3e97a77d8ccb53f0e8f0405cc7f45e983b75
Reviewed-on: https://chrome-internal-review.googlesource.com/219096
Reviewed-by: Luis Lozano <llozano@chromium.org>
Commit-Queue: Caroline Tice <cmtice@google.com>
Tested-by: Caroline Tice <cmtice@google.com>

5 files changed, 75 insertions, 30 deletions

diff --git a/crosperf/experiment.py b/crosperf/experiment.py
index a7a4a7a0..33b3f878 100644
--- a/crosperf/experiment.py
+++ b/crosperf/experiment.py
@@ -46,6 +46,10 @@ class Experiment(object):
     self.num_complete = 0
     self.num_run_complete = 0
     self.share_cache = share_cache
+    # If locks_directory (self.lock_dir) not blank, we will use the file
+    # locking mechanism; if it is blank then we will use the AFE server
+    # locking mechanism.
+    self.locks_dir = locks_directory
 
     # We need one chromeos_root to run the benchmarks in, but it doesn't
     # matter where it is, unless the ABIs are different.
@@ -57,20 +61,12 @@ class Experiment(object):
       raise Exception("No chromeos_root given and could not determine one from "
                       "the image path.")
 
-    # This is a local directory, where the machine manager will keep track of
-    # which machines are available for which benchmark run.   The assumption is
-    # that all of the machines have been globally locked for this experiment,
-    # to keep other people/experiments from accessing them, but we still need the
-    # local locks directory to keep two or more benchmark runs within the same
-    # experiment from trying to use the same machine at the same time.
-    local_locks_directory = os.path.join(self.working_directory,
-                                         "local_locks")
     if test_flag.GetTestMode():
       self.machine_manager = MockMachineManager(chromeos_root, acquire_timeout,
                                                 log_level, locks_directory)
     else:
       self.machine_manager = MachineManager(chromeos_root, acquire_timeout,
-                                            log_level, local_locks_directory)
+                                            log_level, locks_directory)
     self.l = logger.GetLogger(log_dir)
 
     for machine in remote:
@@ -145,12 +141,18 @@ class Experiment(object):
 
   def Cleanup(self):
     """Make sure all machines are unlocked."""
-    all_machines = self.remote
-    for l in self.labels:
-      all_machines += l.remote
-    lock_mgr = afe_lock_machine.AFELockManager(all_machines, "",
-                                               self.labels[0].chromeos_root, None)
-    machine_states = lock_mgr.GetMachineStates("unlock")
-    for k, state in machine_states.iteritems():
-      if state["locked"]:
-        lock_mgr.UpdateLockInAFE(False, k)
+    if self.locks_dir:
+      # We are using the file locks mechanism, so call machine_manager.Cleanup
+      # to unlock everything.
+      self.machine_manager.Cleanup()
+    else:
+      all_machines = self.remote
+      for l in self.labels:
+        all_machines += l.remote
+      lock_mgr = afe_lock_machine.AFELockManager(all_machines, "",
+                                                 self.labels[0].chromeos_root,
+                                                 None)
+      machine_states = lock_mgr.GetMachineStates("unlock")
+      for k, state in machine_states.iteritems():
+        if state["locked"]:
+          lock_mgr.UpdateLockInAFE(False, k)
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index 85322bf5..390f9ad8 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -17,6 +17,7 @@ from label import Label
 from label import MockLabel
 from results_cache import CacheConditions
 import test_flag
+import file_lock_machine
 
 # Users may want to run Telemetry tests either individually, or in
 # specified sets.  Here we define sets of tests that users may want
@@ -112,7 +113,14 @@ class ExperimentFactory(object):
     config.AddConfig("no_email", global_settings.GetField("no_email"))
     share_cache = global_settings.GetField("share_cache")
     results_dir = global_settings.GetField("results_dir")
+    use_file_locks = global_settings.GetField("use_file_locks")
     locks_dir = global_settings.GetField("locks_dir")
+    # If we pass a blank locks_dir to the Experiment, it will use the AFE server
+    # lock mechanism.  So if the user specified use_file_locks, but did not
+    # specify a locks dir, set the locks  dir to the default locks dir in
+    # file_lock_machine.
+    if use_file_locks and not locks_dir:
+      locks_dir = file_lock_machine.Machine.LOCKS_DIR
     chrome_src = global_settings.GetField("chrome_src")
     show_all_results = global_settings.GetField("show_all_results")
     log_level = global_settings.GetField("logging_level")
diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py
index ec109b5a..dae3c2e3 100644
--- a/crosperf/experiment_runner.py
+++ b/crosperf/experiment_runner.py
@@ -86,7 +86,8 @@ class ExperimentRunner(object):
 
   def _Run(self, experiment):
     try:
-      self._LockAllMachines(experiment)
+      if not experiment.locks_dir:
+        self._LockAllMachines(experiment)
       status = ExperimentStatus(experiment)
       experiment.Run()
       last_status_time = 0
@@ -119,7 +120,8 @@ class ExperimentRunner(object):
         self.l.LogError("Ctrl-c pressed. Cleaning up...")
         experiment.Terminate()
     finally:
-      self._UnlockAllMachines(experiment)
+      if not experiment.locks_dir:
+        self._UnlockAllMachines(experiment)
 
   def _PrintTable(self, experiment):
     self.l.LogOutput(TextResultsReport(experiment).GetReport())
diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py
index 2155d377..e36ae7d5 100644
--- a/crosperf/machine_manager.py
+++ b/crosperf/machine_manager.py
@@ -6,6 +6,7 @@
 
 import hashlib
 import image_chromeos
+import file_lock_machine
 import math
 import os.path
 import re
@@ -193,11 +194,15 @@ class MachineManager(object):
     self.machine_checksum_string = {}
     self.acquire_timeout = acquire_timeout
     self.log_level = log_level
-    self.locks_dir = None
+    self.locks_dir = locks_dir
     self.ce = cmd_exec or command_executer.GetCommandExecuter(
         log_level=self.log_level)
     self.logger = lgr or logger.GetLogger()
 
+    if self.locks_dir and not os.path.isdir(self.locks_dir):
+      raise MissingLocksDirectory("Cannot access locks directory: %s"
+                                   % self.locks_dir)
+
     self._initialized_machines = []
     self.chromeos_root = chromeos_root
 
@@ -276,14 +281,21 @@ class MachineManager(object):
       for m in self._machines:
         if m.name == cros_machine.name:
           return
-
-      self._machines.append(cros_machine)
-      command = "cat %s" % CHECKSUM_FILE
-      ret, out, _ = self.ce.CrosRunCommand(
-          command, return_output=True, chromeos_root=self.chromeos_root,
-          machine=cros_machine.name)
-      if ret == 0:
-        cros_machine.checksum = out.strip()
+      locked = True
+      if self.locks_dir:
+        locked = file_lock_machine.Machine(cros_machine.name,
+                                           self.locks_dir).Lock(True,
+                                                                sys.argv[0])
+      if locked:
+        self._machines.append(cros_machine)
+        command = "cat %s" % CHECKSUM_FILE
+        ret, out, _ = self.ce.CrosRunCommand(
+            command, return_output=True, chromeos_root=self.chromeos_root,
+            machine=cros_machine.name)
+        if ret == 0:
+          cros_machine.checksum = out.strip()
+      elif self.locks_dir:
+        self.logger.LogOutput("Couldn't lock: %s" % cros_machine.name)
 
   # This is called from single threaded mode.
   def AddMachine(self, machine_name):
@@ -309,6 +321,12 @@ class MachineManager(object):
     with self._lock:
       self._machines = [m for m in self._machines
                         if m.name != machine_name]
+      if self.locks_dir:
+        res = file_lock_machine.Machine(machine_name,
+                                        self.locks_dir).Unlock(True)
+        if not res:
+          self.logger.LogError("Could not unlock machine: '%s'."
+                               % machine_name)
 
   def ForceSameImageToAllMachines(self, label):
     machines = self.GetMachines(label)
@@ -410,6 +428,16 @@ class MachineManager(object):
           m.status = "Available"
           break
 
+  def Cleanup(self):
+    with self._lock:
+      # Unlock all machines (via file lock)
+      for m in self._machines:
+        res = file_lock_machine.Machine(m.name, self.locks_dir).Unlock(True)
+
+        if not res:
+          self.logger.LogError("Could not unlock machine: '%s'."
+                               % m.name)
+
   def __str__(self):
     with self._lock:
       l = ["MachineManager Status:"]
diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py
index 6793a608..96e8d90b 100644
--- a/crosperf/settings_factory.py
+++ b/crosperf/settings_factory.py
@@ -99,6 +99,10 @@ class GlobalSettings(Settings):
     self.AddField(BooleanField("same_machine", default=False,
                                description="Ensure cached runs are run on the "
                                "exact the same remote"))
+    self.AddField(BooleanField("use_file_locks", default=False,
+                               description="Whether to use the file locks  "
+                               "mechanism (deprecated) instead of the AFE "
+                               "server lock mechanism."))
     self.AddField(IntegerField("iterations", default=1,
                                description="Number of iterations to run all "
                                "tests."))
@@ -140,7 +144,8 @@ class GlobalSettings(Settings):
                             description="The results dir"))
     self.AddField(TextField("locks_dir", default="",
                             description="An alternate directory to use for "
-                            "storing/checking machine locks.\n"
+                            "storing/checking machine locks. Using this field "
+                            "automatically sets use_file_locks to True.\n"
                             "WARNING:  If you use your own locks directory, "
                             "there is no guarantee that someone else might not "
                             "hold a lock on the same machine in a different "