Add 'build' settings option and xbuddy image format.

This patch updates Crosperf to add the 'build' tag in the
experiment file, and to allow that field to contain xbuddy
syntax for using trybot and/or official builds in the test runs.
It also adds a bit more checking to make sure we have everything
necessary for running 'cros flash' before attempting to use it.

BUG=None
TEST=I have run this using an experiment file that compares a local
image, a trybot image and an official image against each other.  It
all worked.

Change-Id: Ia896799061508fb5718a3201b1599d8622de0b3f
Reviewed-on: https://chrome-internal-review.googlesource.com/154097
Reviewed-by: Yunlian Jiang <yunlian@google.com>
Commit-Queue: Caroline Tice <cmtice@google.com>
Tested-by: Caroline Tice <cmtice@google.com>

10 files changed, 140 insertions, 10 deletions

diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index f6c82051..86118a38 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -108,6 +108,7 @@ class ExperimentFactory(object):
     share_users = global_settings.GetField("share_users")
     results_dir = global_settings.GetField("results_dir")
     chrome_src = global_settings.GetField("chrome_src")
+    build = global_settings.GetField("build")
     use_test_that = global_settings.GetField("use_test_that")
     show_all_results = global_settings.GetField("show_all_results")
     # Default cache hit conditions. The image checksum in the cache and the
@@ -182,9 +183,12 @@ class ExperimentFactory(object):
     all_remote = list(remote)
     for label_settings in all_label_settings:
       label_name = label_settings.name
+      board = label_settings.GetField("board")
       image = label_settings.GetField("chromeos_image")
+      if image == "":
+        build = label_settings.GetField("build")
+        image = label_settings.GetXbuddyPath (build, board)
       chromeos_root = label_settings.GetField("chromeos_root")
-      board = label_settings.GetField("board")
       my_remote = label_settings.GetField("remote")
       new_remote = []
       for i in my_remote:
diff --git a/crosperf/experiment_file.py b/crosperf/experiment_file.py
index 293e822f..0a3c3b98 100644
--- a/crosperf/experiment_file.py
+++ b/crosperf/experiment_file.py
@@ -131,10 +131,13 @@ class ExperimentFile(object):
   def Canonicalize(self):
     """Convert parsed experiment file back into an experiment file."""
     res = ""
+    board = ""
     for field_name in self.global_settings.fields:
       field = self.global_settings.fields[field_name]
       if field.assigned:
         res += "%s: %s\n" % (field.name, field.GetString())
+      if field.name == "board":
+        board = field.GetString()
     res += "\n"
 
     for settings in self.all_settings:
@@ -149,6 +152,10 @@ class ExperimentFile(object):
                            (os.path.expanduser(field.GetString())))
               if real_file != field.GetString():
                 res += "\t#actual_image: %s\n" % real_file
+            if field.name == "build":
+              value = field.GetString()
+              xbuddy_path = settings.GetXbuddyPath (value, board)
+              res +=  "\t#actual_image: %s\n" % xbuddy_path
         res += "}\n\n"
 
     return res
diff --git a/crosperf/experiment_files/official-image.exp b/crosperf/experiment_files/official-image.exp
new file mode 100644
index 00000000..72be02bc
--- /dev/null
+++ b/crosperf/experiment_files/official-image.exp
@@ -0,0 +1,31 @@
+# This is an example experiment file for Crosperf, showing how to run
+# a basic test, using a (previously made) trybot image.
+
+name: trybot_example
+# Replace board and remote values below appropriately. e.g. "lumpy" and
+# "123.45.678.901"  or "my-machine.blah.com".
+board: <your-board-goes-here>
+remote: <your-remote-ip-address-here>
+
+# You can replace 'canvasmark' below with the name of the Telemetry
+# benchmakr you want to run.
+benchmark: canvasmark {
+  suite:telemetry_Crosperf
+  iterations: 1
+}
+
+
+# Replace <path-to-your-chroot-goes-here> with the actual directory path
+# to the top of your ChromimumOS chroot.
+trybot_image {
+  chromeos_root:<path-to-your-chroot-goes-here>
+  # Replace <xbuddy-official-image-designation> with the xbuddy syntax
+  # for the official image you want to use (see
+  # http://www.chromium.org/chromium-os/how-tos-and-troubleshooting/using-the-dev-server/xbuddy-for-devserver#TOC-XBuddy-Paths
+  # for xbuddy syntax).  Omit the "http://xbuddy/remote/<board>/" prefix.
+  # For example, if you want to use the "latest-dev" official image,
+  # your build field would look like:
+  # build:latest-dev
+  build:<xbuddy-official-image-designation>
+}
+
diff --git a/crosperf/experiment_files/trybot-image.exp b/crosperf/experiment_files/trybot-image.exp
new file mode 100644
index 00000000..d80f4187
--- /dev/null
+++ b/crosperf/experiment_files/trybot-image.exp
@@ -0,0 +1,32 @@
+# This is an example experiment file for Crosperf, showing how to run
+# a basic test, using a (previously made) trybot image.
+
+name: trybot_example
+# Replace board and remote values below appropriately. e.g. "lumpy" and
+# "123.45.678.901"  or "my-machine.blah.com".
+board: <your-board-goes-here>
+remote: <your-remote-ip-address-here>
+
+# You can replace 'canvasmark' below with the name of the Telemetry
+# benchmakr you want to run.
+benchmark: canvasmark {
+  suite:telemetry_Crosperf
+  iterations: 1
+}
+
+
+# Replace <path-to-your-chroot-goes-here> with the actual directory path
+# to the top of your ChromimumOS chroot.
+trybot_image {
+  chromeos_root:<path-to-your-chroot-goes-here>
+  # Replace <trybot-image-name> with the actual name of the trybot image
+  # that you wish to use.  You can find this by going to the trybot build
+  # log, going  # to the 'Report' stage, and looking for "Build Artifacts'
+  # at the bottom.  You will see something like:
+  # 'lumpy: https://storage.cloud.google.com/chromeos-image-archive/trybot-lumpy-paladin/R34-5393.0.0-b1504/index.html'
+  # From that you can extract the trybot image name and put it in the build
+  # field:
+  # build:trybot-lumpy-paladin/R34-5417.0.0-b1506
+  build:<trybot-image-name>
+}
+
diff --git a/crosperf/image_checksummer.py b/crosperf/image_checksummer.py
index dcc1cb02..eeb4466e 100644
--- a/crosperf/image_checksummer.py
+++ b/crosperf/image_checksummer.py
@@ -22,6 +22,8 @@ class ImageChecksummer(object):
           logger.GetLogger().LogOutput("Acquiring checksum for '%s'." %
                                        self.label.name)
           self._checksum = None
+          if self.label.image_type != "local":
+            raise Exception("Called Checksum on non-local image!")
           if self.label.chromeos_image:
             if os.path.exists(self.label.chromeos_image):
               self._checksum = FileUtils().Md5File(self.label.chromeos_image)
@@ -49,6 +51,8 @@ class ImageChecksummer(object):
       return cls._instance
 
   def Checksum(self, label):
+    if label.image_type != "local":
+      raise Exception("Attempt to call Checksum on non-local image.")
     with self._lock:
       if label.name not in self._per_image_checksummers:
         self._per_image_checksummers[label.name] = (ImageChecksummer.
diff --git a/crosperf/label.py b/crosperf/label.py
index 61698653..c212125b 100644
--- a/crosperf/label.py
+++ b/crosperf/label.py
@@ -14,9 +14,13 @@ from utils import misc
 class Label(object):
   def __init__(self, name, chromeos_image, chromeos_root, board, remote,
                image_args, image_md5sum, cache_dir, chrome_src=None):
+
+    self.image_type = self._GetImageType(chromeos_image)
+
     # Expand ~
     chromeos_root = os.path.expanduser(chromeos_root)
-    chromeos_image = os.path.expanduser(chromeos_image)
+    if self.image_type == "local":
+      chromeos_image = os.path.expanduser(chromeos_image)
 
     self.name = name
     self.chromeos_image = chromeos_image
@@ -27,7 +31,8 @@ class Label(object):
     self.cache_dir = cache_dir
 
     if not chromeos_root:
-      chromeos_root = FileUtils().ChromeOSRootFromImage(chromeos_image)
+      if self.image_type == "local":
+        chromeos_root = FileUtils().ChromeOSRootFromImage(chromeos_image)
       if not chromeos_root:
         raise Exception("No ChromeOS root given for label '%s' and could not "
                         "determine one from image path: '%s'." %
@@ -52,6 +57,15 @@ class Label(object):
                         % (name, chrome_src))
       self.chrome_src = chromeos_src
 
+  def _GetImageType(self, chromeos_image):
+    image_type = None
+    if chromeos_image.find("xbuddy://") < 0:
+      image_type = "local"
+    elif chromeos_image.find("trybot") >= 0:
+      image_type = "trybot"
+    else:
+      image_type = "official"
+    return image_type
 
 class MockLabel(object):
   def __init__(self, name, chromeos_image, chromeos_root, board, remote,
@@ -67,3 +81,4 @@ class MockLabel(object):
       self.chromeos_root = chromeos_root
     self.image_args = image_args
     self.image_md5sum = image_md5sum
+    self.chrome_src = chrome_src
diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py
index 282bbc5b..51e10554 100644
--- a/crosperf/machine_manager.py
+++ b/crosperf/machine_manager.py
@@ -180,8 +180,14 @@ class MachineManager(object):
     self.chromeos_root = chromeos_root
 
   def ImageMachine(self, machine, label):
-    checksum = ImageChecksummer().Checksum(label)
-    if machine.checksum == checksum:
+    if label.image_type == "local":
+      checksum = ImageChecksummer().Checksum(label)
+    elif label.image_type == "trybot":
+      checksum = machine._GetMD5Checksum(label.chromeos_image)
+    else:
+      checksum = None
+
+    if checksum and (machine.checksum == checksum):
       return
     chromeos_root = label.chromeos_root
     if not chromeos_root:
@@ -271,7 +277,12 @@ class MachineManager(object):
                                     % m.name)
 
   def AcquireMachine(self, chromeos_image, label):
-    image_checksum = ImageChecksummer().Checksum(label)
+    if label.image_type == "local":
+      image_checksum = ImageChecksummer().Checksum(label)
+    elif label.image_type == "trybot":
+      image_checksum = hashlib.md5(chromeos_image).hexdigest()
+    else:
+      image_checksum = None
     machines = self.GetMachines(label)
     check_interval_time = 120
     with self._lock:
@@ -305,7 +316,7 @@ class MachineManager(object):
 ###          return None
       for m in [machine for machine in self.GetAvailableMachines(label)
                 if not machine.locked]:
-        if m.checksum == image_checksum:
+        if image_checksum and (m.checksum == image_checksum):
           m.locked = True
           m.test_run = threading.current_thread()
           return m
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index 4c1cf4c0..fb66e5c4 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -539,6 +539,10 @@ class ResultsCache(object):
       machine_checksum = self.machine_manager.machine_checksum[self.label.name]
     if read and CacheConditions.CHECKSUMS_MATCH not in self.cache_conditions:
       checksum = "*"
+    elif self.label.image_type == "trybot":
+      checksum = hashlib.md5(self.label.chromeos_image).hexdigest()
+    elif self.label.image_type == "official":
+      checksum = "*"
     else:
       checksum = ImageChecksummer().Checksum(self.label)
 
diff --git a/crosperf/settings.py b/crosperf/settings.py
index e407a143..2ed18c3e 100644
--- a/crosperf/settings.py
+++ b/crosperf/settings.py
@@ -60,3 +60,11 @@ class Settings(object):
     for name in self.fields:
       if not self.fields[name].assigned and self.fields[name].required:
         raise Exception("Field %s is invalid." % name)
+
+  def GetXbuddyPath(self, path_str, board):
+    prefix = "xbuddy://remote"
+    if path_str.find("trybot") < 0:
+      xbuddy_path = "%s/%s/%s" % (prefix, board, path_str)
+    else:
+      xbuddy_path = "%s/%s" % (prefix, path_str)
+    return xbuddy_path
diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py
index ba3fcee1..6be34816 100644
--- a/crosperf/settings_factory.py
+++ b/crosperf/settings_factory.py
@@ -54,9 +54,10 @@ class BenchmarkSettings(Settings):
 class LabelSettings(Settings):
   def __init__(self, name):
     super(LabelSettings, self).__init__(name, "label")
-    self.AddField(TextField("chromeos_image", required=True,
+    self.AddField(TextField("chromeos_image", required=False,
                             description="The path to the image to run tests "
-                            "on."))
+                            "on, for local/custom-built images. See 'build' "
+                            "option for official or trybot images."))
     self.AddField(TextField("chromeos_root",
                             description="The path to a chromeos checkout which "
                             "contains a src/scripts directory. Defaults to "
@@ -80,6 +81,13 @@ class LabelSettings(Settings):
                             "This is used to run telemetry benchmarks. "
                             "The default one is the src inside chroot.",
                             required=False, default=""))
+    self.AddField(TextField("build",
+                            description="The xbuddy specification for an "
+                            "official or trybot image to use for tests. "
+                            "'/remote' is assumed, and the board is given "
+                            "elsewhere, so omit the '/remote/<board>/' xbuddy"
+                            "prefix.",
+                            required=False, default=""))
 
 
 class GlobalSettings(Settings):
@@ -153,7 +161,13 @@ class GlobalSettings(Settings):
                             description="The path to the source of chrome. "
                             "This is used to run telemetry benchmarks. "
                             "The default one is the src inside chroot.",
-
+                            required=False, default=""))
+    self.AddField(TextField("build",
+                            description="The xbuddy specification for an "
+                            "official or trybot image to use for tests. "
+                            "'/remote' is assumed, and the board is given "
+                            "elsewhere, so omit the '/remote/<board>/' xbuddy"
+                            "prefix.",
                             required=False, default=""))