New script for using buildbots for nightly tests.

This script could eventually replace the current test_toolchains.py
script.  Instead of downloading and building ChromeOS locally, it
launches a buildbot to build ChromeOS with the latest mobile
toolchain compiler.  It then downloads the trybot image (and the
corresponding vanilla image), generates the crosperf experiment file,
and launches crosperf.

I have been running this as a second cron job on chrotomation2,
running daisy tests, and it has been working just fine.

BUG=None
TEST=Ran in cron jobs on chrotomation2.

Change-Id: I3b17d0373e2fae359fc8f4e0188dcec2943c7b71
Reviewed-on: https://chrome-internal-review.googlesource.com/187735
Reviewed-by: Caroline Tice <cmtice@google.com>
Trybot-Ready: Caroline Tice <cmtice@google.com>
Tested-by: Caroline Tice <cmtice@google.com>
Commit-Queue: Caroline Tice <cmtice@google.com>

1 files changed, 247 insertions, 0 deletions

diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py
new file mode 100644
index 00000000..bc2c0566
--- /dev/null
+++ b/buildbot_test_toolchains.py
@@ -0,0 +1,247 @@
+#!/usr/bin/python
+"""
+Script for running nightly compiler tests on ChromeOS.
+
+This script launches a buildbot to build ChromeOS with the latest compiler on
+a particular board; then it finds and downloads the trybot image and the
+corresponding official image, and runs crosperf performance tests comparing
+the two.  It then generates a report, emails it to the c-compiler-chrome, as
+well as copying the images into the seven-day reports directory.
+"""
+
+# Script to test different toolchains against ChromeOS benchmarks.
+import optparse
+import os
+import sys
+import time
+import urllib2
+
+from utils import command_executer
+from utils import logger
+
+from utils import buildbot_utils
+
+# CL that updated GCC ebuilds to use 'next_gcc'.
+USE_NEXT_GCC_PATCH ="230260"
+
+WEEKLY_REPORTS_ROOT = "/usr/local/google/crostc/weekly_test_data"
+ROLE_ACCOUNT = "mobiletc-prebuild"
+TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+class ToolchainComparator():
+  """
+  Class for doing the nightly tests work.
+  """
+
+  def __init__(self, board, remotes, chromeos_root, weekday):
+    self._board = board
+    self._remotes = remotes
+    self._chromeos_root = chromeos_root
+    self._base_dir = os.getcwd()
+    self._ce = command_executer.GetCommandExecuter()
+    self._l = logger.GetLogger()
+    self._build = "%s-release" % board
+    if not weekday:
+      self._weekday = time.strftime("%a")
+    else:
+      self._weekday = weekday
+
+  def _ParseVanillaImage(self, trybot_image):
+    """
+    Parse a trybot artifact name to get corresponding vanilla image.
+
+    This function takes an artifact name, such as
+    'trybot-daisy-release/R40-6394.0.0-b1389', and returns the
+    corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
+    """
+    start_pos = trybot_image.find(self._build)
+    end_pos = trybot_image.rfind("-b")
+    vanilla_image = trybot_image[start_pos:end_pos]
+    return vanilla_image
+
+
+  def _FinishSetup(self):
+    """
+    Make sure testing_rsa file is properly set up.
+    """
+    # Fix protections on ssh key
+    command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
+               "/chrome-src-internal/src/third_party/chromite/ssh_keys"
+               "/testing_rsa")
+    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
+    if ret_val:
+      raise Exception("chmod for testing_rsa failed")
+
+  def _TestImages(self, trybot_image, vanilla_image):
+    """
+    Create crosperf experiment file.
+
+    Given the names of the trybot and vanilla images, create the
+    appropriate crosperf experiment file and launch crosperf on it.
+    """
+    experiment_file_dir = os.path.join (self._chromeos_root, "..",
+                                        self._weekday)
+    experiment_file_name = "%s_toolchain_experiment.txt" % self._board
+    experiment_file = os.path.join (experiment_file_dir,
+                                    experiment_file_name)
+    experiment_header = """
+    board: %s
+    remote: %s
+    """ % (self._board, self._remotes)
+    experiment_tests = """
+    benchmark: all_perfv2 {
+      suite: telemetry_Crosperf
+      iterations: 3
+    }
+    """
+    with open(experiment_file, "w") as f:
+      print >> f, experiment_header
+      print >> f, experiment_tests
+
+      # Now add vanilla to test file.
+      official_image = """
+          vanilla_image {
+            chromeos_root: %s
+            build: %s
+          }
+          """ % (self._chromeos_root, vanilla_image)
+      print >> f, official_image
+
+      experiment_image = """
+          test_image {
+            chromeos_root: %s
+            build: %s
+          }
+          """ % (self._chromeos_root, trybot_image)
+      print >> f, experiment_image
+
+    crosperf = os.path.join(TOOLCHAIN_DIR,
+                            "crosperf",
+                            "crosperf")
+    command = "%s --email=c-compiler-chrome %s" % (crosperf, experiment_file)
+    ret = self._ce.RunCommand(command)
+    if ret:
+      raise Exception("Couldn't run crosperf!")
+    return ret
+
+
+  def _CopyWeeklyReportFiles(self, trybot_image, vanilla_image):
+    """
+    Put files in place for running seven-day reports.
+
+    Create tar files of the custom and official images and copy them
+    to the weekly reports directory, so they exist when the weekly report
+    gets generated.  IMPORTANT NOTE: This function must run *after*
+    crosperf has been run; otherwise the vanilla images will not be there.
+    """
+
+    dry_run = False
+    if (os.getlogin() != ROLE_ACCOUNT):
+      self._l.LogOutput("Running this from non-role account; not copying "
+                        "tar files for weekly reports.")
+      dry_run = True
+
+    images_path = os.path.join(os.path.realpath(self._chromeos_root),
+                               "chroot/tmp")
+
+    data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
+    dest_dir = os.path.join (data_dir, self._weekday)
+    if not os.path.exists(dest_dir):
+      os.makedirs(dest_dir)
+
+    # Make sure dest_dir is empty (clean out last week's data).
+    cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
+    if dry_run:
+      print "CMD: %s" % cmd
+    else:
+      self._ce.RunCommand(cmd)
+
+    # Now create new tar files and copy them over.
+    labels = [ "test", "vanilla" ]
+    for label_name in labels:
+      if label_name == "test":
+        test_path = trybot_image
+      else:
+        test_path = vanilla_image
+      tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
+      cmd = "cd %s; tar -cvf %s %s/*; cp %s %s/." % (images_path,
+                                                     tar_file_name,
+                                                     test_path,
+                                                     tar_file_name,
+                                                     dest_dir)
+      if dry_run:
+        print "CMD: %s" % cmd
+        tar_ret = 0
+      else:
+        tar_ret = self._ce.RunCommand(cmd)
+      if tar_ret:
+        self._l.LogOutput("Error while creating/copying test tar file(%s)."
+                          % tar_file_name)
+
+
+  def DoAll(self):
+    """
+    Main function inside ToolchainComparator class.
+
+    Launch trybot, get image names, create crosperf experiment file, run
+    crosperf, and copy images into seven-day report directories.
+    """
+
+    description = "master_%s_%s" % (USE_NEXT_GCC_PATCH, self._build)
+    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
+                                                 self._build,
+                                                 [ USE_NEXT_GCC_PATCH ],
+                                                 description)
+
+    vanilla_image = self._ParseVanillaImage(trybot_image)
+
+    print ("trybot_image: %s" % trybot_image)
+    print ("vanilla_image: %s" % vanilla_image)
+    if os.getlogin() == ROLE_ACCOUNT:
+      self._FinishSetup()
+
+    if not self._TestImages(trybot_image, vanilla_image):
+      # Only try to copy the image files if the test runs ran successfully.
+      self._CopyWeeklyReportFiles(trybot_image, vanilla_image)
+    return 0
+
+
+def Main(argv):
+  """The main function."""
+
+  # Common initializations
+  command_executer.InitCommandExecuter()
+  parser = optparse.OptionParser()
+  parser.add_option("--remote",
+                    dest="remote",
+                    help="Remote machines to run tests on.")
+  parser.add_option("--board",
+                    dest="board",
+                    default="x86-zgb",
+                    help="The target board.")
+  parser.add_option("--chromeos_root",
+                    dest="chromeos_root",
+                    help="The chromeos root from which to run tests.")
+  parser.add_option("--weekday", default="",
+                    dest="weekday",
+                    help="The day of the week for which to run tests.")
+  options, _ = parser.parse_args(argv)
+  if not options.board:
+    print "Please give a board."
+    return 1
+  if not options.remote:
+    print "Please give at least one remote machine."
+    return 1
+  if not options.chromeos_root:
+    print "Please specify the ChromeOS root directory."
+    return 1
+
+  fc = ToolchainComparator(options.board, options.remote,
+                           options.chromeos_root, options.weekday)
+  return fc.DoAll()
+
+
+if __name__ == "__main__":
+  retval = Main(sys.argv)
+  sys.exit(retval)