Handle exceptions in crosperf.

Change error message in buildbot.
Catch exceptions in crosperf exceution to do a graceful exit.

BUG=chromium:622875
TEST=ran run_tests

Change-Id: I03093ab27edcc85e318ad43097ab8cd316792dcb
Reviewed-on: https://chrome-internal-review.googlesource.com/299275
Commit-Ready: Manoj Gupta <manojgupta@google.com>
Tested-by: Manoj Gupta <manojgupta@google.com>
Reviewed-by: Caroline Tice <cmtice@google.com>
Reviewed-by: Luis Lozano <llozano@chromium.org>

3 files changed, 133 insertions, 108 deletions

diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py
index b9f6c500..6bc89fda 100755
--- a/buildbot_test_toolchains.py
+++ b/buildbot_test_toolchains.py
@@ -45,13 +45,15 @@ IMAGE_FS = (r'{board}-{image_type}/{chrome_version}-{tip}\.' +
             r'{branch}\.{branch_branch}')
 TRYBOT_IMAGE_FS = 'trybot-' + IMAGE_FS + '-{build_id}'
 PFQ_IMAGE_FS = IMAGE_FS + '-rc1'
-IMAGE_RE_GROUPS = {'board': r'(?P<board>\S+)',
-                   'image_type': r'(?P<image_type>\S+)',
-                   'chrome_version': r'(?P<chrome_version>R\d+)',
-                   'tip': r'(?P<tip>\d+)',
-                   'branch': r'(?P<branch>\d+)',
-                   'branch_branch': r'(?P<branch_branch>\d+)',
-                   'build_id': r'(?P<build_id>b\d+)'}
+IMAGE_RE_GROUPS = {
+    'board': r'(?P<board>\S+)',
+    'image_type': r'(?P<image_type>\S+)',
+    'chrome_version': r'(?P<chrome_version>R\d+)',
+    'tip': r'(?P<tip>\d+)',
+    'branch': r'(?P<branch>\d+)',
+    'branch_branch': r'(?P<branch_branch>\d+)',
+    'build_id': r'(?P<build_id>b\d+)'
+}
 TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS)
 
 
@@ -82,8 +84,9 @@ class ToolchainComparator(object):
       self._weekday = weekday
     timestamp = datetime.datetime.strftime(datetime.datetime.now(),
                                            '%Y-%m-%d_%H:%M:%S')
-    self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
-                                     '%s.%s' % (timestamp, board),)
+    self._reports_dir = os.path.join(
+        NIGHTLY_TESTS_DIR,
+        '%s.%s' % (timestamp, board),)
 
   def _GetVanillaImageName(self, trybot_image):
     """Given a trybot artifact name, get corresponding vanilla image name.
@@ -216,7 +219,7 @@ class ToolchainComparator(object):
 
     ret = self._ce.RunCommand(command)
     if ret != 0:
-      raise RuntimeError("Couldn't run crosperf!")
+      raise RuntimeError('Crosperf execution error!')
     else:
       # Copy json report to pending archives directory.
       command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
@@ -292,12 +295,13 @@ class ToolchainComparator(object):
     date_str = datetime.date.today()
     description = 'master_%s_%s_%s' % (self._patches_string, self._build,
                                        date_str)
-    trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
-                                                 self._build,
-                                                 self._patches,
-                                                 description,
-                                                 other_flags=['--notests'],
-                                                 build_toolchain=True)
+    trybot_image = buildbot_utils.GetTrybotImage(
+        self._chromeos_root,
+        self._build,
+        self._patches,
+        description,
+        other_flags=['--notests'],
+        build_toolchain=True)
 
     if len(trybot_image) == 0:
       self._l.LogError('Unable to find trybot_image for %s!' % description)
@@ -331,30 +335,31 @@ def Main(argv):
   # Common initializations
   command_executer.InitCommandExecuter()
   parser = argparse.ArgumentParser()
-  parser.add_argument('--remote',
-                      dest='remote',
-                      help='Remote machines to run tests on.')
-  parser.add_argument('--board',
-                      dest='board',
-                      default='x86-zgb',
-                      help='The target board.')
-  parser.add_argument('--chromeos_root',
-                      dest='chromeos_root',
-                      help='The chromeos root from which to run tests.')
-  parser.add_argument('--weekday',
-                      default='',
-                      dest='weekday',
-                      help='The day of the week for which to run tests.')
-  parser.add_argument('--patch',
-                      dest='patches',
-                      help='The patches to use for the testing, '
-                      "seprate the patch numbers with ',' "
-                      'for more than one patches.')
-  parser.add_argument('--noschedv2',
-                      dest='noschedv2',
-                      action='store_true',
-                      default=False,
-                      help='Pass --noschedv2 to crosperf.')
+  parser.add_argument(
+      '--remote', dest='remote', help='Remote machines to run tests on.')
+  parser.add_argument(
+      '--board', dest='board', default='x86-zgb', help='The target board.')
+  parser.add_argument(
+      '--chromeos_root',
+      dest='chromeos_root',
+      help='The chromeos root from which to run tests.')
+  parser.add_argument(
+      '--weekday',
+      default='',
+      dest='weekday',
+      help='The day of the week for which to run tests.')
+  parser.add_argument(
+      '--patch',
+      dest='patches',
+      help='The patches to use for the testing, '
+      "seprate the patch numbers with ',' "
+      'for more than one patches.')
+  parser.add_argument(
+      '--noschedv2',
+      dest='noschedv2',
+      action='store_true',
+      default=False,
+      help='Pass --noschedv2 to crosperf.')
 
   options = parser.parse_args(argv[1:])
   if not options.board:
diff --git a/crosperf/crosperf.py b/crosperf/crosperf.py
index 0a32cb8b..b78c8b9e 100755
--- a/crosperf/crosperf.py
+++ b/crosperf/crosperf.py
@@ -26,21 +26,23 @@ import test_flag
 
 def SetupParserOptions(parser):
   """Add all options to the parser."""
-  parser.add_argument('--dry_run',
-                      dest='dry_run',
-                      help=('Parse the experiment file and '
-                            'show what will be done'),
-                      action='store_true',
-                      default=False)
+  parser.add_argument(
+      '--dry_run',
+      dest='dry_run',
+      help=('Parse the experiment file and '
+            'show what will be done'),
+      action='store_true',
+      default=False)
   # Allow each of the global fields to be overridden by passing in
   # options. Add each global field as an option.
   option_settings = GlobalSettings('')
   for field_name in option_settings.fields:
     field = option_settings.fields[field_name]
-    parser.add_argument('--%s' % field.name,
-                        dest=field.name,
-                        help=field.description,
-                        action='store')
+    parser.add_argument(
+        '--%s' % field.name,
+        dest=field.name,
+        help=field.description,
+        action='store')
 
 
 def ConvertOptionsToSettings(options):
@@ -68,20 +70,22 @@ def CallExitHandler(signum, _):
   sys.exit(128 + signum)
 
 
-def Main(argv):
+def RunCrosperf(argv):
   parser = argparse.ArgumentParser()
 
-  parser.add_argument('--noschedv2',
-                      dest='noschedv2',
-                      default=False,
-                      action='store_true',
-                      help=('Do not use new scheduler. '
-                            'Use original scheduler instead.'))
-  parser.add_argument('-l',
-                      '--log_dir',
-                      dest='log_dir',
-                      default='',
-                      help='The log_dir, default is under <crosperf_logs>/logs')
+  parser.add_argument(
+      '--noschedv2',
+      dest='noschedv2',
+      default=False,
+      action='store_true',
+      help=('Do not use new scheduler. '
+            'Use original scheduler instead.'))
+  parser.add_argument(
+      '-l',
+      '--log_dir',
+      dest='log_dir',
+      default='',
+      help='The log_dir, default is under <crosperf_logs>/logs')
 
   SetupParserOptions(parser)
   options, args = parser.parse_known_args(argv)
@@ -117,12 +121,24 @@ def Main(argv):
   if options.dry_run:
     runner = MockExperimentRunner(experiment, json_report)
   else:
-    runner = ExperimentRunner(experiment,
-                              json_report,
-                              using_schedv2=(not options.noschedv2))
+    runner = ExperimentRunner(
+        experiment, json_report, using_schedv2=(not options.noschedv2))
 
   runner.Run()
 
 
+def Main(argv):
+  try:
+    RunCrosperf(argv)
+  except Exception as ex:
+    # Flush buffers before exiting to avoid out of order printing
+    sys.stdout.flush()
+    sys.stderr.flush()
+    print('Crosperf error: %s' % repr(ex))
+    sys.stdout.flush()
+    sys.stderr.flush()
+    sys.exit(1)
+
+
 if __name__ == '__main__':
   Main(sys.argv)
diff --git a/test_toolchains.py b/test_toolchains.py
index 4c03d456..eccf2f55 100755
--- a/test_toolchains.py
+++ b/test_toolchains.py
@@ -91,9 +91,10 @@ class ChromeOSCheckout(object):
   def _BuildAndImage(self, label=''):
     if (not label or
         not misc.DoesLabelExist(self._chromeos_root, self._board, label)):
-      build_chromeos_args = [build_chromeos.__file__,
-                             '--chromeos_root=%s' % self._chromeos_root,
-                             '--board=%s' % self._board, '--rebuild']
+      build_chromeos_args = [
+          build_chromeos.__file__, '--chromeos_root=%s' % self._chromeos_root,
+          '--board=%s' % self._board, '--rebuild'
+      ]
       if self._public:
         build_chromeos_args.append('--env=USE=-chrome_internal')
 
@@ -117,10 +118,8 @@ class ChromeOSCheckout(object):
 
   def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag):
     env_string = misc.GetEnvStringFromDict(env_dict)
-    command = ('%s %s' % (env_string,
-                          misc.GetSetupBoardCommand(self._board,
-                                                    usepkg=usepkg_flag,
-                                                    force=clobber_flag)))
+    command = ('%s %s' % (env_string, misc.GetSetupBoardCommand(
+        self._board, usepkg=usepkg_flag, force=clobber_flag)))
     ret = self._ce.ChrootRunCommand(self._chromeos_root, command)
     error_str = "Could not setup board: '%s'" % command
     assert ret == 0, error_str
@@ -148,9 +147,11 @@ class ChromeOSCheckout(object):
     # Now uninstall the vanilla compiler and setup/build our custom
     # compiler.
     self._UnInstallToolchain()
-    envdict = {'USE': 'git_gcc',
-               'GCC_GITHASH': config.gcc_config.githash,
-               'EMERGE_DEFAULT_OPTS': '--exclude=gcc'}
+    envdict = {
+        'USE': 'git_gcc',
+        'GCC_GITHASH': config.gcc_config.githash,
+        'EMERGE_DEFAULT_OPTS': '--exclude=gcc'
+    }
     self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False)
 
 
@@ -176,8 +177,9 @@ class ToolchainComparator(ChromeOSCheckout):
     self._l = logger.GetLogger()
     timestamp = datetime.datetime.strftime(datetime.datetime.now(),
                                            '%Y-%m-%d_%H:%M:%S')
-    self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
-                                     '%s.%s' % (timestamp, board),)
+    self._reports_dir = os.path.join(
+        NIGHTLY_TESTS_DIR,
+        '%s.%s' % (timestamp, board),)
     self._noschedv2 = noschedv2
     ChromeOSCheckout.__init__(self, board, self._chromeos_root)
 
@@ -264,7 +266,7 @@ class ToolchainComparator(ChromeOSCheckout):
 
     ret = self._ce.RunCommand(command)
     if ret != 0:
-      raise RuntimeError("Couldn't run crosperf!")
+      raise RuntimeError('Crosperf execution error!')
     else:
       # Copy json report to pending archives directory.
       command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
@@ -346,36 +348,38 @@ def Main(argv):
   ###  command_executer.InitCommandExecuter(True)
   command_executer.InitCommandExecuter()
   parser = argparse.ArgumentParser()
-  parser.add_argument('--remote',
-                      dest='remote',
-                      help='Remote machines to run tests on.')
-  parser.add_argument('--board',
-                      dest='board',
-                      default='x86-alex',
-                      help='The target board.')
-  parser.add_argument('--githashes',
-                      dest='githashes',
-                      default='master',
-                      help='The gcc githashes to test.')
-  parser.add_argument('--clean',
-                      dest='clean',
-                      default=False,
-                      action='store_true',
-                      help='Clean the chroot after testing.')
-  parser.add_argument('--public',
-                      dest='public',
-                      default=False,
-                      action='store_true',
-                      help='Use the public checkout/build.')
-  parser.add_argument('--force-mismatch',
-                      dest='force_mismatch',
-                      default='',
-                      help='Force the image regardless of board mismatch')
-  parser.add_argument('--noschedv2',
-                      dest='noschedv2',
-                      action='store_true',
-                      default=False,
-                      help='Pass --noschedv2 to crosperf.')
+  parser.add_argument(
+      '--remote', dest='remote', help='Remote machines to run tests on.')
+  parser.add_argument(
+      '--board', dest='board', default='x86-alex', help='The target board.')
+  parser.add_argument(
+      '--githashes',
+      dest='githashes',
+      default='master',
+      help='The gcc githashes to test.')
+  parser.add_argument(
+      '--clean',
+      dest='clean',
+      default=False,
+      action='store_true',
+      help='Clean the chroot after testing.')
+  parser.add_argument(
+      '--public',
+      dest='public',
+      default=False,
+      action='store_true',
+      help='Use the public checkout/build.')
+  parser.add_argument(
+      '--force-mismatch',
+      dest='force_mismatch',
+      default='',
+      help='Force the image regardless of board mismatch')
+  parser.add_argument(
+      '--noschedv2',
+      dest='noschedv2',
+      action='store_true',
+      default=False,
+      help='Pass --noschedv2 to crosperf.')
   options = parser.parse_args(argv)
   if not options.board:
     print('Please give a board.')