crosperf: add support to run tast benchmarks

Crosperf used to only support autotest server/client tests, and was
limiting us from running and collecting tast benchmarks.

This patch introduces a simple support for crosperf to run tast. Note
that currently we do not support passing arguments or running tast with
local changes. It basically launches `tast run <ip> <tast_test>` and
collects chart json result and parses it.

BUG=chromium:1071937
TEST=tested with tast.platform.ReportDiskUsage

Change-Id: I1980a48c4ca8e02548e5313d32b8c5ddcf5e3e60
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2160128
Commit-Queue: Zhizhou Yang <zhizhouy@google.com>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Auto-Submit: Zhizhou Yang <zhizhouy@google.com>
Reviewed-by: George Burgess <gbiv@chromium.org>

4 files changed, 81 insertions, 16 deletions

diff --git a/crosperf/default-telemetry-results.json b/crosperf/default-telemetry-results.json
index c4fe0d44..3dd22f86 100644
--- a/crosperf/default-telemetry-results.json
+++ b/crosperf/default-telemetry-results.json
@@ -169,5 +169,8 @@
   "rendering.desktop@@aquarium_20k$": [
     "avg_surface_fps",
     "exp_avg_surface_fps"
+  ],
+  "platform.ReportDiskUsage": [
+    "bytes_rootfs_prod__summary"
   ]
 }
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index a7fd4169..85c3c6e3 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -268,7 +268,10 @@ class Result(object):
     return [samples, u'samples']
 
   def GetResultsDir(self):
-    mo = re.search(r'Results placed in (\S+)', self.out)
+    if self.suite == 'tast':
+      mo = re.search(r'Writing results to (\S+)', self.out)
+    else:
+      mo = re.search(r'Results placed in (\S+)', self.out)
     if mo:
       result = mo.group(1)
       return result
diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py
index 62a85cc6..17e1ad73 100644
--- a/crosperf/suite_runner.py
+++ b/crosperf/suite_runner.py
@@ -17,7 +17,7 @@ import time
 from cros_utils import command_executer
 
 TEST_THAT_PATH = '/usr/bin/test_that'
-# TODO: Need to check whether Skylab is installed and set up correctly.
+TAST_PATH = '/usr/bin/tast'
 SKYLAB_PATH = '/usr/local/bin/skylab'
 GS_UTIL = 'src/chromium/depot_tools/gsutil.py'
 AUTOTEST_DIR = '/mnt/host/source/src/third_party/autotest/files'
@@ -78,8 +78,11 @@ class SuiteRunner(object):
       if label.skylab:
         ret_tup = self.Skylab_Run(label, benchmark, test_args, profiler_args)
       else:
-        ret_tup = self.Test_That_Run(machine_name, label, benchmark, test_args,
-                                     profiler_args)
+        if benchmark.suite == 'tast':
+          ret_tup = self.Tast_Run(machine_name, label, benchmark)
+        else:
+          ret_tup = self.Test_That_Run(machine_name, label, benchmark,
+                                       test_args, profiler_args)
       if ret_tup[0] != 0:
         self.logger.LogOutput('benchmark %s failed. Retries left: %s' %
                               (benchmark.name, benchmark.retries - i))
@@ -127,6 +130,24 @@ class SuiteRunner(object):
 
     return args_list
 
+  # TODO(zhizhouy): Currently do not support passing arguments or running
+  # customized tast tests, as we do not have such requirements.
+  def Tast_Run(self, machine, label, benchmark):
+    # Remove existing tast results
+    command = 'rm -rf /usr/local/autotest/results/*'
+    self._ce.CrosRunCommand(
+        command, machine=machine, chromeos_root=label.chromeos_root)
+
+    command = ' '.join(
+        [TAST_PATH, 'run', '-build=False', machine, benchmark.test_name])
+
+    if self.log_level != 'verbose':
+      self.logger.LogOutput('Running test.')
+      self.logger.LogOutput('CMD: %s' % command)
+
+    return self._ce.ChrootRunCommandWOutput(
+        label.chromeos_root, command, command_terminator=self._ct)
+
   def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
     """Run the test_that test.."""
 
diff --git a/crosperf/suite_runner_unittest.py b/crosperf/suite_runner_unittest.py
index 73fcb45b..86e1ef19 100755
--- a/crosperf/suite_runner_unittest.py
+++ b/crosperf/suite_runner_unittest.py
@@ -26,8 +26,6 @@ from machine_manager import MockCrosMachine
 
 class SuiteRunnerTest(unittest.TestCase):
   """Class of SuiteRunner test."""
-  real_logger = logger.GetLogger()
-
   mock_json = mock.Mock(spec=json)
   mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
   mock_cmd_term = mock.Mock(spec=command_executer.CommandTerminator)
@@ -55,12 +53,23 @@ class SuiteRunnerTest(unittest.TestCase):
       '',  # perf_args
       'crosperf_Wrapper')  # suite
 
+  tast_bench = Benchmark(
+      'b3_test',  # name
+      'platform.ReportDiskUsage',  # test_name
+      '',  # test_args
+      1,  # iterations
+      False,  # rm_chroot_tmp
+      '',  # perf_args
+      'tast')  # suite
+
   def __init__(self, *args, **kwargs):
     super(SuiteRunnerTest, self).__init__(*args, **kwargs)
     self.skylab_run_args = []
     self.test_that_args = []
+    self.tast_args = []
     self.call_skylab_run = False
     self.call_test_that_run = False
+    self.call_tast_run = False
 
   def setUp(self):
     self.runner = suite_runner.SuiteRunner(
@@ -90,8 +99,10 @@ class SuiteRunnerTest(unittest.TestCase):
     def reset():
       self.test_that_args = []
       self.skylab_run_args = []
+      self.tast_args = []
       self.call_test_that_run = False
       self.call_skylab_run = False
+      self.call_tast_run = False
 
     def FakeSkylabRun(test_label, benchmark, test_args, profiler_args):
       self.skylab_run_args = [test_label, benchmark, test_args, profiler_args]
@@ -106,8 +117,14 @@ class SuiteRunnerTest(unittest.TestCase):
       self.call_test_that_run = True
       return 'Ran FakeTestThatRun'
 
+    def FakeTastRun(machine, test_label, benchmark):
+      self.tast_args = [machine, test_label, benchmark]
+      self.call_tast_run = True
+      return 'Ran FakeTastRun'
+
     self.runner.Skylab_Run = FakeSkylabRun
     self.runner.Test_That_Run = FakeTestThatRun
+    self.runner.Tast_Run = FakeTastRun
 
     self.runner.dut_config['enable_aslr'] = False
     self.runner.dut_config['cooldown_time'] = 0
@@ -158,6 +175,15 @@ class SuiteRunnerTest(unittest.TestCase):
         'fake_machine', self.mock_label, self.telemetry_crosperf_bench, '', ''
     ])
 
+    # Test tast run for tast benchmarks.
+    reset()
+    self.runner.Run(cros_machine, self.mock_label, self.tast_bench, '', '')
+    self.assertTrue(self.call_tast_run)
+    self.assertFalse(self.call_test_that_run)
+    self.assertFalse(self.call_skylab_run)
+    self.assertEqual(self.tast_args,
+                     ['fake_machine', self.mock_label, self.tast_bench])
+
   def test_gen_test_args(self):
     test_args = '--iterations=2'
     perf_args = 'record -a -e cycles'
@@ -180,16 +206,29 @@ class SuiteRunnerTest(unittest.TestCase):
   @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
   @mock.patch.object(command_executer.CommandExecuter,
                      'ChrootRunCommandWOutput')
-  def test_test_that_run(self, mock_chroot_runcmd, mock_cros_runcmd):
-
-    def FakeLogMsg(fd, termfd, msg, flush=True):
-      if fd or termfd or msg or flush:
-        pass
-
-    save_log_msg = self.real_logger.LogMsg
-    self.real_logger.LogMsg = FakeLogMsg
-    self.runner.logger = self.real_logger
+  def test_tast_run(self, mock_chroot_runcmd, mock_cros_runcmd):
+    mock_chroot_runcmd.return_value = 0
+    self.mock_cmd_exec.ChrootRunCommandWOutput = mock_chroot_runcmd
+    self.mock_cmd_exec.CrosRunCommand = mock_cros_runcmd
+    res = self.runner.Tast_Run('lumpy1.cros', self.mock_label, self.tast_bench)
+    self.assertEqual(mock_cros_runcmd.call_count, 1)
+    self.assertEqual(mock_chroot_runcmd.call_count, 1)
+    self.assertEqual(res, 0)
+    self.assertEqual(mock_cros_runcmd.call_args_list[0][0],
+                     ('rm -rf /usr/local/autotest/results/*',))
+    args_list = mock_chroot_runcmd.call_args_list[0][0]
+    args_dict = mock_chroot_runcmd.call_args_list[0][1]
+    self.assertEqual(len(args_list), 2)
+    self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
 
+  @mock.patch.object(command_executer.CommandExecuter, 'CrosRunCommand')
+  @mock.patch.object(command_executer.CommandExecuter,
+                     'ChrootRunCommandWOutput')
+  @mock.patch.object(logger.Logger, 'LogFatal')
+  def test_test_that_run(self, mock_log_fatal, mock_chroot_runcmd,
+                         mock_cros_runcmd):
+    mock_log_fatal.side_effect = SystemExit()
+    self.runner.logger.LogFatal = mock_log_fatal
     # Test crosperf_Wrapper benchmarks cannot take perf_args
     raised_exception = False
     try:
@@ -215,7 +254,6 @@ class SuiteRunnerTest(unittest.TestCase):
     args_dict = mock_chroot_runcmd.call_args_list[0][1]
     self.assertEqual(len(args_list), 2)
     self.assertEqual(args_dict['command_terminator'], self.mock_cmd_term)
-    self.real_logger.LogMsg = save_log_msg
 
   @mock.patch.object(command_executer.CommandExecuter, 'RunCommandWOutput')
   @mock.patch.object(json, 'loads')