Merging 18 commit(s) from Chromium's toolchain-utils am: 0ae38c8498 am: 2a19d36a82 am: e617e3393dr_aml_301500702android-mainline-12.0.0_r55android-mainline-11.0.0_r9android-mainline-11.0.0_r8android-mainline-11.0.0_r7android-mainline-11.0.0_r6android-mainline-11.0.0_r5android-mainline-11.0.0_r45android-mainline-11.0.0_r44android-mainline-11.0.0_r43android-mainline-11.0.0_r42android-mainline-11.0.0_r41android-mainline-11.0.0_r40android-mainline-11.0.0_r4android-mainline-11.0.0_r39android-mainline-11.0.0_r38android-mainline-11.0.0_r37android-mainline-11.0.0_r36android-mainline-11.0.0_r35android-mainline-11.0.0_r34android-mainline-11.0.0_r33android-mainline-11.0.0_r32android-mainline-11.0.0_r31android-mainline-11.0.0_r30android-mainline-11.0.0_r3android-mainline-11.0.0_r29android-mainline-11.0.0_r28android-mainline-11.0.0_r27android-mainline-11.0.0_r26android-mainline-11.0.0_r25android-mainline-11.0.0_r24android-mainline-11.0.0_r23android-mainline-11.0.0_r22android-mainline-11.0.0_r21android-mainline-11.0.0_r20android-mainline-11.0.0_r2android-mainline-11.0.0_r19android-mainline-11.0.0_r18android-mainline-11.0.0_r17android-mainline-11.0.0_r16android-mainline-11.0.0_r15android-mainline-11.0.0_r14android-mainline-11.0.0_r13android-mainline-11.0.0_r12android-mainline-11.0.0_r10android-mainline-11.0.0_r1android-11.0.0_r48android-11.0.0_r47android-11.0.0_r46android-11.0.0_r45android-11.0.0_r44android-11.0.0_r43android-11.0.0_r42android-11.0.0_r41android-11.0.0_r40android-11.0.0_r39android-11.0.0_r38android-11.0.0_r37android-11.0.0_r36android-11.0.0_r35android-11.0.0_r34android-11.0.0_r33android-11.0.0_r32android-11.0.0_r31android-11.0.0_r30android-11.0.0_r29android-11.0.0_r28android-11.0.0_r27android-11.0.0_r26android-11.0.0_r24android-11.0.0_r23android-11.0.0_r22android-11.0.0_r21android-11.0.0_r20android-11.0.0_r19android-11.0.0_r18android-11.0.0_r16android11-qpr3-s1-releaseandroid11-qpr3-releaseandroid11-qpr2-releaseandroid11-qpr1-s2-releaseandroid11-qpr1-s1-releaseandroid11-qpr1-releaseandroid11-qpr1-d-s1-releaseandroid11-qpr1-d-releaseandroid11-qpr1-c-releaseandroid11-mainline-tethering-releaseandroid11-mainline-sparse-2021-jan-releaseandroid11-mainline-sparse-2020-dec-releaseandroid11-mainline-releaseandroid11-mainline-permission-releaseandroid11-mainline-os-statsd-releaseandroid11-mainline-networkstack-releaseandroid11-mainline-media-swcodec-releaseandroid11-mainline-media-releaseandroid11-mainline-extservices-releaseandroid11-mainline-documentsui-releaseandroid11-mainline-conscrypt-releaseandroid11-mainline-cellbroadcast-releaseandroid11-mainline-captiveportallogin-releaseandroid11-devandroid11-d2-releaseandroid11-d1-b-release

Change-Id: I3f25c7ee034b2e20e37ed941b8eae24eec7043eb

1 files changed, 79 insertions, 127 deletions

diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py
index 71ca7e7f..79ace20d 100644
--- a/crosperf/suite_runner.py
+++ b/crosperf/suite_runner.py
@@ -15,7 +15,6 @@ import shlex
 import time
 
 from cros_utils import command_executer
-from cros_utils.device_setup_utils import DutWrapper
 
 TEST_THAT_PATH = '/usr/bin/test_that'
 # TODO: Need to check whether Skylab is installed and set up correctly.
@@ -75,34 +74,12 @@ class SuiteRunner(object):
 
   def Run(self, cros_machine, label, benchmark, test_args, profiler_args):
     machine_name = cros_machine.name
-    if benchmark.suite != 'telemetry_Crosperf':
-      # Initialize command executer on DUT for test_that runs.
-      run_on_dut = DutWrapper(
-          label.chromeos_root,
-          machine_name,
-          logger=self.logger,
-          log_level=self.log_level,
-          ce=self._ce,
-          dut_config=self.dut_config)
     for i in range(0, benchmark.retries + 1):
-      # TODO: For telemetry_Crosperf run, device setup has been moved into
-      # server test script; for client runs, need to figure out wrapper to do
-      # it before running, now it is still setup here.
-      if benchmark.suite != 'telemetry_Crosperf':
-        wait_time = run_on_dut.SetupDevice()
-        # This is for accumulating wait time for test_that runs only,
-        # for telemetry_Cropserf runs, please refer to result_cache.
-        cros_machine.AddCooldownWaitTime(wait_time)
-
       if label.skylab:
         ret_tup = self.Skylab_Run(label, benchmark, test_args, profiler_args)
-      elif benchmark.suite == 'telemetry_Crosperf':
-        ret_tup = self.Telemetry_Crosperf_Run(machine_name, label, benchmark,
-                                              test_args, profiler_args)
       else:
         ret_tup = self.Test_That_Run(machine_name, label, benchmark, test_args,
                                      profiler_args)
-
       if ret_tup[0] != 0:
         self.logger.LogOutput('benchmark %s failed. Retries left: %s' %
                               (benchmark.name, benchmark.retries - i))
@@ -116,43 +93,94 @@ class SuiteRunner(object):
         break
     return ret_tup
 
+  def RemoveTelemetryTempFile(self, machine, chromeos_root):
+    filename = 'telemetry@%s' % machine
+    fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename)
+    if os.path.exists(fullname):
+      os.remove(fullname)
+
+  def GenTestArgs(self, benchmark, test_args, profiler_args):
+    args_list = []
+
+    if benchmark.suite != 'telemetry_Crosperf' and profiler_args:
+      self.logger.LogFatal('Tests other than telemetry_Crosperf do not '
+                           'support profiler.')
+
+    if test_args:
+      # Strip double quotes off args (so we can wrap them in single
+      # quotes, to pass through to Telemetry).
+      if test_args[0] == '"' and test_args[-1] == '"':
+        test_args = test_args[1:-1]
+      args_list.append("test_args='%s'" % test_args)
+
+    args_list.append(GetDutConfigArgs(self.dut_config))
+
+    if not (benchmark.suite == 'telemetry_Crosperf' or
+            benchmark.suite == 'crosperf_Wrapper'):
+      self.logger.LogWarning('Please make sure the server test has stage for '
+                             'device setup.\n')
+    else:
+      args_list.append('test=%s' % benchmark.test_name)
+      if benchmark.suite == 'telemetry_Crosperf':
+        args_list.append('run_local=%s' % benchmark.run_local)
+        args_list.append(GetProfilerArgs(profiler_args))
+
+    return args_list
+
   def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
     """Run the test_that test.."""
-    options = []
-    if label.board:
-      options.append('--board=%s' % label.board)
-    if test_args:
-      options.append(test_args)
-    if profiler_args:
-      self.logger.LogFatal('test_that does not support profiler.')
+
+    # Remove existing test_that results
     command = 'rm -rf /usr/local/autotest/results/*'
     self._ce.CrosRunCommand(
         command, machine=machine, chromeos_root=label.chromeos_root)
 
-    autotest_dir = AUTOTEST_DIR
-    if label.autotest_path != '':
-      autotest_dir = label.autotest_path
+    if benchmark.suite == 'telemetry_Crosperf':
+      if not os.path.isdir(label.chrome_src):
+        self.logger.LogFatal('Cannot find chrome src dir to '
+                             'run telemetry: %s' % label.chrome_src)
+      # Check for and remove temporary file that may have been left by
+      # previous telemetry runs (and which might prevent this run from
+      # working).
+      self.RemoveTelemetryTempFile(machine, label.chromeos_root)
+
+    # --autotest_dir specifies which autotest directory to use.
+    autotest_dir_arg = '--autotest_dir=%s' % (
+        label.autotest_path if label.autotest_path else AUTOTEST_DIR)
+
+    # --fast avoids unnecessary copies of syslogs.
+    fast_arg = '--fast'
+    board_arg = '--board=%s' % label.board
+
+    args_list = self.GenTestArgs(benchmark, test_args, profiler_args)
+    args_arg = '--args=%s' % pipes.quote(' '.join(args_list))
+
+    command = ' '.join([
+        TEST_THAT_PATH, autotest_dir_arg, fast_arg, board_arg, args_arg,
+        machine, benchmark.suite if
+        (benchmark.suite == 'telemetry_Crosperf' or
+         benchmark.suite == 'crosperf_Wrapper') else benchmark.test_name
+    ])
 
-    autotest_dir_arg = '--autotest_dir %s' % autotest_dir
-    # For non-telemetry tests, specify an autotest directory only if the
-    # specified directory is different from default (crosbug.com/679001).
-    if autotest_dir == AUTOTEST_DIR:
-      autotest_dir_arg = ''
+    # Use --no-ns-pid so that cros_sdk does not create a different
+    # process namespace and we can kill process created easily by their
+    # process group.
+    chrome_root_options = ('--no-ns-pid '
+                           '--chrome_root={} --chrome_root_mount={} '
+                           'FEATURES="-usersandbox" '
+                           'CHROME_ROOT={}'.format(label.chrome_src,
+                                                   CHROME_MOUNT_DIR,
+                                                   CHROME_MOUNT_DIR))
 
-    command = (('%s %s --fast '
-                '%s %s %s') % (TEST_THAT_PATH, autotest_dir_arg,
-                               ' '.join(options), machine, benchmark.test_name))
     if self.log_level != 'verbose':
       self.logger.LogOutput('Running test.')
       self.logger.LogOutput('CMD: %s' % command)
-    # Use --no-ns-pid so that cros_sdk does not create a different
-    # process namespace and we can kill process created easily by
-    # their process group.
+
     return self._ce.ChrootRunCommandWOutput(
         label.chromeos_root,
         command,
         command_terminator=self._ct,
-        cros_sdk_options='--no-ns-pid')
+        cros_sdk_options=chrome_root_options)
 
   def DownloadResult(self, label, task_id):
     gsutil_cmd = os.path.join(label.chromeos_root, GS_UTIL)
@@ -200,26 +228,9 @@ class SuiteRunner(object):
     # TODO: now only put toolchain pool here, user need to be able to specify
     # which pool to use. Need to request feature to not use this option at all.
     options.append('-pool=toolchain')
-    test_args_list = []
-    if benchmark.suite == 'telemetry_Crosperf':
-      if test_args:
-        # Strip double quotes off args (so we can wrap them in single
-        # quotes, to pass through to Telemetry).
-        if test_args[0] == '"' and test_args[-1] == '"':
-          test_args_list.append(test_args[1:-1])
-      if profiler_args:
-        test_args_list.append(GetProfilerArgs(profiler_args))
-      if self.dut_config:
-        test_args_list.append(GetDutConfigArgs(self.dut_config))
-      test_args_list.append('run_local=%s' % benchmark.run_local)
-      test_args_list.append('test=%s' % benchmark.test_name)
-    else:
-      if profiler_args:
-        self.logger.LogFatal('Client tests do not support profiler.')
-      if test_args:
-        test_args_list.append(test_args)
-    if test_args_list:
-      options.append('-test-args=%s' % pipes.quote(' '.join(test_args_list)))
+
+    args_list = self.GenTestArgs(benchmark, test_args, profiler_args)
+    options.append('-test-args=%s' % pipes.quote(' '.join(args_list)))
 
     dimensions = []
     for dut in label.remote:
@@ -227,7 +238,9 @@ class SuiteRunner(object):
 
     command = (('%s create-test %s %s %s') % \
               (SKYLAB_PATH, ' '.join(dimensions), ' '.join(options),
-               'telemetry_Crosperf' if benchmark.suite == 'telemetry_Crosperf'
+               benchmark.suite if
+               (benchmark.suite == 'telemetry_Crosperf' or
+                benchmark.suite == 'crosperf_Wrapper')
                else benchmark.test_name))
 
     if self.log_level != 'verbose':
@@ -279,67 +292,6 @@ class SuiteRunner(object):
         return (ret_tup[0], result_dir, ret_tup[2])
     return ret_tup
 
-  def RemoveTelemetryTempFile(self, machine, chromeos_root):
-    filename = 'telemetry@%s' % machine
-    fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename)
-    if os.path.exists(fullname):
-      os.remove(fullname)
-
-  def Telemetry_Crosperf_Run(self, machine, label, benchmark, test_args,
-                             profiler_args):
-    if not os.path.isdir(label.chrome_src):
-      self.logger.LogFatal('Cannot find chrome src dir to'
-                           ' run telemetry: %s' % label.chrome_src)
-
-    # Check for and remove temporary file that may have been left by
-    # previous telemetry runs (and which might prevent this run from
-    # working).
-    self.RemoveTelemetryTempFile(machine, label.chromeos_root)
-
-    # For telemetry runs, we can use the autotest copy from the source
-    # location. No need to have one under /build/<board>.
-    autotest_dir_arg = '--autotest_dir %s' % AUTOTEST_DIR
-    if label.autotest_path != '':
-      autotest_dir_arg = '--autotest_dir %s' % label.autotest_path
-
-    profiler_args = GetProfilerArgs(profiler_args)
-    dut_config_args = GetDutConfigArgs(self.dut_config)
-    # --fast avoids unnecessary copies of syslogs.
-    fast_arg = '--fast'
-    args_string = ''
-    if test_args:
-      # Strip double quotes off args (so we can wrap them in single
-      # quotes, to pass through to Telemetry).
-      if test_args[0] == '"' and test_args[-1] == '"':
-        test_args = test_args[1:-1]
-      args_string = "test_args='%s'" % test_args
-
-    args = '{} run_local={} test={} {} {}'.format(
-        args_string, benchmark.run_local, benchmark.test_name, dut_config_args,
-        profiler_args)
-
-    cmd = ('{} {} {} --board={} --args={} {} telemetry_Crosperf'.format(
-        TEST_THAT_PATH, autotest_dir_arg, fast_arg, label.board,
-        pipes.quote(args), machine))
-
-    # Use --no-ns-pid so that cros_sdk does not create a different
-    # process namespace and we can kill process created easily by their
-    # process group.
-    chrome_root_options = ('--no-ns-pid '
-                           '--chrome_root={} --chrome_root_mount={} '
-                           'FEATURES="-usersandbox" '
-                           'CHROME_ROOT={}'.format(label.chrome_src,
-                                                   CHROME_MOUNT_DIR,
-                                                   CHROME_MOUNT_DIR))
-    if self.log_level != 'verbose':
-      self.logger.LogOutput('Running test.')
-      self.logger.LogOutput('CMD: %s' % cmd)
-    return self._ce.ChrootRunCommandWOutput(
-        label.chromeos_root,
-        cmd,
-        command_terminator=self._ct,
-        cros_sdk_options=chrome_root_options)
-
   def CommandTerminator(self):
     return self._ct