crosperf: migrate device setup functions for telemetry_Crosperf

This is a patch working together with crrev.com/c/1949606 to migrate
device setup functions from suite_runner to autotest telmetry_Crosperf.

In this patch, we modified the logic of different suite types along with
skylab mode for suite_runner.

In result_cache, we handle the wait time log file to accumulate cool
down time per machine.

We also move intel_pstate kernel updating code into SetupDevice() since
it doesn't increase overhead by checking every iteration.

TEST=Passed all unittests; tested with different benchmark modes (Note
that currently skylab server tests cannot be tested because of
regression: crbug.com/984103).
BUG=chromium:1020655

Cq-Depend: chromium:1949606
Change-Id: If8e5099d096d2568d4f54584a8fcfd3c0f99c8f8
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1949626
Reviewed-by: George Burgess <gbiv@chromium.org>
Reviewed-by: Denis Nikitin <denik@chromium.org>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Auto-Submit: Zhizhou Yang <zhizhouy@google.com>
Commit-Queue: Denis Nikitin <denik@chromium.org>

2 files changed, 171 insertions, 8 deletions

diff --git a/cros_utils/device_setup_utils.py b/cros_utils/device_setup_utils.py
index 93e681bd..ea705263 100755
--- a/cros_utils/device_setup_utils.py
+++ b/cros_utils/device_setup_utils.py
@@ -18,15 +18,13 @@ __author__ = 'zhizhouy@google.com (Zhizhou Yang)'
 import re
 import time
 
+from contextlib import contextmanager
+
 from cros_utils import command_executer
 
 
 class DutWrapper(object):
-  """Wrap DUT parameters inside.
-
-  Eventially CommandExecuter will reqiure only one
-  argument - command.
-  """
+  """Wrap DUT parameters inside."""
 
   def __init__(self,
                chromeos_root,
@@ -43,6 +41,7 @@ class DutWrapper(object):
     self.dut_config = dut_config
 
   def RunCommandOnDut(self, command, ignore_status=False):
+    """Helper function to run command on DUT."""
     ret, msg, err_msg = self.ce.CrosRunCommandWOutput(
         command, machine=self.remote, chromeos_root=self.chromeos_root)
 
@@ -60,15 +59,17 @@ class DutWrapper(object):
     return ret, msg, err_msg
 
   def DisableASLR(self):
+    """Disable ASLR on DUT."""
     disable_aslr = ('set -e; '
                     'if [[ -e /proc/sys/kernel/randomize_va_space ]]; then '
                     '  echo 0 > /proc/sys/kernel/randomize_va_space; '
                     'fi')
     if self.log_level == 'average':
       self.logger.LogOutput('Disable ASLR.')
-    self.RunCommandOnDut(disable_aslr, ignore_status=False)
+    self.RunCommandOnDut(disable_aslr)
 
   def SetCpuGovernor(self, governor, ignore_status=False):
+    """Setup CPU Governor on DUT."""
     set_gov_cmd = (
         'for f in `ls -d /sys/devices/system/cpu/cpu*/cpufreq 2>/dev/null`; do '
         # Skip writing scaling_governor if cpu is offline.
@@ -85,6 +86,7 @@ class DutWrapper(object):
     return ret
 
   def DisableTurbo(self):
+    """Disable Turbo on DUT."""
     dis_turbo_cmd = (
         'if [[ -e /sys/devices/system/cpu/intel_pstate/no_turbo ]]; then '
         '  if grep -q 0 /sys/devices/system/cpu/intel_pstate/no_turbo;  then '
@@ -284,6 +286,7 @@ class DutWrapper(object):
                            (avail_ngt_target, max_freq_path, min_freq_path))
 
   def WaitCooldown(self):
+    """Wait for DUT to cool down to certain temperature."""
     waittime = 0
     timeout_in_sec = int(self.dut_config['cooldown_time']) * 60
     # Temperature from sensors come in uCelsius units.
@@ -327,11 +330,13 @@ class DutWrapper(object):
       self.RunCommandOnDut(sed_command + FILE)
 
   def StopUI(self):
+    """Stop UI on DUT."""
     # Added "ignore_status" for the case when crosperf stops ui service which
     # was already stopped. Command is going to fail with 1.
     self.RunCommandOnDut('stop ui', ignore_status=True)
 
   def StartUI(self):
+    """Start UI on DUT."""
     # Similar to StopUI, `start ui` fails if the service is already started.
     self.RunCommandOnDut('start ui', ignore_status=True)
 
@@ -456,3 +461,78 @@ class DutWrapper(object):
           'Final verification failed with status %d' % ret_code)
 
     self.logger.LogOutput('Kernel cmdline updated successfully.')
+
+  @contextmanager
+  def PauseUI(self):
+    """Stop UI before and Start UI after the context block.
+
+    Context manager will make sure UI is always resumed at the end.
+    """
+    self.StopUI()
+    try:
+      yield
+
+    finally:
+      self.StartUI()
+
+  def SetupDevice(self):
+    """Setup device to get it ready for testing.
+
+    @Returns Wait time of cool down for this benchmark run.
+    """
+    self.logger.LogOutput('Update kernel cmdline if necessary and reboot')
+    intel_pstate = self.dut_config['intel_pstate']
+    if intel_pstate and self.KerncmdUpdateNeeded(intel_pstate):
+      self.UpdateKerncmdIntelPstate(intel_pstate)
+
+    wait_time = 0
+    # Pause UI while configuring the DUT.
+    # This will accelerate setup (waiting for cooldown has x10 drop)
+    # and help to reset a Chrome state left after the previous test.
+    with self.PauseUI():
+      # Unless the user turns on ASLR in the flag, we first disable ASLR
+      # before running the benchmarks
+      if not self.dut_config['enable_aslr']:
+        self.DisableASLR()
+
+      # CPU usage setup comes first where we enable/disable cores.
+      self.SetupCpuUsage()
+      cpu_online_status = self.GetCpuOnline()
+      # List of online cores of type int (core number).
+      online_cores = [
+          core for core, status in cpu_online_status.items() if status
+      ]
+      if self.dut_config['cooldown_time']:
+        # Setup power conservative mode for effective cool down.
+        # Set ignore status since powersave may no be available
+        # on all platforms and we are going to handle it.
+        ret = self.SetCpuGovernor('powersave', ignore_status=True)
+        if ret:
+          # "powersave" is not available, use "ondemand".
+          # Still not a fatal error if it fails.
+          ret = self.SetCpuGovernor('ondemand', ignore_status=True)
+        # TODO(denik): Run comparison test for 'powersave' and 'ondemand'
+        # on scarlet and kevin64.
+        # We might have to consider reducing freq manually to the min
+        # if it helps to reduce waiting time.
+        wait_time = self.WaitCooldown()
+
+      # Setup CPU governor for the benchmark run.
+      # It overwrites the previous governor settings.
+      governor = self.dut_config['governor']
+      # FIXME(denik): Pass online cores to governor setup.
+      self.SetCpuGovernor(governor)
+
+      # Disable Turbo and Setup CPU freq should ALWAYS proceed governor setup
+      # since governor may change:
+      # - frequency;
+      # - turbo/boost.
+      self.DisableTurbo()
+      self.SetupCpuFreq(online_cores)
+
+      self.DecreaseWaitTime()
+      # FIXME(denik): Currently we are not recovering the previous cpufreq
+      # settings since we do reboot/setup every time anyway.
+      # But it may change in the future and then we have to recover the
+      # settings.
+    return wait_time
diff --git a/cros_utils/device_setup_utils_unittest.py b/cros_utils/device_setup_utils_unittest.py
index b2f76f7d..24fe8b3f 100755
--- a/cros_utils/device_setup_utils_unittest.py
+++ b/cros_utils/device_setup_utils_unittest.py
@@ -162,8 +162,7 @@ class DutWrapperTest(unittest.TestCase):
                    'if [[ -e /proc/sys/kernel/randomize_va_space ]]; then '
                    '  echo 0 > /proc/sys/kernel/randomize_va_space; '
                    'fi')
-    self.dw.RunCommandOnDut.assert_called_once_with(
-        set_cpu_cmd, ignore_status=False)
+    self.dw.RunCommandOnDut.assert_called_once_with(set_cpu_cmd)
 
   def test_set_cpu_governor(self):
     self.dw.RunCommandOnDut = mock.Mock(return_value=(0, '', ''))
@@ -615,6 +614,90 @@ class DutWrapperTest(unittest.TestCase):
     self.dw.RunCommandOnDut.assert_called_once_with(
         'start ui', ignore_status=True)
 
+  def test_setup_device(self):
+
+    def FakeRunner(command, ignore_status=False):
+      # pylint fix for unused variable.
+      del command, ignore_status
+      return 0, '', ''
+
+    def SetupMockFunctions():
+      self.dw.RunCommandOnDut = mock.Mock(return_value=FakeRunner)
+      self.dw.KerncmdUpdateNeeded = mock.Mock(return_value=True)
+      self.dw.UpdateKerncmdIntelPstate = mock.Mock(return_value=0)
+      self.dw.DisableASLR = mock.Mock(return_value=0)
+      self.dw.SetupCpuUsage = mock.Mock(return_value=0)
+      self.dw.SetupCpuFreq = mock.Mock(return_value=0)
+      self.dw.GetCpuOnline = mock.Mock(return_value={0: 1, 1: 1, 2: 0})
+      self.dw.SetCpuGovernor = mock.Mock(return_value=0)
+      self.dw.DisableTurbo = mock.Mock(return_value=0)
+      self.dw.StopUI = mock.Mock(return_value=0)
+      self.dw.StartUI = mock.Mock(return_value=0)
+      self.dw.WaitCooldown = mock.Mock(return_value=0)
+      self.dw.DecreaseWaitTime = mock.Mock(return_value=0)
+
+    self.dw.dut_config['enable_aslr'] = False
+    self.dw.dut_config['cooldown_time'] = 0
+    self.dw.dut_config['governor'] = 'fake_governor'
+    self.dw.dut_config['cpu_freq_pct'] = 65
+    self.dw.dut_config['intel_pstate'] = 'no_hwp'
+
+    SetupMockFunctions()
+    self.dw.SetupDevice()
+
+    self.dw.KerncmdUpdateNeeded.assert_called_once()
+    self.dw.UpdateKerncmdIntelPstate.assert_called_once()
+    self.dw.DisableASLR.assert_called_once()
+    self.dw.SetupCpuUsage.assert_called_once_with()
+    self.dw.SetupCpuFreq.assert_called_once_with([0, 1])
+    self.dw.GetCpuOnline.assert_called_once_with()
+    self.dw.SetCpuGovernor.assert_called_once_with('fake_governor')
+    self.dw.DisableTurbo.assert_called_once_with()
+    self.dw.DecreaseWaitTime.assert_called_once_with()
+    self.dw.StopUI.assert_called_once_with()
+    self.dw.StartUI.assert_called_once_with()
+    self.dw.WaitCooldown.assert_not_called()
+
+    # Test SetupDevice with cooldown
+    self.dw.dut_config['cooldown_time'] = 10
+
+    SetupMockFunctions()
+    self.dw.GetCpuOnline = mock.Mock(return_value={0: 0, 1: 1})
+
+    self.dw.SetupDevice()
+
+    self.dw.WaitCooldown.assert_called_once_with()
+    self.dw.DisableASLR.assert_called_once()
+    self.dw.DisableTurbo.assert_called_once_with()
+    self.dw.SetupCpuUsage.assert_called_once_with()
+    self.dw.SetupCpuFreq.assert_called_once_with([1])
+    self.dw.SetCpuGovernor.assert_called()
+    self.dw.GetCpuOnline.assert_called_once_with()
+    self.dw.StopUI.assert_called_once_with()
+    self.dw.StartUI.assert_called_once_with()
+    self.assertGreater(self.dw.SetCpuGovernor.call_count, 1)
+    self.assertEqual(self.dw.SetCpuGovernor.call_args,
+                     mock.call('fake_governor'))
+
+    # Test SetupDevice with cooldown
+    SetupMockFunctions()
+    self.dw.SetupCpuUsage = mock.Mock(side_effect=RuntimeError())
+
+    with self.assertRaises(RuntimeError):
+      self.dw.SetupDevice()
+
+    # This call injected an exception.
+    self.dw.SetupCpuUsage.assert_called_once_with()
+    # Calls following the expeption are skipped.
+    self.dw.WaitCooldown.assert_not_called()
+    self.dw.DisableTurbo.assert_not_called()
+    self.dw.SetupCpuFreq.assert_not_called()
+    self.dw.SetCpuGovernor.assert_not_called()
+    self.dw.GetCpuOnline.assert_not_called()
+    # Check that Stop/Start UI are always called.
+    self.dw.StopUI.assert_called_once_with()
+    self.dw.StartUI.assert_called_once_with()
+
 
 if __name__ == '__main__':
   unittest.main()