Snap for 10843824 from 0348d10214299073ea59bda4987dd8813a9e7812 to 24Q1-releaseandroid-14.0.0_r37android-14.0.0_r36android-14.0.0_r35android-14.0.0_r34android-14.0.0_r33android-14.0.0_r32android-14.0.0_r31android-14.0.0_r30android-14.0.0_r29android14-qpr2-s5-releaseandroid14-qpr2-s4-releaseandroid14-qpr2-s3-releaseandroid14-qpr2-s2-releaseandroid14-qpr2-s1-releaseandroid14-qpr2-release

Change-Id: Ia39ecafdc4b00a0b2505b638c14b80c02e34d4b6

16 files changed, 572 insertions, 438 deletions

diff --git a/crosperf/README.md b/crosperf/README.md
index 18601b67..f1429513 100644
--- a/crosperf/README.md
+++ b/crosperf/README.md
@@ -3,8 +3,7 @@
 To use these experiment files, replace the board, remote and images
 placeholders and run crosperf on them.
 
-Further information about crosperf:
-https://sites.google.com/a/google.com/chromeos-toolchain-team-home2/home/team-tools-and-scripts/crosperf-cros-image-performance-comparison-tool
+Further information about crosperf: https://goto.google.com/crostc-crosperf
 
 The final experiment file should look something like the following (but with
 different actual values for the fields):
diff --git a/crosperf/benchmark.py b/crosperf/benchmark.py
index f9de0cf3..eb8661e9 100644
--- a/crosperf/benchmark.py
+++ b/crosperf/benchmark.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2013 The ChromiumOS Authors
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -7,10 +6,10 @@
 
 
 import math
+import statistics
+from typing import Any
 
-# FIXME(denik): Fix the import in chroot.
-# pylint: disable=import-error
-from scipy import stats
+import numpy as np
 
 
 # See crbug.com/673558 for how these are estimated.
@@ -26,10 +25,49 @@ _estimated_stddev = {
     "loading.desktop": 0.021,  # Copied from page_cycler initially
 }
 
+# Numpy makes it hard to know the real type of some inputs
+# and outputs, so this type alias is just for docs.
+FloatLike = Any
+
+
+def isf(x: FloatLike, mu=0.0, sigma=1.0, pitch=0.01) -> FloatLike:
+    """Compute the inverse survival function for value x.
+
+    In the abscence of using scipy.stats.norm's isf(), this function
+    attempts to re-implement the inverse survival function by calculating
+    the numerical inverse of the survival function, interpolating between
+    table values. See bug b/284489250 for details.
+
+    Survival function as defined by:
+    https://en.wikipedia.org/wiki/Survival_function
+
+    Examples:
+        >>> -2.0e-16 < isf(0.5) <  2.0e-16
+        True
+
+    Args:
+        x: float or numpy array-like to compute the ISF for.
+        mu: Center of the underlying normal distribution.
+        sigma: Spread of the underlying normal distribution.
+        pitch: Absolute spacing between y-value interpolation points.
+
+    Returns:
+        float or numpy array-like representing the ISF of `x`.
+    """
+    norm = statistics.NormalDist(mu, sigma)
+    # np.interp requires a monotonically increasing x table.
+    # Because the survival table is monotonically decreasing, we have to
+    # reverse the y_vals too.
+    y_vals = np.flip(np.arange(-4.0, 4.0, pitch))
+    survival_table = np.fromiter(
+        (1.0 - norm.cdf(y) for y in y_vals), y_vals.dtype
+    )
+    return np.interp(x, survival_table, y_vals)
+
 
 # Get #samples needed to guarantee a given confidence interval, assuming the
 # samples follow normal distribution.
-def _samples(b):
+def _samples(b: str) -> int:
     # TODO: Make this an option
     # CI = (0.9, 0.02), i.e., 90% chance that |sample mean - true mean| < 2%.
     p = 0.9
@@ -39,7 +77,7 @@ def _samples(b):
     d = _estimated_stddev[b]
     # Get at least 2 samples so as to calculate standard deviation, which is
     # needed in T-test for p-value.
-    n = int(math.ceil((stats.norm.isf((1 - p) / 2) * d / e) ** 2))
+    n = int(math.ceil((isf((1 - p) / 2) * d / e) ** 2))
     return n if n > 1 else 2
 
 
diff --git a/crosperf/crosperf_unittest.py b/crosperf/crosperf_unittest.py
index 7b52f2e0..bbcb1712 100755
--- a/crosperf/crosperf_unittest.py
+++ b/crosperf/crosperf_unittest.py
@@ -67,7 +67,7 @@ class CrosperfTest(unittest.TestCase):
         settings = crosperf.ConvertOptionsToSettings(options)
         self.assertIsNotNone(settings)
         self.assertIsInstance(settings, settings_factory.GlobalSettings)
-        self.assertEqual(len(settings.fields), 40)
+        self.assertEqual(len(settings.fields), 42)
         self.assertTrue(settings.GetField("rerun"))
         argv = ["crosperf/crosperf.py", "temp.exp"]
         options, _ = parser.parse_known_args(argv)
diff --git a/crosperf/default_remotes b/crosperf/default_remotes
index 714385e7..80563664 100644
--- a/crosperf/default_remotes
+++ b/crosperf/default_remotes
@@ -1,5 +1,5 @@
 bob           : chromeos8-row12-rack16-host2
-chell         : chromeos2-row1-rack10-host2    chromeos2-row1-rack10-host4
+chell         : chromeos6-row16-rack5-host6    chromeos6-row16-rack5-host7
 coral         : chromeos6-row5-rack6-host1     chromeos6-row5-rack6-host3    chromeos6-row5-rack6-host5
 elm           : chromeos6-row14-rack15-host21
 nautilus      : chromeos6-row5-rack10-host1    chromeos6-row5-rack10-host3
diff --git a/crosperf/download_images.py b/crosperf/download_images.py
index 9a46280d..38d5d68a 100644
--- a/crosperf/download_images.py
+++ b/crosperf/download_images.py
@@ -8,8 +8,10 @@
 
 import ast
 import os
+import shlex
 
 from cros_utils import command_executer
+from cros_utils import misc
 import test_flag
 
 
@@ -79,7 +81,9 @@ class ImageDownloader(object):
             )
 
         # Make sure the directory for downloading the image exists.
-        download_path = os.path.join(chromeos_root, "chroot/tmp", build_id)
+        download_path = misc.GetOutsideChrootPath(
+            chromeos_root, os.path.join("/tmp", build_id)
+        )
         image_path = os.path.join(download_path, "chromiumos_test_image.bin")
         if not os.path.exists(download_path):
             os.makedirs(download_path)
@@ -105,18 +109,19 @@ class ImageDownloader(object):
 
     def UncompressImage(self, chromeos_root, build_id):
         # Check to see if the file has already been uncompresssed, etc.
-        if os.path.exists(
+        download_path = misc.GetOutsideChrootPath(
+            chromeos_root,
             os.path.join(
-                chromeos_root,
-                "chroot/tmp",
+                "/tmp",
                 build_id,
-                "chromiumos_test_image.bin",
-            )
+            ),
+        )
+        if os.path.exists(
+            os.path.join(download_path, "chromiumos_test_image.bin")
         ):
             return
 
         # Uncompress and untar the downloaded image.
-        download_path = os.path.join(chromeos_root, "chroot/tmp", build_id)
         command = (
             "cd %s ; tar -Jxf chromiumos_test_image.tar.xz " % download_path
         )
@@ -178,7 +183,9 @@ class ImageDownloader(object):
             )
 
         # Make sure the directory for downloading the package exists.
-        download_path = os.path.join(chromeos_root, "chroot/tmp", build_id)
+        download_path = misc.GetOutsideChrootPath(
+            chromeos_root, os.path.join("/tmp", build_id)
+        )
         package_path = os.path.join(download_path, package_file_name)
         if not os.path.exists(download_path):
             os.makedirs(download_path)
@@ -204,7 +211,9 @@ class ImageDownloader(object):
         self, chromeos_root, build_id, package_file_name, uncompress_cmd
     ):
         # Uncompress file
-        download_path = os.path.join(chromeos_root, "chroot/tmp", build_id)
+        download_path = misc.GetOutsideChrootPath(
+            chromeos_root, os.path.join("/tmp", build_id)
+        )
         command = "cd %s ; %s %s" % (
             download_path,
             uncompress_cmd,
@@ -252,13 +261,13 @@ class ImageDownloader(object):
         autotest_server_package_name = "autotest_server_package.tar.bz2"
         autotest_control_files_name = "control_files.tar"
 
-        download_path = os.path.join(chromeos_root, "chroot/tmp", build_id)
+        download_path = misc.GetOutsideChrootPath(
+            chromeos_root, os.path.join("/tmp", build_id)
+        )
         # Autotest directory relative path wrt chroot
         autotest_rel_path = os.path.join("/tmp", build_id, "autotest_files")
         # Absolute Path to download files
-        autotest_path = os.path.join(
-            chromeos_root, "chroot/tmp", build_id, "autotest_files"
-        )
+        autotest_path = os.path.join(download_path, "autotest_files")
 
         if not os.path.exists(autotest_path):
             # Quickly verify if the files are present on server
@@ -315,12 +324,14 @@ class ImageDownloader(object):
         # Download autest package files (3 files)
         debug_archive_name = "debug.tgz"
 
-        download_path = os.path.join(chromeos_root, "chroot/tmp", build_id)
+        download_path = misc.GetOutsideChrootPath(
+            chromeos_root, os.path.join("/tmp", build_id)
+        )
         # Debug directory relative path wrt chroot
         debug_rel_path = os.path.join("/tmp", build_id, "debug_files")
         # Debug path to download files
-        debug_path = os.path.join(
-            chromeos_root, "chroot/tmp", build_id, "debug_files"
+        debug_path = misc.GetOutsideChrootPath(
+            chromeos_root, os.path.join("/tmp", build_id, "debug_files")
         )
 
         if not os.path.exists(debug_path):
@@ -338,15 +349,11 @@ class ImageDownloader(object):
             # File exists on server, download and uncompress it
             self.DownloadSingleFile(chromeos_root, build_id, debug_archive_name)
 
-            self.UncompressSingleFile(
-                chromeos_root, build_id, debug_archive_name, "tar -xf "
-            )
             # Extract and move debug files into the proper location.
-            debug_dir = "debug_files/usr/lib"
-            command = "cd %s ; mkdir -p %s; mv debug %s" % (
-                download_path,
-                debug_dir,
-                debug_dir,
+            debug_dir = "debug_files/usr/lib/debug"
+            command = (
+                f"cd {shlex.quote(download_path)}; "
+                f"mkdir -p {shlex.quote(debug_dir)}"
             )
             if self.log_level != "verbose":
                 self._logger.LogOutput("CMD: %s" % command)
@@ -357,6 +364,12 @@ class ImageDownloader(object):
                     "Could not create directory %s"
                     % os.path.join(debug_dir, "debug")
                 )
+            self.UncompressSingleFile(
+                chromeos_root,
+                build_id,
+                debug_archive_name,
+                f"tar -C {shlex.quote(debug_dir)} -xf ",
+            )
 
         return debug_rel_path
 
diff --git a/crosperf/download_images_unittest.py b/crosperf/download_images_unittest.py
index 6a640f80..0e47e757 100755
--- a/crosperf/download_images_unittest.py
+++ b/crosperf/download_images_unittest.py
@@ -8,6 +8,7 @@
 
 
 import os
+import re
 import unittest
 import unittest.mock as mock
 
@@ -20,6 +21,16 @@ import test_flag
 MOCK_LOGGER = logger.GetLogger(log_dir="", mock=True)
 
 
+class RegexMatcher:
+    """A regex matcher, for passing to mocks."""
+
+    def __init__(self, regex):
+        self._regex = re.compile(regex)
+
+    def __eq__(self, string):
+        return self._regex.search(string) is not None
+
+
 class ImageDownloaderTestcast(unittest.TestCase):
     """The image downloader test class."""
 
@@ -34,7 +45,6 @@ class ImageDownloaderTestcast(unittest.TestCase):
     @mock.patch.object(os, "makedirs")
     @mock.patch.object(os.path, "exists")
     def test_download_image(self, mock_path_exists, mock_mkdirs):
-
         # Set mock and test values.
         mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
         test_chroot = "/usr/local/home/chromeos"
@@ -59,50 +69,63 @@ class ImageDownloaderTestcast(unittest.TestCase):
             image_path,
         )
 
-        # Verify os.path.exists was called twice, with proper arguments.
-        self.assertEqual(mock_path_exists.call_count, 2)
-        mock_path_exists.assert_called_with(
-            "/usr/local/home/chromeos/chroot/tmp/lumpy-release/"
-            "R36-5814.0.0/chromiumos_test_image.bin"
+        # Verify os.path.exists was called thrice, with proper arguments.
+        self.assertEqual(mock_path_exists.call_count, 3)
+        mock_path_exists.assert_any_call(
+            RegexMatcher(
+                "/usr/local/home/chromeos/.*tmp/lumpy-release/"
+                "R36-5814.0.0/chromiumos_test_image.bin"
+            )
         )
         mock_path_exists.assert_any_call(
-            "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0"
+            RegexMatcher(
+                "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0"
+            )
         )
+        mock_path_exists.assert_any_call("/etc/cros_chroot_version")
 
         # Verify we called os.mkdirs
         self.assertEqual(mock_mkdirs.call_count, 1)
         mock_mkdirs.assert_called_with(
-            "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0"
+            RegexMatcher(
+                "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0"
+            )
         )
 
         # Verify we called RunCommand once, with proper arguments.
         self.assertEqual(mock_cmd_exec.RunCommand.call_count, 1)
-        expected_args = (
+        expected_args = RegexMatcher(
             "/usr/local/home/chromeos/src/chromium/depot_tools/gsutil.py "
             "cp gs://chromeos-image-archive/lumpy-release/R36-5814.0.0/"
             "chromiumos_test_image.tar.xz "
-            "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0"
+            "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0"
         )
 
         mock_cmd_exec.RunCommand.assert_called_with(expected_args)
 
-        # Reset the velues in the mocks; set os.path.exists to always return True.
+        # Reset the velues in the mocks; set os.path.exists to always return
+        # True (except for "inside chroot" check).
         mock_path_exists.reset_mock()
         mock_cmd_exec.reset_mock()
-        mock_path_exists.return_value = True
+        mock_path_exists.side_effect = lambda x: x != "/etc/cros_chroot_version"
 
         # Run downloader
         downloader.DownloadImage(test_chroot, test_build_id, image_path)
 
-        # Verify os.path.exists was called twice, with proper arguments.
-        self.assertEqual(mock_path_exists.call_count, 2)
+        # Verify os.path.exists was called thrice, with proper arguments.
+        self.assertEqual(mock_path_exists.call_count, 3)
         mock_path_exists.assert_called_with(
-            "/usr/local/home/chromeos/chroot/tmp/lumpy-release/"
-            "R36-5814.0.0/chromiumos_test_image.bin"
+            RegexMatcher(
+                "/usr/local/home/chromeos/.*tmp/lumpy-release/"
+                "R36-5814.0.0/chromiumos_test_image.bin"
+            )
         )
         mock_path_exists.assert_any_call(
-            "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0"
+            RegexMatcher(
+                "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0"
+            )
         )
+        mock_path_exists.assert_any_call("/etc/cros_chroot_version")
 
         # Verify we made no RunCommand or ChrootRunCommand calls (since
         # os.path.exists returned True, there was no work do be done).
@@ -111,7 +134,6 @@ class ImageDownloaderTestcast(unittest.TestCase):
 
     @mock.patch.object(os.path, "exists")
     def test_uncompress_image(self, mock_path_exists):
-
         # set mock and test values.
         mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
         test_chroot = "/usr/local/home/chromeos"
@@ -130,12 +152,15 @@ class ImageDownloaderTestcast(unittest.TestCase):
             test_build_id,
         )
 
-        # Verify os.path.exists was called once, with correct arguments.
-        self.assertEqual(mock_path_exists.call_count, 1)
+        # Verify os.path.exists was called twice, with correct arguments.
+        self.assertEqual(mock_path_exists.call_count, 2)
         mock_path_exists.assert_called_with(
-            "/usr/local/home/chromeos/chroot/tmp/lumpy-release/"
-            "R36-5814.0.0/chromiumos_test_image.bin"
+            RegexMatcher(
+                "/usr/local/home/chromeos/.*tmp/lumpy-release/"
+                "R36-5814.0.0/chromiumos_test_image.bin"
+            )
         )
+        mock_path_exists.assert_any_call("/etc/cros_chroot_version")
 
         # Verify RunCommand was called twice with correct arguments.
         self.assertEqual(mock_cmd_exec.RunCommand.call_count, 2)
@@ -143,8 +168,10 @@ class ImageDownloaderTestcast(unittest.TestCase):
         self.assertEqual(len(mock_cmd_exec.RunCommand.call_args_list[0]), 2)
         actual_arg = mock_cmd_exec.RunCommand.call_args_list[0][0]
         expected_arg = (
-            "cd /usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0 ; "
-            "tar -Jxf chromiumos_test_image.tar.xz ",
+            RegexMatcher(
+                "cd /usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0 ; "
+                "tar -Jxf chromiumos_test_image.tar.xz "
+            ),
         )
         self.assertEqual(expected_arg, actual_arg)
         # 2nd arg must be exception handler
@@ -158,8 +185,10 @@ class ImageDownloaderTestcast(unittest.TestCase):
         self.assertEqual(len(mock_cmd_exec.RunCommand.call_args_list[1]), 2)
         actual_arg = mock_cmd_exec.RunCommand.call_args_list[1][0]
         expected_arg = (
-            "cd /usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0 ; "
-            "rm -f chromiumos_test_image.bin ",
+            RegexMatcher(
+                "cd /usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0 ; "
+                "rm -f chromiumos_test_image.bin "
+            ),
         )
         self.assertEqual(expected_arg, actual_arg)
         # 2nd arg must be empty
@@ -167,24 +196,27 @@ class ImageDownloaderTestcast(unittest.TestCase):
             "{}" in repr(mock_cmd_exec.RunCommand.call_args_list[1][1])
         )
 
-        # Set os.path.exists to always return True and run uncompress.
+        # Set os.path.exists to always return True (except for "inside chroot"
+        # check) and run uncompress.
         mock_path_exists.reset_mock()
         mock_cmd_exec.reset_mock()
-        mock_path_exists.return_value = True
+        mock_path_exists.side_effect = lambda x: x != "/etc/cros_chroot_version"
         downloader.UncompressImage(test_chroot, test_build_id)
 
         # Verify os.path.exists was called once, with correct arguments.
-        self.assertEqual(mock_path_exists.call_count, 1)
+        self.assertEqual(mock_path_exists.call_count, 2)
         mock_path_exists.assert_called_with(
-            "/usr/local/home/chromeos/chroot/tmp/lumpy-release/"
-            "R36-5814.0.0/chromiumos_test_image.bin"
+            RegexMatcher(
+                "/usr/local/home/chromeos/.*tmp/lumpy-release/"
+                "R36-5814.0.0/chromiumos_test_image.bin"
+            )
         )
+        mock_path_exists.assert_any_call("/etc/cros_chroot_version")
 
         # Verify RunCommand was not called.
         self.assertEqual(mock_cmd_exec.RunCommand.call_count, 0)
 
     def test_run(self):
-
         # Set test arguments
         test_chroot = "/usr/local/home/chromeos"
         test_build_id = "remote/lumpy/latest-dev"
diff --git a/crosperf/experiment.py b/crosperf/experiment.py
index 9973f7e9..7d35e319 100644
--- a/crosperf/experiment.py
+++ b/crosperf/experiment.py
@@ -44,6 +44,7 @@ class Experiment(object):
         ignore_min_max,
         crosfleet,
         dut_config,
+        keep_stateful: bool,
         no_lock: bool,
     ):
         self.name = name
@@ -101,7 +102,11 @@ class Experiment(object):
         if test_flag.GetTestMode():
             machine_manager_fn = MockMachineManager
         self.machine_manager = machine_manager_fn(
-            chromeos_root, acquire_timeout, log_level, locks_directory
+            chromeos_root,
+            acquire_timeout,
+            log_level,
+            locks_directory,
+            keep_stateful=keep_stateful,
         )
         self.l = logger.GetLogger(log_dir)
 
@@ -137,7 +142,6 @@ class Experiment(object):
         for label in self.labels:
             for benchmark in self.benchmarks:
                 for iteration in range(1, benchmark.iterations + 1):
-
                     benchmark_run_name = "%s: %s (%s)" % (
                         label.name,
                         benchmark.name,
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index c71981ab..e89adb87 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -28,33 +28,8 @@ import config
 # specified sets.  Here we define sets of tests that users may want
 # to run together.
 
-telemetry_perfv2_tests = [
-    "kraken",
-    "octane",
-]
-
-telemetry_pagecycler_tests = [
-    "page_cycler_v2.intl_ar_fa_he",
-    "page_cycler_v2.intl_es_fr_pt-BR",
-    "page_cycler_v2.intl_hi_ru",
-    "page_cycler_v2.intl_ja_zh",
-    "page_cycler_v2.intl_ko_th_vi",
-    "page_cycler_v2.typical_25",
-]
-
-telemetry_toolchain_old_perf_tests = [
-    "page_cycler_v2.intl_es_fr_pt-BR",
-    "page_cycler_v2.intl_hi_ru",
-    "page_cycler_v2.intl_ja_zh",
-    "page_cycler_v2.intl_ko_th_vi",
-    "page_cycler_v2.netsim.top_10",
-    "page_cycler_v2.typical_25",
-    "spaceport",
-    "tab_switching.top_10",
-]
 telemetry_toolchain_perf_tests = [
     "octane",
-    "kraken",
     "speedometer",
     "speedometer2",
     "jetstream2",
@@ -68,13 +43,10 @@ graphics_perf_tests = [
 ]
 # TODO: disable rendering.desktop by default as the benchmark is
 # currently in a bad state
-# page_cycler_v2.typical_25 is deprecated and the recommend replacement is
-# loading.desktop@@typical (crbug.com/916340)
 telemetry_crosbolt_perf_tests = [
     "octane",
-    "kraken",
     "speedometer2",
-    "jetstream",
+    "jetstream2",
     "loading.desktop",
     # 'rendering.desktop',
 ]
@@ -84,12 +56,6 @@ crosbolt_perf_tests = [
     "tast.video.PlaybackPerfVP91080P30FPS",
 ]
 
-#    'cheets_AntutuTest',
-#    'cheets_PerfBootServer',
-#    'cheets_CandyCrushTest',
-#    'cheets_LinpackTest',
-# ]
-
 dso_list = [
     "all",
     "chrome",
@@ -211,6 +177,7 @@ class ExperimentFactory(object):
             "turbostat": global_settings.GetField("turbostat"),
             "top_interval": global_settings.GetField("top_interval"),
         }
+        keep_stateful = global_settings.GetField("keep_stateful")
 
         # Default cache hit conditions. The image checksum in the cache and the
         # computed checksum of the image must match. Also a cache file must exist.
@@ -220,7 +187,9 @@ class ExperimentFactory(object):
         ]
         if global_settings.GetField("rerun_if_failed"):
             cache_conditions.append(CacheConditions.RUN_SUCCEEDED)
-        if global_settings.GetField("rerun"):
+        if global_settings.GetField("rerun") or global_settings.GetField(
+            "ignore_cache"
+        ):
             cache_conditions.append(CacheConditions.FALSE)
         if global_settings.GetField("same_machine"):
             cache_conditions.append(CacheConditions.SAME_MACHINE_MATCH)
@@ -305,37 +274,7 @@ class ExperimentFactory(object):
                 )
 
             if suite == "telemetry_Crosperf":
-                if test_name == "all_perfv2":
-                    self.AppendBenchmarkSet(
-                        benchmarks,
-                        telemetry_perfv2_tests,
-                        test_args,
-                        iterations,
-                        rm_chroot_tmp,
-                        perf_args,
-                        suite,
-                        show_all_results,
-                        retries,
-                        run_local,
-                        cwp_dso,
-                        weight,
-                    )
-                elif test_name == "all_pagecyclers":
-                    self.AppendBenchmarkSet(
-                        benchmarks,
-                        telemetry_pagecycler_tests,
-                        test_args,
-                        iterations,
-                        rm_chroot_tmp,
-                        perf_args,
-                        suite,
-                        show_all_results,
-                        retries,
-                        run_local,
-                        cwp_dso,
-                        weight,
-                    )
-                elif test_name == "all_crosbolt_perf":
+                if test_name == "all_crosbolt_perf":
                     self.AppendBenchmarkSet(
                         benchmarks,
                         telemetry_crosbolt_perf_tests,
@@ -381,21 +320,6 @@ class ExperimentFactory(object):
                     )
                     # Add non-telemetry toolchain-perf benchmarks:
 
-                    # Tast test platform.ReportDiskUsage for image size.
-                    benchmarks.append(
-                        Benchmark(
-                            "platform.ReportDiskUsage",
-                            "platform.ReportDiskUsage",
-                            "",
-                            1,  # This is not a performance benchmark, only run once.
-                            rm_chroot_tmp,
-                            "",
-                            "tast",  # Specify the suite to be 'tast'
-                            show_all_results,
-                            retries,
-                        )
-                    )
-
                     # TODO: crbug.com/1057755 Do not enable graphics_WebGLAquarium until
                     # it gets fixed.
                     #
@@ -413,21 +337,6 @@ class ExperimentFactory(object):
                     #         run_local=False,
                     #         cwp_dso=cwp_dso,
                     #         weight=weight))
-                elif test_name == "all_toolchain_perf_old":
-                    self.AppendBenchmarkSet(
-                        benchmarks,
-                        telemetry_toolchain_old_perf_tests,
-                        test_args,
-                        iterations,
-                        rm_chroot_tmp,
-                        perf_args,
-                        suite,
-                        show_all_results,
-                        retries,
-                        run_local,
-                        cwp_dso,
-                        weight,
-                    )
                 else:
                     benchmark = Benchmark(
                         benchmark_name,
@@ -613,6 +522,7 @@ class ExperimentFactory(object):
             ignore_min_max,
             crosfleet,
             dut_config,
+            keep_stateful,
             no_lock=no_lock,
         )
 
@@ -623,7 +533,7 @@ class ExperimentFactory(object):
             os.path.dirname(__file__), "default_remotes"
         )
         try:
-            with open(default_remotes_file) as f:
+            with open(default_remotes_file, encoding="utf-8") as f:
                 for line in f:
                     key, v = line.split(":")
                     if key.strip() == board:
@@ -632,15 +542,15 @@ class ExperimentFactory(object):
                             return remotes
                         else:
                             raise RuntimeError(
-                                "There is no remote for {0}".format(board)
+                                f"There is no remote for {board}"
                             )
         except IOError:
             # TODO: rethrow instead of throwing different exception.
             raise RuntimeError(
-                "IOError while reading file {0}".format(default_remotes_file)
+                f"IOError while reading file {default_remotes_file}"
             )
         else:
-            raise RuntimeError("There is no remote for {0}".format(board))
+            raise RuntimeError(f"There is no remote for {board}")
 
     def CheckRemotesInCrosfleet(self, remote):
         # TODO: (AI:zhizhouy) need to check whether a remote is a local or lab
diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py
index 0541bb9b..87e8c4f9 100755
--- a/crosperf/experiment_factory_unittest.py
+++ b/crosperf/experiment_factory_unittest.py
@@ -20,6 +20,7 @@ from cros_utils.file_utils import FileUtils
 import experiment_factory
 from experiment_factory import ExperimentFactory
 from experiment_file import ExperimentFile
+from results_cache import CacheConditions
 import settings_factory
 import test_flag
 
@@ -258,7 +259,7 @@ class ExperimentFactoryTest(unittest.TestCase):
         bench_list = []
         ef.AppendBenchmarkSet(
             bench_list,
-            experiment_factory.telemetry_perfv2_tests,
+            experiment_factory.telemetry_crosbolt_perf_tests,
             "",
             1,
             False,
@@ -271,14 +272,15 @@ class ExperimentFactoryTest(unittest.TestCase):
             0,
         )
         self.assertEqual(
-            len(bench_list), len(experiment_factory.telemetry_perfv2_tests)
+            len(bench_list),
+            len(experiment_factory.telemetry_crosbolt_perf_tests),
         )
         self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark))
 
         bench_list = []
         ef.AppendBenchmarkSet(
             bench_list,
-            experiment_factory.telemetry_pagecycler_tests,
+            experiment_factory.telemetry_toolchain_perf_tests,
             "",
             1,
             False,
@@ -291,7 +293,8 @@ class ExperimentFactoryTest(unittest.TestCase):
             0,
         )
         self.assertEqual(
-            len(bench_list), len(experiment_factory.telemetry_pagecycler_tests)
+            len(bench_list),
+            len(experiment_factory.telemetry_toolchain_perf_tests),
         )
         self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark))
 
@@ -318,7 +321,6 @@ class ExperimentFactoryTest(unittest.TestCase):
 
     @mock.patch.object(socket, "gethostname")
     def test_get_experiment(self, mock_socket):
-
         test_flag.SetTestMode(False)
         self.append_benchmark_call_args = []
 
@@ -418,10 +420,13 @@ class ExperimentFactoryTest(unittest.TestCase):
         )
         self.assertEqual(exp.labels[0].autotest_path, "/tmp/autotest")
         self.assertEqual(exp.labels[0].board, "lumpy")
+        self.assertEqual(exp.machine_manager.keep_stateful, False)
 
         # Second test: Remotes listed in labels.
         test_flag.SetTestMode(True)
         label_settings.SetField("remote", "chromeos1.cros chromeos2.cros")
+        # Also verify keep_stateful.
+        global_settings.SetField("keep_stateful", "true")
         exp = ef.GetExperiment(mock_experiment_file, "", "")
         self.assertCountEqual(
             exp.remote,
@@ -432,6 +437,9 @@ class ExperimentFactoryTest(unittest.TestCase):
                 "chromeos2.cros",
             ],
         )
+        # keep_stateful is propagated to machine_manager which flashes the
+        # images.
+        self.assertEqual(exp.machine_manager.keep_stateful, True)
 
         # Third test: Automatic fixing of bad  logging_level param:
         global_settings.SetField("logging_level", "really loud!")
@@ -450,7 +458,47 @@ class ExperimentFactoryTest(unittest.TestCase):
         label_settings.SetField("remote", "")
         global_settings.SetField("remote", "123.45.67.89")
         exp = ef.GetExperiment(mock_experiment_file, "", "")
-        self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1])
+        self.assertEqual(
+            exp.cache_conditions,
+            [
+                CacheConditions.CACHE_FILE_EXISTS,
+                CacheConditions.CHECKSUMS_MATCH,
+                CacheConditions.RUN_SUCCEEDED,
+                CacheConditions.FALSE,
+                CacheConditions.SAME_MACHINE_MATCH,
+                CacheConditions.MACHINES_MATCH,
+            ],
+        )
+
+        # Check the alias option to ignore cache.
+        global_settings.SetField("rerun", "false")
+        global_settings.SetField("ignore_cache", "true")
+        exp = ef.GetExperiment(mock_experiment_file, "", "")
+        self.assertEqual(
+            exp.cache_conditions,
+            [
+                CacheConditions.CACHE_FILE_EXISTS,
+                CacheConditions.CHECKSUMS_MATCH,
+                CacheConditions.RUN_SUCCEEDED,
+                CacheConditions.FALSE,
+                CacheConditions.SAME_MACHINE_MATCH,
+                CacheConditions.MACHINES_MATCH,
+            ],
+        )
+        # Check without cache use.
+        global_settings.SetField("rerun", "false")
+        global_settings.SetField("ignore_cache", "false")
+        exp = ef.GetExperiment(mock_experiment_file, "", "")
+        self.assertEqual(
+            exp.cache_conditions,
+            [
+                CacheConditions.CACHE_FILE_EXISTS,
+                CacheConditions.CHECKSUMS_MATCH,
+                CacheConditions.RUN_SUCCEEDED,
+                CacheConditions.SAME_MACHINE_MATCH,
+                CacheConditions.MACHINES_MATCH,
+            ],
+        )
 
         # Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all
         # remotes (Call GetDefaultRemotes).
diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py
index 1f78dcc0..c41459a1 100644
--- a/crosperf/experiment_runner.py
+++ b/crosperf/experiment_runner.py
@@ -298,7 +298,6 @@ class ExperimentRunner(object):
         experiment_file_path = os.path.join(results_directory, "experiment.exp")
         FileUtils().WriteFile(experiment_file_path, experiment.experiment_file)
 
-        has_failure = False
         all_failed = True
 
         topstats_file = os.path.join(results_directory, "topstats.log")
@@ -306,17 +305,21 @@ class ExperimentRunner(object):
             "Storing top statistics of each benchmark run into %s."
             % topstats_file
         )
+        # Track if any iterations for a given benchmark has passed for each
+        # label.
+        benchmarks_passes = {}
         with open(topstats_file, "w") as top_fd:
             for benchmark_run in experiment.benchmark_runs:
+                benchmarks_passes.setdefault(
+                    benchmark_run.label.name,
+                    {benchmark_run.benchmark.name: False},
+                )
                 if benchmark_run.result:
-                    # FIXME: Pylint has a bug suggesting the following change, which
-                    # should be fixed in pylint 2.0. Resolve this after pylint >= 2.0.
-                    # Bug: https://github.com/PyCQA/pylint/issues/1984
-                    # pylint: disable=simplifiable-if-statement
-                    if benchmark_run.result.retval:
-                        has_failure = True
-                    else:
+                    if not benchmark_run.result.retval:
                         all_failed = False
+                        benchmarks_passes[benchmark_run.label.name][
+                            benchmark_run.benchmark.name
+                        ] = True
                     # Header with benchmark run name.
                     top_fd.write("%s\n" % str(benchmark_run))
                     # Formatted string with top statistics.
@@ -325,6 +328,11 @@ class ExperimentRunner(object):
 
         if all_failed:
             return self.ALL_FAILED
+        # Set has_passes if atleast one iteration of all benchmarks has passed
+        # for every label.
+        has_passes = True
+        for benchmarks in benchmarks_passes.values():
+            has_passes = has_passes and all(benchmarks.values())
 
         self.l.LogOutput("Storing results of each benchmark run.")
         for benchmark_run in experiment.benchmark_runs:
@@ -339,8 +347,10 @@ class ExperimentRunner(object):
                     benchmark_run.result.CompressResultsTo(benchmark_run_path)
                 else:
                     benchmark_run.result.CopyResultsTo(benchmark_run_path)
+                # Don't remove benchmark tmp if it was a cache hit.
                 benchmark_run.result.CleanUp(
                     benchmark_run.benchmark.rm_chroot_tmp
+                    and not benchmark_run.cache_hit
                 )
 
         self.l.LogOutput("Storing results report in %s." % results_directory)
@@ -367,7 +377,7 @@ class ExperimentRunner(object):
         msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report
         FileUtils().WriteFile(msg_file_path, msg_body)
 
-        return self.SUCCEEDED if not has_failure else self.HAS_FAILURE
+        return self.SUCCEEDED if has_passes else self.HAS_FAILURE
 
     def Run(self):
         try:
diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py
index ffb0b5e6..17db64f5 100644
--- a/crosperf/machine_manager.py
+++ b/crosperf/machine_manager.py
@@ -221,6 +221,7 @@ class MachineManager(object):
         locks_dir,
         cmd_exec=None,
         lgr=None,
+        keep_stateful: bool = False,
     ):
         self._lock = threading.RLock()
         self._all_machines = []
@@ -233,6 +234,7 @@ class MachineManager(object):
         self.acquire_timeout = acquire_timeout
         self.log_level = log_level
         self.locks_dir = locks_dir
+        self.keep_stateful = keep_stateful
         self.ce = cmd_exec or command_executer.GetCommandExecuter(
             log_level=self.log_level
         )
@@ -282,14 +284,16 @@ class MachineManager(object):
         image_chromeos_args = [
             image_chromeos.__file__,
             "--no_lock",
-            "--chromeos_root=%s" % chromeos_root,
-            "--image=%s" % label.chromeos_image,
-            "--image_args=%s" % label.image_args,
-            "--remote=%s" % machine.name,
-            "--logging_level=%s" % self.log_level,
+            f"--chromeos_root={chromeos_root}",
+            f"--image={label.chromeos_image}",
+            f"--image_args={label.image_args}",
+            f"--remote={machine.name}",
+            f"--logging_level={self.log_level}",
         ]
         if label.board:
-            image_chromeos_args.append("--board=%s" % label.board)
+            image_chromeos_args.append(f"--board={label.board}")
+        if self.keep_stateful:
+            image_chromeos_args.append("--keep_stateful")
 
         # Currently can't image two machines at once.
         # So have to serialized on this lock.
@@ -729,9 +733,20 @@ power management:
 class MockMachineManager(MachineManager):
     """Mock machine manager class."""
 
-    def __init__(self, chromeos_root, acquire_timeout, log_level, locks_dir):
+    def __init__(
+        self,
+        chromeos_root,
+        acquire_timeout,
+        log_level,
+        locks_dir,
+        keep_stateful: bool = False,
+    ):
         super(MockMachineManager, self).__init__(
-            chromeos_root, acquire_timeout, log_level, locks_dir
+            chromeos_root,
+            acquire_timeout,
+            log_level,
+            locks_dir,
+            keep_stateful=keep_stateful,
         )
 
     def _TryToLockMachine(self, cros_machine):
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index 043da990..b08fde48 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -185,13 +185,16 @@ class Result(object):
                 # Otherwise get the base filename and create the correct
                 # path for it.
                 _, f_base = misc.GetRoot(f)
-                data_filename = os.path.join(
-                    self.chromeos_root, "chroot/tmp", self.temp_dir, f_base
+                data_filename = misc.GetOutsideChrootPath(
+                    self.chromeos_root,
+                    os.path.join("/tmp", self.temp_dir, f_base),
                 )
             if data_filename.find(".json") > 0:
                 raw_dict = dict()
                 if os.path.exists(data_filename):
-                    with open(data_filename, "r") as data_file:
+                    with open(
+                        data_filename, "r", encoding="utf-8"
+                    ) as data_file:
                         raw_dict = json.load(data_file)
 
                 if "charts" in raw_dict:
@@ -220,7 +223,9 @@ class Result(object):
                         units_dict[key] = result_dict["units"]
             else:
                 if os.path.exists(data_filename):
-                    with open(data_filename, "r") as data_file:
+                    with open(
+                        data_filename, "r", encoding="utf-8"
+                    ) as data_file:
                         lines = data_file.readlines()
                         for line in lines:
                             tmp_dict = json.loads(line)
@@ -258,22 +263,24 @@ class Result(object):
         return results_dict
 
     def GetKeyvals(self):
-        results_in_chroot = os.path.join(self.chromeos_root, "chroot", "tmp")
+        results_in_chroot = misc.GetOutsideChrootPath(
+            self.chromeos_root, "/tmp"
+        )
         if not self.temp_dir:
             self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot)
             command = f"cp -r {self.results_dir}/* {self.temp_dir}"
             self.ce.RunCommand(command, print_to_console=False)
 
+        tmp_dir_in_chroot = misc.GetInsideChrootPath(
+            self.chromeos_root, self.temp_dir
+        )
         command = "./generate_test_report --no-color --csv %s" % (
-            os.path.join("/tmp", os.path.basename(self.temp_dir))
+            tmp_dir_in_chroot
         )
         _, out, _ = self.ce.ChrootRunCommandWOutput(
             self.chromeos_root, command, print_to_console=False
         )
         keyvals_dict = {}
-        tmp_dir_in_chroot = misc.GetInsideChrootPath(
-            self.chromeos_root, self.temp_dir
-        )
         for line in out.splitlines():
             tokens = re.split("=|,", line)
             key = tokens[-2]
@@ -297,8 +304,8 @@ class Result(object):
             chroot_perf_data_file = misc.GetInsideChrootPath(
                 self.chromeos_root, perf_data_file
             )
-            perf_path = os.path.join(
-                self.chromeos_root, "chroot", "usr/bin/perf"
+            perf_path = misc.GetOutsideChrootPath(
+                self.chromeos_root, "/usr/bin/perf"
             )
             perf_file = "/usr/sbin/perf"
             if os.path.exists(perf_path):
@@ -308,20 +315,19 @@ class Result(object):
             # We specify exact match for known DSO type, and every sample for `all`.
             exact_match = ""
             if self.cwp_dso == "all":
-                exact_match = '""'
+                exact_match = ""
             elif self.cwp_dso == "chrome":
-                exact_match = '" chrome "'
+                exact_match = "chrome"
             elif self.cwp_dso == "kallsyms":
-                exact_match = '"[kernel.kallsyms]"'
+                exact_match = "[kernel.kallsyms]"
             else:
                 # This will need to be updated once there are more DSO types supported,
                 # if user want an exact match for the field they want.
-                exact_match = '"%s"' % self.cwp_dso
+                exact_match = self.cwp_dso
 
-            command = "%s report -n -s dso -i %s 2> /dev/null | grep %s" % (
-                perf_file,
-                chroot_perf_data_file,
-                exact_match,
+            command = (
+                f"{perf_file} report -n -s dso -i "
+                f"{chroot_perf_data_file} 2> /dev/null"
             )
             _, result, _ = self.ce.ChrootRunCommandWOutput(
                 self.chromeos_root, command
@@ -335,6 +341,8 @@ class Result(object):
                 for line in result.split("\n"):
                     attr = line.split()
                     if len(attr) == 3 and "%" in attr[0]:
+                        if exact_match and exact_match != attr[2]:
+                            continue
                         samples += int(attr[1])
             except:
                 raise RuntimeError("Cannot parse perf dso result")
@@ -357,11 +365,12 @@ class Result(object):
                     "default_idle",
                     "cpu_idle_loop",
                     "do_idle",
+                    "cpuidle_enter_state",
                 ),
             }
             idle_samples = 0
 
-            with open(perf_report_file) as f:
+            with open(perf_report_file, encoding="utf-8") as f:
                 try:
                     for line in f:
                         line = line.strip()
@@ -462,10 +471,7 @@ class Result(object):
         return self.FindFilesInResultsDir("-name wait_time.log").split("\n")[0]
 
     def _CheckDebugPath(self, option, path):
-        relative_path = path[1:]
-        out_chroot_path = os.path.join(
-            self.chromeos_root, "chroot", relative_path
-        )
+        out_chroot_path = misc.GetOutsideChrootPath(self.chromeos_root, path)
         if os.path.exists(out_chroot_path):
             if option == "kallsyms":
                 path = os.path.join(path, "System.map-*")
@@ -493,8 +499,8 @@ class Result(object):
             chroot_perf_report_file = misc.GetInsideChrootPath(
                 self.chromeos_root, perf_report_file
             )
-            perf_path = os.path.join(
-                self.chromeos_root, "chroot", "usr/bin/perf"
+            perf_path = misc.GetOutsideChrootPath(
+                self.chromeos_root, "/usr/bin/perf"
             )
 
             perf_file = "/usr/sbin/perf"
@@ -562,7 +568,7 @@ class Result(object):
     def GatherPerfResults(self):
         report_id = 0
         for perf_report_file in self.perf_report_files:
-            with open(perf_report_file, "r") as f:
+            with open(perf_report_file, "r", encoding="utf-8") as f:
                 report_contents = f.read()
                 for group in re.findall(
                     r"Events: (\S+) (\S+)", report_contents
@@ -612,7 +618,7 @@ class Result(object):
             raise IOError("%s does not exist" % filename)
 
         keyvals = {}
-        with open(filename, "r") as f:
+        with open(filename, "r", encoding="utf-8") as f:
             raw_dict = json.load(f)
             if "charts" in raw_dict:
                 raw_dict = raw_dict["charts"]
@@ -660,7 +666,7 @@ class Result(object):
         """
         cpustats = {}
         read_data = ""
-        with open(self.turbostat_log_file) as f:
+        with open(self.turbostat_log_file, encoding="utf-8") as f:
             read_data = f.readlines()
 
         if not read_data:
@@ -732,7 +738,7 @@ class Result(object):
           121 root      20   0       0      0      0 S   1.0   0.0   0:00.45 spi5
         """
         all_data = ""
-        with open(self.top_log_file) as f:
+        with open(self.top_log_file, encoding="utf-8") as f:
             all_data = f.read()
 
         if not all_data:
@@ -874,7 +880,7 @@ class Result(object):
 
         cpustats = {}
         read_data = ""
-        with open(self.cpustats_log_file) as f:
+        with open(self.cpustats_log_file, encoding="utf-8") as f:
             read_data = f.readlines()
 
         if not read_data:
@@ -935,7 +941,7 @@ class Result(object):
             raise IOError("%s does not exist" % filename)
 
         keyvals = {}
-        with open(filename) as f:
+        with open(filename, encoding="utf-8") as f:
             histograms = json.load(f)
             value_map = {}
             # Gets generic set values.
@@ -1129,7 +1135,7 @@ class Result(object):
         if self.perf_data_files and self.top_cmds:
             self.VerifyPerfDataPID()
         if self.wait_time_log_file:
-            with open(self.wait_time_log_file) as f:
+            with open(self.wait_time_log_file, encoding="utf-8") as f:
                 wait_time = f.readline().strip()
                 try:
                     wait_time = float(wait_time)
@@ -1162,7 +1168,7 @@ class Result(object):
         chrome_version = ""
         keys_file = os.path.join(cache_dir, CACHE_KEYS_FILE)
         if os.path.exists(keys_file):
-            with open(keys_file, "r") as f:
+            with open(keys_file, "r", encoding="utf-8") as f:
                 lines = f.readlines()
                 for l in lines:
                     if l.startswith("Google Chrome "):
@@ -1184,7 +1190,7 @@ class Result(object):
 
         # Untar the tarball to a temporary directory
         self.temp_dir = tempfile.mkdtemp(
-            dir=os.path.join(self.chromeos_root, "chroot", "tmp")
+            dir=misc.GetOutsideChrootPath(self.chromeos_root, "/tmp")
         )
 
         command = "cd %s && tar xf %s" % (
@@ -1202,7 +1208,11 @@ class Result(object):
         self.ProcessResults(use_cache=True)
 
     def CleanUp(self, rm_chroot_tmp):
-        if rm_chroot_tmp and self.results_dir:
+        if (
+            rm_chroot_tmp
+            and self.results_dir
+            and self.results_dir != self.temp_dir
+        ):
             dirname, basename = misc.GetRoot(self.results_dir)
             if basename.find("test_that_results_") != -1:
                 command = "rm -rf %s" % self.results_dir
@@ -1240,7 +1250,9 @@ class Result(object):
             pickle.dump(self.retval, f)
 
         if not test_flag.GetTestMode():
-            with open(os.path.join(temp_dir, CACHE_KEYS_FILE), "w") as f:
+            with open(
+                os.path.join(temp_dir, CACHE_KEYS_FILE), "w", encoding="utf-8"
+            ) as f:
                 f.write("%s\n" % self.label.name)
                 f.write("%s\n" % self.label.chrome_version)
                 f.write("%s\n" % self.machine.checksum_string)
@@ -1255,7 +1267,9 @@ class Result(object):
         # Store machine info.
         # TODO(asharif): Make machine_manager a singleton, and don't pass it into
         # this function.
-        with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f:
+        with open(
+            os.path.join(temp_dir, MACHINE_FILE), "w", encoding="utf-8"
+        ) as f:
             f.write(machine_manager.machine_checksum_string[self.label.name])
 
         if os.path.exists(cache_dir):
diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py
index cad149e0..06a8b94d 100755
--- a/crosperf/results_cache_unittest.py
+++ b/crosperf/results_cache_unittest.py
@@ -11,6 +11,7 @@
 import io
 import os
 import pickle
+import re
 import shutil
 import tempfile
 import unittest
@@ -449,6 +450,16 @@ class MockResult(Result):
         return keyvals
 
 
+class RegexMatcher:
+    """A regex matcher, for passing to mocks."""
+
+    def __init__(self, regex):
+        self._regex = re.compile(regex)
+
+    def __eq__(self, string):
+        return self._regex.search(string) is not None
+
+
 class ResultTest(unittest.TestCase):
     """Result test class."""
 
@@ -490,7 +501,9 @@ class ResultTest(unittest.TestCase):
             None,
         )
 
-    def testCreateFromRun(self):
+    @mock.patch.object(os.path, "exists")
+    def testCreateFromRun(self, mock_path_exists):
+        mock_path_exists.side_effect = lambda x: x != "/etc/cros_chroot_version"
         result = MockResult.CreateFromRun(
             logger.GetLogger(),
             "average",
@@ -508,7 +521,7 @@ class ResultTest(unittest.TestCase):
         )
         self.assertEqual(
             result.results_dir,
-            "/tmp/chroot/tmp/test_that.PO1234567/platform_LibCBench",
+            RegexMatcher("/tmp/.*tmp/test_that.PO1234567/platform_LibCBench"),
         )
         self.assertEqual(result.retval, 0)
 
@@ -517,12 +530,12 @@ class ResultTest(unittest.TestCase):
             self.mock_logger, self.mock_label, "average", self.mock_cmd_exec
         )
         self.result.chromeos_root = "/tmp/chromeos"
+        self.orig_exists = os.path.exists
 
     @mock.patch.object(os.path, "isdir")
     @mock.patch.object(command_executer.CommandExecuter, "RunCommand")
     @mock.patch.object(command_executer.CommandExecuter, "CopyFiles")
     def test_copy_files_to(self, mock_copyfiles, mock_runcmd, mock_isdir):
-
         files = ["src_file_1", "src_file_2", "src_file_3"]
         dest_dir = "/tmp/test"
         self.mock_cmd_exec.RunCommand = mock_runcmd
@@ -614,118 +627,118 @@ class ResultTest(unittest.TestCase):
         self.assertEqual(
             kv_dict2,
             {
-                u"Box2D__Box2D": 4775,
-                u"Mandreel__Mandreel": 6620,
-                u"Gameboy__Gameboy": 9901,
-                u"Crypto__Crypto": 8737,
-                u"telemetry_page_measurement_results__num_errored": 0,
-                u"telemetry_page_measurement_results__num_failed": 0,
-                u"PdfJS__PdfJS": 6455,
-                u"Total__Score": 7918,
-                u"EarleyBoyer__EarleyBoyer": 14340,
-                u"MandreelLatency__MandreelLatency": 5188,
-                u"CodeLoad__CodeLoad": 6271,
-                u"DeltaBlue__DeltaBlue": 14401,
-                u"Typescript__Typescript": 9815,
-                u"SplayLatency__SplayLatency": 7653,
-                u"zlib__zlib": 16094,
-                u"Richards__Richards": 10358,
-                u"RegExp__RegExp": 1765,
-                u"NavierStokes__NavierStokes": 9815,
-                u"Splay__Splay": 4425,
-                u"RayTrace__RayTrace": 16600,
+                "Box2D__Box2D": 4775,
+                "Mandreel__Mandreel": 6620,
+                "Gameboy__Gameboy": 9901,
+                "Crypto__Crypto": 8737,
+                "telemetry_page_measurement_results__num_errored": 0,
+                "telemetry_page_measurement_results__num_failed": 0,
+                "PdfJS__PdfJS": 6455,
+                "Total__Score": 7918,
+                "EarleyBoyer__EarleyBoyer": 14340,
+                "MandreelLatency__MandreelLatency": 5188,
+                "CodeLoad__CodeLoad": 6271,
+                "DeltaBlue__DeltaBlue": 14401,
+                "Typescript__Typescript": 9815,
+                "SplayLatency__SplayLatency": 7653,
+                "zlib__zlib": 16094,
+                "Richards__Richards": 10358,
+                "RegExp__RegExp": 1765,
+                "NavierStokes__NavierStokes": 9815,
+                "Splay__Splay": 4425,
+                "RayTrace__RayTrace": 16600,
             },
         )
         self.assertEqual(
             udict,
             {
-                u"Box2D__Box2D": u"score",
-                u"Mandreel__Mandreel": u"score",
-                u"Gameboy__Gameboy": u"score",
-                u"Crypto__Crypto": u"score",
-                u"telemetry_page_measurement_results__num_errored": u"count",
-                u"telemetry_page_measurement_results__num_failed": u"count",
-                u"PdfJS__PdfJS": u"score",
-                u"Total__Score": u"score",
-                u"EarleyBoyer__EarleyBoyer": u"score",
-                u"MandreelLatency__MandreelLatency": u"score",
-                u"CodeLoad__CodeLoad": u"score",
-                u"DeltaBlue__DeltaBlue": u"score",
-                u"Typescript__Typescript": u"score",
-                u"SplayLatency__SplayLatency": u"score",
-                u"zlib__zlib": u"score",
-                u"Richards__Richards": u"score",
-                u"RegExp__RegExp": u"score",
-                u"NavierStokes__NavierStokes": u"score",
-                u"Splay__Splay": u"score",
-                u"RayTrace__RayTrace": u"score",
+                "Box2D__Box2D": "score",
+                "Mandreel__Mandreel": "score",
+                "Gameboy__Gameboy": "score",
+                "Crypto__Crypto": "score",
+                "telemetry_page_measurement_results__num_errored": "count",
+                "telemetry_page_measurement_results__num_failed": "count",
+                "PdfJS__PdfJS": "score",
+                "Total__Score": "score",
+                "EarleyBoyer__EarleyBoyer": "score",
+                "MandreelLatency__MandreelLatency": "score",
+                "CodeLoad__CodeLoad": "score",
+                "DeltaBlue__DeltaBlue": "score",
+                "Typescript__Typescript": "score",
+                "SplayLatency__SplayLatency": "score",
+                "zlib__zlib": "score",
+                "Richards__Richards": "score",
+                "RegExp__RegExp": "score",
+                "NavierStokes__NavierStokes": "score",
+                "Splay__Splay": "score",
+                "RayTrace__RayTrace": "score",
             },
         )
 
     def test_append_telemetry_units(self):
         kv_dict = {
-            u"Box2D__Box2D": 4775,
-            u"Mandreel__Mandreel": 6620,
-            u"Gameboy__Gameboy": 9901,
-            u"Crypto__Crypto": 8737,
-            u"PdfJS__PdfJS": 6455,
-            u"Total__Score": 7918,
-            u"EarleyBoyer__EarleyBoyer": 14340,
-            u"MandreelLatency__MandreelLatency": 5188,
-            u"CodeLoad__CodeLoad": 6271,
-            u"DeltaBlue__DeltaBlue": 14401,
-            u"Typescript__Typescript": 9815,
-            u"SplayLatency__SplayLatency": 7653,
-            u"zlib__zlib": 16094,
-            u"Richards__Richards": 10358,
-            u"RegExp__RegExp": 1765,
-            u"NavierStokes__NavierStokes": 9815,
-            u"Splay__Splay": 4425,
-            u"RayTrace__RayTrace": 16600,
+            "Box2D__Box2D": 4775,
+            "Mandreel__Mandreel": 6620,
+            "Gameboy__Gameboy": 9901,
+            "Crypto__Crypto": 8737,
+            "PdfJS__PdfJS": 6455,
+            "Total__Score": 7918,
+            "EarleyBoyer__EarleyBoyer": 14340,
+            "MandreelLatency__MandreelLatency": 5188,
+            "CodeLoad__CodeLoad": 6271,
+            "DeltaBlue__DeltaBlue": 14401,
+            "Typescript__Typescript": 9815,
+            "SplayLatency__SplayLatency": 7653,
+            "zlib__zlib": 16094,
+            "Richards__Richards": 10358,
+            "RegExp__RegExp": 1765,
+            "NavierStokes__NavierStokes": 9815,
+            "Splay__Splay": 4425,
+            "RayTrace__RayTrace": 16600,
         }
         units_dict = {
-            u"Box2D__Box2D": u"score",
-            u"Mandreel__Mandreel": u"score",
-            u"Gameboy__Gameboy": u"score",
-            u"Crypto__Crypto": u"score",
-            u"PdfJS__PdfJS": u"score",
-            u"Total__Score": u"score",
-            u"EarleyBoyer__EarleyBoyer": u"score",
-            u"MandreelLatency__MandreelLatency": u"score",
-            u"CodeLoad__CodeLoad": u"score",
-            u"DeltaBlue__DeltaBlue": u"score",
-            u"Typescript__Typescript": u"score",
-            u"SplayLatency__SplayLatency": u"score",
-            u"zlib__zlib": u"score",
-            u"Richards__Richards": u"score",
-            u"RegExp__RegExp": u"score",
-            u"NavierStokes__NavierStokes": u"score",
-            u"Splay__Splay": u"score",
-            u"RayTrace__RayTrace": u"score",
+            "Box2D__Box2D": "score",
+            "Mandreel__Mandreel": "score",
+            "Gameboy__Gameboy": "score",
+            "Crypto__Crypto": "score",
+            "PdfJS__PdfJS": "score",
+            "Total__Score": "score",
+            "EarleyBoyer__EarleyBoyer": "score",
+            "MandreelLatency__MandreelLatency": "score",
+            "CodeLoad__CodeLoad": "score",
+            "DeltaBlue__DeltaBlue": "score",
+            "Typescript__Typescript": "score",
+            "SplayLatency__SplayLatency": "score",
+            "zlib__zlib": "score",
+            "Richards__Richards": "score",
+            "RegExp__RegExp": "score",
+            "NavierStokes__NavierStokes": "score",
+            "Splay__Splay": "score",
+            "RayTrace__RayTrace": "score",
         }
 
         results_dict = self.result.AppendTelemetryUnits(kv_dict, units_dict)
         self.assertEqual(
             results_dict,
             {
-                u"Box2D__Box2D": [4775, u"score"],
-                u"Splay__Splay": [4425, u"score"],
-                u"Gameboy__Gameboy": [9901, u"score"],
-                u"Crypto__Crypto": [8737, u"score"],
-                u"PdfJS__PdfJS": [6455, u"score"],
-                u"Total__Score": [7918, u"score"],
-                u"EarleyBoyer__EarleyBoyer": [14340, u"score"],
-                u"MandreelLatency__MandreelLatency": [5188, u"score"],
-                u"DeltaBlue__DeltaBlue": [14401, u"score"],
-                u"SplayLatency__SplayLatency": [7653, u"score"],
-                u"Mandreel__Mandreel": [6620, u"score"],
-                u"Richards__Richards": [10358, u"score"],
-                u"zlib__zlib": [16094, u"score"],
-                u"CodeLoad__CodeLoad": [6271, u"score"],
-                u"Typescript__Typescript": [9815, u"score"],
-                u"RegExp__RegExp": [1765, u"score"],
-                u"RayTrace__RayTrace": [16600, u"score"],
-                u"NavierStokes__NavierStokes": [9815, u"score"],
+                "Box2D__Box2D": [4775, "score"],
+                "Splay__Splay": [4425, "score"],
+                "Gameboy__Gameboy": [9901, "score"],
+                "Crypto__Crypto": [8737, "score"],
+                "PdfJS__PdfJS": [6455, "score"],
+                "Total__Score": [7918, "score"],
+                "EarleyBoyer__EarleyBoyer": [14340, "score"],
+                "MandreelLatency__MandreelLatency": [5188, "score"],
+                "DeltaBlue__DeltaBlue": [14401, "score"],
+                "SplayLatency__SplayLatency": [7653, "score"],
+                "Mandreel__Mandreel": [6620, "score"],
+                "Richards__Richards": [10358, "score"],
+                "zlib__zlib": [16094, "score"],
+                "CodeLoad__CodeLoad": [6271, "score"],
+                "Typescript__Typescript": [9815, "score"],
+                "RegExp__RegExp": [1765, "score"],
+                "RayTrace__RayTrace": [16600, "score"],
+                "NavierStokes__NavierStokes": [9815, "score"],
             },
         )
 
@@ -738,7 +751,6 @@ class ResultTest(unittest.TestCase):
     def test_get_keyvals(
         self, mock_chrootruncmd, mock_runcmd, mock_mkdtemp, mock_getpath
     ):
-
         self.kv_dict = {}
         self.callGetNewKeyvals = False
 
@@ -821,9 +833,9 @@ class ResultTest(unittest.TestCase):
 
         # Test 3. suite != telemetry_Crosperf.  Normally this would be for
         # running non-Telemetry autotests, such as BootPerfServer.  In this test
-        # case, the keyvals we have set up were returned from a Telemetry test run;
-        # so this pass is basically testing that we don't append the units to the
-        # test results (which we do for Telemetry autotest runs).
+        # case, the keyvals we have set up were returned from a Telemetry test
+        # run; so this pass is basically testing that we don't append the units
+        # to the test results (which we do for Telemetry autotest runs).
         reset()
         self.result.suite = ""
         res = self.result.GetKeyvals()
@@ -839,10 +851,15 @@ class ResultTest(unittest.TestCase):
     ):
         self.result.perf_data_files = ["/tmp/results/perf.data"]
         self.result.board = "samus"
+        self.result.cwp_dso = "kallsyms"
         mock_getpath.return_value = "/usr/chromeos/chroot/tmp/results/perf.data"
         mock_get_total_samples.return_value = [
             "",
-            "45.42%        237210  chrome ",
+            (
+                "45.42%        53721   chrome \n"
+                "10.01%        12345   [kernel.kallsyms] \n"
+                "1.42%         1234    ssh "
+            ),
             "",
         ]
         mock_exists.return_value = True
@@ -855,10 +872,9 @@ class ResultTest(unittest.TestCase):
 
         with mock.patch("builtins.open", return_value=io.StringIO(content)):
             samples = self.result.GetSamples()
-        self.assertEqual(samples, [237210 - 60, u"samples"])
+        self.assertEqual(samples, [12345 - 60, "samples"])
 
     def test_get_results_dir(self):
-
         self.result.out = ""
         self.assertRaises(Exception, self.result.GetResultsDir)
 
@@ -868,7 +884,6 @@ class ResultTest(unittest.TestCase):
 
     @mock.patch.object(command_executer.CommandExecuter, "RunCommandGeneric")
     def test_find_files_in_results_dir(self, mock_runcmd):
-
         self.result.results_dir = None
         res = self.result.FindFilesInResultsDir("-name perf.data")
         self.assertEqual(res, "")
@@ -1149,11 +1164,12 @@ class ResultTest(unittest.TestCase):
         """Normal case when log exists and contains valid data."""
         self.result.turbostat_log_file = "/tmp/somelogfile.log"
         with mock.patch(
-            "builtins.open", mock.mock_open(read_data=TURBOSTAT_LOG_OUTPUT)
+            "builtins.open",
+            mock.mock_open(read_data=TURBOSTAT_LOG_OUTPUT),
         ) as mo:
             cpustats = self.result.ProcessTurbostatResults()
             # Check that the log got opened and data were read/parsed.
-            calls = [mock.call("/tmp/somelogfile.log")]
+            calls = [mock.call("/tmp/somelogfile.log", encoding="utf-8")]
             mo.assert_has_calls(calls)
             self.assertEqual(cpustats, TURBOSTAT_DATA)
 
@@ -1162,9 +1178,9 @@ class ResultTest(unittest.TestCase):
         self.result.turbostat_log_file = "/tmp/emptylogfile.log"
         with mock.patch("builtins.open", mock.mock_open(read_data="")) as mo:
             cpustats = self.result.ProcessTurbostatResults()
-            # Check that the log got opened and parsed successfully and empty data
-            # returned.
-            calls = [mock.call("/tmp/emptylogfile.log")]
+            # Check that the log got opened and parsed successfully and empty
+            # data returned.
+            calls = [mock.call("/tmp/emptylogfile.log", encoding="utf-8")]
             mo.assert_has_calls(calls)
             self.assertEqual(cpustats, {})
 
@@ -1189,11 +1205,12 @@ class ResultTest(unittest.TestCase):
         """
         self.result.cpustats_log_file = "/tmp/somelogfile.log"
         with mock.patch(
-            "builtins.open", mock.mock_open(read_data=CPUSTATS_UNIQ_OUTPUT)
+            "builtins.open",
+            mock.mock_open(read_data=CPUSTATS_UNIQ_OUTPUT),
         ) as mo:
             cpustats = self.result.ProcessCpustatsResults()
             # Check that the log got opened and data were read/parsed.
-            calls = [mock.call("/tmp/somelogfile.log")]
+            calls = [mock.call("/tmp/somelogfile.log", encoding="utf-8")]
             mo.assert_has_calls(calls)
             self.assertEqual(cpustats, CPUSTATS_UNIQ_DATA)
 
@@ -1207,11 +1224,12 @@ class ResultTest(unittest.TestCase):
         """
         self.result.cpustats_log_file = "/tmp/somelogfile.log"
         with mock.patch(
-            "builtins.open", mock.mock_open(read_data=CPUSTATS_DUPL_OUTPUT)
+            "builtins.open",
+            mock.mock_open(read_data=CPUSTATS_DUPL_OUTPUT),
         ) as mo:
             cpustats = self.result.ProcessCpustatsResults()
             # Check that the log got opened and data were read/parsed.
-            calls = [mock.call("/tmp/somelogfile.log")]
+            calls = [mock.call("/tmp/somelogfile.log", encoding="utf-8")]
             mo.assert_has_calls(calls)
             self.assertEqual(cpustats, CPUSTATS_DUPL_DATA)
 
@@ -1220,9 +1238,9 @@ class ResultTest(unittest.TestCase):
         self.result.cpustats_log_file = "/tmp/emptylogfile.log"
         with mock.patch("builtins.open", mock.mock_open(read_data="")) as mo:
             cpustats = self.result.ProcessCpustatsResults()
-            # Check that the log got opened and parsed successfully and empty data
-            # returned.
-            calls = [mock.call("/tmp/emptylogfile.log")]
+            # Check that the log got opened and parsed successfully and empty
+            # data returned.
+            calls = [mock.call("/tmp/emptylogfile.log", encoding="utf-8")]
             mo.assert_has_calls(calls)
             self.assertEqual(cpustats, {})
 
@@ -1235,7 +1253,7 @@ class ResultTest(unittest.TestCase):
         ) as mo:
             topproc = self.result.ProcessTopResults()
             # Check that the log got opened and data were read/parsed.
-            calls = [mock.call("/tmp/fakelogfile.log")]
+            calls = [mock.call("/tmp/fakelogfile.log", encoding="utf-8")]
             mo.assert_has_calls(calls)
             self.assertEqual(topproc, TOP_DATA)
 
@@ -1244,9 +1262,9 @@ class ResultTest(unittest.TestCase):
         self.result.top_log_file = "/tmp/emptylogfile.log"
         with mock.patch("builtins.open", mock.mock_open(read_data="")) as mo:
             topcalls = self.result.ProcessTopResults()
-            # Check that the log got opened and parsed successfully and empty data
-            # returned.
-            calls = [mock.call("/tmp/emptylogfile.log")]
+            # Check that the log got opened and parsed successfully and empty
+            # data returned.
+            calls = [mock.call("/tmp/emptylogfile.log", encoding="utf-8")]
             mo.assert_has_calls(calls)
             self.assertEqual(topcalls, [])
 
@@ -1342,10 +1360,18 @@ class ResultTest(unittest.TestCase):
             ),
         )
 
+    @mock.patch.object(os.path, "exists")
     @mock.patch.object(misc, "GetInsideChrootPath")
     @mock.patch.object(command_executer.CommandExecuter, "ChrootRunCommand")
-    def test_generate_perf_report_files(self, mock_chrootruncmd, mock_getpath):
-        fake_file = "/usr/chromeos/chroot/tmp/results/fake_file"
+    def test_generate_perf_report_files(
+        self, mock_chrootruncmd, mock_getpath, mock_pathexists
+    ):
+        mock_pathexists.side_effect = (
+            lambda x: self.orig_exists(x)
+            if x != "/etc/cros_chroot_version"
+            else False
+        )
+        fake_file = "/tmp/results/perf.data.report"
         self.result.perf_data_files = ["/tmp/results/perf.data"]
         self.result.board = "lumpy"
         mock_getpath.return_value = fake_file
@@ -1354,7 +1380,8 @@ class ResultTest(unittest.TestCase):
         # Debug path not found
         self.result.label.debug_path = ""
         tmp = self.result.GeneratePerfReportFiles()
-        self.assertEqual(tmp, ["/tmp/chromeos/chroot%s" % fake_file])
+        self.assertEqual(len(tmp), 1)
+        self.assertEqual(tmp[0], RegexMatcher("/tmp/chromeos.*%s" % fake_file))
         self.assertEqual(
             mock_chrootruncmd.call_args_list[0][0],
             (
@@ -1364,12 +1391,18 @@ class ResultTest(unittest.TestCase):
             ),
         )
 
+    @mock.patch.object(os.path, "exists")
     @mock.patch.object(misc, "GetInsideChrootPath")
     @mock.patch.object(command_executer.CommandExecuter, "ChrootRunCommand")
     def test_generate_perf_report_files_debug(
-        self, mock_chrootruncmd, mock_getpath
+        self, mock_chrootruncmd, mock_getpath, mock_pathexists
     ):
-        fake_file = "/usr/chromeos/chroot/tmp/results/fake_file"
+        mock_pathexists.side_effect = (
+            lambda x: self.orig_exists(x)
+            if x != "/etc/cros_chroot_version"
+            else False
+        )
+        fake_file = "/tmp/results/perf.data.report"
         self.result.perf_data_files = ["/tmp/results/perf.data"]
         self.result.board = "lumpy"
         mock_getpath.return_value = fake_file
@@ -1378,7 +1411,8 @@ class ResultTest(unittest.TestCase):
         # Debug path found
         self.result.label.debug_path = "/tmp/debug"
         tmp = self.result.GeneratePerfReportFiles()
-        self.assertEqual(tmp, ["/tmp/chromeos/chroot%s" % fake_file])
+        self.assertEqual(len(tmp), 1)
+        self.assertEqual(tmp[0], RegexMatcher("/tmp/chromeos.*%s" % fake_file))
         self.assertEqual(
             mock_chrootruncmd.call_args_list[0][0],
             (
@@ -1521,27 +1555,27 @@ class ResultTest(unittest.TestCase):
         # format
         self.result.suite = "telemetry_Crosperf"
         self.result.results_file = [tempfile.mkdtemp() + "/histograms.json"]
-        with open(self.result.results_file[0], "w") as f:
+        with open(self.result.results_file[0], "w", encoding="utf-8") as f:
             f.write(HISTOGRAMSET)
         self.result.ProcessResults()
         shutil.rmtree(os.path.dirname(self.result.results_file[0]))
         # Verify the summary for the story is correct
         self.assertEqual(
             self.result.keyvals["timeToFirstContentfulPaint__typical"],
-            [880.000, u"ms_smallerIsBetter"],
+            [880.000, "ms_smallerIsBetter"],
         )
         # Veirfy the summary for a certain stroy tag is correct
         self.assertEqual(
             self.result.keyvals[
                 "timeToFirstContentfulPaint__cache_temperature:cold"
             ],
-            [1000.000, u"ms_smallerIsBetter"],
+            [1000.000, "ms_smallerIsBetter"],
         )
         self.assertEqual(
             self.result.keyvals[
                 "timeToFirstContentfulPaint__cache_temperature:warm"
             ],
-            [800.000, u"ms_smallerIsBetter"],
+            [800.000, "ms_smallerIsBetter"],
         )
 
     @mock.patch.object(Result, "ProcessCpustatsResults")
@@ -1664,7 +1698,6 @@ class ResultTest(unittest.TestCase):
         command_executer.CommandExecuter, "ChrootRunCommandWOutput"
     )
     def test_populate_from_cache_dir(self, mock_runchrootcmd, mock_getpath):
-
         # pylint: disable=redefined-builtin
         def FakeMkdtemp(dir=None):
             if dir:
@@ -1672,7 +1705,7 @@ class ResultTest(unittest.TestCase):
             return self.tmpdir
 
         def FakeGetSamples():
-            return [1, u"samples"]
+            return [1, "samples"]
 
         current_path = os.getcwd()
         cache_dir = os.path.join(current_path, "test_cache/test_input")
@@ -1696,46 +1729,46 @@ class ResultTest(unittest.TestCase):
         self.assertEqual(
             self.result.keyvals,
             {
-                u"Total__Total": [444.0, u"ms"],
-                u"regexp-dna__regexp-dna": [16.2, u"ms"],
-                u"telemetry_page_measurement_results__num_failed": [
+                "Total__Total": [444.0, "ms"],
+                "regexp-dna__regexp-dna": [16.2, "ms"],
+                "telemetry_page_measurement_results__num_failed": [
                     0,
-                    u"count",
+                    "count",
                 ],
-                u"telemetry_page_measurement_results__num_errored": [
+                "telemetry_page_measurement_results__num_errored": [
                     0,
-                    u"count",
+                    "count",
                 ],
-                u"string-fasta__string-fasta": [23.2, u"ms"],
-                u"crypto-sha1__crypto-sha1": [11.6, u"ms"],
-                u"bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [
+                "string-fasta__string-fasta": [23.2, "ms"],
+                "crypto-sha1__crypto-sha1": [11.6, "ms"],
+                "bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [
                     3.2,
-                    u"ms",
+                    "ms",
                 ],
-                u"access-nsieve__access-nsieve": [7.9, u"ms"],
-                u"bitops-nsieve-bits__bitops-nsieve-bits": [9.4, u"ms"],
-                u"string-validate-input__string-validate-input": [19.3, u"ms"],
-                u"3d-raytrace__3d-raytrace": [24.7, u"ms"],
-                u"3d-cube__3d-cube": [28.0, u"ms"],
-                u"string-unpack-code__string-unpack-code": [46.7, u"ms"],
-                u"date-format-tofte__date-format-tofte": [26.3, u"ms"],
-                u"math-partial-sums__math-partial-sums": [22.0, u"ms"],
+                "access-nsieve__access-nsieve": [7.9, "ms"],
+                "bitops-nsieve-bits__bitops-nsieve-bits": [9.4, "ms"],
+                "string-validate-input__string-validate-input": [19.3, "ms"],
+                "3d-raytrace__3d-raytrace": [24.7, "ms"],
+                "3d-cube__3d-cube": [28.0, "ms"],
+                "string-unpack-code__string-unpack-code": [46.7, "ms"],
+                "date-format-tofte__date-format-tofte": [26.3, "ms"],
+                "math-partial-sums__math-partial-sums": [22.0, "ms"],
                 "\telemetry_Crosperf": ["PASS", ""],
-                u"crypto-aes__crypto-aes": [15.2, u"ms"],
-                u"bitops-bitwise-and__bitops-bitwise-and": [8.4, u"ms"],
-                u"crypto-md5__crypto-md5": [10.5, u"ms"],
-                u"string-tagcloud__string-tagcloud": [52.8, u"ms"],
-                u"access-nbody__access-nbody": [8.5, u"ms"],
+                "crypto-aes__crypto-aes": [15.2, "ms"],
+                "bitops-bitwise-and__bitops-bitwise-and": [8.4, "ms"],
+                "crypto-md5__crypto-md5": [10.5, "ms"],
+                "string-tagcloud__string-tagcloud": [52.8, "ms"],
+                "access-nbody__access-nbody": [8.5, "ms"],
                 "retval": 0,
-                u"math-spectral-norm__math-spectral-norm": [6.6, u"ms"],
-                u"math-cordic__math-cordic": [8.7, u"ms"],
-                u"access-binary-trees__access-binary-trees": [4.5, u"ms"],
-                u"controlflow-recursive__controlflow-recursive": [4.4, u"ms"],
-                u"access-fannkuch__access-fannkuch": [17.8, u"ms"],
-                u"string-base64__string-base64": [16.0, u"ms"],
-                u"date-format-xparb__date-format-xparb": [20.9, u"ms"],
-                u"3d-morph__3d-morph": [22.1, u"ms"],
-                u"bitops-bits-in-byte__bitops-bits-in-byte": [9.1, u"ms"],
+                "math-spectral-norm__math-spectral-norm": [6.6, "ms"],
+                "math-cordic__math-cordic": [8.7, "ms"],
+                "access-binary-trees__access-binary-trees": [4.5, "ms"],
+                "controlflow-recursive__controlflow-recursive": [4.4, "ms"],
+                "access-fannkuch__access-fannkuch": [17.8, "ms"],
+                "string-base64__string-base64": [16.0, "ms"],
+                "date-format-xparb__date-format-xparb": [20.9, "ms"],
+                "3d-morph__3d-morph": [22.1, "ms"],
+                "bitops-bits-in-byte__bitops-bits-in-byte": [9.1, "ms"],
             },
         )
 
@@ -1746,47 +1779,47 @@ class ResultTest(unittest.TestCase):
         self.assertEqual(
             self.result.keyvals,
             {
-                u"Total__Total": [444.0, u"ms"],
-                u"regexp-dna__regexp-dna": [16.2, u"ms"],
-                u"telemetry_page_measurement_results__num_failed": [
+                "Total__Total": [444.0, "ms"],
+                "regexp-dna__regexp-dna": [16.2, "ms"],
+                "telemetry_page_measurement_results__num_failed": [
                     0,
-                    u"count",
+                    "count",
                 ],
-                u"telemetry_page_measurement_results__num_errored": [
+                "telemetry_page_measurement_results__num_errored": [
                     0,
-                    u"count",
+                    "count",
                 ],
-                u"string-fasta__string-fasta": [23.2, u"ms"],
-                u"crypto-sha1__crypto-sha1": [11.6, u"ms"],
-                u"bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [
+                "string-fasta__string-fasta": [23.2, "ms"],
+                "crypto-sha1__crypto-sha1": [11.6, "ms"],
+                "bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [
                     3.2,
-                    u"ms",
+                    "ms",
                 ],
-                u"access-nsieve__access-nsieve": [7.9, u"ms"],
-                u"bitops-nsieve-bits__bitops-nsieve-bits": [9.4, u"ms"],
-                u"string-validate-input__string-validate-input": [19.3, u"ms"],
-                u"3d-raytrace__3d-raytrace": [24.7, u"ms"],
-                u"3d-cube__3d-cube": [28.0, u"ms"],
-                u"string-unpack-code__string-unpack-code": [46.7, u"ms"],
-                u"date-format-tofte__date-format-tofte": [26.3, u"ms"],
-                u"math-partial-sums__math-partial-sums": [22.0, u"ms"],
+                "access-nsieve__access-nsieve": [7.9, "ms"],
+                "bitops-nsieve-bits__bitops-nsieve-bits": [9.4, "ms"],
+                "string-validate-input__string-validate-input": [19.3, "ms"],
+                "3d-raytrace__3d-raytrace": [24.7, "ms"],
+                "3d-cube__3d-cube": [28.0, "ms"],
+                "string-unpack-code__string-unpack-code": [46.7, "ms"],
+                "date-format-tofte__date-format-tofte": [26.3, "ms"],
+                "math-partial-sums__math-partial-sums": [22.0, "ms"],
                 "\telemetry_Crosperf": ["PASS", ""],
-                u"crypto-aes__crypto-aes": [15.2, u"ms"],
-                u"bitops-bitwise-and__bitops-bitwise-and": [8.4, u"ms"],
-                u"crypto-md5__crypto-md5": [10.5, u"ms"],
-                u"string-tagcloud__string-tagcloud": [52.8, u"ms"],
-                u"access-nbody__access-nbody": [8.5, u"ms"],
+                "crypto-aes__crypto-aes": [15.2, "ms"],
+                "bitops-bitwise-and__bitops-bitwise-and": [8.4, "ms"],
+                "crypto-md5__crypto-md5": [10.5, "ms"],
+                "string-tagcloud__string-tagcloud": [52.8, "ms"],
+                "access-nbody__access-nbody": [8.5, "ms"],
                 "retval": 0,
-                u"math-spectral-norm__math-spectral-norm": [6.6, u"ms"],
-                u"math-cordic__math-cordic": [8.7, u"ms"],
-                u"access-binary-trees__access-binary-trees": [4.5, u"ms"],
-                u"controlflow-recursive__controlflow-recursive": [4.4, u"ms"],
-                u"access-fannkuch__access-fannkuch": [17.8, u"ms"],
-                u"string-base64__string-base64": [16.0, u"ms"],
-                u"date-format-xparb__date-format-xparb": [20.9, u"ms"],
-                u"3d-morph__3d-morph": [22.1, u"ms"],
-                u"bitops-bits-in-byte__bitops-bits-in-byte": [9.1, u"ms"],
-                u"samples": [1, u"samples"],
+                "math-spectral-norm__math-spectral-norm": [6.6, "ms"],
+                "math-cordic__math-cordic": [8.7, "ms"],
+                "access-binary-trees__access-binary-trees": [4.5, "ms"],
+                "controlflow-recursive__controlflow-recursive": [4.4, "ms"],
+                "access-fannkuch__access-fannkuch": [17.8, "ms"],
+                "string-base64__string-base64": [16.0, "ms"],
+                "date-format-xparb__date-format-xparb": [20.9, "ms"],
+                "3d-morph__3d-morph": [22.1, "ms"],
+                "bitops-bits-in-byte__bitops-bits-in-byte": [9.1, "ms"],
+                "samples": [1, "samples"],
             },
         )
 
@@ -1798,7 +1831,6 @@ class ResultTest(unittest.TestCase):
     @mock.patch.object(misc, "GetRoot")
     @mock.patch.object(command_executer.CommandExecuter, "RunCommand")
     def test_cleanup(self, mock_runcmd, mock_getroot):
-
         # Test 1. 'rm_chroot_tmp' is True; self.results_dir exists;
         # self.temp_dir exists; results_dir name contains 'test_that_results_'.
         mock_getroot.return_value = [
@@ -2031,7 +2063,6 @@ class TelemetryResultTest(unittest.TestCase):
         self.assertEqual(self.result.retval, 3)
 
     def test_populate_from_cache_dir_and_process_results(self):
-
         self.result = TelemetryResult(
             self.mock_logger, self.mock_label, "average", self.mock_machine
         )
@@ -2244,7 +2275,6 @@ class ResultsCacheTest(unittest.TestCase):
     @mock.patch.object(os.path, "isdir")
     @mock.patch.object(Result, "CreateFromCacheHit")
     def test_read_result(self, mock_create, mock_isdir, mock_runcmd):
-
         self.fakeCacheReturnResult = None
 
         def FakeGetCacheDirForRead():
@@ -2303,7 +2333,8 @@ class ResultsCacheTest(unittest.TestCase):
         self.assertIsNone(res)
 
         # Test 5. os.path.isdir returns true, but mock_create now returns None
-        # (the call to CreateFromCacheHit returns None), so overal result is None.
+        # (the call to CreateFromCacheHit returns None), so overal result is
+        # None.
         mock_isdir.return_value = True
         mock_create.return_value = None
         res = self.results_cache.ReadResult()
diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py
index 6382bba7..b34f0b16 100644
--- a/crosperf/settings_factory.py
+++ b/crosperf/settings_factory.py
@@ -206,7 +206,7 @@ class GlobalSettings(Settings):
         self.AddField(
             BooleanField(
                 "rerun_if_failed",
-                description="Whether to re-run failed test runs " "or not.",
+                description="Whether to re-run failed test runs or not.",
                 default=False,
             )
         )
@@ -235,6 +235,13 @@ class GlobalSettings(Settings):
         )
         self.AddField(
             BooleanField(
+                "ignore_cache",
+                description='Alias of "rerun" to ignore cache.',
+                default=False,
+            )
+        )
+        self.AddField(
+            BooleanField(
                 "same_specs",
                 default=True,
                 description="Ensure cached runs are run on the "
@@ -246,7 +253,7 @@ class GlobalSettings(Settings):
             BooleanField(
                 "same_machine",
                 default=False,
-                description="Ensure cached runs are run on the " "same remote.",
+                description="Ensure cached runs are run on the same remote.",
             )
         )
         self.AddField(
@@ -552,6 +559,15 @@ class GlobalSettings(Settings):
                 " Useful when lock is held externally, say with crosfleet.",
             )
         )
+        self.AddField(
+            BooleanField(
+                "keep_stateful",
+                default=False,
+                description="When flashing a ChromeOS image keep the stateful"
+                " partition, i.e. don't use --clobber-stateful. This option"
+                " is useful to keep ssh keys, wi-fi settings and so on.",
+            )
+        )
 
 
 class SettingsFactory(object):
diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py
index 93d3bd6d..a6771c03 100755
--- a/crosperf/settings_factory_unittest.py
+++ b/crosperf/settings_factory_unittest.py
@@ -49,7 +49,7 @@ class GlobalSettingsTest(unittest.TestCase):
     def test_init(self):
         res = settings_factory.GlobalSettings("g_settings")
         self.assertIsNotNone(res)
-        self.assertEqual(len(res.fields), 40)
+        self.assertEqual(len(res.fields), 42)
         self.assertEqual(res.GetField("name"), "")
         self.assertEqual(res.GetField("board"), "")
         self.assertEqual(res.GetField("crosfleet"), False)
@@ -58,6 +58,7 @@ class GlobalSettingsTest(unittest.TestCase):
         self.assertEqual(res.GetField("rm_chroot_tmp"), False)
         self.assertEqual(res.GetField("email"), None)
         self.assertEqual(res.GetField("rerun"), False)
+        self.assertEqual(res.GetField("ignore_cache"), False)
         self.assertEqual(res.GetField("same_specs"), True)
         self.assertEqual(res.GetField("same_machine"), False)
         self.assertEqual(res.GetField("iterations"), 0)
@@ -114,7 +115,7 @@ class SettingsFactoryTest(unittest.TestCase):
             "global", "global"
         )
         self.assertIsInstance(g_settings, settings_factory.GlobalSettings)
-        self.assertEqual(len(g_settings.fields), 40)
+        self.assertEqual(len(g_settings.fields), 42)
 
 
 if __name__ == "__main__":
diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py
index e777a57f..f5566f51 100644
--- a/crosperf/suite_runner.py
+++ b/crosperf/suite_runner.py
@@ -17,6 +17,7 @@ import subprocess
 import time
 
 from cros_utils import command_executer
+from cros_utils import misc
 
 
 # sshwatcher path, relative to ChromiumOS source root.
@@ -152,7 +153,9 @@ class SuiteRunner(object):
 
     def RemoveTelemetryTempFile(self, machine, chromeos_root):
         filename = "telemetry@%s" % machine
-        fullname = os.path.join(chromeos_root, "chroot", "tmp", filename)
+        fullname = misc.GetOutsideChrootPath(
+            chromeos_root, os.path.join("/tmp", filename)
+        )
         if os.path.exists(fullname):
             os.remove(fullname)
 
@@ -286,7 +289,7 @@ class SuiteRunner(object):
     def DownloadResult(self, label, task_id):
         gsutil_cmd = os.path.join(label.chromeos_root, GS_UTIL)
         result_dir = "gs://chromeos-autotest-results/swarming-%s" % task_id
-        download_path = os.path.join(label.chromeos_root, "chroot/tmp")
+        download_path = misc.GetOutsideChrootPath(label.chromeos_root, "/tmp")
         ls_command = "%s ls %s" % (
             gsutil_cmd,
             os.path.join(result_dir, "autoserv_test"),