diff options
author | Yabin Cui <yabinc@google.com> | 2023-08-21 22:16:21 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2023-08-21 22:16:21 +0000 |
commit | 0348d10214299073ea59bda4987dd8813a9e7812 (patch) | |
tree | 3e4cabab5a57ae288c245cff9a1a6ff7926a3207 /crosperf | |
parent | 40214b48188358a80b7478bfff21d4814dd9177c (diff) | |
parent | 8eed62767c6a80c3c26359d38d4961069da83813 (diff) | |
download | toolchain-utils-0348d10214299073ea59bda4987dd8813a9e7812.tar.gz |
Upgrade toolchain-utils to 2942bd8023e3e0a79ed78fd4fff22c1f961e9f73 am: 1c2bb62c89 am: e0c166ae0c am: 1bcc74be81 am: 34bad74c3c am: 8eed62767c
Original change: https://android-review.googlesource.com/c/platform/external/toolchain-utils/+/2716013
Change-Id: I782720da8f7650817d1ea90c4d12fe1474ef9ae4
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
Diffstat (limited to 'crosperf')
-rw-r--r-- | crosperf/README.md | 3 | ||||
-rw-r--r-- | crosperf/benchmark.py | 50 | ||||
-rwxr-xr-x | crosperf/crosperf_unittest.py | 2 | ||||
-rw-r--r-- | crosperf/default_remotes | 2 | ||||
-rw-r--r-- | crosperf/download_images.py | 61 | ||||
-rwxr-xr-x | crosperf/download_images_unittest.py | 96 | ||||
-rw-r--r-- | crosperf/experiment.py | 8 | ||||
-rw-r--r-- | crosperf/experiment_factory.py | 112 | ||||
-rwxr-xr-x | crosperf/experiment_factory_unittest.py | 60 | ||||
-rw-r--r-- | crosperf/experiment_runner.py | 28 | ||||
-rw-r--r-- | crosperf/machine_manager.py | 31 | ||||
-rw-r--r-- | crosperf/results_cache.py | 90 | ||||
-rwxr-xr-x | crosperf/results_cache_unittest.py | 435 | ||||
-rw-r--r-- | crosperf/settings_factory.py | 20 | ||||
-rwxr-xr-x | crosperf/settings_factory_unittest.py | 5 | ||||
-rw-r--r-- | crosperf/suite_runner.py | 7 |
16 files changed, 572 insertions, 438 deletions
diff --git a/crosperf/README.md b/crosperf/README.md index 18601b67..f1429513 100644 --- a/crosperf/README.md +++ b/crosperf/README.md @@ -3,8 +3,7 @@ To use these experiment files, replace the board, remote and images placeholders and run crosperf on them. -Further information about crosperf: -https://sites.google.com/a/google.com/chromeos-toolchain-team-home2/home/team-tools-and-scripts/crosperf-cros-image-performance-comparison-tool +Further information about crosperf: https://goto.google.com/crostc-crosperf The final experiment file should look something like the following (but with different actual values for the fields): diff --git a/crosperf/benchmark.py b/crosperf/benchmark.py index f9de0cf3..eb8661e9 100644 --- a/crosperf/benchmark.py +++ b/crosperf/benchmark.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2013 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -7,10 +6,10 @@ import math +import statistics +from typing import Any -# FIXME(denik): Fix the import in chroot. -# pylint: disable=import-error -from scipy import stats +import numpy as np # See crbug.com/673558 for how these are estimated. @@ -26,10 +25,49 @@ _estimated_stddev = { "loading.desktop": 0.021, # Copied from page_cycler initially } +# Numpy makes it hard to know the real type of some inputs +# and outputs, so this type alias is just for docs. +FloatLike = Any + + +def isf(x: FloatLike, mu=0.0, sigma=1.0, pitch=0.01) -> FloatLike: + """Compute the inverse survival function for value x. + + In the abscence of using scipy.stats.norm's isf(), this function + attempts to re-implement the inverse survival function by calculating + the numerical inverse of the survival function, interpolating between + table values. See bug b/284489250 for details. + + Survival function as defined by: + https://en.wikipedia.org/wiki/Survival_function + + Examples: + >>> -2.0e-16 < isf(0.5) < 2.0e-16 + True + + Args: + x: float or numpy array-like to compute the ISF for. + mu: Center of the underlying normal distribution. + sigma: Spread of the underlying normal distribution. + pitch: Absolute spacing between y-value interpolation points. + + Returns: + float or numpy array-like representing the ISF of `x`. + """ + norm = statistics.NormalDist(mu, sigma) + # np.interp requires a monotonically increasing x table. + # Because the survival table is monotonically decreasing, we have to + # reverse the y_vals too. + y_vals = np.flip(np.arange(-4.0, 4.0, pitch)) + survival_table = np.fromiter( + (1.0 - norm.cdf(y) for y in y_vals), y_vals.dtype + ) + return np.interp(x, survival_table, y_vals) + # Get #samples needed to guarantee a given confidence interval, assuming the # samples follow normal distribution. -def _samples(b): +def _samples(b: str) -> int: # TODO: Make this an option # CI = (0.9, 0.02), i.e., 90% chance that |sample mean - true mean| < 2%. p = 0.9 @@ -39,7 +77,7 @@ def _samples(b): d = _estimated_stddev[b] # Get at least 2 samples so as to calculate standard deviation, which is # needed in T-test for p-value. - n = int(math.ceil((stats.norm.isf((1 - p) / 2) * d / e) ** 2)) + n = int(math.ceil((isf((1 - p) / 2) * d / e) ** 2)) return n if n > 1 else 2 diff --git a/crosperf/crosperf_unittest.py b/crosperf/crosperf_unittest.py index 7b52f2e0..bbcb1712 100755 --- a/crosperf/crosperf_unittest.py +++ b/crosperf/crosperf_unittest.py @@ -67,7 +67,7 @@ class CrosperfTest(unittest.TestCase): settings = crosperf.ConvertOptionsToSettings(options) self.assertIsNotNone(settings) self.assertIsInstance(settings, settings_factory.GlobalSettings) - self.assertEqual(len(settings.fields), 40) + self.assertEqual(len(settings.fields), 42) self.assertTrue(settings.GetField("rerun")) argv = ["crosperf/crosperf.py", "temp.exp"] options, _ = parser.parse_known_args(argv) diff --git a/crosperf/default_remotes b/crosperf/default_remotes index 714385e7..80563664 100644 --- a/crosperf/default_remotes +++ b/crosperf/default_remotes @@ -1,5 +1,5 @@ bob : chromeos8-row12-rack16-host2 -chell : chromeos2-row1-rack10-host2 chromeos2-row1-rack10-host4 +chell : chromeos6-row16-rack5-host6 chromeos6-row16-rack5-host7 coral : chromeos6-row5-rack6-host1 chromeos6-row5-rack6-host3 chromeos6-row5-rack6-host5 elm : chromeos6-row14-rack15-host21 nautilus : chromeos6-row5-rack10-host1 chromeos6-row5-rack10-host3 diff --git a/crosperf/download_images.py b/crosperf/download_images.py index 9a46280d..38d5d68a 100644 --- a/crosperf/download_images.py +++ b/crosperf/download_images.py @@ -8,8 +8,10 @@ import ast import os +import shlex from cros_utils import command_executer +from cros_utils import misc import test_flag @@ -79,7 +81,9 @@ class ImageDownloader(object): ) # Make sure the directory for downloading the image exists. - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id) + ) image_path = os.path.join(download_path, "chromiumos_test_image.bin") if not os.path.exists(download_path): os.makedirs(download_path) @@ -105,18 +109,19 @@ class ImageDownloader(object): def UncompressImage(self, chromeos_root, build_id): # Check to see if the file has already been uncompresssed, etc. - if os.path.exists( + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join( - chromeos_root, - "chroot/tmp", + "/tmp", build_id, - "chromiumos_test_image.bin", - ) + ), + ) + if os.path.exists( + os.path.join(download_path, "chromiumos_test_image.bin") ): return # Uncompress and untar the downloaded image. - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) command = ( "cd %s ; tar -Jxf chromiumos_test_image.tar.xz " % download_path ) @@ -178,7 +183,9 @@ class ImageDownloader(object): ) # Make sure the directory for downloading the package exists. - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id) + ) package_path = os.path.join(download_path, package_file_name) if not os.path.exists(download_path): os.makedirs(download_path) @@ -204,7 +211,9 @@ class ImageDownloader(object): self, chromeos_root, build_id, package_file_name, uncompress_cmd ): # Uncompress file - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id) + ) command = "cd %s ; %s %s" % ( download_path, uncompress_cmd, @@ -252,13 +261,13 @@ class ImageDownloader(object): autotest_server_package_name = "autotest_server_package.tar.bz2" autotest_control_files_name = "control_files.tar" - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id) + ) # Autotest directory relative path wrt chroot autotest_rel_path = os.path.join("/tmp", build_id, "autotest_files") # Absolute Path to download files - autotest_path = os.path.join( - chromeos_root, "chroot/tmp", build_id, "autotest_files" - ) + autotest_path = os.path.join(download_path, "autotest_files") if not os.path.exists(autotest_path): # Quickly verify if the files are present on server @@ -315,12 +324,14 @@ class ImageDownloader(object): # Download autest package files (3 files) debug_archive_name = "debug.tgz" - download_path = os.path.join(chromeos_root, "chroot/tmp", build_id) + download_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id) + ) # Debug directory relative path wrt chroot debug_rel_path = os.path.join("/tmp", build_id, "debug_files") # Debug path to download files - debug_path = os.path.join( - chromeos_root, "chroot/tmp", build_id, "debug_files" + debug_path = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", build_id, "debug_files") ) if not os.path.exists(debug_path): @@ -338,15 +349,11 @@ class ImageDownloader(object): # File exists on server, download and uncompress it self.DownloadSingleFile(chromeos_root, build_id, debug_archive_name) - self.UncompressSingleFile( - chromeos_root, build_id, debug_archive_name, "tar -xf " - ) # Extract and move debug files into the proper location. - debug_dir = "debug_files/usr/lib" - command = "cd %s ; mkdir -p %s; mv debug %s" % ( - download_path, - debug_dir, - debug_dir, + debug_dir = "debug_files/usr/lib/debug" + command = ( + f"cd {shlex.quote(download_path)}; " + f"mkdir -p {shlex.quote(debug_dir)}" ) if self.log_level != "verbose": self._logger.LogOutput("CMD: %s" % command) @@ -357,6 +364,12 @@ class ImageDownloader(object): "Could not create directory %s" % os.path.join(debug_dir, "debug") ) + self.UncompressSingleFile( + chromeos_root, + build_id, + debug_archive_name, + f"tar -C {shlex.quote(debug_dir)} -xf ", + ) return debug_rel_path diff --git a/crosperf/download_images_unittest.py b/crosperf/download_images_unittest.py index 6a640f80..0e47e757 100755 --- a/crosperf/download_images_unittest.py +++ b/crosperf/download_images_unittest.py @@ -8,6 +8,7 @@ import os +import re import unittest import unittest.mock as mock @@ -20,6 +21,16 @@ import test_flag MOCK_LOGGER = logger.GetLogger(log_dir="", mock=True) +class RegexMatcher: + """A regex matcher, for passing to mocks.""" + + def __init__(self, regex): + self._regex = re.compile(regex) + + def __eq__(self, string): + return self._regex.search(string) is not None + + class ImageDownloaderTestcast(unittest.TestCase): """The image downloader test class.""" @@ -34,7 +45,6 @@ class ImageDownloaderTestcast(unittest.TestCase): @mock.patch.object(os, "makedirs") @mock.patch.object(os.path, "exists") def test_download_image(self, mock_path_exists, mock_mkdirs): - # Set mock and test values. mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) test_chroot = "/usr/local/home/chromeos" @@ -59,50 +69,63 @@ class ImageDownloaderTestcast(unittest.TestCase): image_path, ) - # Verify os.path.exists was called twice, with proper arguments. - self.assertEqual(mock_path_exists.call_count, 2) - mock_path_exists.assert_called_with( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/" - "R36-5814.0.0/chromiumos_test_image.bin" + # Verify os.path.exists was called thrice, with proper arguments. + self.assertEqual(mock_path_exists.call_count, 3) + mock_path_exists.assert_any_call( + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/" + "R36-5814.0.0/chromiumos_test_image.bin" + ) ) mock_path_exists.assert_any_call( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0" + ) ) + mock_path_exists.assert_any_call("/etc/cros_chroot_version") # Verify we called os.mkdirs self.assertEqual(mock_mkdirs.call_count, 1) mock_mkdirs.assert_called_with( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0" + ) ) # Verify we called RunCommand once, with proper arguments. self.assertEqual(mock_cmd_exec.RunCommand.call_count, 1) - expected_args = ( + expected_args = RegexMatcher( "/usr/local/home/chromeos/src/chromium/depot_tools/gsutil.py " "cp gs://chromeos-image-archive/lumpy-release/R36-5814.0.0/" "chromiumos_test_image.tar.xz " - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0" + "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0" ) mock_cmd_exec.RunCommand.assert_called_with(expected_args) - # Reset the velues in the mocks; set os.path.exists to always return True. + # Reset the velues in the mocks; set os.path.exists to always return + # True (except for "inside chroot" check). mock_path_exists.reset_mock() mock_cmd_exec.reset_mock() - mock_path_exists.return_value = True + mock_path_exists.side_effect = lambda x: x != "/etc/cros_chroot_version" # Run downloader downloader.DownloadImage(test_chroot, test_build_id, image_path) - # Verify os.path.exists was called twice, with proper arguments. - self.assertEqual(mock_path_exists.call_count, 2) + # Verify os.path.exists was called thrice, with proper arguments. + self.assertEqual(mock_path_exists.call_count, 3) mock_path_exists.assert_called_with( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/" - "R36-5814.0.0/chromiumos_test_image.bin" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/" + "R36-5814.0.0/chromiumos_test_image.bin" + ) ) mock_path_exists.assert_any_call( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0" + ) ) + mock_path_exists.assert_any_call("/etc/cros_chroot_version") # Verify we made no RunCommand or ChrootRunCommand calls (since # os.path.exists returned True, there was no work do be done). @@ -111,7 +134,6 @@ class ImageDownloaderTestcast(unittest.TestCase): @mock.patch.object(os.path, "exists") def test_uncompress_image(self, mock_path_exists): - # set mock and test values. mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) test_chroot = "/usr/local/home/chromeos" @@ -130,12 +152,15 @@ class ImageDownloaderTestcast(unittest.TestCase): test_build_id, ) - # Verify os.path.exists was called once, with correct arguments. - self.assertEqual(mock_path_exists.call_count, 1) + # Verify os.path.exists was called twice, with correct arguments. + self.assertEqual(mock_path_exists.call_count, 2) mock_path_exists.assert_called_with( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/" - "R36-5814.0.0/chromiumos_test_image.bin" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/" + "R36-5814.0.0/chromiumos_test_image.bin" + ) ) + mock_path_exists.assert_any_call("/etc/cros_chroot_version") # Verify RunCommand was called twice with correct arguments. self.assertEqual(mock_cmd_exec.RunCommand.call_count, 2) @@ -143,8 +168,10 @@ class ImageDownloaderTestcast(unittest.TestCase): self.assertEqual(len(mock_cmd_exec.RunCommand.call_args_list[0]), 2) actual_arg = mock_cmd_exec.RunCommand.call_args_list[0][0] expected_arg = ( - "cd /usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0 ; " - "tar -Jxf chromiumos_test_image.tar.xz ", + RegexMatcher( + "cd /usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0 ; " + "tar -Jxf chromiumos_test_image.tar.xz " + ), ) self.assertEqual(expected_arg, actual_arg) # 2nd arg must be exception handler @@ -158,8 +185,10 @@ class ImageDownloaderTestcast(unittest.TestCase): self.assertEqual(len(mock_cmd_exec.RunCommand.call_args_list[1]), 2) actual_arg = mock_cmd_exec.RunCommand.call_args_list[1][0] expected_arg = ( - "cd /usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0 ; " - "rm -f chromiumos_test_image.bin ", + RegexMatcher( + "cd /usr/local/home/chromeos/.*tmp/lumpy-release/R36-5814.0.0 ; " + "rm -f chromiumos_test_image.bin " + ), ) self.assertEqual(expected_arg, actual_arg) # 2nd arg must be empty @@ -167,24 +196,27 @@ class ImageDownloaderTestcast(unittest.TestCase): "{}" in repr(mock_cmd_exec.RunCommand.call_args_list[1][1]) ) - # Set os.path.exists to always return True and run uncompress. + # Set os.path.exists to always return True (except for "inside chroot" + # check) and run uncompress. mock_path_exists.reset_mock() mock_cmd_exec.reset_mock() - mock_path_exists.return_value = True + mock_path_exists.side_effect = lambda x: x != "/etc/cros_chroot_version" downloader.UncompressImage(test_chroot, test_build_id) # Verify os.path.exists was called once, with correct arguments. - self.assertEqual(mock_path_exists.call_count, 1) + self.assertEqual(mock_path_exists.call_count, 2) mock_path_exists.assert_called_with( - "/usr/local/home/chromeos/chroot/tmp/lumpy-release/" - "R36-5814.0.0/chromiumos_test_image.bin" + RegexMatcher( + "/usr/local/home/chromeos/.*tmp/lumpy-release/" + "R36-5814.0.0/chromiumos_test_image.bin" + ) ) + mock_path_exists.assert_any_call("/etc/cros_chroot_version") # Verify RunCommand was not called. self.assertEqual(mock_cmd_exec.RunCommand.call_count, 0) def test_run(self): - # Set test arguments test_chroot = "/usr/local/home/chromeos" test_build_id = "remote/lumpy/latest-dev" diff --git a/crosperf/experiment.py b/crosperf/experiment.py index 9973f7e9..7d35e319 100644 --- a/crosperf/experiment.py +++ b/crosperf/experiment.py @@ -44,6 +44,7 @@ class Experiment(object): ignore_min_max, crosfleet, dut_config, + keep_stateful: bool, no_lock: bool, ): self.name = name @@ -101,7 +102,11 @@ class Experiment(object): if test_flag.GetTestMode(): machine_manager_fn = MockMachineManager self.machine_manager = machine_manager_fn( - chromeos_root, acquire_timeout, log_level, locks_directory + chromeos_root, + acquire_timeout, + log_level, + locks_directory, + keep_stateful=keep_stateful, ) self.l = logger.GetLogger(log_dir) @@ -137,7 +142,6 @@ class Experiment(object): for label in self.labels: for benchmark in self.benchmarks: for iteration in range(1, benchmark.iterations + 1): - benchmark_run_name = "%s: %s (%s)" % ( label.name, benchmark.name, diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py index c71981ab..e89adb87 100644 --- a/crosperf/experiment_factory.py +++ b/crosperf/experiment_factory.py @@ -28,33 +28,8 @@ import config # specified sets. Here we define sets of tests that users may want # to run together. -telemetry_perfv2_tests = [ - "kraken", - "octane", -] - -telemetry_pagecycler_tests = [ - "page_cycler_v2.intl_ar_fa_he", - "page_cycler_v2.intl_es_fr_pt-BR", - "page_cycler_v2.intl_hi_ru", - "page_cycler_v2.intl_ja_zh", - "page_cycler_v2.intl_ko_th_vi", - "page_cycler_v2.typical_25", -] - -telemetry_toolchain_old_perf_tests = [ - "page_cycler_v2.intl_es_fr_pt-BR", - "page_cycler_v2.intl_hi_ru", - "page_cycler_v2.intl_ja_zh", - "page_cycler_v2.intl_ko_th_vi", - "page_cycler_v2.netsim.top_10", - "page_cycler_v2.typical_25", - "spaceport", - "tab_switching.top_10", -] telemetry_toolchain_perf_tests = [ "octane", - "kraken", "speedometer", "speedometer2", "jetstream2", @@ -68,13 +43,10 @@ graphics_perf_tests = [ ] # TODO: disable rendering.desktop by default as the benchmark is # currently in a bad state -# page_cycler_v2.typical_25 is deprecated and the recommend replacement is -# loading.desktop@@typical (crbug.com/916340) telemetry_crosbolt_perf_tests = [ "octane", - "kraken", "speedometer2", - "jetstream", + "jetstream2", "loading.desktop", # 'rendering.desktop', ] @@ -84,12 +56,6 @@ crosbolt_perf_tests = [ "tast.video.PlaybackPerfVP91080P30FPS", ] -# 'cheets_AntutuTest', -# 'cheets_PerfBootServer', -# 'cheets_CandyCrushTest', -# 'cheets_LinpackTest', -# ] - dso_list = [ "all", "chrome", @@ -211,6 +177,7 @@ class ExperimentFactory(object): "turbostat": global_settings.GetField("turbostat"), "top_interval": global_settings.GetField("top_interval"), } + keep_stateful = global_settings.GetField("keep_stateful") # Default cache hit conditions. The image checksum in the cache and the # computed checksum of the image must match. Also a cache file must exist. @@ -220,7 +187,9 @@ class ExperimentFactory(object): ] if global_settings.GetField("rerun_if_failed"): cache_conditions.append(CacheConditions.RUN_SUCCEEDED) - if global_settings.GetField("rerun"): + if global_settings.GetField("rerun") or global_settings.GetField( + "ignore_cache" + ): cache_conditions.append(CacheConditions.FALSE) if global_settings.GetField("same_machine"): cache_conditions.append(CacheConditions.SAME_MACHINE_MATCH) @@ -305,37 +274,7 @@ class ExperimentFactory(object): ) if suite == "telemetry_Crosperf": - if test_name == "all_perfv2": - self.AppendBenchmarkSet( - benchmarks, - telemetry_perfv2_tests, - test_args, - iterations, - rm_chroot_tmp, - perf_args, - suite, - show_all_results, - retries, - run_local, - cwp_dso, - weight, - ) - elif test_name == "all_pagecyclers": - self.AppendBenchmarkSet( - benchmarks, - telemetry_pagecycler_tests, - test_args, - iterations, - rm_chroot_tmp, - perf_args, - suite, - show_all_results, - retries, - run_local, - cwp_dso, - weight, - ) - elif test_name == "all_crosbolt_perf": + if test_name == "all_crosbolt_perf": self.AppendBenchmarkSet( benchmarks, telemetry_crosbolt_perf_tests, @@ -381,21 +320,6 @@ class ExperimentFactory(object): ) # Add non-telemetry toolchain-perf benchmarks: - # Tast test platform.ReportDiskUsage for image size. - benchmarks.append( - Benchmark( - "platform.ReportDiskUsage", - "platform.ReportDiskUsage", - "", - 1, # This is not a performance benchmark, only run once. - rm_chroot_tmp, - "", - "tast", # Specify the suite to be 'tast' - show_all_results, - retries, - ) - ) - # TODO: crbug.com/1057755 Do not enable graphics_WebGLAquarium until # it gets fixed. # @@ -413,21 +337,6 @@ class ExperimentFactory(object): # run_local=False, # cwp_dso=cwp_dso, # weight=weight)) - elif test_name == "all_toolchain_perf_old": - self.AppendBenchmarkSet( - benchmarks, - telemetry_toolchain_old_perf_tests, - test_args, - iterations, - rm_chroot_tmp, - perf_args, - suite, - show_all_results, - retries, - run_local, - cwp_dso, - weight, - ) else: benchmark = Benchmark( benchmark_name, @@ -613,6 +522,7 @@ class ExperimentFactory(object): ignore_min_max, crosfleet, dut_config, + keep_stateful, no_lock=no_lock, ) @@ -623,7 +533,7 @@ class ExperimentFactory(object): os.path.dirname(__file__), "default_remotes" ) try: - with open(default_remotes_file) as f: + with open(default_remotes_file, encoding="utf-8") as f: for line in f: key, v = line.split(":") if key.strip() == board: @@ -632,15 +542,15 @@ class ExperimentFactory(object): return remotes else: raise RuntimeError( - "There is no remote for {0}".format(board) + f"There is no remote for {board}" ) except IOError: # TODO: rethrow instead of throwing different exception. raise RuntimeError( - "IOError while reading file {0}".format(default_remotes_file) + f"IOError while reading file {default_remotes_file}" ) else: - raise RuntimeError("There is no remote for {0}".format(board)) + raise RuntimeError(f"There is no remote for {board}") def CheckRemotesInCrosfleet(self, remote): # TODO: (AI:zhizhouy) need to check whether a remote is a local or lab diff --git a/crosperf/experiment_factory_unittest.py b/crosperf/experiment_factory_unittest.py index 0541bb9b..87e8c4f9 100755 --- a/crosperf/experiment_factory_unittest.py +++ b/crosperf/experiment_factory_unittest.py @@ -20,6 +20,7 @@ from cros_utils.file_utils import FileUtils import experiment_factory from experiment_factory import ExperimentFactory from experiment_file import ExperimentFile +from results_cache import CacheConditions import settings_factory import test_flag @@ -258,7 +259,7 @@ class ExperimentFactoryTest(unittest.TestCase): bench_list = [] ef.AppendBenchmarkSet( bench_list, - experiment_factory.telemetry_perfv2_tests, + experiment_factory.telemetry_crosbolt_perf_tests, "", 1, False, @@ -271,14 +272,15 @@ class ExperimentFactoryTest(unittest.TestCase): 0, ) self.assertEqual( - len(bench_list), len(experiment_factory.telemetry_perfv2_tests) + len(bench_list), + len(experiment_factory.telemetry_crosbolt_perf_tests), ) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) bench_list = [] ef.AppendBenchmarkSet( bench_list, - experiment_factory.telemetry_pagecycler_tests, + experiment_factory.telemetry_toolchain_perf_tests, "", 1, False, @@ -291,7 +293,8 @@ class ExperimentFactoryTest(unittest.TestCase): 0, ) self.assertEqual( - len(bench_list), len(experiment_factory.telemetry_pagecycler_tests) + len(bench_list), + len(experiment_factory.telemetry_toolchain_perf_tests), ) self.assertTrue(isinstance(bench_list[0], benchmark.Benchmark)) @@ -318,7 +321,6 @@ class ExperimentFactoryTest(unittest.TestCase): @mock.patch.object(socket, "gethostname") def test_get_experiment(self, mock_socket): - test_flag.SetTestMode(False) self.append_benchmark_call_args = [] @@ -418,10 +420,13 @@ class ExperimentFactoryTest(unittest.TestCase): ) self.assertEqual(exp.labels[0].autotest_path, "/tmp/autotest") self.assertEqual(exp.labels[0].board, "lumpy") + self.assertEqual(exp.machine_manager.keep_stateful, False) # Second test: Remotes listed in labels. test_flag.SetTestMode(True) label_settings.SetField("remote", "chromeos1.cros chromeos2.cros") + # Also verify keep_stateful. + global_settings.SetField("keep_stateful", "true") exp = ef.GetExperiment(mock_experiment_file, "", "") self.assertCountEqual( exp.remote, @@ -432,6 +437,9 @@ class ExperimentFactoryTest(unittest.TestCase): "chromeos2.cros", ], ) + # keep_stateful is propagated to machine_manager which flashes the + # images. + self.assertEqual(exp.machine_manager.keep_stateful, True) # Third test: Automatic fixing of bad logging_level param: global_settings.SetField("logging_level", "really loud!") @@ -450,7 +458,47 @@ class ExperimentFactoryTest(unittest.TestCase): label_settings.SetField("remote", "") global_settings.SetField("remote", "123.45.67.89") exp = ef.GetExperiment(mock_experiment_file, "", "") - self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1]) + self.assertEqual( + exp.cache_conditions, + [ + CacheConditions.CACHE_FILE_EXISTS, + CacheConditions.CHECKSUMS_MATCH, + CacheConditions.RUN_SUCCEEDED, + CacheConditions.FALSE, + CacheConditions.SAME_MACHINE_MATCH, + CacheConditions.MACHINES_MATCH, + ], + ) + + # Check the alias option to ignore cache. + global_settings.SetField("rerun", "false") + global_settings.SetField("ignore_cache", "true") + exp = ef.GetExperiment(mock_experiment_file, "", "") + self.assertEqual( + exp.cache_conditions, + [ + CacheConditions.CACHE_FILE_EXISTS, + CacheConditions.CHECKSUMS_MATCH, + CacheConditions.RUN_SUCCEEDED, + CacheConditions.FALSE, + CacheConditions.SAME_MACHINE_MATCH, + CacheConditions.MACHINES_MATCH, + ], + ) + # Check without cache use. + global_settings.SetField("rerun", "false") + global_settings.SetField("ignore_cache", "false") + exp = ef.GetExperiment(mock_experiment_file, "", "") + self.assertEqual( + exp.cache_conditions, + [ + CacheConditions.CACHE_FILE_EXISTS, + CacheConditions.CHECKSUMS_MATCH, + CacheConditions.RUN_SUCCEEDED, + CacheConditions.SAME_MACHINE_MATCH, + CacheConditions.MACHINES_MATCH, + ], + ) # Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all # remotes (Call GetDefaultRemotes). diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py index 1f78dcc0..c41459a1 100644 --- a/crosperf/experiment_runner.py +++ b/crosperf/experiment_runner.py @@ -298,7 +298,6 @@ class ExperimentRunner(object): experiment_file_path = os.path.join(results_directory, "experiment.exp") FileUtils().WriteFile(experiment_file_path, experiment.experiment_file) - has_failure = False all_failed = True topstats_file = os.path.join(results_directory, "topstats.log") @@ -306,17 +305,21 @@ class ExperimentRunner(object): "Storing top statistics of each benchmark run into %s." % topstats_file ) + # Track if any iterations for a given benchmark has passed for each + # label. + benchmarks_passes = {} with open(topstats_file, "w") as top_fd: for benchmark_run in experiment.benchmark_runs: + benchmarks_passes.setdefault( + benchmark_run.label.name, + {benchmark_run.benchmark.name: False}, + ) if benchmark_run.result: - # FIXME: Pylint has a bug suggesting the following change, which - # should be fixed in pylint 2.0. Resolve this after pylint >= 2.0. - # Bug: https://github.com/PyCQA/pylint/issues/1984 - # pylint: disable=simplifiable-if-statement - if benchmark_run.result.retval: - has_failure = True - else: + if not benchmark_run.result.retval: all_failed = False + benchmarks_passes[benchmark_run.label.name][ + benchmark_run.benchmark.name + ] = True # Header with benchmark run name. top_fd.write("%s\n" % str(benchmark_run)) # Formatted string with top statistics. @@ -325,6 +328,11 @@ class ExperimentRunner(object): if all_failed: return self.ALL_FAILED + # Set has_passes if atleast one iteration of all benchmarks has passed + # for every label. + has_passes = True + for benchmarks in benchmarks_passes.values(): + has_passes = has_passes and all(benchmarks.values()) self.l.LogOutput("Storing results of each benchmark run.") for benchmark_run in experiment.benchmark_runs: @@ -339,8 +347,10 @@ class ExperimentRunner(object): benchmark_run.result.CompressResultsTo(benchmark_run_path) else: benchmark_run.result.CopyResultsTo(benchmark_run_path) + # Don't remove benchmark tmp if it was a cache hit. benchmark_run.result.CleanUp( benchmark_run.benchmark.rm_chroot_tmp + and not benchmark_run.cache_hit ) self.l.LogOutput("Storing results report in %s." % results_directory) @@ -367,7 +377,7 @@ class ExperimentRunner(object): msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report FileUtils().WriteFile(msg_file_path, msg_body) - return self.SUCCEEDED if not has_failure else self.HAS_FAILURE + return self.SUCCEEDED if has_passes else self.HAS_FAILURE def Run(self): try: diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py index ffb0b5e6..17db64f5 100644 --- a/crosperf/machine_manager.py +++ b/crosperf/machine_manager.py @@ -221,6 +221,7 @@ class MachineManager(object): locks_dir, cmd_exec=None, lgr=None, + keep_stateful: bool = False, ): self._lock = threading.RLock() self._all_machines = [] @@ -233,6 +234,7 @@ class MachineManager(object): self.acquire_timeout = acquire_timeout self.log_level = log_level self.locks_dir = locks_dir + self.keep_stateful = keep_stateful self.ce = cmd_exec or command_executer.GetCommandExecuter( log_level=self.log_level ) @@ -282,14 +284,16 @@ class MachineManager(object): image_chromeos_args = [ image_chromeos.__file__, "--no_lock", - "--chromeos_root=%s" % chromeos_root, - "--image=%s" % label.chromeos_image, - "--image_args=%s" % label.image_args, - "--remote=%s" % machine.name, - "--logging_level=%s" % self.log_level, + f"--chromeos_root={chromeos_root}", + f"--image={label.chromeos_image}", + f"--image_args={label.image_args}", + f"--remote={machine.name}", + f"--logging_level={self.log_level}", ] if label.board: - image_chromeos_args.append("--board=%s" % label.board) + image_chromeos_args.append(f"--board={label.board}") + if self.keep_stateful: + image_chromeos_args.append("--keep_stateful") # Currently can't image two machines at once. # So have to serialized on this lock. @@ -729,9 +733,20 @@ power management: class MockMachineManager(MachineManager): """Mock machine manager class.""" - def __init__(self, chromeos_root, acquire_timeout, log_level, locks_dir): + def __init__( + self, + chromeos_root, + acquire_timeout, + log_level, + locks_dir, + keep_stateful: bool = False, + ): super(MockMachineManager, self).__init__( - chromeos_root, acquire_timeout, log_level, locks_dir + chromeos_root, + acquire_timeout, + log_level, + locks_dir, + keep_stateful=keep_stateful, ) def _TryToLockMachine(self, cros_machine): diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py index 043da990..b08fde48 100644 --- a/crosperf/results_cache.py +++ b/crosperf/results_cache.py @@ -185,13 +185,16 @@ class Result(object): # Otherwise get the base filename and create the correct # path for it. _, f_base = misc.GetRoot(f) - data_filename = os.path.join( - self.chromeos_root, "chroot/tmp", self.temp_dir, f_base + data_filename = misc.GetOutsideChrootPath( + self.chromeos_root, + os.path.join("/tmp", self.temp_dir, f_base), ) if data_filename.find(".json") > 0: raw_dict = dict() if os.path.exists(data_filename): - with open(data_filename, "r") as data_file: + with open( + data_filename, "r", encoding="utf-8" + ) as data_file: raw_dict = json.load(data_file) if "charts" in raw_dict: @@ -220,7 +223,9 @@ class Result(object): units_dict[key] = result_dict["units"] else: if os.path.exists(data_filename): - with open(data_filename, "r") as data_file: + with open( + data_filename, "r", encoding="utf-8" + ) as data_file: lines = data_file.readlines() for line in lines: tmp_dict = json.loads(line) @@ -258,22 +263,24 @@ class Result(object): return results_dict def GetKeyvals(self): - results_in_chroot = os.path.join(self.chromeos_root, "chroot", "tmp") + results_in_chroot = misc.GetOutsideChrootPath( + self.chromeos_root, "/tmp" + ) if not self.temp_dir: self.temp_dir = tempfile.mkdtemp(dir=results_in_chroot) command = f"cp -r {self.results_dir}/* {self.temp_dir}" self.ce.RunCommand(command, print_to_console=False) + tmp_dir_in_chroot = misc.GetInsideChrootPath( + self.chromeos_root, self.temp_dir + ) command = "./generate_test_report --no-color --csv %s" % ( - os.path.join("/tmp", os.path.basename(self.temp_dir)) + tmp_dir_in_chroot ) _, out, _ = self.ce.ChrootRunCommandWOutput( self.chromeos_root, command, print_to_console=False ) keyvals_dict = {} - tmp_dir_in_chroot = misc.GetInsideChrootPath( - self.chromeos_root, self.temp_dir - ) for line in out.splitlines(): tokens = re.split("=|,", line) key = tokens[-2] @@ -297,8 +304,8 @@ class Result(object): chroot_perf_data_file = misc.GetInsideChrootPath( self.chromeos_root, perf_data_file ) - perf_path = os.path.join( - self.chromeos_root, "chroot", "usr/bin/perf" + perf_path = misc.GetOutsideChrootPath( + self.chromeos_root, "/usr/bin/perf" ) perf_file = "/usr/sbin/perf" if os.path.exists(perf_path): @@ -308,20 +315,19 @@ class Result(object): # We specify exact match for known DSO type, and every sample for `all`. exact_match = "" if self.cwp_dso == "all": - exact_match = '""' + exact_match = "" elif self.cwp_dso == "chrome": - exact_match = '" chrome "' + exact_match = "chrome" elif self.cwp_dso == "kallsyms": - exact_match = '"[kernel.kallsyms]"' + exact_match = "[kernel.kallsyms]" else: # This will need to be updated once there are more DSO types supported, # if user want an exact match for the field they want. - exact_match = '"%s"' % self.cwp_dso + exact_match = self.cwp_dso - command = "%s report -n -s dso -i %s 2> /dev/null | grep %s" % ( - perf_file, - chroot_perf_data_file, - exact_match, + command = ( + f"{perf_file} report -n -s dso -i " + f"{chroot_perf_data_file} 2> /dev/null" ) _, result, _ = self.ce.ChrootRunCommandWOutput( self.chromeos_root, command @@ -335,6 +341,8 @@ class Result(object): for line in result.split("\n"): attr = line.split() if len(attr) == 3 and "%" in attr[0]: + if exact_match and exact_match != attr[2]: + continue samples += int(attr[1]) except: raise RuntimeError("Cannot parse perf dso result") @@ -357,11 +365,12 @@ class Result(object): "default_idle", "cpu_idle_loop", "do_idle", + "cpuidle_enter_state", ), } idle_samples = 0 - with open(perf_report_file) as f: + with open(perf_report_file, encoding="utf-8") as f: try: for line in f: line = line.strip() @@ -462,10 +471,7 @@ class Result(object): return self.FindFilesInResultsDir("-name wait_time.log").split("\n")[0] def _CheckDebugPath(self, option, path): - relative_path = path[1:] - out_chroot_path = os.path.join( - self.chromeos_root, "chroot", relative_path - ) + out_chroot_path = misc.GetOutsideChrootPath(self.chromeos_root, path) if os.path.exists(out_chroot_path): if option == "kallsyms": path = os.path.join(path, "System.map-*") @@ -493,8 +499,8 @@ class Result(object): chroot_perf_report_file = misc.GetInsideChrootPath( self.chromeos_root, perf_report_file ) - perf_path = os.path.join( - self.chromeos_root, "chroot", "usr/bin/perf" + perf_path = misc.GetOutsideChrootPath( + self.chromeos_root, "/usr/bin/perf" ) perf_file = "/usr/sbin/perf" @@ -562,7 +568,7 @@ class Result(object): def GatherPerfResults(self): report_id = 0 for perf_report_file in self.perf_report_files: - with open(perf_report_file, "r") as f: + with open(perf_report_file, "r", encoding="utf-8") as f: report_contents = f.read() for group in re.findall( r"Events: (\S+) (\S+)", report_contents @@ -612,7 +618,7 @@ class Result(object): raise IOError("%s does not exist" % filename) keyvals = {} - with open(filename, "r") as f: + with open(filename, "r", encoding="utf-8") as f: raw_dict = json.load(f) if "charts" in raw_dict: raw_dict = raw_dict["charts"] @@ -660,7 +666,7 @@ class Result(object): """ cpustats = {} read_data = "" - with open(self.turbostat_log_file) as f: + with open(self.turbostat_log_file, encoding="utf-8") as f: read_data = f.readlines() if not read_data: @@ -732,7 +738,7 @@ class Result(object): 121 root 20 0 0 0 0 S 1.0 0.0 0:00.45 spi5 """ all_data = "" - with open(self.top_log_file) as f: + with open(self.top_log_file, encoding="utf-8") as f: all_data = f.read() if not all_data: @@ -874,7 +880,7 @@ class Result(object): cpustats = {} read_data = "" - with open(self.cpustats_log_file) as f: + with open(self.cpustats_log_file, encoding="utf-8") as f: read_data = f.readlines() if not read_data: @@ -935,7 +941,7 @@ class Result(object): raise IOError("%s does not exist" % filename) keyvals = {} - with open(filename) as f: + with open(filename, encoding="utf-8") as f: histograms = json.load(f) value_map = {} # Gets generic set values. @@ -1129,7 +1135,7 @@ class Result(object): if self.perf_data_files and self.top_cmds: self.VerifyPerfDataPID() if self.wait_time_log_file: - with open(self.wait_time_log_file) as f: + with open(self.wait_time_log_file, encoding="utf-8") as f: wait_time = f.readline().strip() try: wait_time = float(wait_time) @@ -1162,7 +1168,7 @@ class Result(object): chrome_version = "" keys_file = os.path.join(cache_dir, CACHE_KEYS_FILE) if os.path.exists(keys_file): - with open(keys_file, "r") as f: + with open(keys_file, "r", encoding="utf-8") as f: lines = f.readlines() for l in lines: if l.startswith("Google Chrome "): @@ -1184,7 +1190,7 @@ class Result(object): # Untar the tarball to a temporary directory self.temp_dir = tempfile.mkdtemp( - dir=os.path.join(self.chromeos_root, "chroot", "tmp") + dir=misc.GetOutsideChrootPath(self.chromeos_root, "/tmp") ) command = "cd %s && tar xf %s" % ( @@ -1202,7 +1208,11 @@ class Result(object): self.ProcessResults(use_cache=True) def CleanUp(self, rm_chroot_tmp): - if rm_chroot_tmp and self.results_dir: + if ( + rm_chroot_tmp + and self.results_dir + and self.results_dir != self.temp_dir + ): dirname, basename = misc.GetRoot(self.results_dir) if basename.find("test_that_results_") != -1: command = "rm -rf %s" % self.results_dir @@ -1240,7 +1250,9 @@ class Result(object): pickle.dump(self.retval, f) if not test_flag.GetTestMode(): - with open(os.path.join(temp_dir, CACHE_KEYS_FILE), "w") as f: + with open( + os.path.join(temp_dir, CACHE_KEYS_FILE), "w", encoding="utf-8" + ) as f: f.write("%s\n" % self.label.name) f.write("%s\n" % self.label.chrome_version) f.write("%s\n" % self.machine.checksum_string) @@ -1255,7 +1267,9 @@ class Result(object): # Store machine info. # TODO(asharif): Make machine_manager a singleton, and don't pass it into # this function. - with open(os.path.join(temp_dir, MACHINE_FILE), "w") as f: + with open( + os.path.join(temp_dir, MACHINE_FILE), "w", encoding="utf-8" + ) as f: f.write(machine_manager.machine_checksum_string[self.label.name]) if os.path.exists(cache_dir): diff --git a/crosperf/results_cache_unittest.py b/crosperf/results_cache_unittest.py index cad149e0..06a8b94d 100755 --- a/crosperf/results_cache_unittest.py +++ b/crosperf/results_cache_unittest.py @@ -11,6 +11,7 @@ import io import os import pickle +import re import shutil import tempfile import unittest @@ -449,6 +450,16 @@ class MockResult(Result): return keyvals +class RegexMatcher: + """A regex matcher, for passing to mocks.""" + + def __init__(self, regex): + self._regex = re.compile(regex) + + def __eq__(self, string): + return self._regex.search(string) is not None + + class ResultTest(unittest.TestCase): """Result test class.""" @@ -490,7 +501,9 @@ class ResultTest(unittest.TestCase): None, ) - def testCreateFromRun(self): + @mock.patch.object(os.path, "exists") + def testCreateFromRun(self, mock_path_exists): + mock_path_exists.side_effect = lambda x: x != "/etc/cros_chroot_version" result = MockResult.CreateFromRun( logger.GetLogger(), "average", @@ -508,7 +521,7 @@ class ResultTest(unittest.TestCase): ) self.assertEqual( result.results_dir, - "/tmp/chroot/tmp/test_that.PO1234567/platform_LibCBench", + RegexMatcher("/tmp/.*tmp/test_that.PO1234567/platform_LibCBench"), ) self.assertEqual(result.retval, 0) @@ -517,12 +530,12 @@ class ResultTest(unittest.TestCase): self.mock_logger, self.mock_label, "average", self.mock_cmd_exec ) self.result.chromeos_root = "/tmp/chromeos" + self.orig_exists = os.path.exists @mock.patch.object(os.path, "isdir") @mock.patch.object(command_executer.CommandExecuter, "RunCommand") @mock.patch.object(command_executer.CommandExecuter, "CopyFiles") def test_copy_files_to(self, mock_copyfiles, mock_runcmd, mock_isdir): - files = ["src_file_1", "src_file_2", "src_file_3"] dest_dir = "/tmp/test" self.mock_cmd_exec.RunCommand = mock_runcmd @@ -614,118 +627,118 @@ class ResultTest(unittest.TestCase): self.assertEqual( kv_dict2, { - u"Box2D__Box2D": 4775, - u"Mandreel__Mandreel": 6620, - u"Gameboy__Gameboy": 9901, - u"Crypto__Crypto": 8737, - u"telemetry_page_measurement_results__num_errored": 0, - u"telemetry_page_measurement_results__num_failed": 0, - u"PdfJS__PdfJS": 6455, - u"Total__Score": 7918, - u"EarleyBoyer__EarleyBoyer": 14340, - u"MandreelLatency__MandreelLatency": 5188, - u"CodeLoad__CodeLoad": 6271, - u"DeltaBlue__DeltaBlue": 14401, - u"Typescript__Typescript": 9815, - u"SplayLatency__SplayLatency": 7653, - u"zlib__zlib": 16094, - u"Richards__Richards": 10358, - u"RegExp__RegExp": 1765, - u"NavierStokes__NavierStokes": 9815, - u"Splay__Splay": 4425, - u"RayTrace__RayTrace": 16600, + "Box2D__Box2D": 4775, + "Mandreel__Mandreel": 6620, + "Gameboy__Gameboy": 9901, + "Crypto__Crypto": 8737, + "telemetry_page_measurement_results__num_errored": 0, + "telemetry_page_measurement_results__num_failed": 0, + "PdfJS__PdfJS": 6455, + "Total__Score": 7918, + "EarleyBoyer__EarleyBoyer": 14340, + "MandreelLatency__MandreelLatency": 5188, + "CodeLoad__CodeLoad": 6271, + "DeltaBlue__DeltaBlue": 14401, + "Typescript__Typescript": 9815, + "SplayLatency__SplayLatency": 7653, + "zlib__zlib": 16094, + "Richards__Richards": 10358, + "RegExp__RegExp": 1765, + "NavierStokes__NavierStokes": 9815, + "Splay__Splay": 4425, + "RayTrace__RayTrace": 16600, }, ) self.assertEqual( udict, { - u"Box2D__Box2D": u"score", - u"Mandreel__Mandreel": u"score", - u"Gameboy__Gameboy": u"score", - u"Crypto__Crypto": u"score", - u"telemetry_page_measurement_results__num_errored": u"count", - u"telemetry_page_measurement_results__num_failed": u"count", - u"PdfJS__PdfJS": u"score", - u"Total__Score": u"score", - u"EarleyBoyer__EarleyBoyer": u"score", - u"MandreelLatency__MandreelLatency": u"score", - u"CodeLoad__CodeLoad": u"score", - u"DeltaBlue__DeltaBlue": u"score", - u"Typescript__Typescript": u"score", - u"SplayLatency__SplayLatency": u"score", - u"zlib__zlib": u"score", - u"Richards__Richards": u"score", - u"RegExp__RegExp": u"score", - u"NavierStokes__NavierStokes": u"score", - u"Splay__Splay": u"score", - u"RayTrace__RayTrace": u"score", + "Box2D__Box2D": "score", + "Mandreel__Mandreel": "score", + "Gameboy__Gameboy": "score", + "Crypto__Crypto": "score", + "telemetry_page_measurement_results__num_errored": "count", + "telemetry_page_measurement_results__num_failed": "count", + "PdfJS__PdfJS": "score", + "Total__Score": "score", + "EarleyBoyer__EarleyBoyer": "score", + "MandreelLatency__MandreelLatency": "score", + "CodeLoad__CodeLoad": "score", + "DeltaBlue__DeltaBlue": "score", + "Typescript__Typescript": "score", + "SplayLatency__SplayLatency": "score", + "zlib__zlib": "score", + "Richards__Richards": "score", + "RegExp__RegExp": "score", + "NavierStokes__NavierStokes": "score", + "Splay__Splay": "score", + "RayTrace__RayTrace": "score", }, ) def test_append_telemetry_units(self): kv_dict = { - u"Box2D__Box2D": 4775, - u"Mandreel__Mandreel": 6620, - u"Gameboy__Gameboy": 9901, - u"Crypto__Crypto": 8737, - u"PdfJS__PdfJS": 6455, - u"Total__Score": 7918, - u"EarleyBoyer__EarleyBoyer": 14340, - u"MandreelLatency__MandreelLatency": 5188, - u"CodeLoad__CodeLoad": 6271, - u"DeltaBlue__DeltaBlue": 14401, - u"Typescript__Typescript": 9815, - u"SplayLatency__SplayLatency": 7653, - u"zlib__zlib": 16094, - u"Richards__Richards": 10358, - u"RegExp__RegExp": 1765, - u"NavierStokes__NavierStokes": 9815, - u"Splay__Splay": 4425, - u"RayTrace__RayTrace": 16600, + "Box2D__Box2D": 4775, + "Mandreel__Mandreel": 6620, + "Gameboy__Gameboy": 9901, + "Crypto__Crypto": 8737, + "PdfJS__PdfJS": 6455, + "Total__Score": 7918, + "EarleyBoyer__EarleyBoyer": 14340, + "MandreelLatency__MandreelLatency": 5188, + "CodeLoad__CodeLoad": 6271, + "DeltaBlue__DeltaBlue": 14401, + "Typescript__Typescript": 9815, + "SplayLatency__SplayLatency": 7653, + "zlib__zlib": 16094, + "Richards__Richards": 10358, + "RegExp__RegExp": 1765, + "NavierStokes__NavierStokes": 9815, + "Splay__Splay": 4425, + "RayTrace__RayTrace": 16600, } units_dict = { - u"Box2D__Box2D": u"score", - u"Mandreel__Mandreel": u"score", - u"Gameboy__Gameboy": u"score", - u"Crypto__Crypto": u"score", - u"PdfJS__PdfJS": u"score", - u"Total__Score": u"score", - u"EarleyBoyer__EarleyBoyer": u"score", - u"MandreelLatency__MandreelLatency": u"score", - u"CodeLoad__CodeLoad": u"score", - u"DeltaBlue__DeltaBlue": u"score", - u"Typescript__Typescript": u"score", - u"SplayLatency__SplayLatency": u"score", - u"zlib__zlib": u"score", - u"Richards__Richards": u"score", - u"RegExp__RegExp": u"score", - u"NavierStokes__NavierStokes": u"score", - u"Splay__Splay": u"score", - u"RayTrace__RayTrace": u"score", + "Box2D__Box2D": "score", + "Mandreel__Mandreel": "score", + "Gameboy__Gameboy": "score", + "Crypto__Crypto": "score", + "PdfJS__PdfJS": "score", + "Total__Score": "score", + "EarleyBoyer__EarleyBoyer": "score", + "MandreelLatency__MandreelLatency": "score", + "CodeLoad__CodeLoad": "score", + "DeltaBlue__DeltaBlue": "score", + "Typescript__Typescript": "score", + "SplayLatency__SplayLatency": "score", + "zlib__zlib": "score", + "Richards__Richards": "score", + "RegExp__RegExp": "score", + "NavierStokes__NavierStokes": "score", + "Splay__Splay": "score", + "RayTrace__RayTrace": "score", } results_dict = self.result.AppendTelemetryUnits(kv_dict, units_dict) self.assertEqual( results_dict, { - u"Box2D__Box2D": [4775, u"score"], - u"Splay__Splay": [4425, u"score"], - u"Gameboy__Gameboy": [9901, u"score"], - u"Crypto__Crypto": [8737, u"score"], - u"PdfJS__PdfJS": [6455, u"score"], - u"Total__Score": [7918, u"score"], - u"EarleyBoyer__EarleyBoyer": [14340, u"score"], - u"MandreelLatency__MandreelLatency": [5188, u"score"], - u"DeltaBlue__DeltaBlue": [14401, u"score"], - u"SplayLatency__SplayLatency": [7653, u"score"], - u"Mandreel__Mandreel": [6620, u"score"], - u"Richards__Richards": [10358, u"score"], - u"zlib__zlib": [16094, u"score"], - u"CodeLoad__CodeLoad": [6271, u"score"], - u"Typescript__Typescript": [9815, u"score"], - u"RegExp__RegExp": [1765, u"score"], - u"RayTrace__RayTrace": [16600, u"score"], - u"NavierStokes__NavierStokes": [9815, u"score"], + "Box2D__Box2D": [4775, "score"], + "Splay__Splay": [4425, "score"], + "Gameboy__Gameboy": [9901, "score"], + "Crypto__Crypto": [8737, "score"], + "PdfJS__PdfJS": [6455, "score"], + "Total__Score": [7918, "score"], + "EarleyBoyer__EarleyBoyer": [14340, "score"], + "MandreelLatency__MandreelLatency": [5188, "score"], + "DeltaBlue__DeltaBlue": [14401, "score"], + "SplayLatency__SplayLatency": [7653, "score"], + "Mandreel__Mandreel": [6620, "score"], + "Richards__Richards": [10358, "score"], + "zlib__zlib": [16094, "score"], + "CodeLoad__CodeLoad": [6271, "score"], + "Typescript__Typescript": [9815, "score"], + "RegExp__RegExp": [1765, "score"], + "RayTrace__RayTrace": [16600, "score"], + "NavierStokes__NavierStokes": [9815, "score"], }, ) @@ -738,7 +751,6 @@ class ResultTest(unittest.TestCase): def test_get_keyvals( self, mock_chrootruncmd, mock_runcmd, mock_mkdtemp, mock_getpath ): - self.kv_dict = {} self.callGetNewKeyvals = False @@ -821,9 +833,9 @@ class ResultTest(unittest.TestCase): # Test 3. suite != telemetry_Crosperf. Normally this would be for # running non-Telemetry autotests, such as BootPerfServer. In this test - # case, the keyvals we have set up were returned from a Telemetry test run; - # so this pass is basically testing that we don't append the units to the - # test results (which we do for Telemetry autotest runs). + # case, the keyvals we have set up were returned from a Telemetry test + # run; so this pass is basically testing that we don't append the units + # to the test results (which we do for Telemetry autotest runs). reset() self.result.suite = "" res = self.result.GetKeyvals() @@ -839,10 +851,15 @@ class ResultTest(unittest.TestCase): ): self.result.perf_data_files = ["/tmp/results/perf.data"] self.result.board = "samus" + self.result.cwp_dso = "kallsyms" mock_getpath.return_value = "/usr/chromeos/chroot/tmp/results/perf.data" mock_get_total_samples.return_value = [ "", - "45.42% 237210 chrome ", + ( + "45.42% 53721 chrome \n" + "10.01% 12345 [kernel.kallsyms] \n" + "1.42% 1234 ssh " + ), "", ] mock_exists.return_value = True @@ -855,10 +872,9 @@ class ResultTest(unittest.TestCase): with mock.patch("builtins.open", return_value=io.StringIO(content)): samples = self.result.GetSamples() - self.assertEqual(samples, [237210 - 60, u"samples"]) + self.assertEqual(samples, [12345 - 60, "samples"]) def test_get_results_dir(self): - self.result.out = "" self.assertRaises(Exception, self.result.GetResultsDir) @@ -868,7 +884,6 @@ class ResultTest(unittest.TestCase): @mock.patch.object(command_executer.CommandExecuter, "RunCommandGeneric") def test_find_files_in_results_dir(self, mock_runcmd): - self.result.results_dir = None res = self.result.FindFilesInResultsDir("-name perf.data") self.assertEqual(res, "") @@ -1149,11 +1164,12 @@ class ResultTest(unittest.TestCase): """Normal case when log exists and contains valid data.""" self.result.turbostat_log_file = "/tmp/somelogfile.log" with mock.patch( - "builtins.open", mock.mock_open(read_data=TURBOSTAT_LOG_OUTPUT) + "builtins.open", + mock.mock_open(read_data=TURBOSTAT_LOG_OUTPUT), ) as mo: cpustats = self.result.ProcessTurbostatResults() # Check that the log got opened and data were read/parsed. - calls = [mock.call("/tmp/somelogfile.log")] + calls = [mock.call("/tmp/somelogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(cpustats, TURBOSTAT_DATA) @@ -1162,9 +1178,9 @@ class ResultTest(unittest.TestCase): self.result.turbostat_log_file = "/tmp/emptylogfile.log" with mock.patch("builtins.open", mock.mock_open(read_data="")) as mo: cpustats = self.result.ProcessTurbostatResults() - # Check that the log got opened and parsed successfully and empty data - # returned. - calls = [mock.call("/tmp/emptylogfile.log")] + # Check that the log got opened and parsed successfully and empty + # data returned. + calls = [mock.call("/tmp/emptylogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(cpustats, {}) @@ -1189,11 +1205,12 @@ class ResultTest(unittest.TestCase): """ self.result.cpustats_log_file = "/tmp/somelogfile.log" with mock.patch( - "builtins.open", mock.mock_open(read_data=CPUSTATS_UNIQ_OUTPUT) + "builtins.open", + mock.mock_open(read_data=CPUSTATS_UNIQ_OUTPUT), ) as mo: cpustats = self.result.ProcessCpustatsResults() # Check that the log got opened and data were read/parsed. - calls = [mock.call("/tmp/somelogfile.log")] + calls = [mock.call("/tmp/somelogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(cpustats, CPUSTATS_UNIQ_DATA) @@ -1207,11 +1224,12 @@ class ResultTest(unittest.TestCase): """ self.result.cpustats_log_file = "/tmp/somelogfile.log" with mock.patch( - "builtins.open", mock.mock_open(read_data=CPUSTATS_DUPL_OUTPUT) + "builtins.open", + mock.mock_open(read_data=CPUSTATS_DUPL_OUTPUT), ) as mo: cpustats = self.result.ProcessCpustatsResults() # Check that the log got opened and data were read/parsed. - calls = [mock.call("/tmp/somelogfile.log")] + calls = [mock.call("/tmp/somelogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(cpustats, CPUSTATS_DUPL_DATA) @@ -1220,9 +1238,9 @@ class ResultTest(unittest.TestCase): self.result.cpustats_log_file = "/tmp/emptylogfile.log" with mock.patch("builtins.open", mock.mock_open(read_data="")) as mo: cpustats = self.result.ProcessCpustatsResults() - # Check that the log got opened and parsed successfully and empty data - # returned. - calls = [mock.call("/tmp/emptylogfile.log")] + # Check that the log got opened and parsed successfully and empty + # data returned. + calls = [mock.call("/tmp/emptylogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(cpustats, {}) @@ -1235,7 +1253,7 @@ class ResultTest(unittest.TestCase): ) as mo: topproc = self.result.ProcessTopResults() # Check that the log got opened and data were read/parsed. - calls = [mock.call("/tmp/fakelogfile.log")] + calls = [mock.call("/tmp/fakelogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(topproc, TOP_DATA) @@ -1244,9 +1262,9 @@ class ResultTest(unittest.TestCase): self.result.top_log_file = "/tmp/emptylogfile.log" with mock.patch("builtins.open", mock.mock_open(read_data="")) as mo: topcalls = self.result.ProcessTopResults() - # Check that the log got opened and parsed successfully and empty data - # returned. - calls = [mock.call("/tmp/emptylogfile.log")] + # Check that the log got opened and parsed successfully and empty + # data returned. + calls = [mock.call("/tmp/emptylogfile.log", encoding="utf-8")] mo.assert_has_calls(calls) self.assertEqual(topcalls, []) @@ -1342,10 +1360,18 @@ class ResultTest(unittest.TestCase): ), ) + @mock.patch.object(os.path, "exists") @mock.patch.object(misc, "GetInsideChrootPath") @mock.patch.object(command_executer.CommandExecuter, "ChrootRunCommand") - def test_generate_perf_report_files(self, mock_chrootruncmd, mock_getpath): - fake_file = "/usr/chromeos/chroot/tmp/results/fake_file" + def test_generate_perf_report_files( + self, mock_chrootruncmd, mock_getpath, mock_pathexists + ): + mock_pathexists.side_effect = ( + lambda x: self.orig_exists(x) + if x != "/etc/cros_chroot_version" + else False + ) + fake_file = "/tmp/results/perf.data.report" self.result.perf_data_files = ["/tmp/results/perf.data"] self.result.board = "lumpy" mock_getpath.return_value = fake_file @@ -1354,7 +1380,8 @@ class ResultTest(unittest.TestCase): # Debug path not found self.result.label.debug_path = "" tmp = self.result.GeneratePerfReportFiles() - self.assertEqual(tmp, ["/tmp/chromeos/chroot%s" % fake_file]) + self.assertEqual(len(tmp), 1) + self.assertEqual(tmp[0], RegexMatcher("/tmp/chromeos.*%s" % fake_file)) self.assertEqual( mock_chrootruncmd.call_args_list[0][0], ( @@ -1364,12 +1391,18 @@ class ResultTest(unittest.TestCase): ), ) + @mock.patch.object(os.path, "exists") @mock.patch.object(misc, "GetInsideChrootPath") @mock.patch.object(command_executer.CommandExecuter, "ChrootRunCommand") def test_generate_perf_report_files_debug( - self, mock_chrootruncmd, mock_getpath + self, mock_chrootruncmd, mock_getpath, mock_pathexists ): - fake_file = "/usr/chromeos/chroot/tmp/results/fake_file" + mock_pathexists.side_effect = ( + lambda x: self.orig_exists(x) + if x != "/etc/cros_chroot_version" + else False + ) + fake_file = "/tmp/results/perf.data.report" self.result.perf_data_files = ["/tmp/results/perf.data"] self.result.board = "lumpy" mock_getpath.return_value = fake_file @@ -1378,7 +1411,8 @@ class ResultTest(unittest.TestCase): # Debug path found self.result.label.debug_path = "/tmp/debug" tmp = self.result.GeneratePerfReportFiles() - self.assertEqual(tmp, ["/tmp/chromeos/chroot%s" % fake_file]) + self.assertEqual(len(tmp), 1) + self.assertEqual(tmp[0], RegexMatcher("/tmp/chromeos.*%s" % fake_file)) self.assertEqual( mock_chrootruncmd.call_args_list[0][0], ( @@ -1521,27 +1555,27 @@ class ResultTest(unittest.TestCase): # format self.result.suite = "telemetry_Crosperf" self.result.results_file = [tempfile.mkdtemp() + "/histograms.json"] - with open(self.result.results_file[0], "w") as f: + with open(self.result.results_file[0], "w", encoding="utf-8") as f: f.write(HISTOGRAMSET) self.result.ProcessResults() shutil.rmtree(os.path.dirname(self.result.results_file[0])) # Verify the summary for the story is correct self.assertEqual( self.result.keyvals["timeToFirstContentfulPaint__typical"], - [880.000, u"ms_smallerIsBetter"], + [880.000, "ms_smallerIsBetter"], ) # Veirfy the summary for a certain stroy tag is correct self.assertEqual( self.result.keyvals[ "timeToFirstContentfulPaint__cache_temperature:cold" ], - [1000.000, u"ms_smallerIsBetter"], + [1000.000, "ms_smallerIsBetter"], ) self.assertEqual( self.result.keyvals[ "timeToFirstContentfulPaint__cache_temperature:warm" ], - [800.000, u"ms_smallerIsBetter"], + [800.000, "ms_smallerIsBetter"], ) @mock.patch.object(Result, "ProcessCpustatsResults") @@ -1664,7 +1698,6 @@ class ResultTest(unittest.TestCase): command_executer.CommandExecuter, "ChrootRunCommandWOutput" ) def test_populate_from_cache_dir(self, mock_runchrootcmd, mock_getpath): - # pylint: disable=redefined-builtin def FakeMkdtemp(dir=None): if dir: @@ -1672,7 +1705,7 @@ class ResultTest(unittest.TestCase): return self.tmpdir def FakeGetSamples(): - return [1, u"samples"] + return [1, "samples"] current_path = os.getcwd() cache_dir = os.path.join(current_path, "test_cache/test_input") @@ -1696,46 +1729,46 @@ class ResultTest(unittest.TestCase): self.assertEqual( self.result.keyvals, { - u"Total__Total": [444.0, u"ms"], - u"regexp-dna__regexp-dna": [16.2, u"ms"], - u"telemetry_page_measurement_results__num_failed": [ + "Total__Total": [444.0, "ms"], + "regexp-dna__regexp-dna": [16.2, "ms"], + "telemetry_page_measurement_results__num_failed": [ 0, - u"count", + "count", ], - u"telemetry_page_measurement_results__num_errored": [ + "telemetry_page_measurement_results__num_errored": [ 0, - u"count", + "count", ], - u"string-fasta__string-fasta": [23.2, u"ms"], - u"crypto-sha1__crypto-sha1": [11.6, u"ms"], - u"bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [ + "string-fasta__string-fasta": [23.2, "ms"], + "crypto-sha1__crypto-sha1": [11.6, "ms"], + "bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [ 3.2, - u"ms", + "ms", ], - u"access-nsieve__access-nsieve": [7.9, u"ms"], - u"bitops-nsieve-bits__bitops-nsieve-bits": [9.4, u"ms"], - u"string-validate-input__string-validate-input": [19.3, u"ms"], - u"3d-raytrace__3d-raytrace": [24.7, u"ms"], - u"3d-cube__3d-cube": [28.0, u"ms"], - u"string-unpack-code__string-unpack-code": [46.7, u"ms"], - u"date-format-tofte__date-format-tofte": [26.3, u"ms"], - u"math-partial-sums__math-partial-sums": [22.0, u"ms"], + "access-nsieve__access-nsieve": [7.9, "ms"], + "bitops-nsieve-bits__bitops-nsieve-bits": [9.4, "ms"], + "string-validate-input__string-validate-input": [19.3, "ms"], + "3d-raytrace__3d-raytrace": [24.7, "ms"], + "3d-cube__3d-cube": [28.0, "ms"], + "string-unpack-code__string-unpack-code": [46.7, "ms"], + "date-format-tofte__date-format-tofte": [26.3, "ms"], + "math-partial-sums__math-partial-sums": [22.0, "ms"], "\telemetry_Crosperf": ["PASS", ""], - u"crypto-aes__crypto-aes": [15.2, u"ms"], - u"bitops-bitwise-and__bitops-bitwise-and": [8.4, u"ms"], - u"crypto-md5__crypto-md5": [10.5, u"ms"], - u"string-tagcloud__string-tagcloud": [52.8, u"ms"], - u"access-nbody__access-nbody": [8.5, u"ms"], + "crypto-aes__crypto-aes": [15.2, "ms"], + "bitops-bitwise-and__bitops-bitwise-and": [8.4, "ms"], + "crypto-md5__crypto-md5": [10.5, "ms"], + "string-tagcloud__string-tagcloud": [52.8, "ms"], + "access-nbody__access-nbody": [8.5, "ms"], "retval": 0, - u"math-spectral-norm__math-spectral-norm": [6.6, u"ms"], - u"math-cordic__math-cordic": [8.7, u"ms"], - u"access-binary-trees__access-binary-trees": [4.5, u"ms"], - u"controlflow-recursive__controlflow-recursive": [4.4, u"ms"], - u"access-fannkuch__access-fannkuch": [17.8, u"ms"], - u"string-base64__string-base64": [16.0, u"ms"], - u"date-format-xparb__date-format-xparb": [20.9, u"ms"], - u"3d-morph__3d-morph": [22.1, u"ms"], - u"bitops-bits-in-byte__bitops-bits-in-byte": [9.1, u"ms"], + "math-spectral-norm__math-spectral-norm": [6.6, "ms"], + "math-cordic__math-cordic": [8.7, "ms"], + "access-binary-trees__access-binary-trees": [4.5, "ms"], + "controlflow-recursive__controlflow-recursive": [4.4, "ms"], + "access-fannkuch__access-fannkuch": [17.8, "ms"], + "string-base64__string-base64": [16.0, "ms"], + "date-format-xparb__date-format-xparb": [20.9, "ms"], + "3d-morph__3d-morph": [22.1, "ms"], + "bitops-bits-in-byte__bitops-bits-in-byte": [9.1, "ms"], }, ) @@ -1746,47 +1779,47 @@ class ResultTest(unittest.TestCase): self.assertEqual( self.result.keyvals, { - u"Total__Total": [444.0, u"ms"], - u"regexp-dna__regexp-dna": [16.2, u"ms"], - u"telemetry_page_measurement_results__num_failed": [ + "Total__Total": [444.0, "ms"], + "regexp-dna__regexp-dna": [16.2, "ms"], + "telemetry_page_measurement_results__num_failed": [ 0, - u"count", + "count", ], - u"telemetry_page_measurement_results__num_errored": [ + "telemetry_page_measurement_results__num_errored": [ 0, - u"count", + "count", ], - u"string-fasta__string-fasta": [23.2, u"ms"], - u"crypto-sha1__crypto-sha1": [11.6, u"ms"], - u"bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [ + "string-fasta__string-fasta": [23.2, "ms"], + "crypto-sha1__crypto-sha1": [11.6, "ms"], + "bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte": [ 3.2, - u"ms", + "ms", ], - u"access-nsieve__access-nsieve": [7.9, u"ms"], - u"bitops-nsieve-bits__bitops-nsieve-bits": [9.4, u"ms"], - u"string-validate-input__string-validate-input": [19.3, u"ms"], - u"3d-raytrace__3d-raytrace": [24.7, u"ms"], - u"3d-cube__3d-cube": [28.0, u"ms"], - u"string-unpack-code__string-unpack-code": [46.7, u"ms"], - u"date-format-tofte__date-format-tofte": [26.3, u"ms"], - u"math-partial-sums__math-partial-sums": [22.0, u"ms"], + "access-nsieve__access-nsieve": [7.9, "ms"], + "bitops-nsieve-bits__bitops-nsieve-bits": [9.4, "ms"], + "string-validate-input__string-validate-input": [19.3, "ms"], + "3d-raytrace__3d-raytrace": [24.7, "ms"], + "3d-cube__3d-cube": [28.0, "ms"], + "string-unpack-code__string-unpack-code": [46.7, "ms"], + "date-format-tofte__date-format-tofte": [26.3, "ms"], + "math-partial-sums__math-partial-sums": [22.0, "ms"], "\telemetry_Crosperf": ["PASS", ""], - u"crypto-aes__crypto-aes": [15.2, u"ms"], - u"bitops-bitwise-and__bitops-bitwise-and": [8.4, u"ms"], - u"crypto-md5__crypto-md5": [10.5, u"ms"], - u"string-tagcloud__string-tagcloud": [52.8, u"ms"], - u"access-nbody__access-nbody": [8.5, u"ms"], + "crypto-aes__crypto-aes": [15.2, "ms"], + "bitops-bitwise-and__bitops-bitwise-and": [8.4, "ms"], + "crypto-md5__crypto-md5": [10.5, "ms"], + "string-tagcloud__string-tagcloud": [52.8, "ms"], + "access-nbody__access-nbody": [8.5, "ms"], "retval": 0, - u"math-spectral-norm__math-spectral-norm": [6.6, u"ms"], - u"math-cordic__math-cordic": [8.7, u"ms"], - u"access-binary-trees__access-binary-trees": [4.5, u"ms"], - u"controlflow-recursive__controlflow-recursive": [4.4, u"ms"], - u"access-fannkuch__access-fannkuch": [17.8, u"ms"], - u"string-base64__string-base64": [16.0, u"ms"], - u"date-format-xparb__date-format-xparb": [20.9, u"ms"], - u"3d-morph__3d-morph": [22.1, u"ms"], - u"bitops-bits-in-byte__bitops-bits-in-byte": [9.1, u"ms"], - u"samples": [1, u"samples"], + "math-spectral-norm__math-spectral-norm": [6.6, "ms"], + "math-cordic__math-cordic": [8.7, "ms"], + "access-binary-trees__access-binary-trees": [4.5, "ms"], + "controlflow-recursive__controlflow-recursive": [4.4, "ms"], + "access-fannkuch__access-fannkuch": [17.8, "ms"], + "string-base64__string-base64": [16.0, "ms"], + "date-format-xparb__date-format-xparb": [20.9, "ms"], + "3d-morph__3d-morph": [22.1, "ms"], + "bitops-bits-in-byte__bitops-bits-in-byte": [9.1, "ms"], + "samples": [1, "samples"], }, ) @@ -1798,7 +1831,6 @@ class ResultTest(unittest.TestCase): @mock.patch.object(misc, "GetRoot") @mock.patch.object(command_executer.CommandExecuter, "RunCommand") def test_cleanup(self, mock_runcmd, mock_getroot): - # Test 1. 'rm_chroot_tmp' is True; self.results_dir exists; # self.temp_dir exists; results_dir name contains 'test_that_results_'. mock_getroot.return_value = [ @@ -2031,7 +2063,6 @@ class TelemetryResultTest(unittest.TestCase): self.assertEqual(self.result.retval, 3) def test_populate_from_cache_dir_and_process_results(self): - self.result = TelemetryResult( self.mock_logger, self.mock_label, "average", self.mock_machine ) @@ -2244,7 +2275,6 @@ class ResultsCacheTest(unittest.TestCase): @mock.patch.object(os.path, "isdir") @mock.patch.object(Result, "CreateFromCacheHit") def test_read_result(self, mock_create, mock_isdir, mock_runcmd): - self.fakeCacheReturnResult = None def FakeGetCacheDirForRead(): @@ -2303,7 +2333,8 @@ class ResultsCacheTest(unittest.TestCase): self.assertIsNone(res) # Test 5. os.path.isdir returns true, but mock_create now returns None - # (the call to CreateFromCacheHit returns None), so overal result is None. + # (the call to CreateFromCacheHit returns None), so overal result is + # None. mock_isdir.return_value = True mock_create.return_value = None res = self.results_cache.ReadResult() diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py index 6382bba7..b34f0b16 100644 --- a/crosperf/settings_factory.py +++ b/crosperf/settings_factory.py @@ -206,7 +206,7 @@ class GlobalSettings(Settings): self.AddField( BooleanField( "rerun_if_failed", - description="Whether to re-run failed test runs " "or not.", + description="Whether to re-run failed test runs or not.", default=False, ) ) @@ -235,6 +235,13 @@ class GlobalSettings(Settings): ) self.AddField( BooleanField( + "ignore_cache", + description='Alias of "rerun" to ignore cache.', + default=False, + ) + ) + self.AddField( + BooleanField( "same_specs", default=True, description="Ensure cached runs are run on the " @@ -246,7 +253,7 @@ class GlobalSettings(Settings): BooleanField( "same_machine", default=False, - description="Ensure cached runs are run on the " "same remote.", + description="Ensure cached runs are run on the same remote.", ) ) self.AddField( @@ -552,6 +559,15 @@ class GlobalSettings(Settings): " Useful when lock is held externally, say with crosfleet.", ) ) + self.AddField( + BooleanField( + "keep_stateful", + default=False, + description="When flashing a ChromeOS image keep the stateful" + " partition, i.e. don't use --clobber-stateful. This option" + " is useful to keep ssh keys, wi-fi settings and so on.", + ) + ) class SettingsFactory(object): diff --git a/crosperf/settings_factory_unittest.py b/crosperf/settings_factory_unittest.py index 93d3bd6d..a6771c03 100755 --- a/crosperf/settings_factory_unittest.py +++ b/crosperf/settings_factory_unittest.py @@ -49,7 +49,7 @@ class GlobalSettingsTest(unittest.TestCase): def test_init(self): res = settings_factory.GlobalSettings("g_settings") self.assertIsNotNone(res) - self.assertEqual(len(res.fields), 40) + self.assertEqual(len(res.fields), 42) self.assertEqual(res.GetField("name"), "") self.assertEqual(res.GetField("board"), "") self.assertEqual(res.GetField("crosfleet"), False) @@ -58,6 +58,7 @@ class GlobalSettingsTest(unittest.TestCase): self.assertEqual(res.GetField("rm_chroot_tmp"), False) self.assertEqual(res.GetField("email"), None) self.assertEqual(res.GetField("rerun"), False) + self.assertEqual(res.GetField("ignore_cache"), False) self.assertEqual(res.GetField("same_specs"), True) self.assertEqual(res.GetField("same_machine"), False) self.assertEqual(res.GetField("iterations"), 0) @@ -114,7 +115,7 @@ class SettingsFactoryTest(unittest.TestCase): "global", "global" ) self.assertIsInstance(g_settings, settings_factory.GlobalSettings) - self.assertEqual(len(g_settings.fields), 40) + self.assertEqual(len(g_settings.fields), 42) if __name__ == "__main__": diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py index e777a57f..f5566f51 100644 --- a/crosperf/suite_runner.py +++ b/crosperf/suite_runner.py @@ -17,6 +17,7 @@ import subprocess import time from cros_utils import command_executer +from cros_utils import misc # sshwatcher path, relative to ChromiumOS source root. @@ -152,7 +153,9 @@ class SuiteRunner(object): def RemoveTelemetryTempFile(self, machine, chromeos_root): filename = "telemetry@%s" % machine - fullname = os.path.join(chromeos_root, "chroot", "tmp", filename) + fullname = misc.GetOutsideChrootPath( + chromeos_root, os.path.join("/tmp", filename) + ) if os.path.exists(fullname): os.remove(fullname) @@ -286,7 +289,7 @@ class SuiteRunner(object): def DownloadResult(self, label, task_id): gsutil_cmd = os.path.join(label.chromeos_root, GS_UTIL) result_dir = "gs://chromeos-autotest-results/swarming-%s" % task_id - download_path = os.path.join(label.chromeos_root, "chroot/tmp") + download_path = misc.GetOutsideChrootPath(label.chromeos_root, "/tmp") ls_command = "%s ls %s" % ( gsutil_cmd, os.path.join(result_dir, "autoserv_test"), |