diff options
| author | Aurimas Liutikas <aurimas@google.com> | 2021-08-05 14:07:36 -0700 |
|---|---|---|
| committer | Aurimas Liutikas <aurimas@google.com> | 2021-08-05 14:07:36 -0700 |
| commit | 94c21950965ce4f84e6d31e0f026984278b9eeb4 (patch) | |
| tree | d9328f889c1ae1c532e2d729a1febb8086a290c5 | |
| parent | 7a2249aa7a88fc1c89977bff0c60e34b9409faef (diff) | |
| download | platform-tools-94c21950965ce4f84e6d31e0f026984278b9eeb4.tar.gz | |
Upgrade to platform tools 31.0.3androidx-resourceinspection-releaseandroidx-metrics-release
Downloaded from:
https://dl.google.com/android/repository/platform-tools-latest-darwin.zip
Bug: 190206226
Test: ./gradlew tasks
Change-Id: I707ea7fefc502b1f22644741812c04100525da22
73 files changed, 759 insertions, 236 deletions
@@ -10458,6 +10458,7 @@ Notices for file(s): /bin/property_info_checker /bin/signapk /bin/simg2img +/bin/tzdatacheck /bin/unpack_bootimg /bin/zipalign /framework/BugReport.jar @@ -11127,7 +11128,6 @@ freely, subject to the following restrictions: Notices for file(s): /lib64/libgtest.a /lib64/libgtest_host.a -/lib64/libgtest_prod.a ------------------------------------------------------------ Copyright 2008, Google Inc. All rights reserved. Binary files differdiff --git a/dmtracedump b/dmtracedump Binary files differBinary files differBinary files differBinary files differBinary files differindex de7fe49..87fe253 100755 --- a/dmtracedump +++ b/dmtracedump diff --git a/lib64/libc++.dylib b/lib64/libc++.dylib Binary files differBinary files differindex ad0bc2e..f4864a1 100755 --- a/lib64/libc++.dylib +++ b/lib64/libc++.dylib diff --git a/make_f2fs_casefold b/make_f2fs_casefold Binary files differBinary files differBinary files differindex 3f961ea..645b940 100755 --- a/make_f2fs_casefold +++ b/make_f2fs_casefold diff --git a/source.properties b/source.properties index cdbfd93..ae94420 100644 --- a/source.properties +++ b/source.properties @@ -1,2 +1,2 @@ Pkg.UserSrc=false -Pkg.Revision=31.0.2 +Pkg.Revision=31.0.3 Binary files differdiff --git a/systrace/UPSTREAM_REVISION b/systrace/UPSTREAM_REVISION index ab641bd..7988ceb 100644 --- a/systrace/UPSTREAM_REVISION +++ b/systrace/UPSTREAM_REVISION @@ -1 +1 @@ -91735e2e6775c098eb32840a8903e5a9111fad77 +ab9d330fe2a32f84b4b5fe141958c0a0a857c5c9 diff --git a/systrace/catapult/common/py_utils/py_utils/ts_proxy_server.py b/systrace/catapult/common/py_utils/py_utils/ts_proxy_server.py index 0e168a5..ffed090 100644 --- a/systrace/catapult/common/py_utils/py_utils/ts_proxy_server.py +++ b/systrace/catapult/common/py_utils/py_utils/ts_proxy_server.py @@ -194,6 +194,7 @@ class TsProxyServer(object): if not self._proc: return try: + self._IssueCommand('exit', timeout=10) py_utils.WaitFor(lambda: self._proc.poll() is not None, 10) except py_utils.TimeoutException: # signal.SIGINT is not supported on Windows. diff --git a/systrace/catapult/dependency_manager/dependency_manager/__init__.py b/systrace/catapult/dependency_manager/dependency_manager/__init__.py index 3b18f06..4b595c5 100644 --- a/systrace/catapult/dependency_manager/dependency_manager/__init__.py +++ b/systrace/catapult/dependency_manager/dependency_manager/__init__.py @@ -20,8 +20,8 @@ def _AddDirToPythonPath(*path_parts): _AddDirToPythonPath(CATAPULT_PATH, 'common', 'py_utils') _AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'mock') +_AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'six') _AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'pyfakefs') -_AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'zipfile') _AddDirToPythonPath(DEPENDENCY_MANAGER_PATH) diff --git a/systrace/catapult/dependency_manager/dependency_manager/dependency_manager_util.py b/systrace/catapult/dependency_manager/dependency_manager/dependency_manager_util.py index ca0174e..a8e21b8 100644 --- a/systrace/catapult/dependency_manager/dependency_manager/dependency_manager_util.py +++ b/systrace/catapult/dependency_manager/dependency_manager/dependency_manager_util.py @@ -7,7 +7,9 @@ import shutil import stat import subprocess import sys -import zipfile_2_7_13 as zipfile +import zipfile + +import six from dependency_manager import exceptions @@ -17,7 +19,7 @@ def _WinReadOnlyHandler(func, path, execinfo): os.chmod(path, stat.S_IWRITE) func(path) else: - raise execinfo[0], execinfo[1], execinfo[2] + six.reraise(*execinfo) def RemoveDir(dir_path): diff --git a/systrace/catapult/dependency_manager/dependency_manager/local_path_info.py b/systrace/catapult/dependency_manager/dependency_manager/local_path_info.py index 8ac0152..5600966 100644 --- a/systrace/catapult/dependency_manager/dependency_manager/local_path_info.py +++ b/systrace/catapult/dependency_manager/dependency_manager/local_path_info.py @@ -4,6 +4,7 @@ import os +import six class LocalPathInfo(object): @@ -66,4 +67,4 @@ class LocalPathInfo(object): def _ParseLocalPaths(local_paths): if not local_paths: return [] - return [[e] if isinstance(e, basestring) else e for e in local_paths] + return [[e] if isinstance(e, six.string_types) else e for e in local_paths] diff --git a/systrace/catapult/devil/DIR_METADATA b/systrace/catapult/devil/DIR_METADATA new file mode 100644 index 0000000..7608b2f --- /dev/null +++ b/systrace/catapult/devil/DIR_METADATA @@ -0,0 +1,3 @@ +monorail { + component: "Test>Devil" +} diff --git a/systrace/catapult/devil/PRESUBMIT.py b/systrace/catapult/devil/PRESUBMIT.py index 0b49eb8..ec48ddd 100644 --- a/systrace/catapult/devil/PRESUBMIT.py +++ b/systrace/catapult/devil/PRESUBMIT.py @@ -30,14 +30,23 @@ def _RunUnitTests(input_api, output_api): output_api.PresubmitPromptWarning) return input_api.RunTests([ - input_api.Command( - name='devil/bin/run_py_tests', - cmd=[ - input_api.os_path.join(input_api.PresubmitLocalPath(), 'bin', - 'run_py_tests') - ], - kwargs={'env': test_env}, - message=message_type) + input_api.Command(name='devil/bin/run_py_tests', + cmd=[ + input_api.os_path.join( + input_api.PresubmitLocalPath(), 'bin', + 'run_py_tests') + ], + kwargs={'env': test_env}, + message=message_type), + input_api.Command(name='devil/bin/run_py3_tests', + cmd=[ + input_api.os_path.join( + input_api.PresubmitLocalPath(), 'bin', + 'run_py3_tests') + ], + kwargs={'env': test_env}, + message=message_type, + python3=True), ]) diff --git a/systrace/catapult/devil/bin/generate_md_docs b/systrace/catapult/devil/bin/generate_md_docs index d1dbf06..4c6f0f9 100755 --- a/systrace/catapult/devil/bin/generate_md_docs +++ b/systrace/catapult/devil/bin/generate_md_docs @@ -8,7 +8,7 @@ import os import sys _DEVIL_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -_DEVIL_URL = ('https://github.com/catapult-project/catapult/blob/master/devil/') +_DEVIL_URL = ('https://chromium.googlesource.com/catapult.git/+/HEAD/devil/') sys.path.append(_DEVIL_PATH) from devil.utils import cmd_helper diff --git a/systrace/catapult/devil/bin/run_py3_tests b/systrace/catapult/devil/bin/run_py3_tests new file mode 100755 index 0000000..3250ff1 --- /dev/null +++ b/systrace/catapult/devil/bin/run_py3_tests @@ -0,0 +1,113 @@ +#!/usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Runs all python3-compatible tests in devil.""" + +import os +import sys +import unittest + +_DEVIL_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +_SIX_PATH = os.path.join(_DEVIL_PATH, '..', 'third_party', 'six') + +sys.path.append(_DEVIL_PATH) +sys.path.append(_SIX_PATH) +# Import compatible tests here by full module path +import devil.android.apk_helper_test +import devil.android.app_ui_test +import devil.android.battery_utils_test +import devil.android.cpu_temperature_test +import devil.android.decorators_test +import devil.android.device_denylist_test +import devil.android.device_errors_test +import devil.android.device_utils_test +import devil.android.fastboot_utils_test +import devil.android.flag_changer_test +import devil.android.logcat_monitor_test +import devil.android.md5sum_test +import devil.android.perf.perf_control_test +import devil.android.perf.surface_stats_collector_test +import devil.android.sdk.adb_wrapper_test +import devil.android.sdk.shared_prefs_test +import devil.android.tools.device_monitor_test +import devil.android.tools.script_common_test +import devil.android.tools.system_app_test +import devil.devil_env_test +import devil.utils.cmd_helper_test +import devil.utils.decorators_test +import devil.utils.find_usb_devices_test +import devil.utils.geometry_test +import devil.utils.lazy.weak_constant_test +import devil.utils.lsusb_test +import devil.utils.markdown_test +import devil.utils.mock_calls_test +import devil.utils.parallelizer_test +import devil.utils.reraiser_thread_unittest +import devil.utils.timeout_retry_unittest +import devil.utils.zip_utils_test + +PY3_COMPATIBLE_TESTS = [ + # Add full test module path here + devil.android.apk_helper_test, + devil.android.app_ui_test, + devil.android.battery_utils_test, + devil.android.cpu_temperature_test, + devil.android.decorators_test, + devil.android.device_denylist_test, + devil.android.device_errors_test, + devil.android.device_utils_test, + devil.android.fastboot_utils_test, + devil.android.flag_changer_test, + devil.android.logcat_monitor_test, + devil.android.md5sum_test, + devil.android.perf.perf_control_test, + devil.android.perf.surface_stats_collector_test, + devil.android.sdk.adb_wrapper_test, + devil.android.sdk.shared_prefs_test, + devil.android.tools.device_monitor_test, + devil.android.tools.script_common_test, + devil.android.tools.system_app_test, + devil.devil_env_test, + devil.utils.cmd_helper_test, + devil.utils.decorators_test, + devil.utils.find_usb_devices_test, + devil.utils.geometry_test, + devil.utils.lazy.weak_constant_test, + devil.utils.lsusb_test, + devil.utils.markdown_test, + devil.utils.mock_calls_test, + devil.utils.parallelizer_test, + devil.utils.reraiser_thread_unittest, + devil.utils.timeout_retry_unittest, + devil.utils.zip_utils_test, +] + + +def main(): + # TODO(crbug.com/1007101): Use six.PY2 directly once we're using six via + # vpython. + if sys.version_info[0] == 2: + print('Somehow running under python2.') + return 1 + + # Tests mock out internal details of methods, and the ANDROID_SERIAL can + # change which internal methods are called. Since tests don't actually use + # devices, it should be fine to delete the variable. + if 'ANDROID_SERIAL' in os.environ: + del os.environ['ANDROID_SERIAL'] + + # This does not use typ for now, as typ has vpython dependencies that haven't + # yet been updated for python3. + result = unittest.TextTestRunner().run(unittest.TestSuite( + unittest.defaultTestLoader.loadTestsFromModule(test_module) + for test_module in PY3_COMPATIBLE_TESTS + )) + + return 0 if result.wasSuccessful() else 1 + + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/systrace/catapult/devil/devil/android/apk_helper.py b/systrace/catapult/devil/devil/android/apk_helper.py index fdece07..4d723a5 100644 --- a/systrace/catapult/devil/devil/android/apk_helper.py +++ b/systrace/catapult/devil/devil/android/apk_helper.py @@ -11,6 +11,8 @@ import shutil import tempfile import zipfile +import six + from devil import base_error from devil.android.ndk import abis from devil.android.sdk import aapt @@ -69,7 +71,7 @@ def GetInstrumentationName(apk_path): def ToHelper(path_or_helper): """Creates an ApkHelper unless one is already given.""" - if not isinstance(path_or_helper, basestring): + if not isinstance(path_or_helper, six.string_types): return path_or_helper elif path_or_helper.endswith('.apk'): return ApkHelper(path_or_helper) @@ -86,7 +88,7 @@ def ToSplitHelper(path_or_helper, split_apks): if sorted(path_or_helper.split_apk_paths) != sorted(split_apks): raise ApkHelperError('Helper has different split APKs') return path_or_helper - elif (isinstance(path_or_helper, basestring) + elif (isinstance(path_or_helper, six.string_types) and path_or_helper.endswith('.apk')): return SplitApkHelper(path_or_helper, split_apks) diff --git a/systrace/catapult/devil/devil/android/app_ui.py b/systrace/catapult/devil/devil/android/app_ui.py index 399c2ee..4f7af1d 100644 --- a/systrace/catapult/devil/devil/android/app_ui.py +++ b/systrace/catapult/devil/devil/android/app_ui.py @@ -51,7 +51,7 @@ class _UiNode(object): A geometry.Rectangle instance. """ d = _RE_BOUNDS.match(self._GetAttribute('bounds')).groupdict() - return geometry.Rectangle.FromDict({k: int(v) for k, v in d.iteritems()}) + return geometry.Rectangle.FromDict({k: int(v) for k, v in d.items()}) def Tap(self, point=None, dp_units=False): """Send a tap event to the UI node. @@ -150,7 +150,7 @@ class _UiNode(object): and ':id/' not in resource_id): kwargs['resource_id'] = '%s:id/%s' % (self._package, resource_id) - criteria = [(k.replace('_', '-'), v) for k, v in kwargs.iteritems() + criteria = [(k.replace('_', '-'), v) for k, v in kwargs.items() if v is not None] if not criteria: raise TypeError('At least one search criteria should be specified') @@ -193,8 +193,12 @@ class AppUi(object): A UI node instance pointing to the root of the xml screenshot. """ with device_temp_file.DeviceTempFile(self._device.adb) as dtemp: - self._device.RunShellCommand(['uiautomator', 'dump', dtemp.name], - check_return=True) + output = self._device.RunShellCommand( + ['uiautomator', 'dump', dtemp.name], single_line=True, + check_return=True) + if output.startswith('ERROR:'): + raise RuntimeError( + 'uiautomator dump command returned error: {}'.format(output)) xml_node = element_tree.fromstring( self._device.ReadFile(dtemp.name, force_pull=True)) return _UiNode(self._device, xml_node, package=self._package) diff --git a/systrace/catapult/devil/devil/android/app_ui_test.py b/systrace/catapult/devil/devil/android/app_ui_test.py index 938fd40..50f00ca 100644 --- a/systrace/catapult/devil/devil/android/app_ui_test.py +++ b/systrace/catapult/devil/devil/android/app_ui_test.py @@ -80,7 +80,7 @@ class UiAppTest(unittest.TestCase): def assertNodeHasAttribs(self, node, attr): # pylint: disable=protected-access - for key, value in attr.iteritems(): + for key, value in attr.items(): self.assertEquals(node._GetAttribute(key), value) def assertTappedOnceAt(self, x, y): diff --git a/systrace/catapult/devil/devil/android/battery_utils.py b/systrace/catapult/devil/devil/android/battery_utils.py index e8134d2..d680f03 100644 --- a/systrace/catapult/devil/devil/android/battery_utils.py +++ b/systrace/catapult/devil/devil/android/battery_utils.py @@ -300,7 +300,7 @@ class BatteryUtils(object): 'uid': uid, 'data': pwi_entries[uid] } - for p, uid in self._cache['uids'].iteritems() + for p, uid in self._cache['uids'].items() } return {'system_total': system_total, 'per_package': per_package} diff --git a/systrace/catapult/devil/devil/android/cpu_temperature_test.py b/systrace/catapult/devil/devil/android/cpu_temperature_test.py index 8d082bb..47cc28a 100644 --- a/systrace/catapult/devil/devil/android/cpu_temperature_test.py +++ b/systrace/catapult/devil/devil/android/cpu_temperature_test.py @@ -69,9 +69,11 @@ class CpuTemperatureGetThermalDeviceInformationTest(CpuTemperatureTest): 'cpu6': '/sys/class/thermal/thermal_zone17/temp', 'cpu7': '/sys/class/thermal/thermal_zone18/temp' } - self.assertEqual( - cmp(correct_information, - self.cpu_temp.GetDeviceInfoForTesting().get('cpu_temps')), 0) + + self.assertDictEqual( + correct_information, + self.cpu_temp.GetDeviceInfoForTesting().get('cpu_temps') + ) class CpuTemperatureIsSupportedTest(CpuTemperatureTest): diff --git a/systrace/catapult/devil/devil/android/decorators.py b/systrace/catapult/devil/devil/android/decorators.py index 0b3778a..11d2494 100644 --- a/systrace/catapult/devil/devil/android/decorators.py +++ b/systrace/catapult/devil/devil/android/decorators.py @@ -9,6 +9,8 @@ import functools import itertools import sys +import six + from devil.android import device_errors from devil.utils import cmd_helper from devil.utils import reraiser_thread @@ -56,14 +58,19 @@ def _TimeoutRetryWrapper(f, desc = '%s(%s)' % (f.__name__, ', '.join( itertools.chain( (str(a) for a in args), - ('%s=%s' % (k, str(v)) for k, v in kwargs.iteritems())))) + ('%s=%s' % (k, str(v)) for k, v in six.iteritems(kwargs))))) return timeout_retry.Run( impl, timeout, retries, desc=desc, retry_if_func=retry_if_func) except reraiser_thread.TimeoutError as e: - raise device_errors.CommandTimeoutError(str(e)), None, (sys.exc_info()[2]) + six.reraise( + device_errors.CommandTimeoutError, + device_errors.CommandTimeoutError(str(e)), + sys.exc_info()[2]) except cmd_helper.TimeoutError as e: - raise device_errors.CommandTimeoutError( - str(e), output=e.output), None, (sys.exc_info()[2]) + six.reraise( + device_errors.CommandTimeoutError, + device_errors.CommandTimeoutError(str(e), output=e.output), + sys.exc_info()[2]) return timeout_retry_wrapper diff --git a/systrace/catapult/devil/devil/android/device_errors.py b/systrace/catapult/devil/devil/android/device_errors.py index 6e71087..75bf7e3 100644 --- a/systrace/catapult/devil/devil/android/device_errors.py +++ b/systrace/catapult/devil/devil/android/device_errors.py @@ -23,6 +23,8 @@ The class hierarchy for device exceptions is: """ +import six + from devil import base_error from devil.utils import cmd_helper from devil.utils import parallelizer @@ -158,7 +160,7 @@ class AdbShellCommandFailedError(AdbCommandFailedError): segments.append(' exit status: %s\n' % status) if output: segments.append(' output:\n') - if isinstance(output, basestring): + if isinstance(output, six.string_types): output_lines = output.splitlines() else: output_lines = output diff --git a/systrace/catapult/devil/devil/android/device_list.py b/systrace/catapult/devil/devil/android/device_list.py index cd631db..5fb586f 100644 --- a/systrace/catapult/devil/devil/android/device_list.py +++ b/systrace/catapult/devil/devil/android/device_list.py @@ -7,6 +7,8 @@ import json import logging import os +import six + logger = logging.getLogger(__name__) @@ -26,7 +28,7 @@ def GetPersistentDeviceList(file_name): with open(file_name) as f: devices = json.load(f) if not isinstance(devices, list) or not all( - isinstance(d, basestring) for d in devices): + isinstance(d, six.string_types) for d in devices): logger.warning('Unrecognized device file format: %s', devices) return [] return [d for d in devices if d != '(error)'] diff --git a/systrace/catapult/devil/devil/android/device_utils.py b/systrace/catapult/devil/devil/android/device_utils.py index 7b7dad2..093bfc7 100644 --- a/systrace/catapult/devil/devil/android/device_utils.py +++ b/systrace/catapult/devil/devil/android/device_utils.py @@ -24,6 +24,8 @@ import time import threading import uuid +import six + from devil import base_error from devil import devil_env from devil.utils import cmd_helper @@ -105,6 +107,13 @@ _RESTART_ADBD_SCRIPT = """ restart & """ +_UNZIP_AND_CHMOD_SCRIPT = """ + {bin_dir}/unzip {zip_file} && (for dir in {dirs} + do + chmod -R 777 "$dir" || exit 1 + done) +""" + # Not all permissions can be set. _PERMISSIONS_DENYLIST_RE = re.compile('|'.join( fnmatch.translate(p) for p in [ @@ -131,6 +140,7 @@ _PERMISSIONS_DENYLIST_RE = re.compile('|'.join( 'android.permission.INTERNET', 'android.permission.KILL_BACKGROUND_PROCESSES', 'android.permission.MANAGE_ACCOUNTS', + 'android.permission.MANAGE_EXTERNAL_STORAGE', 'android.permission.MODIFY_AUDIO_SETTINGS', 'android.permission.NFC', 'android.permission.QUERY_ALL_PACKAGES', @@ -314,30 +324,6 @@ def GetAVDs(): return avds -@decorators.WithExplicitTimeoutAndRetries(_DEFAULT_TIMEOUT, _DEFAULT_RETRIES) -def RestartServer(): - """Restarts the adb server. - - Raises: - CommandFailedError if we fail to kill or restart the server. - """ - - def adb_killed(): - return not adb_wrapper.AdbWrapper.IsServerOnline() - - def adb_started(): - return adb_wrapper.AdbWrapper.IsServerOnline() - - adb_wrapper.AdbWrapper.KillServer() - if not timeout_retry.WaitFor(adb_killed, wait_period=1, max_tries=5): - # TODO(crbug.com/442319): Switch this to raise an exception if we - # figure out why sometimes not all adb servers on bots get killed. - logger.warning('Failed to kill adb server') - adb_wrapper.AdbWrapper.StartServer() - if not timeout_retry.WaitFor(adb_started, wait_period=1, max_tries=5): - raise device_errors.CommandFailedError('Failed to start adb server') - - def _ParseModeString(mode_str): """Parse a mode string, e.g. 'drwxrwxrwx', into a st_mode value. @@ -376,7 +362,7 @@ def _CreateAdbWrapper(device): def _FormatPartialOutputError(output): - lines = output.splitlines() if isinstance(output, basestring) else output + lines = output.splitlines() if isinstance(output, six.string_types) else output message = ['Partial output found:'] if len(lines) > 11: message.extend('- %s' % line for line in lines[:5]) @@ -468,7 +454,7 @@ class DeviceUtils(object): operation should be retried on failure if no explicit value is provided. """ self.adb = None - if isinstance(device, basestring): + if isinstance(device, six.string_types): self.adb = _CreateAdbWrapper(device) elif isinstance(device, adb_wrapper.AdbWrapper): self.adb = device @@ -1533,7 +1519,7 @@ class DeviceUtils(object): else: raise - if isinstance(cmd, basestring): + if isinstance(cmd, six.string_types): if not shell: # TODO(crbug.com/1029769): Make this an error instead. logger.warning( @@ -1543,7 +1529,7 @@ class DeviceUtils(object): else: cmd = ' '.join(cmd_helper.SingleQuote(s) for s in cmd) if env: - env = ' '.join(env_quote(k, v) for k, v in env.iteritems()) + env = ' '.join(env_quote(k, v) for k, v in env.items()) cmd = '%s %s' % (env, cmd) if cwd: cmd = 'cd %s && %s' % (cmd_helper.SingleQuote(cwd), cmd) @@ -1740,7 +1726,7 @@ class DeviceUtils(object): cmd.append('-w') if raw: cmd.append('-r') - for k, v in extras.iteritems(): + for k, v in extras.items(): cmd.extend(['-e', str(k), str(v)]) cmd.append(component) @@ -2223,13 +2209,19 @@ class DeviceUtils(object): self.adb, suffix='.zip') as device_temp: self.adb.Push(zip_path, device_temp.name) - quoted_dirs = ' '.join(cmd_helper.SingleQuote(d) for d in dirs) - self.RunShellCommand( - 'unzip %s&&chmod -R 777 %s' % (device_temp.name, quoted_dirs), - shell=True, - as_root=True, - env={'PATH': '%s:$PATH' % install_commands.BIN_DIR}, - check_return=True) + with device_temp_file.DeviceTempFile(self.adb, suffix='.sh') as script: + # Read dirs from temp file to avoid potential errors like + # "Argument list too long" (crbug.com/1174331) when the list + # is too long. + self.WriteFile( + script.name, + _UNZIP_AND_CHMOD_SCRIPT.format(bin_dir=install_commands.BIN_DIR, + zip_file=device_temp.name, + dirs=' '.join(dirs))) + + self.RunShellCommand(['source', script.name], + check_return=True, + as_root=True) return True @@ -2263,7 +2255,7 @@ class DeviceUtils(object): DeviceUnreachableError on missing device. """ paths = device_paths - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): paths = (paths, ) if not paths: return True @@ -2322,7 +2314,7 @@ class DeviceUtils(object): args.append('-f') if recursive: args.append('-r') - if isinstance(device_path, basestring): + if isinstance(device_path, six.string_types): args.append(device_path if not rename else _RenamePath(device_path)) else: args.extend( @@ -2596,7 +2588,7 @@ class DeviceUtils(object): """ entries = self._ParseLongLsOutput(device_path, as_root=as_root, **kwargs) for d in entries: - for key, value in d.items(): + for key, value in list(d.items()): if value is None: del d[key] # Remove missing fields. d['st_mode'] = _ParseModeString(d['st_mode']) @@ -2960,7 +2952,7 @@ class DeviceUtils(object): """ assert isinstance( property_name, - basestring), ("property_name is not a string: %r" % property_name) + six.string_types), ("property_name is not a string: %r" % property_name) if cache: # It takes ~120ms to query a single property, and ~130ms to query all @@ -3004,8 +2996,8 @@ class DeviceUtils(object): """ assert isinstance( property_name, - basestring), ("property_name is not a string: %r" % property_name) - assert isinstance(value, basestring), "value is not a string: %r" % value + six.string_types), ("property_name is not a string: %r" % property_name) + assert isinstance(value, six.string_types), "value is not a string: %r" % value self.RunShellCommand(['setprop', property_name, value], check_return=True) prop_cache = self._cache['getprop'] @@ -3084,18 +3076,19 @@ class DeviceUtils(object): Returns: A list of ProcessInfo tuples with |name|, |pid|, and |ppid| fields. """ + # pylint: disable=broad-except process_name = process_name or '' processes = [] for line in self._GetPsOutput(process_name): row = line.split() try: - row = {k: row[i] for k, i in _PS_COLUMNS.iteritems()} + row = {k: row[i] for k, i in _PS_COLUMNS.items()} if row['pid'] == 'PID' or process_name not in row['name']: # Skip over header and non-matching processes. continue row['pid'] = int(row['pid']) row['ppid'] = int(row['ppid']) - except StandardError: # e.g. IndexError, TypeError, ValueError. + except Exception: # e.g. IndexError, TypeError, ValueError. logging.warning('failed to parse ps line: %r', line) continue processes.append(ProcessInfo(**row)) @@ -3550,10 +3543,10 @@ class DeviceUtils(object): # When using a cache across script invokations, verify that apps have # not been uninstalled. self._cache['package_apk_paths_to_verify'] = set( - self._cache['package_apk_paths'].iterkeys()) + self._cache['package_apk_paths']) package_apk_checksums = obj.get('package_apk_checksums', {}) - for k, v in package_apk_checksums.iteritems(): + for k, v in package_apk_checksums.items(): package_apk_checksums[k] = set(v) self._cache['package_apk_checksums'] = package_apk_checksums device_path_checksums = obj.get('device_path_checksums', {}) @@ -3577,27 +3570,27 @@ class DeviceUtils(object): obj['package_apk_paths'] = self._cache['package_apk_paths'] obj['package_apk_checksums'] = self._cache['package_apk_checksums'] # JSON can't handle sets. - for k, v in obj['package_apk_checksums'].iteritems(): + for k, v in obj['package_apk_checksums'].items(): obj['package_apk_checksums'][k] = list(v) obj['device_path_checksums'] = self._cache['device_path_checksums'] return json.dumps(obj, separators=(',', ':')) @classmethod - def parallel(cls, devices, async=False): + def parallel(cls, devices, asyn=False): """Creates a Parallelizer to operate over the provided list of devices. Args: devices: A list of either DeviceUtils instances or objects from from which DeviceUtils instances can be constructed. If None, all attached devices will be used. - async: If true, returns a Parallelizer that runs operations + asyn: If true, returns a Parallelizer that runs operations asynchronously. Returns: A Parallelizer operating over |devices|. """ devices = [d if isinstance(d, cls) else cls(d) for d in devices] - if async: + if asyn: return parallelizer.Parallelizer(devices) else: return parallelizer.SyncParallelizer(devices) @@ -3710,7 +3703,7 @@ class DeviceUtils(object): else: reset_usb.reset_all_android_devices() - for attempt in xrange(retries + 1): + for attempt in range(retries + 1): try: return _get_devices() except device_errors.NoDevicesError: @@ -3728,7 +3721,7 @@ class DeviceUtils(object): 'No devices found. Will try again after restarting adb server ' 'and a short nap of %d s.', sleep_s) time.sleep(sleep_s) - RestartServer() + adb_wrapper.RestartServer() @decorators.WithTimeoutAndRetriesFromInstance() def RestartAdbd(self, timeout=None, retries=None): @@ -3745,6 +3738,17 @@ class DeviceUtils(object): if not permissions: return + # For Andorid-11(R), enable MANAGE_EXTERNAL_STORAGE for testing. + # See https://bit.ly/2MBjBIM for details. + if ('android.permission.MANAGE_EXTERNAL_STORAGE' in permissions + and self.build_version_sdk >= version_codes.R): + script_manage_ext_storage = [ + 'appops set {package} MANAGE_EXTERNAL_STORAGE allow', + 'echo "{sep}MANAGE_EXTERNAL_STORAGE{sep}$?{sep}"', + ] + else: + script_manage_ext_storage = [] + permissions = set(p for p in permissions if not _PERMISSIONS_DENYLIST_RE.match(p)) @@ -3752,10 +3756,15 @@ class DeviceUtils(object): and 'android.permission.READ_EXTERNAL_STORAGE' not in permissions): permissions.add('android.permission.READ_EXTERNAL_STORAGE') - script = ';'.join([ - 'p={package}', 'for q in {permissions}', 'do pm grant "$p" "$q"', - 'echo "{sep}$q{sep}$?{sep}"', 'done' - ]).format( + script_raw = [ + 'p={package}', + 'for q in {permissions}', + 'do pm grant "$p" "$q"', + 'echo "{sep}$q{sep}$?{sep}"', + 'done', + ] + script_manage_ext_storage + + script = ';'.join(script_raw).format( package=cmd_helper.SingleQuote(package), permissions=' '.join( cmd_helper.SingleQuote(p) for p in sorted(permissions)), diff --git a/systrace/catapult/devil/devil/android/device_utils_test.py b/systrace/catapult/devil/devil/android/device_utils_test.py index 62313c5..8e583f0 100755 --- a/systrace/catapult/devil/devil/android/device_utils_test.py +++ b/systrace/catapult/devil/devil/android/device_utils_test.py @@ -11,6 +11,7 @@ Unit tests for the contents of device_utils.py (mostly DeviceUtils). import collections import contextlib +import io import json import logging import os @@ -19,6 +20,8 @@ import stat import sys import unittest +import six + from devil import devil_env from devil.android import device_errors from devil.android import device_signal @@ -117,9 +120,10 @@ class DeviceUtilsInitTest(unittest.TestCase): self.assertEqual(serial_as_str, d.adb.GetDeviceSerial()) def testInitWithUnicode(self): - serial_as_unicode = unicode('fedcba9876543210') - d = device_utils.DeviceUtils(serial_as_unicode) - self.assertEqual(serial_as_unicode, d.adb.GetDeviceSerial()) + if six.PY2: + serial_as_unicode = unicode('fedcba9876543210') + d = device_utils.DeviceUtils(serial_as_unicode) + self.assertEqual(serial_as_unicode, d.adb.GetDeviceSerial()) def testInitWithAdbWrapper(self): serial = '123456789abcdef0' @@ -162,12 +166,12 @@ class DeviceUtilsRestartServerTest(mock_calls.TestCase): ['pgrep', 'adb']), (1, '')), (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutput( ['pgrep', 'adb']), (0, '123\n'))): - device_utils.RestartServer() + adb_wrapper.RestartServer() class MockTempFile(object): def __init__(self, name='/tmp/some/file'): - self.file = mock.MagicMock(spec=file) + self.file = mock.MagicMock(spec=io.BufferedIOBase) self.file.name = name self.file.name_quoted = cmd_helper.SingleQuote(name) @@ -217,6 +221,12 @@ class DeviceUtilsTest(mock_calls.TestCase): self.adb, default_timeout=10, default_retries=0) self.watchMethodCalls(self.call.adb, ignore=['GetDeviceSerial']) + def safeAssertItemsEqual(self, expected, actual): + if six.PY2: + self.assertItemsEqual(expected, actual) + else: + self.assertCountEqual(expected, actual) # pylint: disable=no-member + def AdbCommandError(self, args=None, output=None, status=None, msg=None): if args is None: args = ['[unspecified]'] @@ -2194,11 +2204,16 @@ class DeviceUtilsPushChangedFilesZippedTest(DeviceUtilsTest): self.assertFalse( self.device._PushChangedFilesZipped(test_files, ['/test/dir'])) - def _testPushChangedFilesZipped_spec(self, test_files): + def _testPushChangedFilesZipped_spec(self, test_files, test_dirs): @contextlib.contextmanager def mock_zip_temp_dir(): yield '/test/temp/dir' + expected_cmd = ''.join([ + '\n /data/local/tmp/bin/unzip %s &&', + ' (for dir in %s\n do\n chmod -R 777 "$dir" || exit 1\n', + ' done)\n' + ]) % ('/sdcard/foo123.zip', ' '.join(test_dirs)) with self.assertCalls( (self.call.device._MaybeInstallCommands(), True), (mock.call.py_utils.tempfile_ext.NamedTemporaryDirectory(), @@ -2207,25 +2222,27 @@ class DeviceUtilsPushChangedFilesZippedTest(DeviceUtilsTest): (mock.call.os.path.getsize('/test/temp/dir/tmp.zip'), 123), (self.call.device.NeedsSU(), True), (mock.call.devil.android.device_temp_file.DeviceTempFile( - self.adb, suffix='.zip'), MockTempFile('/test/sdcard/foo123.zip')), - self.call.adb.Push('/test/temp/dir/tmp.zip', '/test/sdcard/foo123.zip'), - self.call.device.RunShellCommand( - 'unzip /test/sdcard/foo123.zip&&chmod -R 777 /test/dir', - shell=True, - as_root=True, - env={'PATH': '/data/local/tmp/bin:$PATH'}, - check_return=True)): + self.adb, suffix='.zip'), MockTempFile('/sdcard/foo123.zip')), + self.call.adb.Push('/test/temp/dir/tmp.zip', '/sdcard/foo123.zip'), + (mock.call.devil.android.device_temp_file.DeviceTempFile( + self.adb, suffix='.sh'), MockTempFile('/sdcard/temp-123.sh')), + self.call.device.WriteFile('/sdcard/temp-123.sh', expected_cmd), + (self.call.device.RunShellCommand(['source', '/sdcard/temp-123.sh'], + check_return=True, + as_root=True))): self.assertTrue( - self.device._PushChangedFilesZipped(test_files, ['/test/dir'])) + self.device._PushChangedFilesZipped(test_files, test_dirs)) def testPushChangedFilesZipped_single(self): - self._testPushChangedFilesZipped_spec([('/test/host/path/file1', - '/test/device/path/file1')]) + self._testPushChangedFilesZipped_spec( + [('/test/host/path/file1', '/test/device/path/file1')], + ['/test/dir1']) def testPushChangedFilesZipped_multiple(self): self._testPushChangedFilesZipped_spec( [('/test/host/path/file1', '/test/device/path/file1'), - ('/test/host/path/file2', '/test/device/path/file2')]) + ('/test/host/path/file2', '/test/device/path/file2')], + ['/test/dir1', '/test/dir2']) class DeviceUtilsPathExistsTest(DeviceUtilsTest): @@ -2374,7 +2391,8 @@ class DeviceUtilsReadFileTest(DeviceUtilsTest): with self.assertCalls( (mock.call.tempfile.mkdtemp(), tmp_host_dir), (self.call.adb.Pull('/path/to/device/file', mock.ANY)), - (mock.call.__builtin__.open(mock.ANY, 'r'), tmp_host), + (mock.call.__builtin__.open(mock.ANY, 'r'), tmp_host) if six.PY2 else \ + (mock.call.builtins.open(mock.ANY, 'r'), tmp_host), (mock.call.os.path.exists(tmp_host_dir), True), (mock.call.shutil.rmtree(tmp_host_dir), None)): self.assertEquals('some interesting contents', @@ -2587,8 +2605,8 @@ class DeviceUtilsStatDirectoryTest(DeviceUtilsTest): self.getStatEntries(path_given='/foo/bar', path_listed='/foo/bar/') def testStatDirectory_fileList(self): - self.assertItemsEqual(self.getStatEntries().keys(), self.FILENAMES) - self.assertItemsEqual(self.getListEntries(), self.FILENAMES) + self.safeAssertItemsEqual(self.getStatEntries().keys(), self.FILENAMES) + self.safeAssertItemsEqual(self.getListEntries(), self.FILENAMES) def testStatDirectory_fileModes(self): expected_modes = ( @@ -2656,7 +2674,7 @@ class DeviceUtilsStatDirectoryTest(DeviceUtilsTest): def testStatDirectory_symbolicLinks(self): entries = self.getStatEntries() self.assertEqual(entries['lnk']['symbolic_link_to'], '/a/path') - for d in entries.itervalues(): + for d in entries.values(): self.assertEqual('symbolic_link_to' in d, stat.S_ISLNK(d['st_mode'])) @@ -3463,7 +3481,7 @@ class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): device_utils.DeviceUtils.HealthyDevices(device_arg=[], retries=0) @mock.patch('time.sleep') - @mock.patch('devil.android.device_utils.RestartServer') + @mock.patch('devil.android.sdk.adb_wrapper.RestartServer') def testHealthyDevices_EmptyListDeviceArg_no_attached_with_retry( self, mock_restart, mock_sleep): with self.assertCalls( @@ -3481,7 +3499,7 @@ class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): mock.call(8), mock.call(16)]) @mock.patch('time.sleep') - @mock.patch('devil.android.device_utils.RestartServer') + @mock.patch('devil.android.sdk.adb_wrapper.RestartServer') def testHealthyDevices_EmptyListDeviceArg_no_attached_with_resets( self, mock_restart, mock_sleep): # The reset_usb import fails on windows. Mock the full import here so it can @@ -3557,7 +3575,7 @@ class DeviceUtilsGrantPermissionsTest(DeviceUtilsTest): def _PmGrantShellCall(self, package, permissions): fragment = 'p=%s;for q in %s;' % (package, ' '.join(sorted(permissions))) results = [] - for permission, result in sorted(permissions.iteritems()): + for permission, result in sorted(permissions.items()): if result: output, status = result + '\n', 1 else: @@ -3610,6 +3628,23 @@ class DeviceUtilsGrantPermissionsTest(DeviceUtilsTest): self.device.GrantPermissions('package', [WRITE]) self.assertEqual(logger.warnings, []) + def testGrantPermissions_ManageExtrnalStorage(self): + with PatchLogger() as logger: + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.R): + with self.assertCalls( + (self.call.device.RunShellCommand( + AnyStringWith('appops set pkg MANAGE_EXTERNAL_STORAGE allow'), + shell=True, + raw_output=True, + large_output=True, + check_return=True), + '{sep}MANAGE_EXTERNAL_STORAGE{sep}0{sep}\n'.format( + sep=device_utils._SHELL_OUTPUT_SEPARATOR))): + self.device.GrantPermissions( + 'pkg', ['android.permission.MANAGE_EXTERNAL_STORAGE']) + self.assertEqual(logger.warnings, []) + def testGrantPermissions_DenyList(self): with PatchLogger() as logger: with self.patch_call( @@ -3866,6 +3901,12 @@ class IterPushableComponentsTest(unittest.TestCase): yield Layout(layout_root, basic_file, symlink, symlink_dir, dir1, dir2) + def safeAssertItemsEqual(self, expected, actual): + if six.PY2: + self.assertItemsEqual(expected, actual) + else: + self.assertCountEqual(expected, actual) # pylint: disable=no-member + def testFile(self): with self.sampleLayout() as layout: device_path = '/sdcard/basic_file' @@ -3873,7 +3914,7 @@ class IterPushableComponentsTest(unittest.TestCase): expected = [(layout.basic_file, device_path, True)] actual = list( device_utils._IterPushableComponents(layout.basic_file, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testSymlinkFile(self): with self.sampleLayout() as layout: @@ -3883,7 +3924,7 @@ class IterPushableComponentsTest(unittest.TestCase): actual = list( device_utils._IterPushableComponents(layout.symlink_file, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testDirectoryWithNoSymlink(self): with self.sampleLayout() as layout: @@ -3893,7 +3934,7 @@ class IterPushableComponentsTest(unittest.TestCase): actual = list( device_utils._IterPushableComponents(layout.dir_without_symlinks, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testDirectoryWithSymlink(self): with self.sampleLayout() as layout: @@ -3910,7 +3951,7 @@ class IterPushableComponentsTest(unittest.TestCase): actual = list( device_utils._IterPushableComponents(layout.dir_with_symlinks, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testSymlinkDirectory(self): with self.sampleLayout() as layout: @@ -3919,7 +3960,7 @@ class IterPushableComponentsTest(unittest.TestCase): expected = [(os.path.realpath(layout.symlink_dir), device_path, False)] actual = list( device_utils._IterPushableComponents(layout.symlink_dir, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testDirectoryWithNestedSymlink(self): with self.sampleLayout() as layout: @@ -3947,7 +3988,7 @@ class IterPushableComponentsTest(unittest.TestCase): ] actual = list( device_utils._IterPushableComponents(layout.root, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) class DeviceUtilsGetTracingPathTest(DeviceUtilsTest): diff --git a/systrace/catapult/devil/devil/android/fastboot_utils_test.py b/systrace/catapult/devil/devil/android/fastboot_utils_test.py index 1ad7319..ed89139 100755 --- a/systrace/catapult/devil/devil/android/fastboot_utils_test.py +++ b/systrace/catapult/devil/devil/android/fastboot_utils_test.py @@ -13,6 +13,8 @@ import io import logging import unittest +import six + from devil import devil_env from devil.android import device_errors from devil.android import device_utils @@ -347,16 +349,22 @@ class FastbootUtilsFastbootMode(FastbootUtilsTest): pass +if six.PY2: + _BUILTIN_OPEN = '__builtin__.open' +else: + _BUILTIN_OPEN = 'builtins.open' + + class FastbootUtilsVerifyBoard(FastbootUtilsTest): def testVerifyBoard_bothValid(self): mock_file = io.StringIO(u'require board=%s\n' % _BOARD) - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_VALID_FILES): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_BothNotValid(self): mock_file = io.StringIO(u'abc') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_INVALID_FILES): self.assertFalse(self.assertFalse(self.fastboot._VerifyBoard('test'))) @@ -366,31 +374,31 @@ class FastbootUtilsVerifyBoard(FastbootUtilsTest): def testVerifyBoard_ZipNotFoundFileValid(self): mock_file = io.StringIO(u'require board=%s\n' % _BOARD) - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=['android-info.txt']): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_zipNotValidFileIs(self): mock_file = io.StringIO(u'require board=%s\n' % _BOARD) - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_INVALID_FILES): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_fileNotValidZipIs(self): mock_file = io.StringIO(u'require board=WrongBoard') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_VALID_FILES): self.assertFalse(self.fastboot._VerifyBoard('test')) def testVerifyBoard_noBoardInFileValidZip(self): mock_file = io.StringIO(u'Regex wont match') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_VALID_FILES): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_noBoardInFileInvalidZip(self): mock_file = io.StringIO(u'Regex wont match') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_INVALID_FILES): self.assertFalse(self.fastboot._VerifyBoard('test')) @@ -419,7 +427,7 @@ class FastbootUtilsFindAndVerifyPartitionsAndImages(FastbootUtilsTest): ] with mock.patch('os.listdir', return_value=files): imgs = self.fastboot._FindAndVerifyPartitionsAndImages(PARTITIONS, 'test') - parts = imgs.keys() + parts = list(imgs.keys()) self.assertDictEqual(imgs, img_check) self.assertListEqual(parts, parts_check) @@ -451,7 +459,7 @@ class FastbootUtilsFindAndVerifyPartitionsAndImages(FastbootUtilsTest): with self.patch_call(self.call.fastboot.supports_ab, return_value=False): imgs = self.fastboot._FindAndVerifyPartitionsAndImages( PARTITIONS, 'test') - parts = imgs.keys() + parts = list(imgs.keys()) self.assertDictEqual(imgs, img_check) self.assertListEqual(parts, parts_check) diff --git a/systrace/catapult/devil/devil/android/flag_changer_test.py b/systrace/catapult/devil/devil/android/flag_changer_test.py index 564ead6..9c155f1 100755 --- a/systrace/catapult/devil/devil/android/flag_changer_test.py +++ b/systrace/catapult/devil/devil/android/flag_changer_test.py @@ -6,6 +6,8 @@ import posixpath import unittest +import six + from devil.android import flag_changer _CMDLINE_FILE = 'chrome-command-line' @@ -109,15 +111,22 @@ class ParseSerializeFlagsTest(unittest.TestCase): def _testParseCmdLine(self, command_line, expected_flags): # Start with a command line, check that flags are parsed as expected. # pylint: disable=protected-access + # pylint: disable=no-member flags = flag_changer._ParseFlags(command_line) - self.assertItemsEqual(flags, expected_flags) + if six.PY2: + self.assertItemsEqual(flags, expected_flags) + else: + self.assertCountEqual(flags, expected_flags) # Check that flags survive a round-trip. # Note: Although new_command_line and command_line may not match, they # should describe the same set of flags. new_command_line = flag_changer._SerializeFlags(flags) new_flags = flag_changer._ParseFlags(new_command_line) - self.assertItemsEqual(new_flags, expected_flags) + if six.PY2: + self.assertItemsEqual(new_flags, expected_flags) + else: + self.assertCountEqual(new_flags, expected_flags) def testParseCmdLine_simple(self): self._testParseCmdLine('chrome --foo --bar="a b" --baz=true --fine="ok"', diff --git a/systrace/catapult/devil/devil/android/logcat_monitor.py b/systrace/catapult/devil/devil/android/logcat_monitor.py index bec7444..df306b0 100644 --- a/systrace/catapult/devil/devil/android/logcat_monitor.py +++ b/systrace/catapult/devil/devil/android/logcat_monitor.py @@ -13,6 +13,8 @@ import tempfile import threading import time +import six + from devil.android import decorators from devil.android import device_errors from devil.android.sdk import adb_wrapper @@ -102,9 +104,9 @@ class LogcatMonitor(object): raise LogcatMonitorCommandError( 'Must be recording logcat when calling |WaitFor|', device_serial=str(self._adb)) - if isinstance(success_regex, basestring): + if isinstance(success_regex, six.string_types): success_regex = re.compile(success_regex) - if isinstance(failure_regex, basestring): + if isinstance(failure_regex, six.string_types): failure_regex = re.compile(failure_regex) logger.debug('Waiting %d seconds for "%s"', timeout, success_regex.pattern) @@ -220,10 +222,14 @@ class LogcatMonitor(object): Clears the logcat if |clear| was set in |__init__|. """ + # pylint: disable=unexpected-keyword-arg if self._clear: self._adb.Logcat(clear=True) if not self._record_file: - self._record_file = tempfile.NamedTemporaryFile(mode='a', bufsize=1) + if six.PY2: + self._record_file = tempfile.NamedTemporaryFile(mode='a', bufsize=1) + else: + self._record_file = tempfile.NamedTemporaryFile(mode='a', buffering=1) self._StartRecording() def Stop(self): diff --git a/systrace/catapult/devil/devil/android/logcat_monitor_test.py b/systrace/catapult/devil/devil/android/logcat_monitor_test.py index 7f2f10a..356fe04 100755 --- a/systrace/catapult/devil/devil/android/logcat_monitor_test.py +++ b/systrace/catapult/devil/devil/android/logcat_monitor_test.py @@ -9,6 +9,8 @@ import itertools import threading import unittest +import six + from devil import devil_env from devil.android import logcat_monitor from devil.android.sdk import adb_wrapper @@ -24,6 +26,13 @@ def _CreateTestLog(raw_logcat=None): return test_log +def zip_longest(expected, actual): + # pylint: disable=no-member + if six.PY2: + return itertools.izip_longest(expected, actual) + else: + return itertools.zip_longest(expected, actual) + class LogcatMonitorTest(unittest.TestCase): _TEST_THREADTIME_LOGCAT_DATA = [ @@ -44,7 +53,7 @@ class LogcatMonitorTest(unittest.TestCase): ] def assertIterEqual(self, expected_iter, actual_iter): - for expected, actual in itertools.izip_longest(expected_iter, actual_iter): + for expected, actual in zip_longest(expected_iter, actual_iter): self.assertIsNotNone( expected, msg='actual has unexpected elements starting with %s' % str(actual)) diff --git a/systrace/catapult/devil/devil/android/md5sum.py b/systrace/catapult/devil/devil/android/md5sum.py index 8adf4ef..e67f3f6 100644 --- a/systrace/catapult/devil/devil/android/md5sum.py +++ b/systrace/catapult/devil/devil/android/md5sum.py @@ -3,10 +3,12 @@ # found in the LICENSE file. import base64 +import io import gzip import os import re -import StringIO + +import six from devil import devil_env from devil.android import device_errors @@ -38,7 +40,7 @@ def CalculateHostMd5Sums(paths): Returns: A dict mapping file paths to their respective md5sum checksums. """ - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): paths = [paths] paths = list(paths) @@ -47,13 +49,17 @@ def CalculateHostMd5Sums(paths): raise IOError('File not built: %s' % md5sum_bin_host_path) out = "" for i in range(0, len(paths), _MAX_PATHS_PER_INVOCATION): - mem_file = StringIO.StringIO() + mem_file = io.BytesIO() compressed = gzip.GzipFile(fileobj=mem_file, mode="wb") - compressed.write(";".join( - [os.path.realpath(p) for p in paths[i:i+_MAX_PATHS_PER_INVOCATION]])) + data = ";".join( + [os.path.realpath(p) for p in paths[i:i+_MAX_PATHS_PER_INVOCATION]]) + if six.PY3: + data = data.encode('utf-8') + compressed.write(data) compressed.close() compressed_paths = base64.b64encode(mem_file.getvalue()) - out += cmd_helper.GetCmdOutput([md5sum_bin_host_path, "-gz", compressed_paths]) + out += cmd_helper.GetCmdOutput( + [md5sum_bin_host_path, "-gz", compressed_paths]) return dict(zip(paths, out.splitlines())) @@ -72,7 +78,7 @@ def CalculateDeviceMd5Sums(paths, device): if not paths: return {} - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): paths = [paths] paths = list(paths) @@ -97,9 +103,12 @@ def CalculateDeviceMd5Sums(paths, device): # Make sure it can find libbase.so md5sum_script += 'export LD_LIBRARY_PATH=%s;' % MD5SUM_DEVICE_LIB_PATH for i in range(0, len(paths), _MAX_PATHS_PER_INVOCATION): - mem_file = StringIO.StringIO() + mem_file = io.BytesIO() compressed = gzip.GzipFile(fileobj=mem_file, mode="wb") - compressed.write(";".join(paths[i:i+_MAX_PATHS_PER_INVOCATION])) + data = ";".join(paths[i:i+_MAX_PATHS_PER_INVOCATION]) + if six.PY3: + data = data.encode('utf-8') + compressed.write(data) compressed.close() compressed_paths = base64.b64encode(mem_file.getvalue()) md5sum_script += '$a -gz %s;' % compressed_paths diff --git a/systrace/catapult/devil/devil/android/md5sum_test.py b/systrace/catapult/devil/devil/android/md5sum_test.py index 548b2d0..9a51313 100755 --- a/systrace/catapult/devil/devil/android/md5sum_test.py +++ b/systrace/catapult/devil/devil/android/md5sum_test.py @@ -112,7 +112,7 @@ class Md5SumTest(unittest.TestCase): def testCalculateDeviceMd5Sums_generator(self): test_path = ('/storage/emulated/legacy/test/file%d.dat' % n - for n in xrange(0, 2)) + for n in range(0, 2)) device = mock.NonCallableMock() device_md5sum_output = [ diff --git a/systrace/catapult/devil/devil/android/perf/perf_control.py b/systrace/catapult/devil/devil/android/perf/perf_control.py index 59485e0..398b27f 100644 --- a/systrace/catapult/devil/devil/android/perf/perf_control.py +++ b/systrace/catapult/devil/devil/android/perf/perf_control.py @@ -203,7 +203,7 @@ class PerfControl(object): if not isinstance(cpu_max_freq, dict): self._SetScalingMaxFreqForCpus(cpu_max_freq, self._cpu_file_list) else: - for key, max_frequency in cpu_max_freq.iteritems(): + for key, max_frequency in cpu_max_freq.items(): # Convert 'X' to 'cpuX' and 'X..Y' to 'cpuX cpu<X+1> .. cpuY'. if '..' in key: range_min, range_max = key.split('..') @@ -211,7 +211,7 @@ class PerfControl(object): else: range_min = range_max = int(key) cpu_files = [ - 'cpu%d' % number for number in xrange(range_min, range_max + 1) + 'cpu%d' % number for number in range(range_min, range_max + 1) ] # Set the |max_frequency| on requested subset of the cores. self._SetScalingMaxFreqForCpus(max_frequency, ' '.join(cpu_files)) diff --git a/systrace/catapult/devil/devil/android/perf/perf_control_test.py b/systrace/catapult/devil/devil/android/perf/perf_control_test.py index bde54d5..a841a0e 100644 --- a/systrace/catapult/devil/devil/android/perf/perf_control_test.py +++ b/systrace/catapult/devil/devil/android/perf/perf_control_test.py @@ -47,7 +47,7 @@ class PerfControlTest(unittest.TestCase): # pylint: disable=no-self-use def testNexus5HighPerfMode(self): # Mock out the device state for PerfControl. - cpu_list = ['cpu%d' % cpu for cpu in xrange(4)] + cpu_list = ['cpu%d' % cpu for cpu in range(4)] mock_device = mock.Mock(spec=device_utils.DeviceUtils) mock_device.product_model = 'Nexus 5' mock_device.adb = mock.Mock(spec=adb_wrapper.AdbWrapper) @@ -70,7 +70,7 @@ class PerfControlTest(unittest.TestCase): def testNexus5XHighPerfMode(self): # Mock out the device state for PerfControl. - cpu_list = ['cpu%d' % cpu for cpu in xrange(6)] + cpu_list = ['cpu%d' % cpu for cpu in range(6)] mock_device = mock.Mock(spec=device_utils.DeviceUtils) mock_device.product_model = 'Nexus 5X' mock_device.adb = mock.Mock(spec=adb_wrapper.AdbWrapper) @@ -93,7 +93,7 @@ class PerfControlTest(unittest.TestCase): def testNexus5XDefaultPerfMode(self): # Mock out the device state for PerfControl. - cpu_list = ['cpu%d' % cpu for cpu in xrange(6)] + cpu_list = ['cpu%d' % cpu for cpu in range(6)] mock_device = mock.Mock(spec=device_utils.DeviceUtils) mock_device.product_model = 'Nexus 5X' mock_device.adb = mock.Mock(spec=adb_wrapper.AdbWrapper) diff --git a/systrace/catapult/devil/devil/android/perf/surface_stats_collector.py b/systrace/catapult/devil/devil/android/perf/surface_stats_collector.py index 6240624..4ddc6f5 100644 --- a/systrace/catapult/devil/devil/android/perf/surface_stats_collector.py +++ b/systrace/catapult/devil/devil/android/perf/surface_stats_collector.py @@ -3,10 +3,16 @@ # found in the LICENSE file. import logging -import Queue import re import threading +import six + +if six.PY3: + import queue # pylint: disable=wrong-import-order +else: + import Queue as queue # pylint: disable=wrong-import-order + # Log marker containing SurfaceTexture timestamps. _SURFACE_TEXTURE_TIMESTAMPS_MESSAGE = 'SurfaceTexture update timestamps' _SURFACE_TEXTURE_TIMESTAMP_RE = r'\d+' @@ -37,7 +43,7 @@ class SurfaceStatsCollector(object): if self._ClearSurfaceFlingerLatencyData(): self._get_data_event = threading.Event() self._stop_event = threading.Event() - self._data_queue = Queue.Queue() + self._data_queue = queue.Queue() self._collector_thread = threading.Thread(target=self._CollectorThread) self._collector_thread.start() else: @@ -148,6 +154,10 @@ class SurfaceStatsCollector(object): return ParseFrameData(output, parse_timestamps=bool(window_name)) +def to_long_int(val): + """Cast val to a long int type.""" + return long(val) if six.PY2 else int(val) + def ParseFrameData(lines, parse_timestamps): # adb shell dumpsys SurfaceFlinger --latency <window name> # prints some information about the last 128 frames displayed in @@ -174,6 +184,7 @@ def ParseFrameData(lines, parse_timestamps): # (each time the number above changes, we have a "jank"). # If this happens a lot during an animation, the animation appears # janky, even if it runs at 60 fps in average. + # pylint: disable=redefined-variable-type results = [] for line in lines: # Skip over lines with anything other than digits and whitespace. @@ -186,7 +197,8 @@ def ParseFrameData(lines, parse_timestamps): timestamps = [] nanoseconds_per_millisecond = 1e6 - refresh_period = long(results[0]) / nanoseconds_per_millisecond + refresh_period = to_long_int(results[0]) / nanoseconds_per_millisecond + if not parse_timestamps: return refresh_period, timestamps @@ -201,7 +213,8 @@ def ParseFrameData(lines, parse_timestamps): if len(fields) != 3: logging.warning('Unexpected line: %s', line) continue - timestamp = long(fields[1]) + timestamp = to_long_int(fields[1]) + if timestamp == pending_fence_timestamp: continue timestamp /= nanoseconds_per_millisecond diff --git a/systrace/catapult/devil/devil/android/ports.py b/systrace/catapult/devil/devil/android/ports.py index f25c8bf..4a7c294 100644 --- a/systrace/catapult/devil/devil/android/ports.py +++ b/systrace/catapult/devil/devil/android/ports.py @@ -159,7 +159,7 @@ def IsHttpServerConnectable(host, message the server returns when connect status is false. """ assert tries >= 1 - for i in xrange(0, tries): + for i in range(0, tries): client_error = None try: with contextlib.closing( diff --git a/systrace/catapult/devil/devil/android/sdk/aapt.py b/systrace/catapult/devil/devil/android/sdk/aapt.py index 76c7ef6..fd35407 100644 --- a/systrace/catapult/devil/devil/android/sdk/aapt.py +++ b/systrace/catapult/devil/devil/android/sdk/aapt.py @@ -3,6 +3,8 @@ # found in the LICENSE file. """This module wraps the Android Asset Packaging Tool.""" +import six + from devil.android.sdk import build_tools from devil.utils import cmd_helper from devil.utils import lazy @@ -36,6 +38,6 @@ def Dump(what, apk, assets=None): assets: List of assets in apk you want to dump information for. """ assets = assets or [] - if isinstance(assets, basestring): + if isinstance(assets, six.string_types): assets = [assets] return _RunAaptCmd(['dump', what, apk] + assets).splitlines() diff --git a/systrace/catapult/devil/devil/android/sdk/adb_wrapper.py b/systrace/catapult/devil/devil/android/sdk/adb_wrapper.py index 71928d5..d899224 100644 --- a/systrace/catapult/devil/devil/android/sdk/adb_wrapper.py +++ b/systrace/catapult/devil/devil/android/sdk/adb_wrapper.py @@ -18,6 +18,8 @@ import posixpath import re import subprocess +import six + from devil import base_error from devil import devil_env from devil.android import decorators @@ -127,6 +129,30 @@ def _IsExtraneousLine(line, send_cmd): return send_cmd.rstrip() in line +@decorators.WithExplicitTimeoutAndRetries(timeout=60, retries=3) +def RestartServer(): + """Restarts the adb server. + + Raises: + CommandFailedError if we fail to kill or restart the server. + """ + + def adb_killed(): + return not AdbWrapper.IsServerOnline() + + def adb_started(): + return AdbWrapper.IsServerOnline() + + AdbWrapper.KillServer() + if not timeout_retry.WaitFor(adb_killed, wait_period=1, max_tries=5): + # TODO(crbug.com/442319): Switch this to raise an exception if we + # figure out why sometimes not all adb servers on bots get killed. + logger.warning('Failed to kill adb server') + AdbWrapper.StartServer() + if not timeout_retry.WaitFor(adb_started, wait_period=1, max_tries=5): + raise device_errors.CommandFailedError('Failed to start adb server') + + class AdbWrapper(object): """A wrapper around a local Android Debug Bridge executable.""" @@ -1105,7 +1131,7 @@ class AdbWrapper(object): Returns: The output of the emulator console command. """ - if isinstance(cmd, basestring): + if isinstance(cmd, six.string_types): cmd = [cmd] return self._RunDeviceAdbCmd(['emu'] + cmd, timeout, retries) diff --git a/systrace/catapult/devil/devil/android/sdk/dexdump.py b/systrace/catapult/devil/devil/android/sdk/dexdump.py index 2a59e9b..c71442c 100644 --- a/systrace/catapult/devil/devil/android/sdk/dexdump.py +++ b/systrace/catapult/devil/devil/android/sdk/dexdump.py @@ -2,6 +2,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import six + from devil.android.sdk import build_tools from devil.utils import cmd_helper from devil.utils import lazy @@ -20,7 +22,7 @@ def DexDump(dexfiles, file_summary=False): An iterable over the output lines. """ # TODO(jbudorick): Add support for more options as necessary. - if isinstance(dexfiles, basestring): + if isinstance(dexfiles, six.string_types): dexfiles = [dexfiles] args = [_dexdump_path.read()] + dexfiles if file_summary: diff --git a/systrace/catapult/devil/devil/android/sdk/intent.py b/systrace/catapult/devil/devil/android/sdk/intent.py index 69c7d90..2ea38c3 100644 --- a/systrace/catapult/devil/devil/android/sdk/intent.py +++ b/systrace/catapult/devil/devil/android/sdk/intent.py @@ -116,7 +116,7 @@ class Intent(object): if self.flags: args.extend(['-f', self.flags]) if self.extras: - for key, value in self.extras.iteritems(): + for key, value in self.extras.items(): if value is None: args.extend(['--esn', key]) elif isinstance(value, str): diff --git a/systrace/catapult/devil/devil/android/sdk/shared_prefs.py b/systrace/catapult/devil/devil/android/sdk/shared_prefs.py index 7b12bf5..32b5bc4 100644 --- a/systrace/catapult/devil/devil/android/sdk/shared_prefs.py +++ b/systrace/catapult/devil/devil/android/sdk/shared_prefs.py @@ -11,6 +11,8 @@ import logging import posixpath from xml.etree import ElementTree +import six + from devil.android import device_errors from devil.android.sdk import version_codes @@ -43,7 +45,10 @@ class BasePref(object): def __str__(self): """Get the underlying xml element as a string.""" - return ElementTree.tostring(self._elem) + if six.PY2: + return ElementTree.tostring(self._elem) + else: + return ElementTree.tostring(self._elem, encoding="unicode") def get(self): """Get the value of this preference.""" @@ -231,7 +236,11 @@ class SharedPrefs(object): def __str__(self): """Get the underlying xml document as a string.""" - return _XML_DECLARATION + ElementTree.tostring(self.xml) + if six.PY2: + return _XML_DECLARATION + ElementTree.tostring(self.xml) + else: + return _XML_DECLARATION + \ + ElementTree.tostring(self.xml, encoding="unicode") @property def package(self): diff --git a/systrace/catapult/devil/devil/android/sdk/shared_prefs_test.py b/systrace/catapult/devil/devil/android/sdk/shared_prefs_test.py index 7374c89..b50d9e0 100755 --- a/systrace/catapult/devil/devil/android/sdk/shared_prefs_test.py +++ b/systrace/catapult/devil/devil/android/sdk/shared_prefs_test.py @@ -132,6 +132,8 @@ class SharedPrefsTest(unittest.TestCase): }) # data survived roundtrip def testForceCommit(self): + type(self.device).build_version_sdk = mock.PropertyMock( + return_value=version_codes.LOLLIPOP_MR1) prefs = shared_prefs.SharedPrefs(self.device, 'com.some.package', 'prefs.xml') prefs.Load() diff --git a/systrace/catapult/devil/devil/android/sdk/version_codes.py b/systrace/catapult/devil/devil/android/sdk/version_codes.py index 943b9d3..564f30d 100644 --- a/systrace/catapult/devil/devil/android/sdk/version_codes.py +++ b/systrace/catapult/devil/devil/android/sdk/version_codes.py @@ -20,3 +20,4 @@ OREO = 26 OREO_MR1 = 27 PIE = 28 Q = 29 +R = 30 diff --git a/systrace/catapult/devil/devil/android/tools/device_monitor.py b/systrace/catapult/devil/devil/android/tools/device_monitor.py index 26e89a2..730df14 100755 --- a/systrace/catapult/devil/devil/android/tools/device_monitor.py +++ b/systrace/catapult/devil/devil/android/tools/device_monitor.py @@ -184,7 +184,7 @@ def get_all_status(denylist): } if denylist: - for device, reason in denylist.Read().iteritems(): + for device, reason in denylist.Read().items(): status_dict['devices'][device] = { 'state': reason.get('reason', 'denylisted') } diff --git a/systrace/catapult/devil/devil/android/tools/device_monitor_test.py b/systrace/catapult/devil/devil/android/tools/device_monitor_test.py index 8082d26..1bb5680 100755 --- a/systrace/catapult/devil/devil/android/tools/device_monitor_test.py +++ b/systrace/catapult/devil/devil/android/tools/device_monitor_test.py @@ -7,6 +7,8 @@ import os import sys import unittest +import six + if __name__ == '__main__': sys.path.append( os.path.abspath( @@ -53,7 +55,7 @@ class DeviceMonitorTest(unittest.TestCase): } def mock_run_shell(cmd, **_kwargs): - args = cmd.split() if isinstance(cmd, basestring) else cmd + args = cmd.split() if isinstance(cmd, six.string_types) else cmd try: return self.cmd_outputs[args[0]] except KeyError: diff --git a/systrace/catapult/devil/devil/android/tools/system_app.py b/systrace/catapult/devil/devil/android/tools/system_app.py index 62bf5a5..50d8559 100755 --- a/systrace/catapult/devil/devil/android/tools/system_app.py +++ b/systrace/catapult/devil/devil/android/tools/system_app.py @@ -22,6 +22,7 @@ from devil.android import decorators from devil.android import device_errors from devil.android import device_temp_file from devil.android.sdk import version_codes +from devil.android.sdk import adb_wrapper from devil.android.tools import script_common from devil.utils import cmd_helper from devil.utils import parallelizer @@ -128,7 +129,15 @@ _ENABLE_MODIFICATION_PROP = 'devil.modify_sys_apps' def _ShouldRetryModification(exc): - return not isinstance(exc, device_errors.CommandTimeoutError) + try: + if isinstance(exc, device_errors.CommandTimeoutError): + logger.info('Restarting the adb server') + adb_wrapper.RestartServer() + return True + except Exception: # pylint: disable=broad-except + logger.exception(('Caught an exception when deciding' + ' to retry system modification')) + return False # timeout and retries are both required by the decorator, but neither @@ -172,7 +181,7 @@ def _SetUpSystemAppModification(device, timeout=None, retries=None): # Point the user to documentation, since there's a good chance they can # workaround this on an emulator. docs_url = ('https://chromium.googlesource.com/chromium/src/+/' - 'master/docs/android_emulator.md#writable-system-partition') + 'HEAD/docs/android_emulator.md#writable-system-partition') logger.error( 'Did you start the emulator with "-writable-system?"\n' 'See %s\n', docs_url) @@ -187,6 +196,22 @@ def _TearDownSystemAppModification(device, timeout=None, retries=None): try: + # The function may be re-entered after the the device loses root + # privilege. For instance if the adb server is restarted before + # re-entering the function then the device may lose root privilege. + # Therefore we need to do a sanity check for root privilege + # on the device and then re-enable root privilege if the device + # does not have it. + if not device.HasRoot(): + logger.warning('Need to re-enable root.') + device.EnableRoot() + + if not device.HasRoot(): + raise device_errors.CommandFailedError( + ('Failed to tear down modification of ' + 'system apps on non-rooted device.'), + str(device)) + device.SetProp(_ENABLE_MODIFICATION_PROP, '0') device.Reboot() device.WaitUntilFullyBooted() diff --git a/systrace/catapult/devil/devil/android/tools/video_recorder.py b/systrace/catapult/devil/devil/android/tools/video_recorder.py index 984931f..4004c46 100755 --- a/systrace/catapult/devil/devil/android/tools/video_recorder.py +++ b/systrace/catapult/devil/devil/android/tools/video_recorder.py @@ -164,7 +164,7 @@ def main(): parallel_devices = device_utils.DeviceUtils.parallel(script_common.GetDevices( args.devices, args.denylist_file), - async=True) + asyn=True) stop_recording = threading.Event() running_recording = parallel_devices.pMap(record_video, stop_recording) print 'Recording. Press Enter to stop.', diff --git a/systrace/catapult/devil/devil/base_error.py b/systrace/catapult/devil/devil/base_error.py index b7b33cc..dd9ed49 100644 --- a/systrace/catapult/devil/devil/base_error.py +++ b/systrace/catapult/devil/devil/base_error.py @@ -9,6 +9,7 @@ class BaseError(Exception): def __init__(self, message, is_infra_error=False): super(BaseError, self).__init__(message) self._is_infra_error = is_infra_error + self.message = message def __eq__(self, other): return (self.message == other.message diff --git a/systrace/catapult/devil/devil/devil_env.py b/systrace/catapult/devil/devil/devil_env.py index b61d1b0..2d57688 100644 --- a/systrace/catapult/devil/devil/devil_env.py +++ b/systrace/catapult/devil/devil/devil_env.py @@ -16,6 +16,7 @@ CATAPULT_ROOT_PATH = os.path.abspath( DEPENDENCY_MANAGER_PATH = os.path.join(CATAPULT_ROOT_PATH, 'dependency_manager') PYMOCK_PATH = os.path.join(CATAPULT_ROOT_PATH, 'third_party', 'mock') PY_UTILS_PATH = os.path.join(CATAPULT_ROOT_PATH, 'common', 'py_utils') +SIX_PATH = os.path.join(CATAPULT_ROOT_PATH, 'third_party', 'six') @contextlib.contextmanager @@ -31,6 +32,9 @@ def SysPath(path): with SysPath(DEPENDENCY_MANAGER_PATH): import dependency_manager # pylint: disable=import-error +with SysPath(SIX_PATH): + import six + _ANDROID_BUILD_TOOLS = {'aapt', 'dexdump', 'split-select'} _DEVIL_DEFAULT_CONFIG = os.path.abspath( @@ -53,7 +57,7 @@ def EmptyConfig(): def LocalConfigItem(dependency_name, dependency_platform, dependency_path): - if isinstance(dependency_path, basestring): + if isinstance(dependency_path, six.string_types): dependency_path = [dependency_path] return { dependency_name: { @@ -69,7 +73,7 @@ def LocalConfigItem(dependency_name, dependency_platform, dependency_path): def _GetEnvironmentVariableConfig(): env_config = EmptyConfig() path_config = ((os.environ.get(k), v) - for k, v in _LEGACY_ENVIRONMENT_VARIABLES.iteritems()) + for k, v in six.iteritems(_LEGACY_ENVIRONMENT_VARIABLES)) path_config = ((p, c) for p, c in path_config if p) for p, c in path_config: env_config['dependencies'].update( @@ -115,7 +119,8 @@ class _Environment(object): # TODO(jbudorick): Remove this recursion if/when dependency_manager # supports loading configurations directly from a dict. if configs: - with tempfile.NamedTemporaryFile(delete=False) as next_config_file: + with tempfile.NamedTemporaryFile(mode='w', + delete=False) as next_config_file: try: next_config_file.write(json.dumps(configs[0])) next_config_file.close() @@ -182,7 +187,10 @@ class _Environment(object): def GetPlatform(arch=None, device=None): if arch or device: return 'android_%s' % (arch or device.product_cpu_abi) - return '%s_%s' % (sys.platform, platform.machine()) + # use 'linux2' for linux as this is already used in json file + return '%s_%s' % ( + sys.platform if not sys.platform.startswith('linux') else 'linux2', + platform.machine()) config = _Environment() diff --git a/systrace/catapult/devil/devil/devil_env_test.py b/systrace/catapult/devil/devil/devil_env_test.py index ee7cd8f..65fd704 100755 --- a/systrace/catapult/devil/devil/devil_env_test.py +++ b/systrace/catapult/devil/devil/devil_env_test.py @@ -10,6 +10,7 @@ import sys import unittest from devil import devil_env +from devil.android.ndk import abis _sys_path_before = list(sys.path) with devil_env.SysPath(devil_env.PYMOCK_PATH): @@ -18,6 +19,11 @@ with devil_env.SysPath(devil_env.PYMOCK_PATH): _sys_path_after = list(sys.path) +class _MockDeviceUtils(object): + def __init__(self): + self.product_cpu_abi = abis.ARM_64 + + class DevilEnvTest(unittest.TestCase): def testSysPath(self): self.assertEquals(_sys_path_before, _sys_path_after) @@ -52,6 +58,21 @@ class DevilEnvTest(unittest.TestCase): }, }, env_config.get('dependencies')) + def testGetPlatform(self): + with mock.patch('platform.machine', mock.Mock(return_value='x86_64')): + with mock.patch('sys.platform', mock.Mock(return_value='linux2')): + platform = devil_env.GetPlatform() + self.assertEquals(platform, 'linux2_x86_64') + with mock.patch('sys.platform', mock.Mock(return_value='linux')): + platform = devil_env.GetPlatform() + self.assertEquals(platform, 'linux2_x86_64') + + platform = devil_env.GetPlatform(arch='arm64-v8a') + self.assertEquals(platform, 'android_arm64-v8a') + + device = _MockDeviceUtils() + platform = devil_env.GetPlatform(device=device) + self.assertEquals(platform, 'android_arm64-v8a') if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) diff --git a/systrace/catapult/devil/devil/utils/cmd_helper.py b/systrace/catapult/devil/devil/utils/cmd_helper.py index 634c971..3a95945 100644 --- a/systrace/catapult/devil/devil/utils/cmd_helper.py +++ b/systrace/catapult/devil/devil/utils/cmd_helper.py @@ -10,11 +10,19 @@ import pipes import select import signal import string -import StringIO import subprocess import sys import time +CATAPULT_ROOT_PATH = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..')) +SIX_PATH = os.path.join(CATAPULT_ROOT_PATH, 'third_party', 'six') + +if SIX_PATH not in sys.path: + sys.path.append(SIX_PATH) + +import six + from devil import base_error logger = logging.getLogger(__name__) @@ -23,7 +31,8 @@ _SafeShellChars = frozenset(string.ascii_letters + string.digits + '@%_-+=:,./') # Cache the string-escape codec to ensure subprocess can find it # later. Return value doesn't matter. -codecs.lookup('string-escape') +if six.PY2: + codecs.lookup('string-escape') def SingleQuote(s): @@ -102,6 +111,7 @@ def Popen(args, cwd=None, env=None): # preexec_fn isn't supported on windows. + # pylint: disable=unexpected-keyword-arg if sys.platform == 'win32': close_fds = (stdin is None and stdout is None and stderr is None) preexec_fn = None @@ -109,7 +119,8 @@ def Popen(args, close_fds = True preexec_fn = lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL) - return subprocess.Popen( + if six.PY2: + return subprocess.Popen( args=args, cwd=cwd, stdin=stdin, @@ -118,8 +129,29 @@ def Popen(args, shell=shell, close_fds=close_fds, env=env, - preexec_fn=preexec_fn) - + preexec_fn=preexec_fn + ) + else: + # opens stdout in text mode, so that caller side always get 'str', + # and there will be no type mismatch error. + # Ignore any decoding error, so that caller will not crash due to + # uncaught exception. Decoding errors are unavoidable, as we + # do not know the encoding of the output, and in some output there + # will be multiple encodings (e.g. adb logcat) + return subprocess.Popen( + args=args, + cwd=cwd, + stdin=stdin, + stdout=stdout, + stderr=stderr, + shell=shell, + close_fds=close_fds, + env=env, + preexec_fn=preexec_fn, + universal_newlines=True, + encoding='utf-8', + errors='ignore' + ) def Call(args, stdout=None, stderr=None, shell=None, cwd=None, env=None): pipe = Popen( @@ -165,7 +197,7 @@ def GetCmdOutput(args, cwd=None, shell=False, env=None): def _ValidateAndLogCommand(args, cwd, shell): - if isinstance(args, basestring): + if isinstance(args, six.string_types): if not shell: raise Exception('string args must be run with shell=True') else: @@ -283,6 +315,12 @@ class TimeoutError(base_error.BaseError): return self._output +def _read_and_decode(fd, buffer_size): + data = os.read(fd, buffer_size) + if data and six.PY3: + data = data.decode('utf-8', errors='ignore') + return data + def _IterProcessStdoutFcntl(process, iter_timeout=None, timeout=None, @@ -316,7 +354,7 @@ def _IterProcessStdoutFcntl(process, read_fds, _, _ = select.select([child_fd], [], [], iter_aware_poll_interval) if child_fd in read_fds: - data = os.read(child_fd, buffer_size) + data = _read_and_decode(child_fd, buffer_size) if not data: break yield data @@ -328,7 +366,7 @@ def _IterProcessStdoutFcntl(process, read_fds, _, _ = select.select([child_fd], [], [], iter_aware_poll_interval) if child_fd in read_fds: - data = os.read(child_fd, buffer_size) + data = _read_and_decode(child_fd, buffer_size) if data: yield data continue @@ -365,7 +403,7 @@ def _IterProcessStdoutQueue(process, # TODO(jbudorick): Pick an appropriate read size here. while True: try: - output_chunk = os.read(process.stdout.fileno(), buffer_size) + output_chunk = _read_and_decode(process.stdout.fileno(), buffer_size) except IOError: break stdout_queue.put(output_chunk, True) @@ -452,7 +490,7 @@ def GetCmdStatusAndOutputWithTimeout(args, TimeoutError on timeout. """ _ValidateAndLogCommand(args, cwd, shell) - output = StringIO.StringIO() + output = six.StringIO() process = Popen( args, cwd=cwd, diff --git a/systrace/catapult/devil/devil/utils/cmd_helper_test.py b/systrace/catapult/devil/devil/utils/cmd_helper_test.py index 57abceb..0eeefe1 100755 --- a/systrace/catapult/devil/devil/utils/cmd_helper_test.py +++ b/systrace/catapult/devil/devil/utils/cmd_helper_test.py @@ -33,6 +33,17 @@ class CmdHelperSingleQuoteTest(unittest.TestCase): self.assertEquals(test_string, cmd_helper.GetCmdOutput(cmd, shell=True).rstrip()) +class CmdHelperGetCmdStatusAndOutputTest(unittest.TestCase): + def testGetCmdStatusAndOutput_success(self): + cmd = 'echo "Hello World"' + status, output = cmd_helper.GetCmdStatusAndOutput(cmd, shell=True) + self.assertEqual(status, 0) + self.assertEqual(output.rstrip(), "Hello World") + + def testGetCmdStatusAndOutput_unicode(self): + # pylint: disable=no-self-use + cmd = 'echo "\x80\x31Hello World\n"' + cmd_helper.GetCmdStatusAndOutput(cmd, shell=True) class CmdHelperDoubleQuoteTest(unittest.TestCase): def testDoubleQuote_basic(self): @@ -195,7 +206,7 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): # pylint: disable=protected-access _SIMPLE_OUTPUT_SEQUENCE = [ - _ProcessOutputEvent(read_contents='1\n2\n'), + _ProcessOutputEvent(read_contents=b'1\n2\n'), ] def testIterCmdOutputLines_success(self): @@ -205,6 +216,14 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): cmd_helper._IterCmdOutputLines(mock_proc, 'mock_proc'), 1): self.assertEquals(num, int(line)) + def testIterCmdOutputLines_unicode(self): + output_sequence = [ + _ProcessOutputEvent(read_contents=b'\x80\x31\nHello\n\xE2\x98\xA0') + ] + with _MockProcess(output_sequence=output_sequence) as mock_proc: + lines = list(cmd_helper._IterCmdOutputLines(mock_proc, 'mock_proc')) + self.assertEquals(lines[1], "Hello") + def testIterCmdOutputLines_exitStatusFail(self): with self.assertRaises(subprocess.CalledProcessError): with _MockProcess( @@ -238,9 +257,9 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): def testIterCmdOutputLines_delay(self): output_sequence = [ - _ProcessOutputEvent(read_contents='1\n2\n', ts=1), + _ProcessOutputEvent(read_contents=b'1\n2\n', ts=1), _ProcessOutputEvent(read_contents=None, ts=2), - _ProcessOutputEvent(read_contents='Awake', ts=10), + _ProcessOutputEvent(read_contents=b'Awake', ts=10), ] with _MockProcess(output_sequence=output_sequence) as mock_proc: for num, line in enumerate( diff --git a/systrace/catapult/devil/devil/utils/decorators.py b/systrace/catapult/devil/devil/utils/decorators.py new file mode 100644 index 0000000..5d28610 --- /dev/null +++ b/systrace/catapult/devil/devil/utils/decorators.py @@ -0,0 +1,17 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import functools + + +def Memoize(f): + """Decorator to cache return values of function.""" + memoize_dict = {} + @functools.wraps(f) + def wrapper(*args, **kwargs): + key = repr((args, kwargs)) + if key not in memoize_dict: + memoize_dict[key] = f(*args, **kwargs) + return memoize_dict[key] + return wrapper diff --git a/systrace/catapult/devil/devil/utils/decorators_test.py b/systrace/catapult/devil/devil/utils/decorators_test.py new file mode 100755 index 0000000..f81974a --- /dev/null +++ b/systrace/catapult/devil/devil/utils/decorators_test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unit tests for decorators.py.""" + +import unittest + +from devil.utils import decorators + + +class MemoizeDecoratorTest(unittest.TestCase): + + def testFunctionExceptionNotMemoized(self): + """Tests that |Memoize| decorator does not cache exception results.""" + + class ExceptionType1(Exception): + pass + + class ExceptionType2(Exception): + pass + + @decorators.Memoize + def raiseExceptions(): + if raiseExceptions.count == 0: + raiseExceptions.count += 1 + raise ExceptionType1() + + if raiseExceptions.count == 1: + raise ExceptionType2() + raiseExceptions.count = 0 + + with self.assertRaises(ExceptionType1): + raiseExceptions() + with self.assertRaises(ExceptionType2): + raiseExceptions() + + def testFunctionResultMemoized(self): + """Tests that |Memoize| decorator caches results.""" + + @decorators.Memoize + def memoized(): + memoized.count += 1 + return memoized.count + memoized.count = 0 + + def notMemoized(): + notMemoized.count += 1 + return notMemoized.count + notMemoized.count = 0 + + self.assertEquals(memoized(), 1) + self.assertEquals(memoized(), 1) + self.assertEquals(memoized(), 1) + + self.assertEquals(notMemoized(), 1) + self.assertEquals(notMemoized(), 2) + self.assertEquals(notMemoized(), 3) + + def testFunctionMemoizedBasedOnArgs(self): + """Tests that |Memoize| caches results based on args and kwargs.""" + + @decorators.Memoize + def returnValueBasedOnArgsKwargs(a, k=0): + return a + k + + self.assertEquals(returnValueBasedOnArgsKwargs(1, 1), 2) + self.assertEquals(returnValueBasedOnArgsKwargs(1, 2), 3) + self.assertEquals(returnValueBasedOnArgsKwargs(2, 1), 3) + self.assertEquals(returnValueBasedOnArgsKwargs(3, 3), 6) + + +if __name__ == '__main__': + unittest.main() diff --git a/systrace/catapult/devil/devil/utils/markdown_test.py b/systrace/catapult/devil/devil/utils/markdown_test.py index 621d56b..11cd46e 100755 --- a/systrace/catapult/devil/devil/utils/markdown_test.py +++ b/systrace/catapult/devil/devil/utils/markdown_test.py @@ -103,10 +103,9 @@ class MarkdownTest(unittest.TestCase): def testLink(self): link_text = 'Devil home' link_target = ( - 'https://github.com/catapult-project/catapult/tree/master/devil') - expected = ( - '[Devil home]' - '(https://github.com/catapult-project/catapult/tree/master/devil)') + 'https://chromium.googlesource.com/catapult.git/+/HEAD/devil') + expected = ('[Devil home]' + '(https://chromium.googlesource.com/catapult.git/+/HEAD/devil)') self.assertEquals(expected, markdown.md_link(link_text, link_target)) def testLinkTextContainsBracket(self): diff --git a/systrace/catapult/devil/devil/utils/mock_calls.py b/systrace/catapult/devil/devil/utils/mock_calls.py index 2b35938..ba96658 100644 --- a/systrace/catapult/devil/devil/utils/mock_calls.py +++ b/systrace/catapult/devil/devil/utils/mock_calls.py @@ -51,7 +51,7 @@ class TestCase(unittest.TestCase): (call.parent.name, call.parent) for call, _ in self._expected_calls) self._patched = [ test_case.patch_call(call, side_effect=do_check(call)) - for call in watched.itervalues() + for call in watched.values() ] def __enter__(self): diff --git a/systrace/catapult/devil/devil/utils/parallelizer_test.py b/systrace/catapult/devil/devil/utils/parallelizer_test.py index 7c86148..2d8f72a 100644 --- a/systrace/catapult/devil/devil/utils/parallelizer_test.py +++ b/systrace/catapult/devil/devil/utils/parallelizer_test.py @@ -19,6 +19,7 @@ if __name__ == '__main__': os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) from devil.utils import parallelizer +from devil.base_error import BaseError class ParallelizerTestObject(object): @@ -58,7 +59,7 @@ class ParallelizerTestObject(object): def _write_completion_file(self): if self._completion_file_name and len(self._completion_file_name): with open(self._completion_file_name, 'w+b') as completion_file: - completion_file.write('complete') + completion_file.write(b'complete') def __getitem__(self, index): return self._thing[index] @@ -87,13 +88,13 @@ class ParallelizerTest(unittest.TestCase): self.assertEquals(expected, r) def testMutate(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] self.assertTrue(all(d.doReturnTheThing() for d in devices)) ParallelizerTestObject.parallel(devices).doSetTheThing(False).pFinish(1) self.assertTrue(not any(d.doReturnTheThing() for d in devices)) def testAllReturn(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] results = ParallelizerTestObject.parallel(devices).doReturnTheThing().pGet( 1) self.assertTrue(isinstance(results, list)) @@ -103,7 +104,7 @@ class ParallelizerTest(unittest.TestCase): def testAllRaise(self): devices = [ ParallelizerTestObject(Exception('thing %d' % i)) - for i in xrange(0, 10) + for i in range(0, 10) ] p = ParallelizerTestObject.parallel(devices).doRaiseTheThing() with self.assertRaises(Exception): @@ -117,21 +118,21 @@ class ParallelizerTest(unittest.TestCase): try: completion_files = [ tempfile.NamedTemporaryFile(delete=False) - for _ in xrange(0, parallel_device_count) + for _ in range(0, parallel_device_count) ] devices = [ ParallelizerTestObject( - i if i != exception_index else Exception(exception_msg), + i if i != exception_index else BaseError(exception_msg), completion_files[i].name) - for i in xrange(0, parallel_device_count) + for i in range(0, parallel_device_count) ] for f in completion_files: f.close() p = ParallelizerTestObject.parallel(devices) - with self.assertRaises(Exception) as e: + with self.assertRaises(BaseError) as e: p.doRaiseIfExceptionElseSleepFor(2).pGet(3) self.assertTrue(exception_msg in str(e.exception)) - for i in xrange(0, parallel_device_count): + for i in range(0, parallel_device_count): with open(completion_files[i].name) as f: if i == exception_index: self.assertEquals('', f.read()) @@ -142,7 +143,7 @@ class ParallelizerTest(unittest.TestCase): os.remove(f.name) def testReusable(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] p = ParallelizerTestObject.parallel(devices) results = p.doReturn(True).pGet(1) self.assertTrue(all(results)) @@ -152,23 +153,23 @@ class ParallelizerTest(unittest.TestCase): results = p.doRaise(Exception('reusableTest')).pGet(1) def testContained(self): - devices = [ParallelizerTestObject(i) for i in xrange(0, 10)] + devices = [ParallelizerTestObject(i) for i in range(0, 10)] results = (ParallelizerTestObject.parallel(devices).helper. doReturnStringThing().pGet(1)) self.assertTrue(isinstance(results, list)) self.assertEquals(10, len(results)) - for i in xrange(0, 10): + for i in range(0, 10): self.assertEquals(str(i), results[i]) def testGetItem(self): - devices = [ParallelizerTestObject(range(i, i + 10)) for i in xrange(0, 10)] + devices = [ParallelizerTestObject(range(i, i + 10)) for i in range(0, 10)] results = ParallelizerTestObject.parallel(devices)[9].pGet(1) - self.assertEquals(range(9, 19), results) + self.assertEquals(list(range(9, 19)), results) class SyncParallelizerTest(unittest.TestCase): def testContextManager(self): - in_context = [False for i in xrange(10)] + in_context = [False for i in range(10)] @contextlib.contextmanager def enter_into_context(i): @@ -179,7 +180,7 @@ class SyncParallelizerTest(unittest.TestCase): in_context[i] = False parallelized_context = parallelizer.SyncParallelizer( - [enter_into_context(i) for i in xrange(10)]) + [enter_into_context(i) for i in range(10)]) with parallelized_context: self.assertTrue(all(in_context)) diff --git a/systrace/catapult/devil/devil/utils/reraiser_thread_unittest.py b/systrace/catapult/devil/devil/utils/reraiser_thread_unittest.py index eb37456..f1fabd0 100644 --- a/systrace/catapult/devil/devil/utils/reraiser_thread_unittest.py +++ b/systrace/catapult/devil/devil/utils/reraiser_thread_unittest.py @@ -64,7 +64,7 @@ class TestReraiserThreadGroup(unittest.TestCase): ran[i] = True group = reraiser_thread.ReraiserThreadGroup() - for i in xrange(5): + for i in range(5): group.Add(reraiser_thread.ReraiserThread(f, args=[i])) group.StartAll() group.JoinAll() @@ -76,7 +76,7 @@ class TestReraiserThreadGroup(unittest.TestCase): raise TestException group = reraiser_thread.ReraiserThreadGroup( - [reraiser_thread.ReraiserThread(f) for _ in xrange(5)]) + [reraiser_thread.ReraiserThread(f) for _ in range(5)]) group.StartAll() with self.assertRaises(TestException): group.JoinAll() diff --git a/systrace/catapult/devil/devil/utils/usb_hubs.py b/systrace/catapult/devil/devil/utils/usb_hubs.py index 313cf3f..b83fb61 100644 --- a/systrace/catapult/devil/devil/utils/usb_hubs.py +++ b/systrace/catapult/devil/devil/utils/usb_hubs.py @@ -73,7 +73,7 @@ class HubType(object): A series of (int, USBNode) tuples giving a physical port and the Node connected to it. """ - for (virtual, physical) in mapping.iteritems(): + for (virtual, physical) in mapping.items(): if node.HasPort(virtual): if isinstance(physical, dict): for res in self._GppHelper(node.PortToDevice(virtual), physical): diff --git a/systrace/catapult/devil/docs/adb_wrapper.md b/systrace/catapult/devil/docs/adb_wrapper.md index 3ca8aa6..16a666f 100644 --- a/systrace/catapult/devil/docs/adb_wrapper.md +++ b/systrace/catapult/devil/docs/adb_wrapper.md @@ -1,4 +1,4 @@ -# [devil.android.sdk.adb_wrapper](https://github.com/catapult-project/catapult/blob/master/devil/devil/android/sdk/adb_wrapper.py) +# [devil.android.sdk.adb_wrapper](https://chromium.googlesource.com/catapult.git/+/HEAD/devil/devil/android/sdk/adb_wrapper.py) *This page was autogenerated. Run `devil/bin/generate_md_docs` to update* diff --git a/systrace/catapult/devil/docs/device_utils.md b/systrace/catapult/devil/docs/device_utils.md index 960ca89..ad37d8e 100644 --- a/systrace/catapult/devil/docs/device_utils.md +++ b/systrace/catapult/devil/docs/device_utils.md @@ -1,4 +1,4 @@ -# [devil.android.device_utils](https://github.com/catapult-project/catapult/blob/master/devil/devil/android/device_utils.py) +# [devil.android.device_utils](https://chromium.googlesource.com/catapult.git/+/HEAD/devil/devil/android/device_utils.py) *This page was autogenerated. Run `devil/bin/generate_md_docs` to update* @@ -467,11 +467,12 @@ Get the WebView update command sysdump on the device. WebViewPackages: Dict of installed WebView providers, mapping "package name" to "reason it's valid/invalid." - It may return an empty dictionary if device does not - support the "dumpsys webviewupdate" command. + The returned dictionary may not include all of the above keys: this depends + on the support of the platform's underlying WebViewUpdateService. This may + return an empty dictionary on OS versions which do not support querying the + WebViewUpdateService. Raises: - CommandFailedError on failure. CommandTimeoutError on timeout. DeviceUnreachableError on missing device. ``` @@ -593,6 +594,8 @@ Determines whether a particular package is installed on the device. ``` Args: package: Name of the package. + version_code: The version of the package to check for as an int, if + applicable. Only used for static shared libraries, otherwise ignored. Returns: True if the application is installed, False otherwise. @@ -866,6 +869,10 @@ Reads the contents of a file from the device. Reboot the device. ``` + Note if the device has the root privilege, it will likely lose it after the + reboot. When |block| is True, it will try to restore the root status if + applicable. + Args: block: A boolean indicating if we should wait for the reboot to complete. wifi: A boolean indicating if we should wait for wifi to be enabled after diff --git a/systrace/catapult/devil/docs/markdown.md b/systrace/catapult/devil/docs/markdown.md index b2fec50..605d4a9 100644 --- a/systrace/catapult/devil/docs/markdown.md +++ b/systrace/catapult/devil/docs/markdown.md @@ -1,4 +1,4 @@ -# [devil.utils.markdown](https://github.com/catapult-project/catapult/blob/master/devil/devil/utils/markdown.py) +# [devil.utils.markdown](https://chromium.googlesource.com/catapult.git/+/HEAD/devil/devil/utils/markdown.py) *This page was autogenerated. Run `devil/bin/generate_md_docs` to update* diff --git a/systrace/catapult/devil/docs/persistent_device_list.md b/systrace/catapult/devil/docs/persistent_device_list.md index 626a878..513f640 100644 --- a/systrace/catapult/devil/docs/persistent_device_list.md +++ b/systrace/catapult/devil/docs/persistent_device_list.md @@ -29,7 +29,7 @@ bots that upload data to the perf dashboard. ## Where it is used The persistent device list is used in the -[chromium_android](https://cs.chromium.org/chromium/build/scripts/slave/recipe_modules/chromium_android/api.py?q=known_devices_file) +[chromium_android](https://source.chromium.org/chromium/chromium/tools/build/+/HEAD:recipes/recipe_modules/chromium_android/api.py;l=50;drc=fd928820620dff8989e853accc54b1d61657f236) recipe module, and consumed by the -[device_status.py](https://cs.chromium.org/chromium/src/third_party/catapult/devil/devil/android/tools/device_status.py?q=\-\-known%5C-devices%5C-file) +[device_status.py](https://source.chromium.org/chromium/chromium/src/+/HEAD:third_party/catapult/devil/devil/android/tools/device_status.py;l=230;drc=1e5bef4469199e4daba5d8fd885966112f8a45d5) script among others. diff --git a/systrace/catapult/systrace/systrace/systrace_trace_viewer.html b/systrace/catapult/systrace/systrace/systrace_trace_viewer.html index 473261d..11efa17 100644 --- a/systrace/catapult/systrace/systrace/systrace_trace_viewer.html +++ b/systrace/catapult/systrace/systrace/systrace_trace_viewer.html @@ -5855,7 +5855,7 @@ platformSpecificTotals[rawTotalName]=parseInt(rawTotalValue,16);} if(totals.peakResidentBytes===undefined&&totals.arePeakResidentBytesResettable!==undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Optional field peak_resident_set_bytes found'+' but is_peak_rss_resetable not found in'+' process memory dump for PID='+pid+' and dump ID='+dumpId+'.'});} if(totals.arePeakResidentBytesResettable!==undefined&&totals.peakResidentBytes===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Optional field is_peak_rss_resetable found'+' but peak_resident_set_bytes not found in'+' process memory dump for PID='+pid+' and dump ID='+dumpId+'.'});} processMemoryDump.totals=totals;},parseMemoryDumpVmRegions_(processMemoryDump,dumps,pid,dumpId){const rawProcessMmaps=dumps.process_mmaps;if(rawProcessMmaps===undefined)return;const rawVmRegions=rawProcessMmaps.vm_regions;if(rawVmRegions===undefined)return;if(processMemoryDump.vmRegions!==undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'VM regions provided multiple times for'+' process memory dump for PID='+pid+' and dump ID='+dumpId+'.'});return;} -const vmRegions=new Array(rawVmRegions.length);for(let i=0;i<rawVmRegions.length;i++){const rawVmRegion=rawVmRegions[i];const byteStats={};const rawByteStats=rawVmRegion.bs;for(const rawByteStatName in rawByteStats){const rawByteStatValue=rawByteStats[rawByteStatName];if(rawByteStatValue===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Byte stat \''+rawByteStatName+'\' of VM region '+ +if(rawVmRegions===null)return;const vmRegions=new Array(rawVmRegions.length);for(let i=0;i<rawVmRegions.length;i++){const rawVmRegion=rawVmRegions[i];const byteStats={};const rawByteStats=rawVmRegion.bs;for(const rawByteStatName in rawByteStats){const rawByteStatValue=rawByteStats[rawByteStatName];if(rawByteStatValue===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Byte stat \''+rawByteStatName+'\' of VM region '+ i+' ('+rawVmRegion.mf+') in process memory dump for '+'PID='+pid+' and dump ID='+dumpId+' does not have a value.'});continue;} const byteStatName=BYTE_STAT_NAME_MAP[rawByteStatName];if(byteStatName===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Unknown byte stat name \''+rawByteStatName+'\' ('+ rawByteStatValue+') of VM region '+i+' ('+ @@ -6495,7 +6495,9 @@ for(let i=0;i<importers.length;i++){const subtraces=importers[i].extractSubtrace if(traces.length&&!this.hasEventDataDecoder_(importers)){throw new Error('Could not find an importer for the provided eventData.');} importers.sort(function(x,y){return x.importPriority-y.importPriority;});});addStageForEachImporter('Importing clock sync markers',importer=>importer.importClockSyncMarkers());addStageForEachImporter('Importing',importer=>importer.importEvents());if(this.importOptions_.customizeModelCallback){addImportStage('Customizing',()=>{this.importOptions_.customizeModelCallback(this.model_);});} addStageForEachImporter('Importing sample data',importer=>importer.importSampleData());addImportStage('Autoclosing open slices...',()=>{this.model_.autoCloseOpenSlices();this.model_.createSubSlices();});addStageForEachImporter('Finalizing import',importer=>importer.finalizeImport());addImportStage('Initializing objects (step 1/2)...',()=>this.model_.preInitializeObjects());if(this.importOptions_.pruneEmptyContainers){addImportStage('Pruning empty containers...',()=>this.model_.pruneEmptyContainers());} -addImportStage('Merging kernel with userland...',()=>this.model_.mergeKernelWithUserland());let auditors=[];addImportStage('Adding arbitrary data to model...',()=>{auditors=this.importOptions_.auditorConstructors.map(auditorConstructor=>new auditorConstructor(this.model_));auditors.forEach((auditor)=>{auditor.runAnnotate();auditor.installUserFriendlyCategoryDriverIfNeeded();});});addImportStage('Computing final world bounds...',()=>{this.model_.computeWorldBounds(this.importOptions_.shiftWorldToZero);});addImportStage('Building flow event map...',()=>this.model_.buildFlowEventIntervalTree());addImportStage('Joining object refs...',()=>this.model_.joinRefs());addImportStage('Cleaning up undeleted objects...',()=>this.model_.cleanupUndeletedObjects());addImportStage('Sorting memory dumps...',()=>this.model_.sortMemoryDumps());addImportStage('Finalizing memory dump graphs...',()=>this.model_.finalizeMemoryGraphs());addImportStage('Initializing objects (step 2/2)...',()=>this.model_.initializeObjects());addImportStage('Building event indices...',()=>this.model_.buildEventIndices());addImportStage('Building UserModel...',()=>{const userModelBuilder=new tr.importer.UserModelBuilder(this.model_);userModelBuilder.buildUserModel();});addImportStage('Sorting user expectations...',()=>this.model_.userModel.sortExpectations());addImportStage('Running auditors...',()=>{auditors.forEach(auditor=>auditor.runAudit());});addImportStage('Updating alerts...',()=>this.model_.sortAlerts());addImportStage('Update bounds...',()=>this.model_.updateBounds());addImportStage('Looking for warnings...',()=>{if(!this.model_.isTimeHighResolution){this.model_.importWarning({type:'low_resolution_timer',message:'Trace time is low resolution, trace may be unusable.',showToUser:true});}});lastTask.after(()=>{this.importing_=false;this.model_.stats.traceImportDurationMs=tr.b.Timing.getCurrentTimeMs()-importStartTimeMs;});return importTask;},createImporter_(eventData){const importerConstructor=tr.importer.Importer.findImporterFor(eventData);if(!importerConstructor){throw new Error('Couldn\'t create an importer for the provided '+'eventData.');} +addImportStage('Merging kernel with userland...',()=>this.model_.mergeKernelWithUserland());let auditors=[];addImportStage('Adding arbitrary data to model...',()=>{for(const auditorConstructor of +this.importOptions_.auditorConstructors){try{auditors.push(new auditorConstructor(this.model_));}catch(e){console.error('Failed to construct an auditor:');console.error(e);}} +auditors.forEach((auditor)=>{try{auditor.runAnnotate();auditor.installUserFriendlyCategoryDriverIfNeeded();}catch(e){console.error('Failed to run an auditor:');console.error(e);}});});addImportStage('Computing final world bounds...',()=>{this.model_.computeWorldBounds(this.importOptions_.shiftWorldToZero);});addImportStage('Building flow event map...',()=>this.model_.buildFlowEventIntervalTree());addImportStage('Joining object refs...',()=>this.model_.joinRefs());addImportStage('Cleaning up undeleted objects...',()=>this.model_.cleanupUndeletedObjects());addImportStage('Sorting memory dumps...',()=>this.model_.sortMemoryDumps());addImportStage('Finalizing memory dump graphs...',()=>this.model_.finalizeMemoryGraphs());addImportStage('Initializing objects (step 2/2)...',()=>this.model_.initializeObjects());addImportStage('Building event indices...',()=>this.model_.buildEventIndices());addImportStage('Building UserModel...',()=>{try{const userModelBuilder=new tr.importer.UserModelBuilder(this.model_);userModelBuilder.buildUserModel();}catch(e){console.error('Failed to build user model');console.error(e);}});addImportStage('Sorting user expectations...',()=>this.model_.userModel.sortExpectations());addImportStage('Running auditors...',()=>{auditors.forEach(auditor=>auditor.runAudit());});addImportStage('Updating alerts...',()=>this.model_.sortAlerts());addImportStage('Update bounds...',()=>this.model_.updateBounds());addImportStage('Looking for warnings...',()=>{if(!this.model_.isTimeHighResolution){this.model_.importWarning({type:'low_resolution_timer',message:'Trace time is low resolution, trace may be unusable.',showToUser:true});}});lastTask.after(()=>{this.importing_=false;this.model_.stats.traceImportDurationMs=tr.b.Timing.getCurrentTimeMs()-importStartTimeMs;});return importTask;},createImporter_(eventData){const importerConstructor=tr.importer.Importer.findImporterFor(eventData);if(!importerConstructor){throw new Error('Couldn\'t create an importer for the provided '+'eventData.');} return new importerConstructor(this.model_,eventData);},hasEventDataDecoder_(importers){for(let i=0;i<importers.length;++i){if(!importers[i].isTraceDataContainer())return true;} return false;}};return{ImportOptions,Import,};});'use strict';tr.exportTo('tr.e.v8',function(){const ThreadSlice=tr.model.ThreadSlice;function V8GCStatsThreadSlice(){ThreadSlice.apply(this,arguments);this.liveObjects_=JSON.parse(this.args.live);delete this.args.live;this.deadObjects_=JSON.parse(this.args.dead);delete this.args.dead;} V8GCStatsThreadSlice.prototype={__proto__:ThreadSlice.prototype,get liveObjects(){return this.liveObjects_;},get deadObjects(){return this.deadObjects_;}};ThreadSlice.subTypes.register(V8GCStatsThreadSlice,{categoryParts:['disabled-by-default-v8.gc_stats'],name:'v8 gc stats slice',pluralName:'v8 gc stats slices'});return{V8GCStatsThreadSlice,};});'use strict';tr.exportTo('tr.e.v8',function(){const ThreadSlice=tr.model.ThreadSlice;function V8ICStatsThreadSlice(){ThreadSlice.apply(this,arguments);this.icStats_=undefined;if(this.args['ic-stats']){this.icStats_=this.args['ic-stats'].data;delete this.args['ic-stats'];}} @@ -8221,12 +8223,21 @@ const firstCounter=countBlinkObjects(dumps[0],pid);const lastCounter=countBlinkO return diffCounter;} function countBlinkObjects(dump,pid){const counter=new Map();const processesMemoryDumps=dump.processMemoryDumps;if(processesMemoryDumps[pid]===undefined)return counter;const blinkObjectsDump=processesMemoryDumps[pid].memoryAllocatorDumps.find(dump=>dump.fullName==='blink_objects');for(const v of blinkObjectsDump.children){counter.set(v.name,v.numerics.object_count.value);} return counter;} -return{leakDetectionMetric,};});'use strict';tr.exportTo('tr.metrics.console',function(){const COUNT_BOUNDARIES=tr.v.HistogramBinBoundaries.createExponential(1,1e4,30);const SUMMARY_OPTIONS=tr.v.Histogram.AVERAGE_ONLY_SUMMARY_OPTIONS;const SOURCES=['all','js','network'];function consoleErrorMetric(histograms,model){const counts={};for(const source of SOURCES){counts[source]={count:0,details:[]};} +return{leakDetectionMetric,};});'use strict';tr.exportTo('tr.metrics',function(){function blinkResourceMetric(histograms,model,opt_options){const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!chromeHelper){return;} +const CATEGORY='blink';const NAME='ResourceFetcher::requestResource';let count=0;for(const helper of Object.values(chromeHelper.rendererHelpers)){if(helper.isChromeTracingUI)continue;const events=tr.e.chrome.EventFinderUtils.getMainThreadEvents(helper,NAME,CATEGORY);for(const event of events){count++;}} +histograms.createHistogram('blinkRequestResourceCount',tr.b.Unit.byName.count,count);} +tr.metrics.MetricRegistry.register(blinkResourceMetric,{supportsRangeOfInterest:false,});return{blinkResourceMetric,};});'use strict';tr.exportTo('tr.metrics.console',function(){const COUNT_BOUNDARIES=tr.v.HistogramBinBoundaries.createExponential(1,1e4,30);const SUMMARY_OPTIONS=tr.v.Histogram.AVERAGE_ONLY_SUMMARY_OPTIONS;const SOURCES=['all','js','network'];function consoleErrorMetric(histograms,model){const counts={};for(const source of SOURCES){counts[source]={count:0,details:[]};} for(const slice of model.getDescendantEvents()){if(slice.category==='blink.console'&&slice.title==='ConsoleMessage::Error'){const source=slice.args.source.toLowerCase();counts.all.count++;if(slice.args.message){counts.all.details.push({pid:slice.getProcess().pid,...slice.args.message});} if(source in counts){counts[source].count++;if(slice.args.message){counts[source].details.push({pid:slice.getProcess().pid,...slice.args.message});}}} if(slice.category==='v8.console'&&(slice.title==='V8ConsoleMessage::Exception'||slice.title==='V8ConsoleMessage::Error'||slice.title==='V8ConsoleMessage::Assert')){counts.all.count++;counts.js.count++;}} for(const source of SOURCES){const hist=histograms.createHistogram(`console:error:${source}`,tr.b.Unit.byName.count_smallerIsBetter,{value:counts[source].count,diagnostics:{details:new tr.v.d.GenericSet(counts[source].details)}},{description:`Number of ${source} console error messages`,summaryOptions:SUMMARY_OPTIONS,});}} -tr.metrics.MetricRegistry.register(consoleErrorMetric);return{consoleErrorMetric,};});'use strict';tr.exportTo('tr.metrics.sh',function(){function getCpuSnapshotsFromModel(model){const snapshots=[];for(const pid in model.processes){const snapshotInstances=model.processes[pid].objects.getAllInstancesNamed('CPUSnapshots');if(!snapshotInstances)continue;for(const object of snapshotInstances[0].snapshots){snapshots.push(object.args.processes);}} +tr.metrics.MetricRegistry.register(consoleErrorMetric);return{consoleErrorMetric,};});'use strict';tr.exportTo('tr.metrics',function(){function countSumMetric(histograms,model,opt_options){const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!chromeHelper){return;} +const CATEGORY='benchmark';const NAME='count_sum';const counts=new Map();const sums=new Map();for(const pid in chromeHelper.rendererHelpers){const helper=chromeHelper.rendererHelpers[pid];if(helper.isChromeTracingUI)continue;const events=tr.e.chrome.EventFinderUtils.getMainThreadEvents(helper,NAME,CATEGORY);for(const event of events){const c=event.args.counter;if(!c){continue;} +if(!counts.get(c)){counts.set(c,0);} +counts.set(c,counts.get(c)+1);if(event.args.value){if(!sums.get(c)){sums.set(c,0);} +sums.set(c,sums.get(c)+event.args.value);}}} +counts.forEach((value,key)=>{histograms.createHistogram('count_'+key,tr.b.Unit.byName.count,value);});sums.forEach((value,key)=>{histograms.createHistogram('sum_'+key,tr.b.Unit.byName.unitlessNumber,value);});} +tr.metrics.MetricRegistry.register(countSumMetric,{supportsRangeOfInterest:false,});return{countSumMetric,};});'use strict';tr.exportTo('tr.metrics.sh',function(){function getCpuSnapshotsFromModel(model){const snapshots=[];for(const pid in model.processes){const snapshotInstances=model.processes[pid].objects.getAllInstancesNamed('CPUSnapshots');if(!snapshotInstances)continue;for(const object of snapshotInstances[0].snapshots){snapshots.push(object.args.processes);}} return snapshots;} function getProcessSumsFromSnapshot(snapshot){const processSums=new Map();for(const processData of snapshot){const processName=processData.name;if(!(processSums.has(processName))){processSums.set(processName,{sum:0.0,paths:new Set()});} processSums.get(processName).sum+=parseFloat(processData.pCpu);if(processData.path){processSums.get(processName).paths.add(processData.path);}} @@ -8291,11 +8302,10 @@ processVideoFreezing(freezing){if(this.freezing_===undefined||this.freezing_>fre addMetricToHistograms(histograms){this.addSample_(histograms,'time_to_video_play',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.timeToVideoPlay);this.addSample_(histograms,'time_to_audio_play',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.timeToAudioPlay);this.addSample_(histograms,'dropped_frame_count',tr.b.Unit.byName.count_smallerIsBetter,this.droppedFrameCount);for(const[key,value]of this.seekTimes.entries()){const keyString=key.toString().replace('.','_');this.addSample_(histograms,'pipeline_seek_time_'+keyString,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,value.pipelineSeekTime);this.addSample_(histograms,'seek_time_'+keyString,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,value.seekTime);} this.addSample_(histograms,'buffering_time',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.bufferingTime);this.addSample_(histograms,'roughness',tr.b.Unit.byName.count_smallerIsBetter,this.roughness);this.addSample_(histograms,'freezing',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.freezing);} addSample_(histograms,name,unit,sample){if(sample===undefined)return;const histogram=histograms.getHistogramNamed(name);if(histogram===undefined){histograms.createHistogram(name,unit,sample);}else{histogram.addSample(sample);}}} -tr.metrics.MetricRegistry.register(mediaMetric);return{mediaMetric,};});'use strict';tr.exportTo('tr.metrics',function(){function memoryAblationMetric(histograms,model){const modelHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!modelHelper.gpuHelper)return;const gpuProcess=modelHelper.gpuHelper.process;const events=[...gpuProcess.findTopmostSlicesNamed('Memory.GPU.PeakMemoryUsage.AblationTimes')];const allocHistogram=histograms.createHistogram('Ablation Alloc',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,[],{binBoundaries:tr.v.HistogramBinBoundaries.createLinear(0,10000,20),description:'The amount of time spent allocating the ablation '+'memory',summaryOptions:tr.metrics.rendering.SUMMARY_OPTIONS,});const deallocHistogram=histograms.createHistogram('Ablation Dealloc',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,[],{binBoundaries:tr.v.HistogramBinBoundaries.createLinear(0,10000,20),description:'The amount of time spent deallocating the ablation '+'memory',summaryOptions:tr.metrics.rendering.SUMMARY_OPTIONS,});for(let i=0;i<events.length;i++){allocHistogram.addSample(events[i].args.alloc);deallocHistogram.addSample(events[i].args.dealloc);}} -tr.metrics.MetricRegistry.register(memoryAblationMetric,{requiredCategories:['gpu.memory'],});return{memoryAblationMetric,};});'use strict';tr.exportTo('tr.metrics.pa',function(){function pcscanMetric(histograms,model){function createNumericForProcess(name,processName,desc){function createNumericForEventTime(name,desc){const n=new tr.v.Histogram(name,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter);n.description=desc;n.customizeSummaryOptions({avg:true,count:true,max:true,min:true,std:true,sum:true});return n;} -const scheme=['pa','pcscan',processName];if(name)scheme.push(name);return createNumericForEventTime(scheme.join(':'),desc);} -function createHistsForProcess(processName){return{scan:createNumericForProcess('scan',processName,'Time for scanning heap for quarantine pointers'),sweep:createNumericForProcess('sweep',processName,'Time for sweeping quarantine'),clear:createNumericForProcess('clear',processName,'Time for clearing quarantine entries'),total:createNumericForProcess('',processName,'Total time for PCScan execution')};} -function addSample(hists,slice){if(!(slice instanceof tr.model.ThreadSlice))return;if(slice.category!=='partition_alloc')return;if(slice.title==='PCScan.Scan'){hists.scan.addSample(slice.duration);}else if(slice.title==='PCScan.Sweep'){hists.sweep.addSample(slice.duration);}else if(slice.title==='PCScan.Clear'){hists.clear.addSample(slice.duration);}else if(slice.title==='PCScan'){hists.total.addSample(slice.duration);}} +tr.metrics.MetricRegistry.register(mediaMetric);return{mediaMetric,};});'use strict';tr.exportTo('tr.metrics.pa',function(){function pcscanMetric(histograms,model){function createNumericForProcess(name,processName,context,desc){function createNumericForEventTime(name,desc){const n=new tr.v.Histogram(name,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter);n.description=desc;n.customizeSummaryOptions({avg:true,count:true,max:true,min:true,std:true,sum:true});return n;} +const scheme=['pa','pcscan',processName,context];if(name)scheme.push(name);return createNumericForEventTime(scheme.join(':'),desc);} +function createHistsForProcess(processName){return{scanner_scan:createNumericForProcess('scan',processName,'scanner','Time for scanning heap for quarantine pointers on concurrent threads'),scanner_sweep:createNumericForProcess('sweep',processName,'scanner','Time for sweeping quarantine'),scanner_clear:createNumericForProcess('clear',processName,'scanner','Time for clearing quarantine entries'),scanner_total:createNumericForProcess('',processName,'scanner','Total time for PCScan execution on concurrent threads'),mutator_scan:createNumericForProcess('scan',processName,'mutator','Time for scanning heap for quarantine pointers on mutator threads'),mutator_clear:createNumericForProcess('clear',processName,'mutator','Time for clearing heap quarantine entries on mutator threads'),mutator_total:createNumericForProcess('',processName,'mutator','Total time for PCScan execution on mutator threads (inside safepoints)'),};} +function addSample(hists,slice){if(!(slice instanceof tr.model.ThreadSlice))return;if(slice.category!=='partition_alloc')return;if(slice.title==='PCScan.Scanner.Scan'){hists.scanner_scan.addSample(slice.duration);}else if(slice.title==='PCScan.Scanner.Sweep'){hists.scanner_sweep.addSample(slice.duration);}else if(slice.title==='PCScan.Scanner.Clear'){hists.scanner_clear.addSample(slice.duration);}else if(slice.title==='PCScan.Scanner'){hists.scanner_total.addSample(slice.duration);}else if(slice.title==='PCScan.Mutator.Scan'){hists.mutator_scan.addSample(slice.duration);}else if(slice.title==='PCScan.Mutator.Clear'){hists.mutator_clear.addSample(slice.duration);}else if(slice.title==='PCScan.Mutator'){hists.mutator_total.addSample(slice.duration);}} function addHistsForProcess(processHists,processHelpers){for(const helper of Object.values(processHelpers)){const processName=tr.e.chrome.chrome_processes.canonicalizeProcessName(helper.process.name);if(!processHists.has(processName)){processHists.set(processName,createHistsForProcess(processName));} for(const slice of helper.process.getDescendantEvents()){addSample(processHists.get(processName),slice);}}} const helper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);const processHists=new Map();addHistsForProcess(processHists,helper.browserHelpers);addHistsForProcess(processHists,helper.rendererHelpers);for(const hists of processHists.values()){for(const hist of Object.values(hists)){histograms.addHistogram(hist);}}} @@ -8477,7 +8487,9 @@ function collectTimeToEvent(rendererHelper,timeToXEntries){const samples=[];for( return samples;} function collectTimeToEventInCpuTime(rendererHelper,timeToXEntries){const samples=[];for(const{targetEvent,navigationStartEvent,url}of timeToXEntries){const navStartToEventRange=tr.b.math.Range.fromExplicitRange(navigationStartEvent.start,targetEvent.start);const mainThreadCpuTime=rendererHelper.mainThread.getCpuTimeForRange(navStartToEventRange);const breakdownTree=tr.metrics.sh.generateCpuTimeBreakdownTree(rendererHelper.mainThread,navStartToEventRange);samples.push({value:mainThreadCpuTime,breakdownTree,diagnostics:{breakdown:createBreakdownDiagnostic(breakdownTree),start:new RelatedEventSet(navigationStartEvent),end:new RelatedEventSet(targetEvent),infos:new tr.v.d.GenericSet([{pid:rendererHelper.pid,start:navigationStartEvent.start,event:targetEvent.start,}]),}});} return samples;} -function findLayoutShiftSamples(rendererHelper){let sample;EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'LayoutShift','loading').forEach((events)=>{const evData=events.pop().args.data;if(evData.is_main_frame){sample={value:evData.cumulative_score};}});return sample?[sample]:[];} +function findMainFrameLayoutShiftSamples(rendererHelper){let sample;EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'LayoutShift','loading').forEach((events)=>{const evData=events.pop().args.data;if(evData.is_main_frame){sample={value:evData.cumulative_score};}});return sample?[sample]:[];} +function findAllLayoutShiftSamples(chromeHelper){let total=0;let foundMainFrame=false;for(const pid in chromeHelper.rendererHelpers){const rendererHelper=chromeHelper.rendererHelpers[pid];if(rendererHelper.isChromeTracingUI)continue;tr.e.chrome.EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'LayoutShift','loading').forEach((events)=>{for(const event of events){const evData=event.args.data;if(evData.is_main_frame){total+=evData.score;foundMainFrame=true;}else{total+=evData.weighted_score_delta;}}});} +return foundMainFrame?[{value:total}]:[];} function addFirstMeaningfulPaintSample(samples,rendererHelper,navigationStart,fmpMarkerEvent,url){const navStartToFMPRange=tr.b.math.Range.fromExplicitRange(navigationStart.start,fmpMarkerEvent.start);const networkEvents=EventFinderUtils.getNetworkEventsInRange(rendererHelper.process,navStartToFMPRange);const timeToFirstMeaningfulPaint=navStartToFMPRange.duration;const breakdownTree=tr.metrics.sh.generateWallClockTimeBreakdownTree(rendererHelper.mainThread,networkEvents,navStartToFMPRange);samples.push({value:timeToFirstMeaningfulPaint,breakdownTree,diagnostics:{breakdown:createBreakdownDiagnostic(breakdownTree),start:new RelatedEventSet(navigationStart),end:new RelatedEventSet(fmpMarkerEvent),infos:new tr.v.d.GenericSet([{url,pid:rendererHelper.pid,start:navigationStart.start,fmp:fmpMarkerEvent.start,}]),}});} function addFirstMeaningfulPaintCpuTimeSample(samples,rendererHelper,navigationStart,fmpMarkerEvent,url){const navStartToFMPRange=tr.b.math.Range.fromExplicitRange(navigationStart.start,fmpMarkerEvent.start);const mainThreadCpuTime=rendererHelper.mainThread.getCpuTimeForRange(navStartToFMPRange);const breakdownTree=tr.metrics.sh.generateCpuTimeBreakdownTree(rendererHelper.mainThread,navStartToFMPRange);samples.push({value:mainThreadCpuTime,breakdownTree,diagnostics:{breakdown:createBreakdownDiagnostic(breakdownTree),start:new RelatedEventSet(navigationStart),end:new RelatedEventSet(fmpMarkerEvent),infos:new tr.v.d.GenericSet([{url,pid:rendererHelper.pid,start:navigationStart.start,fmp:fmpMarkerEvent.start,}]),}});} function decorateInteractivitySampleWithDiagnostics_(rendererHelper,eventTimestamp,navigationStartEvent,firstContentfulPaintTime,domContentLoadedEndTime,url){if(eventTimestamp===undefined)return undefined;const navigationStartTime=navigationStartEvent.start;const navStartToEventTimeRange=tr.b.math.Range.fromExplicitRange(navigationStartTime,eventTimestamp);const networkEvents=EventFinderUtils.getNetworkEventsInRange(rendererHelper.process,navStartToEventTimeRange);const breakdownTree=tr.metrics.sh.generateWallClockTimeBreakdownTree(rendererHelper.mainThread,networkEvents,navStartToEventTimeRange);const breakdownDiagnostic=createBreakdownDiagnostic(breakdownTree);return{value:navStartToEventTimeRange.duration,diagnostics:tr.v.d.DiagnosticMap.fromObject({'Start':new RelatedEventSet(navigationStartEvent),'Navigation infos':new tr.v.d.GenericSet([{url,pid:rendererHelper.pid,navigationStartTime,firstContentfulPaintTime,domContentLoadedEndTime,eventTimestamp,}]),'Breakdown of [navStart, eventTimestamp]':breakdownDiagnostic,}),};} @@ -8490,7 +8502,7 @@ return lastCandidates;} function findLargestTextPaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents){const timeToPaintEntries=findTimeToXEntries('loading','LargestTextPaint::Candidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const timeToPaintBlockingEntries=findTimeToXEntries('loading','LargestTextPaint::NoCandidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const lastCandidateEvents=findLastCandidateForEachNavigation(timeToPaintEntries.concat(timeToPaintBlockingEntries)).filter(event=>event.targetEvent.title!=='LargestTextPaint::NoCandidate');return collectTimeToEvent(rendererHelper,lastCandidateEvents);} function findLargestImagePaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents){const timeToPaintEntries=findTimeToXEntries('loading','LargestImagePaint::Candidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const timeToPaintBlockingEntries=findTimeToXEntries('loading','LargestImagePaint::NoCandidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const lastCandidateEvents=findLastCandidateForEachNavigation(timeToPaintEntries.concat(timeToPaintBlockingEntries)).filter(event=>event.targetEvent.title!=='LargestImagePaint::NoCandidate');return collectTimeToEvent(rendererHelper,lastCandidateEvents);} function findLargestContentfulPaintHistogramSamples(allBrowserEvents){const lcp=new tr.e.chrome.LargestContentfulPaint(allBrowserEvents);const lcpSamples=lcp.findCandidates().map(candidate=>{const{durationInMilliseconds,size,type,inMainFrame,mainFrameTreeNodeId}=candidate;return{value:durationInMilliseconds,diagnostics:{size:new tr.v.d.GenericSet([size]),type:new tr.v.d.GenericSet([type]),inMainFrame:new tr.v.d.GenericSet([inMainFrame]),mainFrameTreeNodeId:new tr.v.d.GenericSet([mainFrameTreeNodeId]),},};});return lcpSamples;} -function collectLoadingMetricsForRenderer(rendererHelper){const frameToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'navigationStart','blink.user_timing');const navIdToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByNavId(rendererHelper,'navigationStart','blink.user_timing');const firstPaintSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('loading','firstPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const timeToFCPEntries=findTimeToXEntries('loading','firstContentfulPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const firstContentfulPaintSamples=collectTimeToEvent(rendererHelper,timeToFCPEntries);const firstContentfulPaintCpuTimeSamples=collectTimeToEventInCpuTime(rendererHelper,timeToFCPEntries);const onLoadSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('blink.user_timing','loadEventStart',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const aboveTheFoldLoadedToVisibleSamples=getAboveTheFoldLoadedToVisibleSamples(rendererHelper);const firstViewportReadySamples=getFirstViewportReadySamples(rendererHelper,navIdToNavStartEvents);const largestImagePaintSamples=findLargestImagePaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const largestTextPaintSamples=findLargestTextPaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const layoutShiftSamples=findLayoutShiftSamples(rendererHelper);const navigationStartSamples=timeToFCPEntries.map(entry=>{return{value:entry.navigationStartEvent.start};});return{frameToNavStartEvents,firstPaintSamples,firstContentfulPaintSamples,firstContentfulPaintCpuTimeSamples,onLoadSamples,aboveTheFoldLoadedToVisibleSamples,firstViewportReadySamples,largestImagePaintSamples,largestTextPaintSamples,layoutShiftSamples,navigationStartSamples,};} +function collectLoadingMetricsForRenderer(rendererHelper){const frameToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'navigationStart','blink.user_timing');const navIdToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByNavId(rendererHelper,'navigationStart','blink.user_timing');const firstPaintSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('loading','firstPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const timeToFCPEntries=findTimeToXEntries('loading','firstContentfulPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const firstContentfulPaintSamples=collectTimeToEvent(rendererHelper,timeToFCPEntries);const firstContentfulPaintCpuTimeSamples=collectTimeToEventInCpuTime(rendererHelper,timeToFCPEntries);const onLoadSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('blink.user_timing','loadEventStart',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const aboveTheFoldLoadedToVisibleSamples=getAboveTheFoldLoadedToVisibleSamples(rendererHelper);const firstViewportReadySamples=getFirstViewportReadySamples(rendererHelper,navIdToNavStartEvents);const largestImagePaintSamples=findLargestImagePaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const largestTextPaintSamples=findLargestTextPaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const mainFrameLayoutShiftSamples=findMainFrameLayoutShiftSamples(rendererHelper);const navigationStartSamples=timeToFCPEntries.map(entry=>{return{value:entry.navigationStartEvent.start};});return{frameToNavStartEvents,firstPaintSamples,firstContentfulPaintSamples,firstContentfulPaintCpuTimeSamples,onLoadSamples,aboveTheFoldLoadedToVisibleSamples,firstViewportReadySamples,largestImagePaintSamples,largestTextPaintSamples,mainFrameLayoutShiftSamples,navigationStartSamples,};} function collectMetricsFromLoadExpectations(model,chromeHelper){const interactiveSamples=[];const firstCpuIdleSamples=[];const firstMeaningfulPaintSamples=[];const firstMeaningfulPaintCpuTimeSamples=[];const totalBlockingTimeSamples=[];for(const expectation of model.userModel.expectations){if(!(expectation instanceof tr.model.um.LoadExpectation))continue;if(tr.e.chrome.CHROME_INTERNAL_URLS.includes(expectation.url)){continue;} const rendererHelper=chromeHelper.rendererHelpers[expectation.renderProcess.pid];if(expectation.fmpEvent!==undefined){addFirstMeaningfulPaintSample(firstMeaningfulPaintSamples,rendererHelper,expectation.navigationStart,expectation.fmpEvent,expectation.url);addFirstMeaningfulPaintCpuTimeSample(firstMeaningfulPaintCpuTimeSamples,rendererHelper,expectation.navigationStart,expectation.fmpEvent,expectation.url);} if(expectation.firstCpuIdleTime!==undefined){firstCpuIdleSamples.push(decorateInteractivitySampleWithDiagnostics_(rendererHelper,expectation.firstCpuIdleTime,expectation.navigationStart,expectation.fcpEvent.start,expectation.domContentLoadedEndEvent.start,expectation.url));} @@ -8500,7 +8512,7 @@ return{firstMeaningfulPaintSamples,firstMeaningfulPaintCpuTimeSamples,firstCpuId function addSamplesToHistogram(samples,histogram,histograms){for(const sample of samples){histogram.addSample(sample.value,sample.diagnostics);if(histogram.name!=='timeToFirstContentfulPaint')continue;if(!sample.breakdownTree)continue;for(const[category,breakdown]of Object.entries(sample.breakdownTree)){const relatedName=`${histogram.name}:${category}`;let relatedHist=histograms.getHistogramsNamed(relatedName)[0];if(!relatedHist){relatedHist=histograms.createHistogram(relatedName,histogram.unit,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,summaryOptions:{count:false,max:false,min:false,sum:false,},});let relatedNames=histogram.diagnostics.get('breakdown');if(!relatedNames){relatedNames=new tr.v.d.RelatedNameMap();histogram.diagnostics.set('breakdown',relatedNames);} relatedNames.set(category,relatedName);} relatedHist.addSample(breakdown.total,{breakdown:tr.v.d.Breakdown.fromEntries(Object.entries(breakdown.events)),});}}} -function loadingMetric(histograms,model){const firstPaintHistogram=histograms.createHistogram('timeToFirstPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintHistogram=histograms.createHistogram('timeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const onLoadHistogram=histograms.createHistogram('timeToOnload',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to onload. '+'This is temporary metric used for PCv1/v2 sanity checking',summaryOptions:SUMMARY_OPTIONS,});const firstMeaningfulPaintHistogram=histograms.createHistogram('timeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstMeaningfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const timeToInteractiveHistogram=histograms.createHistogram('timeToInteractive',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to Interactive',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const totalBlockingTimeHistogram=histograms.createHistogram('totalBlockingTime',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Total Blocking Time',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const timeToFirstCpuIdleHistogram=histograms.createHistogram('timeToFirstCpuIdle',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to First CPU Idle',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const aboveTheFoldLoadedToVisibleHistogram=histograms.createHistogram('aboveTheFoldLoadedToVisible',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from first visible to load for AMP pages only.',summaryOptions:SUMMARY_OPTIONS,});const firstViewportReadyHistogram=histograms.createHistogram('timeToFirstViewportReady',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from navigation to load for AMP pages only. ',summaryOptions:SUMMARY_OPTIONS,});const largestImagePaintHistogram=histograms.createHistogram('largestImagePaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Image Paint',summaryOptions:SUMMARY_OPTIONS,});const largestTextPaintHistogram=histograms.createHistogram('largestTextPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Text Paint',summaryOptions:SUMMARY_OPTIONS,});const largestContentfulPaintHistogram=histograms.createHistogram('largestContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Contentful Paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const layoutShiftHistogram=histograms.createHistogram('mainFrameCumulativeLayoutShift',unitlessNumber_smallerIsBetter,[],{binBoundaries:LAYOUT_SHIFT_SCORE_BOUNDARIES,description:'Main Frame Document Cumulative Layout Shift Score',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_LAYOUT],});const navigationStartHistogram=histograms.createHistogram('navigationStart',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'navigationStart',summaryOptions:SUMMARY_OPTIONS,});tr.metrics.sh.rectsBasedSpeedIndexMetric(histograms,model);const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);for(const pid in chromeHelper.rendererHelpers){const rendererHelper=chromeHelper.rendererHelpers[pid];if(rendererHelper.isChromeTracingUI)continue;const samplesSet=collectLoadingMetricsForRenderer(rendererHelper);const lcpSamples=findLargestContentfulPaintHistogramSamples(chromeHelper.browserHelper.mainThread.sliceGroup.slices);addSamplesToHistogram(lcpSamples,largestContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstPaintSamples,firstPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintSamples,firstContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintCpuTimeSamples,firstContentfulPaintCpuTimeHistogram,histograms);addSamplesToHistogram(samplesSet.onLoadSamples,onLoadHistogram,histograms);addSamplesToHistogram(samplesSet.aboveTheFoldLoadedToVisibleSamples,aboveTheFoldLoadedToVisibleHistogram,histograms);addSamplesToHistogram(samplesSet.firstViewportReadySamples,firstViewportReadyHistogram,histograms);addSamplesToHistogram(samplesSet.largestImagePaintSamples,largestImagePaintHistogram,histograms);addSamplesToHistogram(samplesSet.largestTextPaintSamples,largestTextPaintHistogram,histograms);addSamplesToHistogram(samplesSet.layoutShiftSamples,layoutShiftHistogram,histograms);addSamplesToHistogram(samplesSet.navigationStartSamples,navigationStartHistogram,histograms);} +function loadingMetric(histograms,model){const firstPaintHistogram=histograms.createHistogram('timeToFirstPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintHistogram=histograms.createHistogram('timeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const onLoadHistogram=histograms.createHistogram('timeToOnload',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to onload. '+'This is temporary metric used for PCv1/v2 sanity checking',summaryOptions:SUMMARY_OPTIONS,});const firstMeaningfulPaintHistogram=histograms.createHistogram('timeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstMeaningfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const timeToInteractiveHistogram=histograms.createHistogram('timeToInteractive',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to Interactive',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const totalBlockingTimeHistogram=histograms.createHistogram('totalBlockingTime',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Total Blocking Time',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const timeToFirstCpuIdleHistogram=histograms.createHistogram('timeToFirstCpuIdle',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to First CPU Idle',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const aboveTheFoldLoadedToVisibleHistogram=histograms.createHistogram('aboveTheFoldLoadedToVisible',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from first visible to load for AMP pages only.',summaryOptions:SUMMARY_OPTIONS,});const firstViewportReadyHistogram=histograms.createHistogram('timeToFirstViewportReady',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from navigation to load for AMP pages only. ',summaryOptions:SUMMARY_OPTIONS,});const largestImagePaintHistogram=histograms.createHistogram('largestImagePaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Image Paint',summaryOptions:SUMMARY_OPTIONS,});const largestTextPaintHistogram=histograms.createHistogram('largestTextPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Text Paint',summaryOptions:SUMMARY_OPTIONS,});const largestContentfulPaintHistogram=histograms.createHistogram('largestContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Contentful Paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const mainFrameLayoutShiftHistogram=histograms.createHistogram('mainFrameCumulativeLayoutShift',unitlessNumber_smallerIsBetter,[],{binBoundaries:LAYOUT_SHIFT_SCORE_BOUNDARIES,description:'Main Frame Document Cumulative Layout Shift Score',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_LAYOUT],});const allLayoutShiftHistogram=histograms.createHistogram('overallCumulativeLayoutShift',unitlessNumber_smallerIsBetter,[],{binBoundaries:LAYOUT_SHIFT_SCORE_BOUNDARIES,description:'Document Cumulative Layout Shift Score with iframes',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_LAYOUT],});const navigationStartHistogram=histograms.createHistogram('navigationStart',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'navigationStart',summaryOptions:SUMMARY_OPTIONS,});tr.metrics.sh.rectsBasedSpeedIndexMetric(histograms,model);const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);const allLayoutShiftSamples=findAllLayoutShiftSamples(chromeHelper);addSamplesToHistogram(allLayoutShiftSamples,allLayoutShiftHistogram,histograms);for(const pid in chromeHelper.rendererHelpers){const rendererHelper=chromeHelper.rendererHelpers[pid];if(rendererHelper.isChromeTracingUI)continue;const samplesSet=collectLoadingMetricsForRenderer(rendererHelper);const lcpSamples=findLargestContentfulPaintHistogramSamples(chromeHelper.browserHelper.mainThread.sliceGroup.slices);addSamplesToHistogram(lcpSamples,largestContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstPaintSamples,firstPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintSamples,firstContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintCpuTimeSamples,firstContentfulPaintCpuTimeHistogram,histograms);addSamplesToHistogram(samplesSet.onLoadSamples,onLoadHistogram,histograms);addSamplesToHistogram(samplesSet.aboveTheFoldLoadedToVisibleSamples,aboveTheFoldLoadedToVisibleHistogram,histograms);addSamplesToHistogram(samplesSet.firstViewportReadySamples,firstViewportReadyHistogram,histograms);addSamplesToHistogram(samplesSet.largestImagePaintSamples,largestImagePaintHistogram,histograms);addSamplesToHistogram(samplesSet.largestTextPaintSamples,largestTextPaintHistogram,histograms);addSamplesToHistogram(samplesSet.mainFrameLayoutShiftSamples,mainFrameLayoutShiftHistogram,histograms);addSamplesToHistogram(samplesSet.navigationStartSamples,navigationStartHistogram,histograms);} const samplesSet=collectMetricsFromLoadExpectations(model,chromeHelper);addSamplesToHistogram(samplesSet.firstMeaningfulPaintSamples,firstMeaningfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstMeaningfulPaintCpuTimeSamples,firstMeaningfulPaintCpuTimeHistogram,histograms);addSamplesToHistogram(samplesSet.interactiveSamples,timeToInteractiveHistogram,histograms);addSamplesToHistogram(samplesSet.firstCpuIdleSamples,timeToFirstCpuIdleHistogram,histograms);addSamplesToHistogram(samplesSet.totalBlockingTimeSamples,totalBlockingTimeHistogram,histograms);} tr.metrics.MetricRegistry.register(loadingMetric);return{loadingMetric,createBreakdownDiagnostic};});'use strict';tr.exportTo('tr.metrics',function(){const SPA_NAVIGATION_START_TO_FIRST_PAINT_DURATION_BIN_BOUNDARY=tr.v.HistogramBinBoundaries.createExponential(1,1000,50);function spaNavigationMetric(histograms,model){const histogram=new tr.v.Histogram('spaNavigationStartToFpDuration',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,SPA_NAVIGATION_START_TO_FIRST_PAINT_DURATION_BIN_BOUNDARY);histogram.description='Latency between the input event causing'+' a SPA navigation and the first paint event after it';histogram.customizeSummaryOptions({count:false,sum:false,});const modelHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!modelHelper){return;} const rendererHelpers=modelHelper.rendererHelpers;if(!rendererHelpers){return;} @@ -8987,6 +8999,7 @@ return tr.v.HistogramBinBoundaries.createExponential(1e-3,1e3,50);} function umaMetric(histograms,model){const histogramValues=new Map();const nameCounts=new Map();for(const process of model.getAllProcesses()){const histogramEvents=new Map();for(const event of process.instantEvents){if(event.title!=='UMAHistogramSamples')continue;const name=event.args.name;const events=histogramEvents.get(name)||[];if(!histogramEvents.has(name))histogramEvents.set(name,events);events.push(event);} let processName=tr.e.chrome.chrome_processes.canonicalizeProcessName(process.name);nameCounts.set(processName,(nameCounts.get(processName)||0)+1);processName=`${processName}_${nameCounts.get(processName)}`;for(const[name,events]of histogramEvents){const values=histogramValues.get(name)||{sum:0,bins:[]};if(!histogramValues.has(name))histogramValues.set(name,values);const endValues=parseBuckets_(events[events.length-1],processName);if(events.length===1){mergeBins_(values,endValues,name);}else{throw new Error('There should be at most one snapshot of UMA '+`histogram for ${name} in each process.`);}}} for(const[name,values]of histogramValues){const histogram=new tr.v.Histogram(name,getHistogramUnit_(name),getHistogramBoundaries_(name));const isLinear=getIsHistogramBinsLinear_(name);let sumOfMiddles=0;let sumOfBinLengths=0;for(const bin of values.bins){sumOfMiddles+=bin.count*(bin.min+bin.max)/2;sumOfBinLengths+=bin.count*(bin.max-bin.min);} +if(name.startsWith('CompositorLatency.Type')){let histogramBoundaries=tr.v.HistogramBinBoundaries.createLinear(0,100,101);let histogramUnit=getHistogramUnit_(name);let presentedCount=values.bins[0]?values.bins[0].count:0;let delayedCount=values.bins[1]?values.bins[1].count:0;let droppedCount=values.bins[2]?values.bins[2].count:0;let inTimeCount=presentedCount-delayedCount;let totalCount=presentedCount+droppedCount;const inTimeHistogram=new tr.v.Histogram(name+'.Percentage_of_in_time_frames',histogramUnit,histogramBoundaries);inTimeHistogram.addSample(100.0*inTimeCount/totalCount);histograms.addHistogram(inTimeHistogram);const delayedHistogram=new tr.v.Histogram(name+'.Percentage_of_delayed_frames',histogramUnit,histogramBoundaries);delayedHistogram.addSample(100.0*delayedCount/totalCount);histograms.addHistogram(delayedHistogram);const droppedHistogram=new tr.v.Histogram(name+'.Percentage_of_dropped_frames',histogramUnit,histogramBoundaries);droppedHistogram.addSample(100.0*droppedCount/totalCount);histograms.addHistogram(droppedHistogram);} const shift=(values.sum-sumOfMiddles)/sumOfBinLengths;if(isLinear&&Math.abs(shift)>0.5){throw new Error(`Samples sum is wrong for ${name}.`);} for(const bin of values.bins){if(bin.count===0)continue;const shiftedValue=(bin.min+bin.max)/2+shift*(bin.max-bin.min);for(const[processName,count]of bin.processes){bin.processes.set(processName,shiftedValue*count/bin.count);} for(let i=0;i<bin.count;i++){histogram.addSample(shiftedValue,{processes:bin.processes,events:bin.events});}} @@ -9008,7 +9021,7 @@ histograms.addHistogram(cpuTotalOptimizeCode);histograms.addHistogram(wallTotalO function computeDeoptimizeCodeMetrics(histograms,model){const cpuTotalDeoptimizeCode=new tr.v.Histogram('v8_deoptimize_code_cpu_total',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);cpuTotalDeoptimizeCode.description='cpu total time spent in code deoptimization';const wallTotalDeoptimizeCode=new tr.v.Histogram('v8_deoptimize_code_wall_total',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);wallTotalDeoptimizeCode.description='wall total time spent in code deoptimization';for(const e of model.findTopmostSlicesNamed('V8.DeoptimizeCode')){cpuTotalDeoptimizeCode.addSample(e.cpuDuration);wallTotalDeoptimizeCode.addSample(e.duration);} histograms.addHistogram(cpuTotalDeoptimizeCode);histograms.addHistogram(wallTotalDeoptimizeCode);} function executionMetric(histograms,model){computeExecuteMetrics(histograms,model);computeParseLazyMetrics(histograms,model);computeCompileIgnitionMetrics(histograms,model);computeCompileFullCodeMetrics(histograms,model);computeRecompileMetrics(histograms,model);computeOptimizeCodeMetrics(histograms,model);computeDeoptimizeCodeMetrics(histograms,model);} -tr.metrics.MetricRegistry.register(executionMetric);return{executionMetric,};});'use strict';tr.exportTo('tr.metrics.v8',function(){const TARGET_FPS=60;const MS_PER_SECOND=1000;const WINDOW_SIZE_MS=MS_PER_SECOND/TARGET_FPS;const EPSILON=1e-6;const METRICS=['v8:gc:cycle:full','v8:gc:cycle:full:cpp','v8:gc:cycle:full:mark','v8:gc:cycle:full:mark:cpp','v8:gc:cycle:full:weak','v8:gc:cycle:full:weak:cpp','v8:gc:cycle:full:sweep','v8:gc:cycle:full:sweep:cpp','v8:gc:cycle:full:compact','v8:gc:cycle:full:compact:cpp','v8:gc:cycle:main_thread:full','v8:gc:cycle:main_thread:full:cpp','v8:gc:cycle:main_thread:full:mark','v8:gc:cycle:main_thread:full:mark:cpp','v8:gc:cycle:main_thread:full:weak','v8:gc:cycle:main_thread:full:weak:cpp','v8:gc:cycle:main_thread:full:sweep','v8:gc:cycle:main_thread:full:sweep:cpp','v8:gc:cycle:main_thread:full:compact','v8:gc:cycle:main_thread:full:compact:cpp','v8:gc:event:main_thread:full:atomic','v8:gc:event:main_thread:full:atomic:cpp','v8:gc:event:main_thread:full:atomic:mark','v8:gc:event:main_thread:full:atomic:mark:cpp','v8:gc:event:main_thread:full:atomic:weak','v8:gc:event:main_thread:full:atomic:weak:cpp','v8:gc:event:main_thread:full:atomic:sweep','v8:gc:event:main_thread:full:atomic:sweep:cpp','v8:gc:event:main_thread:full:atomic:compact','v8:gc:event:main_thread:full:atomic:compact:cpp','v8:gc:event:main_thread:full:incremental','v8:gc:event:main_thread:full:incremental:cpp','v8:gc:event:main_thread:full:incremental:mark','v8:gc:event:main_thread:full:incremental:mark:cpp','v8:gc:event:main_thread:full:incremental:sweep','v8:gc:event:main_thread:full:incremental:sweep:cpp','v8:gc:cycle:young','v8:gc:cycle:main_thread:young',];const V8_FULL_ATOMIC_EVENTS=['V8.GCCompactor','V8.GCFinalizeMC','V8.GCFinalizeMCReduceMemory',];const V8_FULL_MARK_EVENTS=['V8.GC_MC_BACKGROUND_MARKING','V8.GC_MC_MARK','V8.GCIncrementalMarking','V8.GCIncrementalMarkingFinalize','V8.GCIncrementalMarkingStart',];const V8_FULL_COMPACT_EVENTS=['V8.GC_MC_BACKGROUND_EVACUATE_COPY','V8.GC_MC_BACKGROUND_EVACUATE_UPDATE_POINTERS','V8.GC_MC_EVACUATE',];const V8_FULL_SWEEP_EVENTS=['V8.GC_MC_BACKGROUND_SWEEPING','V8.GC_MC_SWEEP',];const V8_FULL_WEAK_EVENTS=['V8.GC_MC_CLEAR',];const V8_YOUNG_EVENTS=['V8.GC_SCAVENGER_BACKGROUND_SCAVENGE_PARALLEL','V8.GCScavenger',];const CPP_GC_FULL_MARK_EVENTS=['BlinkGC.AtomicPauseMarkEpilogue','BlinkGC.AtomicPauseMarkPrologue','BlinkGC.AtomicPauseMarkRoots','BlinkGC.AtomicPauseMarkTransitiveClosure','BlinkGC.ConcurrentMarkingStep','BlinkGC.IncrementalMarkingStartMarking','BlinkGC.IncrementalMarkingStep','BlinkGC.MarkBailOutObjects','BlinkGC.MarkFlushEphemeronPairs','BlinkGC.MarkFlushV8References','BlinkGC.UnifiedMarkingStep','CppGC.AtomicMark','CppGC.IncrementalMark','CppGC.ConcurrentMark',];const CPP_GC_FULL_COMPACT_EVENTS=['BlinkGC.AtomicPauseSweepAndCompact','CppGC.AtomicCompact',];const CPP_GC_FULL_SWEEP_EVENTS=['BlinkGC.CompleteSweep','BlinkGC.ConcurrentSweepingStep','BlinkGC.LazySweepInIdle','BlinkGC.LazySweepOnAllocation','CppGC.AtomicSweep','CppGC.IncrementalSweep','CppGC.ConcurrentSweep',];const CPP_GC_FULL_WEAK_EVENTS=['BlinkGC.MarkWeakProcessing','CppGC.AtomicWeak',];const RULES=[{events:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic',},{events:V8_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark',},{events:CPP_GC_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark:cpp',},{events:V8_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark',},{events:CPP_GC_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark:cpp',},{events:V8_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact',},{events:CPP_GC_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact:cpp',},{events:V8_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep:cpp',},{events:V8_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak',},{events:CPP_GC_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak:cpp',},{events:V8_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep:cpp',},{events:V8_YOUNG_EVENTS,contribute_to:'young:atomic',},];const Granularity={CYCLE:'cycle',EVENT:'event',};const ThreadType={MAIN:'main',BACKGROUND:'background',ALL_THREADS:'all_threads',};class Metric{constructor(name){const parts=name.split(':');this.granularity_=parts[2];assert(this.granularity_===Granularity.CYCLE||this.granularity_===Granularity.EVENT);this.thread_=ThreadType.ALL_THREADS;let phasesIndex=3;if(parts[3]==='main_thread'){this.thread_=ThreadType.MAIN;phasesIndex=4;} +tr.metrics.MetricRegistry.register(executionMetric);return{executionMetric,};});'use strict';tr.exportTo('tr.metrics.v8',function(){const TARGET_FPS=60;const MS_PER_SECOND=1000;const WINDOW_SIZE_MS=MS_PER_SECOND/TARGET_FPS;const EPSILON=1e-6;const METRICS=['v8:gc:cycle:full','v8:gc:cycle:full:cpp','v8:gc:cycle:full:mark','v8:gc:cycle:full:mark:cpp','v8:gc:cycle:full:weak','v8:gc:cycle:full:weak:cpp','v8:gc:cycle:full:sweep','v8:gc:cycle:full:sweep:cpp','v8:gc:cycle:full:compact','v8:gc:cycle:full:compact:cpp','v8:gc:cycle:main_thread:full','v8:gc:cycle:main_thread:full:cpp','v8:gc:cycle:main_thread:full:mark','v8:gc:cycle:main_thread:full:mark:cpp','v8:gc:cycle:main_thread:full:weak','v8:gc:cycle:main_thread:full:weak:cpp','v8:gc:cycle:main_thread:full:sweep','v8:gc:cycle:main_thread:full:sweep:cpp','v8:gc:cycle:main_thread:full:compact','v8:gc:cycle:main_thread:full:compact:cpp','v8:gc:cycle:main_thread:full:atomic','v8:gc:cycle:main_thread:full:atomic:cpp','v8:gc:cycle:main_thread:full:atomic:mark','v8:gc:cycle:main_thread:full:atomic:mark:cpp','v8:gc:cycle:main_thread:full:atomic:weak','v8:gc:cycle:main_thread:full:atomic:weak:cpp','v8:gc:cycle:main_thread:full:atomic:sweep','v8:gc:cycle:main_thread:full:atomic:sweep:cpp','v8:gc:cycle:main_thread:full:atomic:compact','v8:gc:cycle:main_thread:full:atomic:compact:cpp','v8:gc:cycle:main_thread:full:incremental','v8:gc:cycle:main_thread:full:incremental:cpp','v8:gc:cycle:main_thread:full:incremental:mark','v8:gc:cycle:main_thread:full:incremental:mark:cpp','v8:gc:cycle:main_thread:full:incremental:sweep','v8:gc:cycle:main_thread:full:incremental:sweep:cpp','v8:gc:event:main_thread:full:atomic','v8:gc:event:main_thread:full:atomic:cpp','v8:gc:event:main_thread:full:atomic:mark','v8:gc:event:main_thread:full:atomic:mark:cpp','v8:gc:event:main_thread:full:atomic:weak','v8:gc:event:main_thread:full:atomic:weak:cpp','v8:gc:event:main_thread:full:atomic:sweep','v8:gc:event:main_thread:full:atomic:sweep:cpp','v8:gc:event:main_thread:full:atomic:compact','v8:gc:event:main_thread:full:atomic:compact:cpp','v8:gc:event:main_thread:full:incremental','v8:gc:event:main_thread:full:incremental:cpp','v8:gc:event:main_thread:full:incremental:mark','v8:gc:event:main_thread:full:incremental:mark:cpp','v8:gc:event:main_thread:full:incremental:sweep','v8:gc:event:main_thread:full:incremental:sweep:cpp','v8:gc:cycle:young','v8:gc:cycle:main_thread:young',];const V8_FULL_ATOMIC_EVENTS=['V8.GC_MARK_COMPACTOR'];const V8_FULL_MARK_EVENTS=['V8.GC_MC_BACKGROUND_MARKING','V8.GC_MC_MARK','V8.GC_MC_INCREMENTAL','V8.GCIncrementalMarkingFinalize','V8.GCIncrementalMarkingStart',];const V8_FULL_COMPACT_EVENTS=['V8.GC_MC_BACKGROUND_EVACUATE_COPY','V8.GC_MC_BACKGROUND_EVACUATE_UPDATE_POINTERS','V8.GC_MC_EVACUATE',];const V8_FULL_SWEEP_EVENTS=['V8.GC_MC_BACKGROUND_SWEEPING','V8.GC_MC_SWEEP',];const V8_FULL_WEAK_EVENTS=['V8.GC_MC_CLEAR',];const V8_YOUNG_EVENTS=['V8.GC_SCAVENGER_BACKGROUND_SCAVENGE_PARALLEL','V8.GC_SCAVENGER',];const CPP_GC_FULL_MARK_EVENTS=['BlinkGC.AtomicPauseMarkEpilogue','BlinkGC.AtomicPauseMarkPrologue','BlinkGC.AtomicPauseMarkRoots','BlinkGC.AtomicPauseMarkTransitiveClosure','BlinkGC.ConcurrentMarkingStep','BlinkGC.IncrementalMarkingStartMarking','BlinkGC.IncrementalMarkingStep','BlinkGC.MarkBailOutObjects','BlinkGC.MarkFlushEphemeronPairs','BlinkGC.MarkFlushV8References','BlinkGC.UnifiedMarkingStep','CppGC.AtomicMark','CppGC.IncrementalMark','CppGC.ConcurrentMark',];const CPP_GC_FULL_COMPACT_EVENTS=['BlinkGC.AtomicPauseSweepAndCompact','CppGC.AtomicCompact',];const CPP_GC_FULL_SWEEP_EVENTS=['BlinkGC.CompleteSweep','BlinkGC.ConcurrentSweepingStep','BlinkGC.LazySweepInIdle','BlinkGC.LazySweepOnAllocation','CppGC.AtomicSweep','CppGC.IncrementalSweep','CppGC.ConcurrentSweep',];const CPP_GC_FULL_WEAK_EVENTS=['BlinkGC.MarkWeakProcessing','CppGC.AtomicWeak',];const RULES=[{events:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic',},{events:V8_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark',},{events:CPP_GC_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark:cpp',},{events:V8_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark',},{events:CPP_GC_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark:cpp',},{events:V8_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact',},{events:CPP_GC_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact:cpp',},{events:V8_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep:cpp',},{events:V8_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak',},{events:CPP_GC_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak:cpp',},{events:V8_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep:cpp',},{events:V8_YOUNG_EVENTS,contribute_to:'young:atomic',},];const Granularity={CYCLE:'cycle',EVENT:'event',};const ThreadType={MAIN:'main',BACKGROUND:'background',ALL_THREADS:'all_threads',};class Metric{constructor(name){const parts=name.split(':');this.granularity_=parts[2];assert(this.granularity_===Granularity.CYCLE||this.granularity_===Granularity.EVENT);this.thread_=ThreadType.ALL_THREADS;let phasesIndex=3;if(parts[3]==='main_thread'){this.thread_=ThreadType.MAIN;phasesIndex=4;} if(parts[3]==='background_threads'){this.thread_=ThreadType.BACKGROUND;phasesIndex=4;} this.phases_=parts.slice(phasesIndex);const maxValue=this.isPerCycleMetric()?10000:1000;const boundaries=tr.v.HistogramBinBoundaries.createExponential(0.1,maxValue,100);this.histogram=new tr.v.Histogram(name,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,boundaries);this.histogram.customizeSummaryOptions({avg:true,count:true,max:true,min:false,std:false,sum:this.isPerCycleMetric(),});} isPerCycleMetric(){return this.granularity_===Granularity.CYCLE;} @@ -9045,16 +9058,19 @@ function getEpoch(event){function checkEpochConsistency(epoch,event){if(epoch=== const result={v8:null,cpp:null};while(event){if('epoch'in event.args){if(isV8Event(event)){checkEpochConsistency(result.v8,event);result.v8=event.args.epoch;}else{checkEpochConsistency(result.cpp,event);result.cpp=event.args.epoch;}} event=event.parentSlice;} return result;} -const cppToV8=new Map();for(const event of events){const epoch=getEpoch(event);if(epoch.cpp!==null&&epoch.v8!==null){assert(!cppToV8.has(epoch.cpp)||cppToV8.get(epoch.cpp)===epoch.v8,`CppGC epoch ${epoch.cpp} corresponds to two v8 epochs `+`${cppToV8.get(epoch.cpp)} and ${epoch.v8}. `+`Detected at ${event.userFriendlyName}.`);cppToV8.set(epoch.cpp,epoch.v8);}} +function GlobalEpochFromV8(v8Epoch){return 2*v8Epoch;} +function GlobalEpochFromCpp(cppEpoch){return 2*cppEpoch+1;} +const cppToV8=new Map();for(const event of events){const epoch=getEpoch(event);if(epoch.cpp!==null&&epoch.v8!==null){if(!cppToV8.has(epoch.cpp)||cppToV8.get(epoch.cpp)>epoch.v8){cppToV8.set(epoch.cpp,epoch.v8);}}} const result=new Map();for(const event of events){const epoch=getEpoch(event);if(epoch.cpp===null&&epoch.v8===null){continue;} -assert(epoch.cpp===null||cppToV8.has(epoch.cpp),`CppGC epoch ${epoch.cpp} doesn't have the corresponding V8 epoch. `+`Detected at ${event.userFriendlyName}`);const key=epoch.v8===null?cppToV8.get(epoch.cpp):epoch.v8;if(result.has(key)){result.get(key).push(event);}else{result.set(key,[event]);}} +let globalEpoch;if(epoch.v8!==null){globalEpoch=GlobalEpochFromV8(epoch.v8);}else if(cppToV8.has(epoch.cpp)){globalEpoch=GlobalEpochFromV8(cppToV8.get(epoch.cpp));}else{globalEpoch=GlobalEpochFromCpp(epoch.cpp);} +if(result.has(globalEpoch)){result.get(globalEpoch).push(event);}else{result.set(globalEpoch,[event]);}} return result;} function addGarbageCollectionMetrics(metricNames,histograms,model){const metrics=metricNames.map(name=>new Metric(name));const gcEventNames=new Set(eventsMentionedIn(RULES));const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);for(const rendererHelper of Object.values(chromeHelper.rendererHelpers)){if(rendererHelper.isChromeTracingUI)continue;const[threads,threadTypes]=jsExecutionThreadsWithTypes(rendererHelper);const events=[];for(const thread of threads){for(const event of thread.sliceGroup.childEvents()){if(gcEventNames.has(event.title)){events.push(event);}}} for(const cycleEvents of groupByEpoch(events).values()){if(cycleEvents.some(tr.metrics.v8.utils.isForcedGarbageCollectionEvent)){continue;} for(const metric of metrics){metric.apply(RULES,cycleEvents,threadTypes);}}} for(const metric of metrics){histograms.addHistogram(metric.histogram);}} function gcMetric(histograms,model,options){options=options||{};addDurationOfTopEvents(histograms,model);addTotalDurationOfTopEvents(histograms,model);if(options.include_sub_events){addDurationOfSubEvents(histograms,model);} -addPercentageInV8ExecuteOfTopEvents(histograms,model);addTotalPercentageInV8Execute(histograms,model);addMarkCompactorMutatorUtilization(histograms,model);addTotalMarkCompactorTime(histograms,model);addTotalMarkCompactorMarkingTime(histograms,model);addScavengerSurvivedFromStackEvents(histograms,model);} +addPercentageInV8ExecuteOfTopEvents(histograms,model);addTotalPercentageInV8Execute(histograms,model);addMarkCompactorMutatorUtilization(histograms,model);addTotalMarkCompactorTime(histograms,model);addTotalMarkCompactorMarkingTime(histograms,model);addScavengerSurvivedFromStackEvents(histograms,model);addGarbageCollectionMetrics(METRICS,histograms,model);} tr.metrics.MetricRegistry.register(gcMetric);const timeDurationInMs_smallerIsBetter=tr.b.Unit.byName.timeDurationInMs_smallerIsBetter;const percentage_biggerIsBetter=tr.b.Unit.byName.normalizedPercentage_biggerIsBetter;const percentage_smallerIsBetter=tr.b.Unit.byName.normalizedPercentage_smallerIsBetter;const bytes_smallerIsBetter=tr.b.Unit.byName.sizeInBytes_smallerIsBetter;const CUSTOM_BOUNDARIES=tr.v.HistogramBinBoundaries.createLinear(0,20,200).addExponentialBins(200,100);function createNumericForTopEventTime(name){const n=new tr.v.Histogram(name,timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);n.customizeSummaryOptions({avg:true,count:true,max:true,min:false,std:true,sum:true,percentile:[0.90]});return n;} function createNumericForSubEventTime(name){const n=new tr.v.Histogram(name,timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);n.customizeSummaryOptions({avg:true,count:false,max:true,min:false,std:false,sum:false,percentile:[0.90]});return n;} function createNumericForIdleTime(name){const n=new tr.v.Histogram(name,timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);n.customizeSummaryOptions({avg:true,count:false,max:true,min:false,std:false,sum:true,percentile:[]});return n;} @@ -9228,9 +9244,9 @@ for(const tab of this.$.tabs.tabs){tab.nodes=undefined;} this.$.tabs.clearSubViews();if(this.displayedNode_===undefined){this.$.tabs.label='No heap node provided.';return;} for(const[dimension,children]of this.displayedNode_.childNodes){if(!this.dimensionToTab_.has(dimension)){this.dimensionToTab_.set(dimension,document.createElement('tr-ui-a-memory-dump-heap-details-breakdown-view-tab'));} const tab=this.dimensionToTab_.get(dimension);tab.aggregationMode=this.aggregationMode_;tab.dimension=dimension;tab.nodes=children;this.$.tabs.addSubView(tab);tab.rebuild();if(dimension===previouslySelectedDimension){this.$.tabs.selectedSubView=tab;if(previouslySelectedTabFocused){tab.focus();}}} -if(this.$.tabs.tabs.length>0){this.$.tabs.label='Break selected node further by:';}else{this.$.tabs.label='Selected node cannot be broken down any further.';}},onKeyDown_(keyEvent){if(!this.displayedNode_)return;let keyHandled=false;switch(keyEvent.keyCode){case 8:{if(!this.displayedNode_.parentNode)break;const viewEvent=new tr.b.Event('enter-node');viewEvent.node=this.displayedNode_.parentNode;this.dispatchEvent(viewEvent);keyHandled=true;break;} +if(this.$.tabs.tabs.length>0){this.$.tabs.label='Break selected node further by:';}else{this.$.tabs.label='Selected node cannot be broken down any further.';}},onKeyDown_(keyEvent){if(!this.displayedNode_)return;let keyHandled=false;switch(keyEvent.keyCode){case 8:{if(!this.displayedNode_.parentNode)break;const viewEvent=new tr.b.Event('enter-node',true);viewEvent.node=this.displayedNode_.parentNode;this.dispatchEvent(viewEvent);keyHandled=true;break;} case 37:case 39:{const wasFocused=this.$.tabs.selectedSubView.isFocused;keyHandled=keyEvent.keyCode===37?this.$.tabs.selectPreviousTabIfPossible():this.$.tabs.selectNextTabIfPossible();if(wasFocused&&keyHandled){this.$.tabs.selectedSubView.focus();}}} -if(!keyHandled)return;keyEvent.stopPropagation();keyEvent.preventDefault();}});Polymer({is:'tr-ui-a-memory-dump-heap-details-breakdown-view-tab',behaviors:[tr.ui.analysis.RebuildableBehavior],created(){this.dimension_=undefined;this.nodes_=undefined;this.aggregationMode_=undefined;this.displayLongTail_=false;},ready(){this.$.table.addEventListener('step-into',function(tableEvent){const viewEvent=new tr.b.Event('enter-node');viewEvent.node=tableEvent.tableRow;this.dispatchEvent(viewEvent);}.bind(this));},get displayLongTail(){return this.displayLongTail_;},set displayLongTail(newValue){if(this.displayLongTail===newValue)return;this.displayLongTail_=newValue;this.scheduleRebuild_();},get dimension(){return this.dimension_;},set dimension(dimension){this.dimension_=dimension;this.scheduleRebuild_();},get nodes(){return this.nodes_;},set nodes(nodes){this.nodes_=nodes;this.scheduleRebuild_();},get nodes(){return this.nodes_||[];},get dimensionLabel_(){if(this.dimension_===undefined)return'(undefined)';return this.dimension_.label;},get tabLabel(){let nodeCount=0;if(this.nodes_){nodeCount=this.nodes_.length;} +if(!keyHandled)return;keyEvent.stopPropagation();keyEvent.preventDefault();}});Polymer({is:'tr-ui-a-memory-dump-heap-details-breakdown-view-tab',behaviors:[tr.ui.analysis.RebuildableBehavior],created(){this.dimension_=undefined;this.nodes_=undefined;this.aggregationMode_=undefined;this.displayLongTail_=false;},ready(){this.$.table.addEventListener('step-into',function(tableEvent){const viewEvent=new tr.b.Event('enter-node',true);viewEvent.node=tableEvent.tableRow;this.dispatchEvent(viewEvent);}.bind(this));},get displayLongTail(){return this.displayLongTail_;},set displayLongTail(newValue){if(this.displayLongTail===newValue)return;this.displayLongTail_=newValue;this.scheduleRebuild_();},get dimension(){return this.dimension_;},set dimension(dimension){this.dimension_=dimension;this.scheduleRebuild_();},get nodes(){return this.nodes_;},set nodes(nodes){this.nodes_=nodes;this.scheduleRebuild_();},get nodes(){return this.nodes_||[];},get dimensionLabel_(){if(this.dimension_===undefined)return'(undefined)';return this.dimension_.label;},get tabLabel(){let nodeCount=0;if(this.nodes_){nodeCount=this.nodes_.length;} return this.dimensionLabel_+' ('+nodeCount+')';},get tabIcon(){if(this.dimension_===undefined||this.dimension_===tr.ui.analysis.HeapDetailsRowDimension.ROOT){return undefined;} return{text:this.dimension_.symbol,style:'color: '+tr.b.ColorScheme.getColorForReservedNameAsString(this.dimension_.color)+';'};},get aggregationMode(){return this.aggregationMode_;},set aggregationMode(aggregationMode){this.aggregationMode_=aggregationMode;this.scheduleRebuild_();},focus(){this.$.table.focus();},blur(){this.$.table.blur();},get isFocused(){return this.$.table.isFocused;},onRebuild_(){this.$.table.selectionMode=tr.ui.b.TableFormat.SelectionMode.ROW;this.$.table.emptyValue='Cannot break down by '+ this.dimensionLabel_.toLowerCase()+' any further.';const[state,rows]=this.getRows_();const total=this.nodes.length;const displayed=rows.length;const hidden=total-displayed;this.updateInfoBar_(state,[total,displayed,hidden]);this.$.table.tableRows=rows;this.$.table.tableColumns=this.createColumns_(rows);if(this.$.table.sortColumnIndex===undefined){this.$.table.sortColumnIndex=0;this.$.table.sortDescending=false;} diff --git a/systrace/catapult/systrace/systrace/util.py b/systrace/catapult/systrace/systrace/util.py index 063f9ed..797d675 100644 --- a/systrace/catapult/systrace/systrace/util.py +++ b/systrace/catapult/systrace/systrace/util.py @@ -69,8 +69,8 @@ def get_tracing_path(device_serial=None): options, _ = parser.parse_args() device_serial = options.device_serial - adb_output, adb_return_code = run_adb_shell(mount_info_args, device_serial) - if adb_return_code == 0 and 'debugfs' not in adb_output: + adb_output, adb_return_code = run_adb_shell(mount_info_args, device_serial, ) + if adb_return_code == 0 and 'tracefs on /sys/kernel/tracing' in adb_output: return '/sys/kernel/tracing' return '/sys/kernel/debug/tracing' diff --git a/systrace/systrace.py b/systrace/systrace.py index 71da650..7bd4537 100755 --- a/systrace/systrace.py +++ b/systrace/systrace.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # Copyright (c) 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be |