diff options
| author | Treehugger Robot <treehugger-gerrit@google.com> | 2021-04-16 18:16:47 +0000 |
|---|---|---|
| committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2021-04-16 18:16:47 +0000 |
| commit | f579e8f3f29091e364f8d39a03870f89b8dd31fc (patch) | |
| tree | b9a6b60023b853553ae095370bf251a2f1f7640e | |
| parent | 1f6d83531e19d5819437949715a021ea9b2b6dc5 (diff) | |
| parent | ef2f29d2b3cf25a46be203d6b12ad0eceaf270b2 (diff) | |
| download | chromium-trace-f579e8f3f29091e364f8d39a03870f89b8dd31fc.tar.gz | |
Merge changes I195c394b,Iff693cf6 am: b101eec5e1 am: d5085cc096 am: 19f3aab438 am: ef2f29d2b3
Original change: https://android-review.googlesource.com/c/platform/external/chromium-trace/+/1677030
Change-Id: I49cd668d1050a3ea1eb8db8c0644f0c9a278e1d2
62 files changed, 756 insertions, 2050 deletions
diff --git a/UPSTREAM_REVISION b/UPSTREAM_REVISION index ab641bd2..7988ceb2 100644 --- a/UPSTREAM_REVISION +++ b/UPSTREAM_REVISION @@ -1 +1 @@ -91735e2e6775c098eb32840a8903e5a9111fad77 +ab9d330fe2a32f84b4b5fe141958c0a0a857c5c9 diff --git a/catapult/common/py_utils/py_utils/ts_proxy_server.py b/catapult/common/py_utils/py_utils/ts_proxy_server.py index 0e168a53..ffed090f 100644 --- a/catapult/common/py_utils/py_utils/ts_proxy_server.py +++ b/catapult/common/py_utils/py_utils/ts_proxy_server.py @@ -194,6 +194,7 @@ class TsProxyServer(object): if not self._proc: return try: + self._IssueCommand('exit', timeout=10) py_utils.WaitFor(lambda: self._proc.poll() is not None, 10) except py_utils.TimeoutException: # signal.SIGINT is not supported on Windows. diff --git a/catapult/dependency_manager/dependency_manager/__init__.py b/catapult/dependency_manager/dependency_manager/__init__.py index 3b18f06a..4b595c5b 100644 --- a/catapult/dependency_manager/dependency_manager/__init__.py +++ b/catapult/dependency_manager/dependency_manager/__init__.py @@ -20,8 +20,8 @@ def _AddDirToPythonPath(*path_parts): _AddDirToPythonPath(CATAPULT_PATH, 'common', 'py_utils') _AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'mock') +_AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'six') _AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'pyfakefs') -_AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'zipfile') _AddDirToPythonPath(DEPENDENCY_MANAGER_PATH) diff --git a/catapult/dependency_manager/dependency_manager/dependency_manager_util.py b/catapult/dependency_manager/dependency_manager/dependency_manager_util.py index ca0174e0..a8e21b81 100644 --- a/catapult/dependency_manager/dependency_manager/dependency_manager_util.py +++ b/catapult/dependency_manager/dependency_manager/dependency_manager_util.py @@ -7,7 +7,9 @@ import shutil import stat import subprocess import sys -import zipfile_2_7_13 as zipfile +import zipfile + +import six from dependency_manager import exceptions @@ -17,7 +19,7 @@ def _WinReadOnlyHandler(func, path, execinfo): os.chmod(path, stat.S_IWRITE) func(path) else: - raise execinfo[0], execinfo[1], execinfo[2] + six.reraise(*execinfo) def RemoveDir(dir_path): diff --git a/catapult/dependency_manager/dependency_manager/local_path_info.py b/catapult/dependency_manager/dependency_manager/local_path_info.py index 8ac0152f..56009662 100644 --- a/catapult/dependency_manager/dependency_manager/local_path_info.py +++ b/catapult/dependency_manager/dependency_manager/local_path_info.py @@ -4,6 +4,7 @@ import os +import six class LocalPathInfo(object): @@ -66,4 +67,4 @@ class LocalPathInfo(object): def _ParseLocalPaths(local_paths): if not local_paths: return [] - return [[e] if isinstance(e, basestring) else e for e in local_paths] + return [[e] if isinstance(e, six.string_types) else e for e in local_paths] diff --git a/catapult/devil/DIR_METADATA b/catapult/devil/DIR_METADATA new file mode 100644 index 00000000..7608b2f5 --- /dev/null +++ b/catapult/devil/DIR_METADATA @@ -0,0 +1,3 @@ +monorail { + component: "Test>Devil" +} diff --git a/catapult/devil/PRESUBMIT.py b/catapult/devil/PRESUBMIT.py index 0b49eb80..ec48dddf 100644 --- a/catapult/devil/PRESUBMIT.py +++ b/catapult/devil/PRESUBMIT.py @@ -30,14 +30,23 @@ def _RunUnitTests(input_api, output_api): output_api.PresubmitPromptWarning) return input_api.RunTests([ - input_api.Command( - name='devil/bin/run_py_tests', - cmd=[ - input_api.os_path.join(input_api.PresubmitLocalPath(), 'bin', - 'run_py_tests') - ], - kwargs={'env': test_env}, - message=message_type) + input_api.Command(name='devil/bin/run_py_tests', + cmd=[ + input_api.os_path.join( + input_api.PresubmitLocalPath(), 'bin', + 'run_py_tests') + ], + kwargs={'env': test_env}, + message=message_type), + input_api.Command(name='devil/bin/run_py3_tests', + cmd=[ + input_api.os_path.join( + input_api.PresubmitLocalPath(), 'bin', + 'run_py3_tests') + ], + kwargs={'env': test_env}, + message=message_type, + python3=True), ]) diff --git a/catapult/devil/bin/generate_md_docs b/catapult/devil/bin/generate_md_docs index d1dbf06f..4c6f0f91 100755 --- a/catapult/devil/bin/generate_md_docs +++ b/catapult/devil/bin/generate_md_docs @@ -8,7 +8,7 @@ import os import sys _DEVIL_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -_DEVIL_URL = ('https://github.com/catapult-project/catapult/blob/master/devil/') +_DEVIL_URL = ('https://chromium.googlesource.com/catapult.git/+/HEAD/devil/') sys.path.append(_DEVIL_PATH) from devil.utils import cmd_helper diff --git a/catapult/devil/bin/run_py3_tests b/catapult/devil/bin/run_py3_tests new file mode 100755 index 00000000..3250ff15 --- /dev/null +++ b/catapult/devil/bin/run_py3_tests @@ -0,0 +1,113 @@ +#!/usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Runs all python3-compatible tests in devil.""" + +import os +import sys +import unittest + +_DEVIL_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +_SIX_PATH = os.path.join(_DEVIL_PATH, '..', 'third_party', 'six') + +sys.path.append(_DEVIL_PATH) +sys.path.append(_SIX_PATH) +# Import compatible tests here by full module path +import devil.android.apk_helper_test +import devil.android.app_ui_test +import devil.android.battery_utils_test +import devil.android.cpu_temperature_test +import devil.android.decorators_test +import devil.android.device_denylist_test +import devil.android.device_errors_test +import devil.android.device_utils_test +import devil.android.fastboot_utils_test +import devil.android.flag_changer_test +import devil.android.logcat_monitor_test +import devil.android.md5sum_test +import devil.android.perf.perf_control_test +import devil.android.perf.surface_stats_collector_test +import devil.android.sdk.adb_wrapper_test +import devil.android.sdk.shared_prefs_test +import devil.android.tools.device_monitor_test +import devil.android.tools.script_common_test +import devil.android.tools.system_app_test +import devil.devil_env_test +import devil.utils.cmd_helper_test +import devil.utils.decorators_test +import devil.utils.find_usb_devices_test +import devil.utils.geometry_test +import devil.utils.lazy.weak_constant_test +import devil.utils.lsusb_test +import devil.utils.markdown_test +import devil.utils.mock_calls_test +import devil.utils.parallelizer_test +import devil.utils.reraiser_thread_unittest +import devil.utils.timeout_retry_unittest +import devil.utils.zip_utils_test + +PY3_COMPATIBLE_TESTS = [ + # Add full test module path here + devil.android.apk_helper_test, + devil.android.app_ui_test, + devil.android.battery_utils_test, + devil.android.cpu_temperature_test, + devil.android.decorators_test, + devil.android.device_denylist_test, + devil.android.device_errors_test, + devil.android.device_utils_test, + devil.android.fastboot_utils_test, + devil.android.flag_changer_test, + devil.android.logcat_monitor_test, + devil.android.md5sum_test, + devil.android.perf.perf_control_test, + devil.android.perf.surface_stats_collector_test, + devil.android.sdk.adb_wrapper_test, + devil.android.sdk.shared_prefs_test, + devil.android.tools.device_monitor_test, + devil.android.tools.script_common_test, + devil.android.tools.system_app_test, + devil.devil_env_test, + devil.utils.cmd_helper_test, + devil.utils.decorators_test, + devil.utils.find_usb_devices_test, + devil.utils.geometry_test, + devil.utils.lazy.weak_constant_test, + devil.utils.lsusb_test, + devil.utils.markdown_test, + devil.utils.mock_calls_test, + devil.utils.parallelizer_test, + devil.utils.reraiser_thread_unittest, + devil.utils.timeout_retry_unittest, + devil.utils.zip_utils_test, +] + + +def main(): + # TODO(crbug.com/1007101): Use six.PY2 directly once we're using six via + # vpython. + if sys.version_info[0] == 2: + print('Somehow running under python2.') + return 1 + + # Tests mock out internal details of methods, and the ANDROID_SERIAL can + # change which internal methods are called. Since tests don't actually use + # devices, it should be fine to delete the variable. + if 'ANDROID_SERIAL' in os.environ: + del os.environ['ANDROID_SERIAL'] + + # This does not use typ for now, as typ has vpython dependencies that haven't + # yet been updated for python3. + result = unittest.TextTestRunner().run(unittest.TestSuite( + unittest.defaultTestLoader.loadTestsFromModule(test_module) + for test_module in PY3_COMPATIBLE_TESTS + )) + + return 0 if result.wasSuccessful() else 1 + + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/catapult/devil/devil/android/apk_helper.py b/catapult/devil/devil/android/apk_helper.py index fdece072..4d723a56 100644 --- a/catapult/devil/devil/android/apk_helper.py +++ b/catapult/devil/devil/android/apk_helper.py @@ -11,6 +11,8 @@ import shutil import tempfile import zipfile +import six + from devil import base_error from devil.android.ndk import abis from devil.android.sdk import aapt @@ -69,7 +71,7 @@ def GetInstrumentationName(apk_path): def ToHelper(path_or_helper): """Creates an ApkHelper unless one is already given.""" - if not isinstance(path_or_helper, basestring): + if not isinstance(path_or_helper, six.string_types): return path_or_helper elif path_or_helper.endswith('.apk'): return ApkHelper(path_or_helper) @@ -86,7 +88,7 @@ def ToSplitHelper(path_or_helper, split_apks): if sorted(path_or_helper.split_apk_paths) != sorted(split_apks): raise ApkHelperError('Helper has different split APKs') return path_or_helper - elif (isinstance(path_or_helper, basestring) + elif (isinstance(path_or_helper, six.string_types) and path_or_helper.endswith('.apk')): return SplitApkHelper(path_or_helper, split_apks) diff --git a/catapult/devil/devil/android/app_ui.py b/catapult/devil/devil/android/app_ui.py index 399c2ee3..4f7af1d7 100644 --- a/catapult/devil/devil/android/app_ui.py +++ b/catapult/devil/devil/android/app_ui.py @@ -51,7 +51,7 @@ class _UiNode(object): A geometry.Rectangle instance. """ d = _RE_BOUNDS.match(self._GetAttribute('bounds')).groupdict() - return geometry.Rectangle.FromDict({k: int(v) for k, v in d.iteritems()}) + return geometry.Rectangle.FromDict({k: int(v) for k, v in d.items()}) def Tap(self, point=None, dp_units=False): """Send a tap event to the UI node. @@ -150,7 +150,7 @@ class _UiNode(object): and ':id/' not in resource_id): kwargs['resource_id'] = '%s:id/%s' % (self._package, resource_id) - criteria = [(k.replace('_', '-'), v) for k, v in kwargs.iteritems() + criteria = [(k.replace('_', '-'), v) for k, v in kwargs.items() if v is not None] if not criteria: raise TypeError('At least one search criteria should be specified') @@ -193,8 +193,12 @@ class AppUi(object): A UI node instance pointing to the root of the xml screenshot. """ with device_temp_file.DeviceTempFile(self._device.adb) as dtemp: - self._device.RunShellCommand(['uiautomator', 'dump', dtemp.name], - check_return=True) + output = self._device.RunShellCommand( + ['uiautomator', 'dump', dtemp.name], single_line=True, + check_return=True) + if output.startswith('ERROR:'): + raise RuntimeError( + 'uiautomator dump command returned error: {}'.format(output)) xml_node = element_tree.fromstring( self._device.ReadFile(dtemp.name, force_pull=True)) return _UiNode(self._device, xml_node, package=self._package) diff --git a/catapult/devil/devil/android/app_ui_test.py b/catapult/devil/devil/android/app_ui_test.py index 938fd408..50f00ca1 100644 --- a/catapult/devil/devil/android/app_ui_test.py +++ b/catapult/devil/devil/android/app_ui_test.py @@ -80,7 +80,7 @@ class UiAppTest(unittest.TestCase): def assertNodeHasAttribs(self, node, attr): # pylint: disable=protected-access - for key, value in attr.iteritems(): + for key, value in attr.items(): self.assertEquals(node._GetAttribute(key), value) def assertTappedOnceAt(self, x, y): diff --git a/catapult/devil/devil/android/battery_utils.py b/catapult/devil/devil/android/battery_utils.py index e8134d2b..d680f03f 100644 --- a/catapult/devil/devil/android/battery_utils.py +++ b/catapult/devil/devil/android/battery_utils.py @@ -300,7 +300,7 @@ class BatteryUtils(object): 'uid': uid, 'data': pwi_entries[uid] } - for p, uid in self._cache['uids'].iteritems() + for p, uid in self._cache['uids'].items() } return {'system_total': system_total, 'per_package': per_package} diff --git a/catapult/devil/devil/android/cpu_temperature_test.py b/catapult/devil/devil/android/cpu_temperature_test.py index 8d082bb9..47cc28a6 100644 --- a/catapult/devil/devil/android/cpu_temperature_test.py +++ b/catapult/devil/devil/android/cpu_temperature_test.py @@ -69,9 +69,11 @@ class CpuTemperatureGetThermalDeviceInformationTest(CpuTemperatureTest): 'cpu6': '/sys/class/thermal/thermal_zone17/temp', 'cpu7': '/sys/class/thermal/thermal_zone18/temp' } - self.assertEqual( - cmp(correct_information, - self.cpu_temp.GetDeviceInfoForTesting().get('cpu_temps')), 0) + + self.assertDictEqual( + correct_information, + self.cpu_temp.GetDeviceInfoForTesting().get('cpu_temps') + ) class CpuTemperatureIsSupportedTest(CpuTemperatureTest): diff --git a/catapult/devil/devil/android/decorators.py b/catapult/devil/devil/android/decorators.py index 0b3778aa..11d2494b 100644 --- a/catapult/devil/devil/android/decorators.py +++ b/catapult/devil/devil/android/decorators.py @@ -9,6 +9,8 @@ import functools import itertools import sys +import six + from devil.android import device_errors from devil.utils import cmd_helper from devil.utils import reraiser_thread @@ -56,14 +58,19 @@ def _TimeoutRetryWrapper(f, desc = '%s(%s)' % (f.__name__, ', '.join( itertools.chain( (str(a) for a in args), - ('%s=%s' % (k, str(v)) for k, v in kwargs.iteritems())))) + ('%s=%s' % (k, str(v)) for k, v in six.iteritems(kwargs))))) return timeout_retry.Run( impl, timeout, retries, desc=desc, retry_if_func=retry_if_func) except reraiser_thread.TimeoutError as e: - raise device_errors.CommandTimeoutError(str(e)), None, (sys.exc_info()[2]) + six.reraise( + device_errors.CommandTimeoutError, + device_errors.CommandTimeoutError(str(e)), + sys.exc_info()[2]) except cmd_helper.TimeoutError as e: - raise device_errors.CommandTimeoutError( - str(e), output=e.output), None, (sys.exc_info()[2]) + six.reraise( + device_errors.CommandTimeoutError, + device_errors.CommandTimeoutError(str(e), output=e.output), + sys.exc_info()[2]) return timeout_retry_wrapper diff --git a/catapult/devil/devil/android/device_errors.py b/catapult/devil/devil/android/device_errors.py index 6e710876..75bf7e3f 100644 --- a/catapult/devil/devil/android/device_errors.py +++ b/catapult/devil/devil/android/device_errors.py @@ -23,6 +23,8 @@ The class hierarchy for device exceptions is: """ +import six + from devil import base_error from devil.utils import cmd_helper from devil.utils import parallelizer @@ -158,7 +160,7 @@ class AdbShellCommandFailedError(AdbCommandFailedError): segments.append(' exit status: %s\n' % status) if output: segments.append(' output:\n') - if isinstance(output, basestring): + if isinstance(output, six.string_types): output_lines = output.splitlines() else: output_lines = output diff --git a/catapult/devil/devil/android/device_list.py b/catapult/devil/devil/android/device_list.py index cd631dbd..5fb586f6 100644 --- a/catapult/devil/devil/android/device_list.py +++ b/catapult/devil/devil/android/device_list.py @@ -7,6 +7,8 @@ import json import logging import os +import six + logger = logging.getLogger(__name__) @@ -26,7 +28,7 @@ def GetPersistentDeviceList(file_name): with open(file_name) as f: devices = json.load(f) if not isinstance(devices, list) or not all( - isinstance(d, basestring) for d in devices): + isinstance(d, six.string_types) for d in devices): logger.warning('Unrecognized device file format: %s', devices) return [] return [d for d in devices if d != '(error)'] diff --git a/catapult/devil/devil/android/device_utils.py b/catapult/devil/devil/android/device_utils.py index 7b7dad24..093bfc71 100644 --- a/catapult/devil/devil/android/device_utils.py +++ b/catapult/devil/devil/android/device_utils.py @@ -24,6 +24,8 @@ import time import threading import uuid +import six + from devil import base_error from devil import devil_env from devil.utils import cmd_helper @@ -105,6 +107,13 @@ _RESTART_ADBD_SCRIPT = """ restart & """ +_UNZIP_AND_CHMOD_SCRIPT = """ + {bin_dir}/unzip {zip_file} && (for dir in {dirs} + do + chmod -R 777 "$dir" || exit 1 + done) +""" + # Not all permissions can be set. _PERMISSIONS_DENYLIST_RE = re.compile('|'.join( fnmatch.translate(p) for p in [ @@ -131,6 +140,7 @@ _PERMISSIONS_DENYLIST_RE = re.compile('|'.join( 'android.permission.INTERNET', 'android.permission.KILL_BACKGROUND_PROCESSES', 'android.permission.MANAGE_ACCOUNTS', + 'android.permission.MANAGE_EXTERNAL_STORAGE', 'android.permission.MODIFY_AUDIO_SETTINGS', 'android.permission.NFC', 'android.permission.QUERY_ALL_PACKAGES', @@ -314,30 +324,6 @@ def GetAVDs(): return avds -@decorators.WithExplicitTimeoutAndRetries(_DEFAULT_TIMEOUT, _DEFAULT_RETRIES) -def RestartServer(): - """Restarts the adb server. - - Raises: - CommandFailedError if we fail to kill or restart the server. - """ - - def adb_killed(): - return not adb_wrapper.AdbWrapper.IsServerOnline() - - def adb_started(): - return adb_wrapper.AdbWrapper.IsServerOnline() - - adb_wrapper.AdbWrapper.KillServer() - if not timeout_retry.WaitFor(adb_killed, wait_period=1, max_tries=5): - # TODO(crbug.com/442319): Switch this to raise an exception if we - # figure out why sometimes not all adb servers on bots get killed. - logger.warning('Failed to kill adb server') - adb_wrapper.AdbWrapper.StartServer() - if not timeout_retry.WaitFor(adb_started, wait_period=1, max_tries=5): - raise device_errors.CommandFailedError('Failed to start adb server') - - def _ParseModeString(mode_str): """Parse a mode string, e.g. 'drwxrwxrwx', into a st_mode value. @@ -376,7 +362,7 @@ def _CreateAdbWrapper(device): def _FormatPartialOutputError(output): - lines = output.splitlines() if isinstance(output, basestring) else output + lines = output.splitlines() if isinstance(output, six.string_types) else output message = ['Partial output found:'] if len(lines) > 11: message.extend('- %s' % line for line in lines[:5]) @@ -468,7 +454,7 @@ class DeviceUtils(object): operation should be retried on failure if no explicit value is provided. """ self.adb = None - if isinstance(device, basestring): + if isinstance(device, six.string_types): self.adb = _CreateAdbWrapper(device) elif isinstance(device, adb_wrapper.AdbWrapper): self.adb = device @@ -1533,7 +1519,7 @@ class DeviceUtils(object): else: raise - if isinstance(cmd, basestring): + if isinstance(cmd, six.string_types): if not shell: # TODO(crbug.com/1029769): Make this an error instead. logger.warning( @@ -1543,7 +1529,7 @@ class DeviceUtils(object): else: cmd = ' '.join(cmd_helper.SingleQuote(s) for s in cmd) if env: - env = ' '.join(env_quote(k, v) for k, v in env.iteritems()) + env = ' '.join(env_quote(k, v) for k, v in env.items()) cmd = '%s %s' % (env, cmd) if cwd: cmd = 'cd %s && %s' % (cmd_helper.SingleQuote(cwd), cmd) @@ -1740,7 +1726,7 @@ class DeviceUtils(object): cmd.append('-w') if raw: cmd.append('-r') - for k, v in extras.iteritems(): + for k, v in extras.items(): cmd.extend(['-e', str(k), str(v)]) cmd.append(component) @@ -2223,13 +2209,19 @@ class DeviceUtils(object): self.adb, suffix='.zip') as device_temp: self.adb.Push(zip_path, device_temp.name) - quoted_dirs = ' '.join(cmd_helper.SingleQuote(d) for d in dirs) - self.RunShellCommand( - 'unzip %s&&chmod -R 777 %s' % (device_temp.name, quoted_dirs), - shell=True, - as_root=True, - env={'PATH': '%s:$PATH' % install_commands.BIN_DIR}, - check_return=True) + with device_temp_file.DeviceTempFile(self.adb, suffix='.sh') as script: + # Read dirs from temp file to avoid potential errors like + # "Argument list too long" (crbug.com/1174331) when the list + # is too long. + self.WriteFile( + script.name, + _UNZIP_AND_CHMOD_SCRIPT.format(bin_dir=install_commands.BIN_DIR, + zip_file=device_temp.name, + dirs=' '.join(dirs))) + + self.RunShellCommand(['source', script.name], + check_return=True, + as_root=True) return True @@ -2263,7 +2255,7 @@ class DeviceUtils(object): DeviceUnreachableError on missing device. """ paths = device_paths - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): paths = (paths, ) if not paths: return True @@ -2322,7 +2314,7 @@ class DeviceUtils(object): args.append('-f') if recursive: args.append('-r') - if isinstance(device_path, basestring): + if isinstance(device_path, six.string_types): args.append(device_path if not rename else _RenamePath(device_path)) else: args.extend( @@ -2596,7 +2588,7 @@ class DeviceUtils(object): """ entries = self._ParseLongLsOutput(device_path, as_root=as_root, **kwargs) for d in entries: - for key, value in d.items(): + for key, value in list(d.items()): if value is None: del d[key] # Remove missing fields. d['st_mode'] = _ParseModeString(d['st_mode']) @@ -2960,7 +2952,7 @@ class DeviceUtils(object): """ assert isinstance( property_name, - basestring), ("property_name is not a string: %r" % property_name) + six.string_types), ("property_name is not a string: %r" % property_name) if cache: # It takes ~120ms to query a single property, and ~130ms to query all @@ -3004,8 +2996,8 @@ class DeviceUtils(object): """ assert isinstance( property_name, - basestring), ("property_name is not a string: %r" % property_name) - assert isinstance(value, basestring), "value is not a string: %r" % value + six.string_types), ("property_name is not a string: %r" % property_name) + assert isinstance(value, six.string_types), "value is not a string: %r" % value self.RunShellCommand(['setprop', property_name, value], check_return=True) prop_cache = self._cache['getprop'] @@ -3084,18 +3076,19 @@ class DeviceUtils(object): Returns: A list of ProcessInfo tuples with |name|, |pid|, and |ppid| fields. """ + # pylint: disable=broad-except process_name = process_name or '' processes = [] for line in self._GetPsOutput(process_name): row = line.split() try: - row = {k: row[i] for k, i in _PS_COLUMNS.iteritems()} + row = {k: row[i] for k, i in _PS_COLUMNS.items()} if row['pid'] == 'PID' or process_name not in row['name']: # Skip over header and non-matching processes. continue row['pid'] = int(row['pid']) row['ppid'] = int(row['ppid']) - except StandardError: # e.g. IndexError, TypeError, ValueError. + except Exception: # e.g. IndexError, TypeError, ValueError. logging.warning('failed to parse ps line: %r', line) continue processes.append(ProcessInfo(**row)) @@ -3550,10 +3543,10 @@ class DeviceUtils(object): # When using a cache across script invokations, verify that apps have # not been uninstalled. self._cache['package_apk_paths_to_verify'] = set( - self._cache['package_apk_paths'].iterkeys()) + self._cache['package_apk_paths']) package_apk_checksums = obj.get('package_apk_checksums', {}) - for k, v in package_apk_checksums.iteritems(): + for k, v in package_apk_checksums.items(): package_apk_checksums[k] = set(v) self._cache['package_apk_checksums'] = package_apk_checksums device_path_checksums = obj.get('device_path_checksums', {}) @@ -3577,27 +3570,27 @@ class DeviceUtils(object): obj['package_apk_paths'] = self._cache['package_apk_paths'] obj['package_apk_checksums'] = self._cache['package_apk_checksums'] # JSON can't handle sets. - for k, v in obj['package_apk_checksums'].iteritems(): + for k, v in obj['package_apk_checksums'].items(): obj['package_apk_checksums'][k] = list(v) obj['device_path_checksums'] = self._cache['device_path_checksums'] return json.dumps(obj, separators=(',', ':')) @classmethod - def parallel(cls, devices, async=False): + def parallel(cls, devices, asyn=False): """Creates a Parallelizer to operate over the provided list of devices. Args: devices: A list of either DeviceUtils instances or objects from from which DeviceUtils instances can be constructed. If None, all attached devices will be used. - async: If true, returns a Parallelizer that runs operations + asyn: If true, returns a Parallelizer that runs operations asynchronously. Returns: A Parallelizer operating over |devices|. """ devices = [d if isinstance(d, cls) else cls(d) for d in devices] - if async: + if asyn: return parallelizer.Parallelizer(devices) else: return parallelizer.SyncParallelizer(devices) @@ -3710,7 +3703,7 @@ class DeviceUtils(object): else: reset_usb.reset_all_android_devices() - for attempt in xrange(retries + 1): + for attempt in range(retries + 1): try: return _get_devices() except device_errors.NoDevicesError: @@ -3728,7 +3721,7 @@ class DeviceUtils(object): 'No devices found. Will try again after restarting adb server ' 'and a short nap of %d s.', sleep_s) time.sleep(sleep_s) - RestartServer() + adb_wrapper.RestartServer() @decorators.WithTimeoutAndRetriesFromInstance() def RestartAdbd(self, timeout=None, retries=None): @@ -3745,6 +3738,17 @@ class DeviceUtils(object): if not permissions: return + # For Andorid-11(R), enable MANAGE_EXTERNAL_STORAGE for testing. + # See https://bit.ly/2MBjBIM for details. + if ('android.permission.MANAGE_EXTERNAL_STORAGE' in permissions + and self.build_version_sdk >= version_codes.R): + script_manage_ext_storage = [ + 'appops set {package} MANAGE_EXTERNAL_STORAGE allow', + 'echo "{sep}MANAGE_EXTERNAL_STORAGE{sep}$?{sep}"', + ] + else: + script_manage_ext_storage = [] + permissions = set(p for p in permissions if not _PERMISSIONS_DENYLIST_RE.match(p)) @@ -3752,10 +3756,15 @@ class DeviceUtils(object): and 'android.permission.READ_EXTERNAL_STORAGE' not in permissions): permissions.add('android.permission.READ_EXTERNAL_STORAGE') - script = ';'.join([ - 'p={package}', 'for q in {permissions}', 'do pm grant "$p" "$q"', - 'echo "{sep}$q{sep}$?{sep}"', 'done' - ]).format( + script_raw = [ + 'p={package}', + 'for q in {permissions}', + 'do pm grant "$p" "$q"', + 'echo "{sep}$q{sep}$?{sep}"', + 'done', + ] + script_manage_ext_storage + + script = ';'.join(script_raw).format( package=cmd_helper.SingleQuote(package), permissions=' '.join( cmd_helper.SingleQuote(p) for p in sorted(permissions)), diff --git a/catapult/devil/devil/android/device_utils_test.py b/catapult/devil/devil/android/device_utils_test.py index 62313c5b..8e583f01 100755 --- a/catapult/devil/devil/android/device_utils_test.py +++ b/catapult/devil/devil/android/device_utils_test.py @@ -11,6 +11,7 @@ Unit tests for the contents of device_utils.py (mostly DeviceUtils). import collections import contextlib +import io import json import logging import os @@ -19,6 +20,8 @@ import stat import sys import unittest +import six + from devil import devil_env from devil.android import device_errors from devil.android import device_signal @@ -117,9 +120,10 @@ class DeviceUtilsInitTest(unittest.TestCase): self.assertEqual(serial_as_str, d.adb.GetDeviceSerial()) def testInitWithUnicode(self): - serial_as_unicode = unicode('fedcba9876543210') - d = device_utils.DeviceUtils(serial_as_unicode) - self.assertEqual(serial_as_unicode, d.adb.GetDeviceSerial()) + if six.PY2: + serial_as_unicode = unicode('fedcba9876543210') + d = device_utils.DeviceUtils(serial_as_unicode) + self.assertEqual(serial_as_unicode, d.adb.GetDeviceSerial()) def testInitWithAdbWrapper(self): serial = '123456789abcdef0' @@ -162,12 +166,12 @@ class DeviceUtilsRestartServerTest(mock_calls.TestCase): ['pgrep', 'adb']), (1, '')), (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutput( ['pgrep', 'adb']), (0, '123\n'))): - device_utils.RestartServer() + adb_wrapper.RestartServer() class MockTempFile(object): def __init__(self, name='/tmp/some/file'): - self.file = mock.MagicMock(spec=file) + self.file = mock.MagicMock(spec=io.BufferedIOBase) self.file.name = name self.file.name_quoted = cmd_helper.SingleQuote(name) @@ -217,6 +221,12 @@ class DeviceUtilsTest(mock_calls.TestCase): self.adb, default_timeout=10, default_retries=0) self.watchMethodCalls(self.call.adb, ignore=['GetDeviceSerial']) + def safeAssertItemsEqual(self, expected, actual): + if six.PY2: + self.assertItemsEqual(expected, actual) + else: + self.assertCountEqual(expected, actual) # pylint: disable=no-member + def AdbCommandError(self, args=None, output=None, status=None, msg=None): if args is None: args = ['[unspecified]'] @@ -2194,11 +2204,16 @@ class DeviceUtilsPushChangedFilesZippedTest(DeviceUtilsTest): self.assertFalse( self.device._PushChangedFilesZipped(test_files, ['/test/dir'])) - def _testPushChangedFilesZipped_spec(self, test_files): + def _testPushChangedFilesZipped_spec(self, test_files, test_dirs): @contextlib.contextmanager def mock_zip_temp_dir(): yield '/test/temp/dir' + expected_cmd = ''.join([ + '\n /data/local/tmp/bin/unzip %s &&', + ' (for dir in %s\n do\n chmod -R 777 "$dir" || exit 1\n', + ' done)\n' + ]) % ('/sdcard/foo123.zip', ' '.join(test_dirs)) with self.assertCalls( (self.call.device._MaybeInstallCommands(), True), (mock.call.py_utils.tempfile_ext.NamedTemporaryDirectory(), @@ -2207,25 +2222,27 @@ class DeviceUtilsPushChangedFilesZippedTest(DeviceUtilsTest): (mock.call.os.path.getsize('/test/temp/dir/tmp.zip'), 123), (self.call.device.NeedsSU(), True), (mock.call.devil.android.device_temp_file.DeviceTempFile( - self.adb, suffix='.zip'), MockTempFile('/test/sdcard/foo123.zip')), - self.call.adb.Push('/test/temp/dir/tmp.zip', '/test/sdcard/foo123.zip'), - self.call.device.RunShellCommand( - 'unzip /test/sdcard/foo123.zip&&chmod -R 777 /test/dir', - shell=True, - as_root=True, - env={'PATH': '/data/local/tmp/bin:$PATH'}, - check_return=True)): + self.adb, suffix='.zip'), MockTempFile('/sdcard/foo123.zip')), + self.call.adb.Push('/test/temp/dir/tmp.zip', '/sdcard/foo123.zip'), + (mock.call.devil.android.device_temp_file.DeviceTempFile( + self.adb, suffix='.sh'), MockTempFile('/sdcard/temp-123.sh')), + self.call.device.WriteFile('/sdcard/temp-123.sh', expected_cmd), + (self.call.device.RunShellCommand(['source', '/sdcard/temp-123.sh'], + check_return=True, + as_root=True))): self.assertTrue( - self.device._PushChangedFilesZipped(test_files, ['/test/dir'])) + self.device._PushChangedFilesZipped(test_files, test_dirs)) def testPushChangedFilesZipped_single(self): - self._testPushChangedFilesZipped_spec([('/test/host/path/file1', - '/test/device/path/file1')]) + self._testPushChangedFilesZipped_spec( + [('/test/host/path/file1', '/test/device/path/file1')], + ['/test/dir1']) def testPushChangedFilesZipped_multiple(self): self._testPushChangedFilesZipped_spec( [('/test/host/path/file1', '/test/device/path/file1'), - ('/test/host/path/file2', '/test/device/path/file2')]) + ('/test/host/path/file2', '/test/device/path/file2')], + ['/test/dir1', '/test/dir2']) class DeviceUtilsPathExistsTest(DeviceUtilsTest): @@ -2374,7 +2391,8 @@ class DeviceUtilsReadFileTest(DeviceUtilsTest): with self.assertCalls( (mock.call.tempfile.mkdtemp(), tmp_host_dir), (self.call.adb.Pull('/path/to/device/file', mock.ANY)), - (mock.call.__builtin__.open(mock.ANY, 'r'), tmp_host), + (mock.call.__builtin__.open(mock.ANY, 'r'), tmp_host) if six.PY2 else \ + (mock.call.builtins.open(mock.ANY, 'r'), tmp_host), (mock.call.os.path.exists(tmp_host_dir), True), (mock.call.shutil.rmtree(tmp_host_dir), None)): self.assertEquals('some interesting contents', @@ -2587,8 +2605,8 @@ class DeviceUtilsStatDirectoryTest(DeviceUtilsTest): self.getStatEntries(path_given='/foo/bar', path_listed='/foo/bar/') def testStatDirectory_fileList(self): - self.assertItemsEqual(self.getStatEntries().keys(), self.FILENAMES) - self.assertItemsEqual(self.getListEntries(), self.FILENAMES) + self.safeAssertItemsEqual(self.getStatEntries().keys(), self.FILENAMES) + self.safeAssertItemsEqual(self.getListEntries(), self.FILENAMES) def testStatDirectory_fileModes(self): expected_modes = ( @@ -2656,7 +2674,7 @@ class DeviceUtilsStatDirectoryTest(DeviceUtilsTest): def testStatDirectory_symbolicLinks(self): entries = self.getStatEntries() self.assertEqual(entries['lnk']['symbolic_link_to'], '/a/path') - for d in entries.itervalues(): + for d in entries.values(): self.assertEqual('symbolic_link_to' in d, stat.S_ISLNK(d['st_mode'])) @@ -3463,7 +3481,7 @@ class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): device_utils.DeviceUtils.HealthyDevices(device_arg=[], retries=0) @mock.patch('time.sleep') - @mock.patch('devil.android.device_utils.RestartServer') + @mock.patch('devil.android.sdk.adb_wrapper.RestartServer') def testHealthyDevices_EmptyListDeviceArg_no_attached_with_retry( self, mock_restart, mock_sleep): with self.assertCalls( @@ -3481,7 +3499,7 @@ class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): mock.call(8), mock.call(16)]) @mock.patch('time.sleep') - @mock.patch('devil.android.device_utils.RestartServer') + @mock.patch('devil.android.sdk.adb_wrapper.RestartServer') def testHealthyDevices_EmptyListDeviceArg_no_attached_with_resets( self, mock_restart, mock_sleep): # The reset_usb import fails on windows. Mock the full import here so it can @@ -3557,7 +3575,7 @@ class DeviceUtilsGrantPermissionsTest(DeviceUtilsTest): def _PmGrantShellCall(self, package, permissions): fragment = 'p=%s;for q in %s;' % (package, ' '.join(sorted(permissions))) results = [] - for permission, result in sorted(permissions.iteritems()): + for permission, result in sorted(permissions.items()): if result: output, status = result + '\n', 1 else: @@ -3610,6 +3628,23 @@ class DeviceUtilsGrantPermissionsTest(DeviceUtilsTest): self.device.GrantPermissions('package', [WRITE]) self.assertEqual(logger.warnings, []) + def testGrantPermissions_ManageExtrnalStorage(self): + with PatchLogger() as logger: + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.R): + with self.assertCalls( + (self.call.device.RunShellCommand( + AnyStringWith('appops set pkg MANAGE_EXTERNAL_STORAGE allow'), + shell=True, + raw_output=True, + large_output=True, + check_return=True), + '{sep}MANAGE_EXTERNAL_STORAGE{sep}0{sep}\n'.format( + sep=device_utils._SHELL_OUTPUT_SEPARATOR))): + self.device.GrantPermissions( + 'pkg', ['android.permission.MANAGE_EXTERNAL_STORAGE']) + self.assertEqual(logger.warnings, []) + def testGrantPermissions_DenyList(self): with PatchLogger() as logger: with self.patch_call( @@ -3866,6 +3901,12 @@ class IterPushableComponentsTest(unittest.TestCase): yield Layout(layout_root, basic_file, symlink, symlink_dir, dir1, dir2) + def safeAssertItemsEqual(self, expected, actual): + if six.PY2: + self.assertItemsEqual(expected, actual) + else: + self.assertCountEqual(expected, actual) # pylint: disable=no-member + def testFile(self): with self.sampleLayout() as layout: device_path = '/sdcard/basic_file' @@ -3873,7 +3914,7 @@ class IterPushableComponentsTest(unittest.TestCase): expected = [(layout.basic_file, device_path, True)] actual = list( device_utils._IterPushableComponents(layout.basic_file, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testSymlinkFile(self): with self.sampleLayout() as layout: @@ -3883,7 +3924,7 @@ class IterPushableComponentsTest(unittest.TestCase): actual = list( device_utils._IterPushableComponents(layout.symlink_file, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testDirectoryWithNoSymlink(self): with self.sampleLayout() as layout: @@ -3893,7 +3934,7 @@ class IterPushableComponentsTest(unittest.TestCase): actual = list( device_utils._IterPushableComponents(layout.dir_without_symlinks, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testDirectoryWithSymlink(self): with self.sampleLayout() as layout: @@ -3910,7 +3951,7 @@ class IterPushableComponentsTest(unittest.TestCase): actual = list( device_utils._IterPushableComponents(layout.dir_with_symlinks, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testSymlinkDirectory(self): with self.sampleLayout() as layout: @@ -3919,7 +3960,7 @@ class IterPushableComponentsTest(unittest.TestCase): expected = [(os.path.realpath(layout.symlink_dir), device_path, False)] actual = list( device_utils._IterPushableComponents(layout.symlink_dir, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testDirectoryWithNestedSymlink(self): with self.sampleLayout() as layout: @@ -3947,7 +3988,7 @@ class IterPushableComponentsTest(unittest.TestCase): ] actual = list( device_utils._IterPushableComponents(layout.root, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) class DeviceUtilsGetTracingPathTest(DeviceUtilsTest): diff --git a/catapult/devil/devil/android/fastboot_utils_test.py b/catapult/devil/devil/android/fastboot_utils_test.py index 1ad73190..ed891393 100755 --- a/catapult/devil/devil/android/fastboot_utils_test.py +++ b/catapult/devil/devil/android/fastboot_utils_test.py @@ -13,6 +13,8 @@ import io import logging import unittest +import six + from devil import devil_env from devil.android import device_errors from devil.android import device_utils @@ -347,16 +349,22 @@ class FastbootUtilsFastbootMode(FastbootUtilsTest): pass +if six.PY2: + _BUILTIN_OPEN = '__builtin__.open' +else: + _BUILTIN_OPEN = 'builtins.open' + + class FastbootUtilsVerifyBoard(FastbootUtilsTest): def testVerifyBoard_bothValid(self): mock_file = io.StringIO(u'require board=%s\n' % _BOARD) - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_VALID_FILES): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_BothNotValid(self): mock_file = io.StringIO(u'abc') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_INVALID_FILES): self.assertFalse(self.assertFalse(self.fastboot._VerifyBoard('test'))) @@ -366,31 +374,31 @@ class FastbootUtilsVerifyBoard(FastbootUtilsTest): def testVerifyBoard_ZipNotFoundFileValid(self): mock_file = io.StringIO(u'require board=%s\n' % _BOARD) - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=['android-info.txt']): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_zipNotValidFileIs(self): mock_file = io.StringIO(u'require board=%s\n' % _BOARD) - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_INVALID_FILES): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_fileNotValidZipIs(self): mock_file = io.StringIO(u'require board=WrongBoard') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_VALID_FILES): self.assertFalse(self.fastboot._VerifyBoard('test')) def testVerifyBoard_noBoardInFileValidZip(self): mock_file = io.StringIO(u'Regex wont match') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_VALID_FILES): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_noBoardInFileInvalidZip(self): mock_file = io.StringIO(u'Regex wont match') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_INVALID_FILES): self.assertFalse(self.fastboot._VerifyBoard('test')) @@ -419,7 +427,7 @@ class FastbootUtilsFindAndVerifyPartitionsAndImages(FastbootUtilsTest): ] with mock.patch('os.listdir', return_value=files): imgs = self.fastboot._FindAndVerifyPartitionsAndImages(PARTITIONS, 'test') - parts = imgs.keys() + parts = list(imgs.keys()) self.assertDictEqual(imgs, img_check) self.assertListEqual(parts, parts_check) @@ -451,7 +459,7 @@ class FastbootUtilsFindAndVerifyPartitionsAndImages(FastbootUtilsTest): with self.patch_call(self.call.fastboot.supports_ab, return_value=False): imgs = self.fastboot._FindAndVerifyPartitionsAndImages( PARTITIONS, 'test') - parts = imgs.keys() + parts = list(imgs.keys()) self.assertDictEqual(imgs, img_check) self.assertListEqual(parts, parts_check) diff --git a/catapult/devil/devil/android/flag_changer_test.py b/catapult/devil/devil/android/flag_changer_test.py index 564ead6e..9c155f13 100755 --- a/catapult/devil/devil/android/flag_changer_test.py +++ b/catapult/devil/devil/android/flag_changer_test.py @@ -6,6 +6,8 @@ import posixpath import unittest +import six + from devil.android import flag_changer _CMDLINE_FILE = 'chrome-command-line' @@ -109,15 +111,22 @@ class ParseSerializeFlagsTest(unittest.TestCase): def _testParseCmdLine(self, command_line, expected_flags): # Start with a command line, check that flags are parsed as expected. # pylint: disable=protected-access + # pylint: disable=no-member flags = flag_changer._ParseFlags(command_line) - self.assertItemsEqual(flags, expected_flags) + if six.PY2: + self.assertItemsEqual(flags, expected_flags) + else: + self.assertCountEqual(flags, expected_flags) # Check that flags survive a round-trip. # Note: Although new_command_line and command_line may not match, they # should describe the same set of flags. new_command_line = flag_changer._SerializeFlags(flags) new_flags = flag_changer._ParseFlags(new_command_line) - self.assertItemsEqual(new_flags, expected_flags) + if six.PY2: + self.assertItemsEqual(new_flags, expected_flags) + else: + self.assertCountEqual(new_flags, expected_flags) def testParseCmdLine_simple(self): self._testParseCmdLine('chrome --foo --bar="a b" --baz=true --fine="ok"', diff --git a/catapult/devil/devil/android/logcat_monitor.py b/catapult/devil/devil/android/logcat_monitor.py index bec74440..df306b0a 100644 --- a/catapult/devil/devil/android/logcat_monitor.py +++ b/catapult/devil/devil/android/logcat_monitor.py @@ -13,6 +13,8 @@ import tempfile import threading import time +import six + from devil.android import decorators from devil.android import device_errors from devil.android.sdk import adb_wrapper @@ -102,9 +104,9 @@ class LogcatMonitor(object): raise LogcatMonitorCommandError( 'Must be recording logcat when calling |WaitFor|', device_serial=str(self._adb)) - if isinstance(success_regex, basestring): + if isinstance(success_regex, six.string_types): success_regex = re.compile(success_regex) - if isinstance(failure_regex, basestring): + if isinstance(failure_regex, six.string_types): failure_regex = re.compile(failure_regex) logger.debug('Waiting %d seconds for "%s"', timeout, success_regex.pattern) @@ -220,10 +222,14 @@ class LogcatMonitor(object): Clears the logcat if |clear| was set in |__init__|. """ + # pylint: disable=unexpected-keyword-arg if self._clear: self._adb.Logcat(clear=True) if not self._record_file: - self._record_file = tempfile.NamedTemporaryFile(mode='a', bufsize=1) + if six.PY2: + self._record_file = tempfile.NamedTemporaryFile(mode='a', bufsize=1) + else: + self._record_file = tempfile.NamedTemporaryFile(mode='a', buffering=1) self._StartRecording() def Stop(self): diff --git a/catapult/devil/devil/android/logcat_monitor_test.py b/catapult/devil/devil/android/logcat_monitor_test.py index 7f2f10a6..356fe041 100755 --- a/catapult/devil/devil/android/logcat_monitor_test.py +++ b/catapult/devil/devil/android/logcat_monitor_test.py @@ -9,6 +9,8 @@ import itertools import threading import unittest +import six + from devil import devil_env from devil.android import logcat_monitor from devil.android.sdk import adb_wrapper @@ -24,6 +26,13 @@ def _CreateTestLog(raw_logcat=None): return test_log +def zip_longest(expected, actual): + # pylint: disable=no-member + if six.PY2: + return itertools.izip_longest(expected, actual) + else: + return itertools.zip_longest(expected, actual) + class LogcatMonitorTest(unittest.TestCase): _TEST_THREADTIME_LOGCAT_DATA = [ @@ -44,7 +53,7 @@ class LogcatMonitorTest(unittest.TestCase): ] def assertIterEqual(self, expected_iter, actual_iter): - for expected, actual in itertools.izip_longest(expected_iter, actual_iter): + for expected, actual in zip_longest(expected_iter, actual_iter): self.assertIsNotNone( expected, msg='actual has unexpected elements starting with %s' % str(actual)) diff --git a/catapult/devil/devil/android/md5sum.py b/catapult/devil/devil/android/md5sum.py index 8adf4ef7..e67f3f60 100644 --- a/catapult/devil/devil/android/md5sum.py +++ b/catapult/devil/devil/android/md5sum.py @@ -3,10 +3,12 @@ # found in the LICENSE file. import base64 +import io import gzip import os import re -import StringIO + +import six from devil import devil_env from devil.android import device_errors @@ -38,7 +40,7 @@ def CalculateHostMd5Sums(paths): Returns: A dict mapping file paths to their respective md5sum checksums. """ - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): paths = [paths] paths = list(paths) @@ -47,13 +49,17 @@ def CalculateHostMd5Sums(paths): raise IOError('File not built: %s' % md5sum_bin_host_path) out = "" for i in range(0, len(paths), _MAX_PATHS_PER_INVOCATION): - mem_file = StringIO.StringIO() + mem_file = io.BytesIO() compressed = gzip.GzipFile(fileobj=mem_file, mode="wb") - compressed.write(";".join( - [os.path.realpath(p) for p in paths[i:i+_MAX_PATHS_PER_INVOCATION]])) + data = ";".join( + [os.path.realpath(p) for p in paths[i:i+_MAX_PATHS_PER_INVOCATION]]) + if six.PY3: + data = data.encode('utf-8') + compressed.write(data) compressed.close() compressed_paths = base64.b64encode(mem_file.getvalue()) - out += cmd_helper.GetCmdOutput([md5sum_bin_host_path, "-gz", compressed_paths]) + out += cmd_helper.GetCmdOutput( + [md5sum_bin_host_path, "-gz", compressed_paths]) return dict(zip(paths, out.splitlines())) @@ -72,7 +78,7 @@ def CalculateDeviceMd5Sums(paths, device): if not paths: return {} - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): paths = [paths] paths = list(paths) @@ -97,9 +103,12 @@ def CalculateDeviceMd5Sums(paths, device): # Make sure it can find libbase.so md5sum_script += 'export LD_LIBRARY_PATH=%s;' % MD5SUM_DEVICE_LIB_PATH for i in range(0, len(paths), _MAX_PATHS_PER_INVOCATION): - mem_file = StringIO.StringIO() + mem_file = io.BytesIO() compressed = gzip.GzipFile(fileobj=mem_file, mode="wb") - compressed.write(";".join(paths[i:i+_MAX_PATHS_PER_INVOCATION])) + data = ";".join(paths[i:i+_MAX_PATHS_PER_INVOCATION]) + if six.PY3: + data = data.encode('utf-8') + compressed.write(data) compressed.close() compressed_paths = base64.b64encode(mem_file.getvalue()) md5sum_script += '$a -gz %s;' % compressed_paths diff --git a/catapult/devil/devil/android/md5sum_test.py b/catapult/devil/devil/android/md5sum_test.py index 548b2d02..9a51313e 100755 --- a/catapult/devil/devil/android/md5sum_test.py +++ b/catapult/devil/devil/android/md5sum_test.py @@ -112,7 +112,7 @@ class Md5SumTest(unittest.TestCase): def testCalculateDeviceMd5Sums_generator(self): test_path = ('/storage/emulated/legacy/test/file%d.dat' % n - for n in xrange(0, 2)) + for n in range(0, 2)) device = mock.NonCallableMock() device_md5sum_output = [ diff --git a/catapult/devil/devil/android/perf/perf_control.py b/catapult/devil/devil/android/perf/perf_control.py index 59485e0e..398b27fa 100644 --- a/catapult/devil/devil/android/perf/perf_control.py +++ b/catapult/devil/devil/android/perf/perf_control.py @@ -203,7 +203,7 @@ class PerfControl(object): if not isinstance(cpu_max_freq, dict): self._SetScalingMaxFreqForCpus(cpu_max_freq, self._cpu_file_list) else: - for key, max_frequency in cpu_max_freq.iteritems(): + for key, max_frequency in cpu_max_freq.items(): # Convert 'X' to 'cpuX' and 'X..Y' to 'cpuX cpu<X+1> .. cpuY'. if '..' in key: range_min, range_max = key.split('..') @@ -211,7 +211,7 @@ class PerfControl(object): else: range_min = range_max = int(key) cpu_files = [ - 'cpu%d' % number for number in xrange(range_min, range_max + 1) + 'cpu%d' % number for number in range(range_min, range_max + 1) ] # Set the |max_frequency| on requested subset of the cores. self._SetScalingMaxFreqForCpus(max_frequency, ' '.join(cpu_files)) diff --git a/catapult/devil/devil/android/perf/perf_control_test.py b/catapult/devil/devil/android/perf/perf_control_test.py index bde54d5a..a841a0e2 100644 --- a/catapult/devil/devil/android/perf/perf_control_test.py +++ b/catapult/devil/devil/android/perf/perf_control_test.py @@ -47,7 +47,7 @@ class PerfControlTest(unittest.TestCase): # pylint: disable=no-self-use def testNexus5HighPerfMode(self): # Mock out the device state for PerfControl. - cpu_list = ['cpu%d' % cpu for cpu in xrange(4)] + cpu_list = ['cpu%d' % cpu for cpu in range(4)] mock_device = mock.Mock(spec=device_utils.DeviceUtils) mock_device.product_model = 'Nexus 5' mock_device.adb = mock.Mock(spec=adb_wrapper.AdbWrapper) @@ -70,7 +70,7 @@ class PerfControlTest(unittest.TestCase): def testNexus5XHighPerfMode(self): # Mock out the device state for PerfControl. - cpu_list = ['cpu%d' % cpu for cpu in xrange(6)] + cpu_list = ['cpu%d' % cpu for cpu in range(6)] mock_device = mock.Mock(spec=device_utils.DeviceUtils) mock_device.product_model = 'Nexus 5X' mock_device.adb = mock.Mock(spec=adb_wrapper.AdbWrapper) @@ -93,7 +93,7 @@ class PerfControlTest(unittest.TestCase): def testNexus5XDefaultPerfMode(self): # Mock out the device state for PerfControl. - cpu_list = ['cpu%d' % cpu for cpu in xrange(6)] + cpu_list = ['cpu%d' % cpu for cpu in range(6)] mock_device = mock.Mock(spec=device_utils.DeviceUtils) mock_device.product_model = 'Nexus 5X' mock_device.adb = mock.Mock(spec=adb_wrapper.AdbWrapper) diff --git a/catapult/devil/devil/android/perf/surface_stats_collector.py b/catapult/devil/devil/android/perf/surface_stats_collector.py index 6240624f..4ddc6f5d 100644 --- a/catapult/devil/devil/android/perf/surface_stats_collector.py +++ b/catapult/devil/devil/android/perf/surface_stats_collector.py @@ -3,10 +3,16 @@ # found in the LICENSE file. import logging -import Queue import re import threading +import six + +if six.PY3: + import queue # pylint: disable=wrong-import-order +else: + import Queue as queue # pylint: disable=wrong-import-order + # Log marker containing SurfaceTexture timestamps. _SURFACE_TEXTURE_TIMESTAMPS_MESSAGE = 'SurfaceTexture update timestamps' _SURFACE_TEXTURE_TIMESTAMP_RE = r'\d+' @@ -37,7 +43,7 @@ class SurfaceStatsCollector(object): if self._ClearSurfaceFlingerLatencyData(): self._get_data_event = threading.Event() self._stop_event = threading.Event() - self._data_queue = Queue.Queue() + self._data_queue = queue.Queue() self._collector_thread = threading.Thread(target=self._CollectorThread) self._collector_thread.start() else: @@ -148,6 +154,10 @@ class SurfaceStatsCollector(object): return ParseFrameData(output, parse_timestamps=bool(window_name)) +def to_long_int(val): + """Cast val to a long int type.""" + return long(val) if six.PY2 else int(val) + def ParseFrameData(lines, parse_timestamps): # adb shell dumpsys SurfaceFlinger --latency <window name> # prints some information about the last 128 frames displayed in @@ -174,6 +184,7 @@ def ParseFrameData(lines, parse_timestamps): # (each time the number above changes, we have a "jank"). # If this happens a lot during an animation, the animation appears # janky, even if it runs at 60 fps in average. + # pylint: disable=redefined-variable-type results = [] for line in lines: # Skip over lines with anything other than digits and whitespace. @@ -186,7 +197,8 @@ def ParseFrameData(lines, parse_timestamps): timestamps = [] nanoseconds_per_millisecond = 1e6 - refresh_period = long(results[0]) / nanoseconds_per_millisecond + refresh_period = to_long_int(results[0]) / nanoseconds_per_millisecond + if not parse_timestamps: return refresh_period, timestamps @@ -201,7 +213,8 @@ def ParseFrameData(lines, parse_timestamps): if len(fields) != 3: logging.warning('Unexpected line: %s', line) continue - timestamp = long(fields[1]) + timestamp = to_long_int(fields[1]) + if timestamp == pending_fence_timestamp: continue timestamp /= nanoseconds_per_millisecond diff --git a/catapult/devil/devil/android/ports.py b/catapult/devil/devil/android/ports.py index f25c8bfd..4a7c2945 100644 --- a/catapult/devil/devil/android/ports.py +++ b/catapult/devil/devil/android/ports.py @@ -159,7 +159,7 @@ def IsHttpServerConnectable(host, message the server returns when connect status is false. """ assert tries >= 1 - for i in xrange(0, tries): + for i in range(0, tries): client_error = None try: with contextlib.closing( diff --git a/catapult/devil/devil/android/sdk/aapt.py b/catapult/devil/devil/android/sdk/aapt.py index 76c7ef68..fd354079 100644 --- a/catapult/devil/devil/android/sdk/aapt.py +++ b/catapult/devil/devil/android/sdk/aapt.py @@ -3,6 +3,8 @@ # found in the LICENSE file. """This module wraps the Android Asset Packaging Tool.""" +import six + from devil.android.sdk import build_tools from devil.utils import cmd_helper from devil.utils import lazy @@ -36,6 +38,6 @@ def Dump(what, apk, assets=None): assets: List of assets in apk you want to dump information for. """ assets = assets or [] - if isinstance(assets, basestring): + if isinstance(assets, six.string_types): assets = [assets] return _RunAaptCmd(['dump', what, apk] + assets).splitlines() diff --git a/catapult/devil/devil/android/sdk/adb_wrapper.py b/catapult/devil/devil/android/sdk/adb_wrapper.py index 71928d58..d8992242 100644 --- a/catapult/devil/devil/android/sdk/adb_wrapper.py +++ b/catapult/devil/devil/android/sdk/adb_wrapper.py @@ -18,6 +18,8 @@ import posixpath import re import subprocess +import six + from devil import base_error from devil import devil_env from devil.android import decorators @@ -127,6 +129,30 @@ def _IsExtraneousLine(line, send_cmd): return send_cmd.rstrip() in line +@decorators.WithExplicitTimeoutAndRetries(timeout=60, retries=3) +def RestartServer(): + """Restarts the adb server. + + Raises: + CommandFailedError if we fail to kill or restart the server. + """ + + def adb_killed(): + return not AdbWrapper.IsServerOnline() + + def adb_started(): + return AdbWrapper.IsServerOnline() + + AdbWrapper.KillServer() + if not timeout_retry.WaitFor(adb_killed, wait_period=1, max_tries=5): + # TODO(crbug.com/442319): Switch this to raise an exception if we + # figure out why sometimes not all adb servers on bots get killed. + logger.warning('Failed to kill adb server') + AdbWrapper.StartServer() + if not timeout_retry.WaitFor(adb_started, wait_period=1, max_tries=5): + raise device_errors.CommandFailedError('Failed to start adb server') + + class AdbWrapper(object): """A wrapper around a local Android Debug Bridge executable.""" @@ -1105,7 +1131,7 @@ class AdbWrapper(object): Returns: The output of the emulator console command. """ - if isinstance(cmd, basestring): + if isinstance(cmd, six.string_types): cmd = [cmd] return self._RunDeviceAdbCmd(['emu'] + cmd, timeout, retries) diff --git a/catapult/devil/devil/android/sdk/dexdump.py b/catapult/devil/devil/android/sdk/dexdump.py index 2a59e9bf..c71442ce 100644 --- a/catapult/devil/devil/android/sdk/dexdump.py +++ b/catapult/devil/devil/android/sdk/dexdump.py @@ -2,6 +2,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import six + from devil.android.sdk import build_tools from devil.utils import cmd_helper from devil.utils import lazy @@ -20,7 +22,7 @@ def DexDump(dexfiles, file_summary=False): An iterable over the output lines. """ # TODO(jbudorick): Add support for more options as necessary. - if isinstance(dexfiles, basestring): + if isinstance(dexfiles, six.string_types): dexfiles = [dexfiles] args = [_dexdump_path.read()] + dexfiles if file_summary: diff --git a/catapult/devil/devil/android/sdk/intent.py b/catapult/devil/devil/android/sdk/intent.py index 69c7d900..2ea38c33 100644 --- a/catapult/devil/devil/android/sdk/intent.py +++ b/catapult/devil/devil/android/sdk/intent.py @@ -116,7 +116,7 @@ class Intent(object): if self.flags: args.extend(['-f', self.flags]) if self.extras: - for key, value in self.extras.iteritems(): + for key, value in self.extras.items(): if value is None: args.extend(['--esn', key]) elif isinstance(value, str): diff --git a/catapult/devil/devil/android/sdk/shared_prefs.py b/catapult/devil/devil/android/sdk/shared_prefs.py index 7b12bf54..32b5bc4d 100644 --- a/catapult/devil/devil/android/sdk/shared_prefs.py +++ b/catapult/devil/devil/android/sdk/shared_prefs.py @@ -11,6 +11,8 @@ import logging import posixpath from xml.etree import ElementTree +import six + from devil.android import device_errors from devil.android.sdk import version_codes @@ -43,7 +45,10 @@ class BasePref(object): def __str__(self): """Get the underlying xml element as a string.""" - return ElementTree.tostring(self._elem) + if six.PY2: + return ElementTree.tostring(self._elem) + else: + return ElementTree.tostring(self._elem, encoding="unicode") def get(self): """Get the value of this preference.""" @@ -231,7 +236,11 @@ class SharedPrefs(object): def __str__(self): """Get the underlying xml document as a string.""" - return _XML_DECLARATION + ElementTree.tostring(self.xml) + if six.PY2: + return _XML_DECLARATION + ElementTree.tostring(self.xml) + else: + return _XML_DECLARATION + \ + ElementTree.tostring(self.xml, encoding="unicode") @property def package(self): diff --git a/catapult/devil/devil/android/sdk/shared_prefs_test.py b/catapult/devil/devil/android/sdk/shared_prefs_test.py index 7374c892..b50d9e0b 100755 --- a/catapult/devil/devil/android/sdk/shared_prefs_test.py +++ b/catapult/devil/devil/android/sdk/shared_prefs_test.py @@ -132,6 +132,8 @@ class SharedPrefsTest(unittest.TestCase): }) # data survived roundtrip def testForceCommit(self): + type(self.device).build_version_sdk = mock.PropertyMock( + return_value=version_codes.LOLLIPOP_MR1) prefs = shared_prefs.SharedPrefs(self.device, 'com.some.package', 'prefs.xml') prefs.Load() diff --git a/catapult/devil/devil/android/sdk/version_codes.py b/catapult/devil/devil/android/sdk/version_codes.py index 943b9d3f..564f30da 100644 --- a/catapult/devil/devil/android/sdk/version_codes.py +++ b/catapult/devil/devil/android/sdk/version_codes.py @@ -20,3 +20,4 @@ OREO = 26 OREO_MR1 = 27 PIE = 28 Q = 29 +R = 30 diff --git a/catapult/devil/devil/android/tools/device_monitor.py b/catapult/devil/devil/android/tools/device_monitor.py index 26e89a24..730df141 100755 --- a/catapult/devil/devil/android/tools/device_monitor.py +++ b/catapult/devil/devil/android/tools/device_monitor.py @@ -184,7 +184,7 @@ def get_all_status(denylist): } if denylist: - for device, reason in denylist.Read().iteritems(): + for device, reason in denylist.Read().items(): status_dict['devices'][device] = { 'state': reason.get('reason', 'denylisted') } diff --git a/catapult/devil/devil/android/tools/device_monitor_test.py b/catapult/devil/devil/android/tools/device_monitor_test.py index 8082d26e..1bb5680a 100755 --- a/catapult/devil/devil/android/tools/device_monitor_test.py +++ b/catapult/devil/devil/android/tools/device_monitor_test.py @@ -7,6 +7,8 @@ import os import sys import unittest +import six + if __name__ == '__main__': sys.path.append( os.path.abspath( @@ -53,7 +55,7 @@ class DeviceMonitorTest(unittest.TestCase): } def mock_run_shell(cmd, **_kwargs): - args = cmd.split() if isinstance(cmd, basestring) else cmd + args = cmd.split() if isinstance(cmd, six.string_types) else cmd try: return self.cmd_outputs[args[0]] except KeyError: diff --git a/catapult/devil/devil/android/tools/system_app.py b/catapult/devil/devil/android/tools/system_app.py index 62bf5a59..50d85595 100755 --- a/catapult/devil/devil/android/tools/system_app.py +++ b/catapult/devil/devil/android/tools/system_app.py @@ -22,6 +22,7 @@ from devil.android import decorators from devil.android import device_errors from devil.android import device_temp_file from devil.android.sdk import version_codes +from devil.android.sdk import adb_wrapper from devil.android.tools import script_common from devil.utils import cmd_helper from devil.utils import parallelizer @@ -128,7 +129,15 @@ _ENABLE_MODIFICATION_PROP = 'devil.modify_sys_apps' def _ShouldRetryModification(exc): - return not isinstance(exc, device_errors.CommandTimeoutError) + try: + if isinstance(exc, device_errors.CommandTimeoutError): + logger.info('Restarting the adb server') + adb_wrapper.RestartServer() + return True + except Exception: # pylint: disable=broad-except + logger.exception(('Caught an exception when deciding' + ' to retry system modification')) + return False # timeout and retries are both required by the decorator, but neither @@ -172,7 +181,7 @@ def _SetUpSystemAppModification(device, timeout=None, retries=None): # Point the user to documentation, since there's a good chance they can # workaround this on an emulator. docs_url = ('https://chromium.googlesource.com/chromium/src/+/' - 'master/docs/android_emulator.md#writable-system-partition') + 'HEAD/docs/android_emulator.md#writable-system-partition') logger.error( 'Did you start the emulator with "-writable-system?"\n' 'See %s\n', docs_url) @@ -187,6 +196,22 @@ def _TearDownSystemAppModification(device, timeout=None, retries=None): try: + # The function may be re-entered after the the device loses root + # privilege. For instance if the adb server is restarted before + # re-entering the function then the device may lose root privilege. + # Therefore we need to do a sanity check for root privilege + # on the device and then re-enable root privilege if the device + # does not have it. + if not device.HasRoot(): + logger.warning('Need to re-enable root.') + device.EnableRoot() + + if not device.HasRoot(): + raise device_errors.CommandFailedError( + ('Failed to tear down modification of ' + 'system apps on non-rooted device.'), + str(device)) + device.SetProp(_ENABLE_MODIFICATION_PROP, '0') device.Reboot() device.WaitUntilFullyBooted() diff --git a/catapult/devil/devil/android/tools/video_recorder.py b/catapult/devil/devil/android/tools/video_recorder.py index 984931f3..4004c467 100755 --- a/catapult/devil/devil/android/tools/video_recorder.py +++ b/catapult/devil/devil/android/tools/video_recorder.py @@ -164,7 +164,7 @@ def main(): parallel_devices = device_utils.DeviceUtils.parallel(script_common.GetDevices( args.devices, args.denylist_file), - async=True) + asyn=True) stop_recording = threading.Event() running_recording = parallel_devices.pMap(record_video, stop_recording) print 'Recording. Press Enter to stop.', diff --git a/catapult/devil/devil/base_error.py b/catapult/devil/devil/base_error.py index b7b33ccf..dd9ed496 100644 --- a/catapult/devil/devil/base_error.py +++ b/catapult/devil/devil/base_error.py @@ -9,6 +9,7 @@ class BaseError(Exception): def __init__(self, message, is_infra_error=False): super(BaseError, self).__init__(message) self._is_infra_error = is_infra_error + self.message = message def __eq__(self, other): return (self.message == other.message diff --git a/catapult/devil/devil/devil_env.py b/catapult/devil/devil/devil_env.py index b61d1b0e..2d576885 100644 --- a/catapult/devil/devil/devil_env.py +++ b/catapult/devil/devil/devil_env.py @@ -16,6 +16,7 @@ CATAPULT_ROOT_PATH = os.path.abspath( DEPENDENCY_MANAGER_PATH = os.path.join(CATAPULT_ROOT_PATH, 'dependency_manager') PYMOCK_PATH = os.path.join(CATAPULT_ROOT_PATH, 'third_party', 'mock') PY_UTILS_PATH = os.path.join(CATAPULT_ROOT_PATH, 'common', 'py_utils') +SIX_PATH = os.path.join(CATAPULT_ROOT_PATH, 'third_party', 'six') @contextlib.contextmanager @@ -31,6 +32,9 @@ def SysPath(path): with SysPath(DEPENDENCY_MANAGER_PATH): import dependency_manager # pylint: disable=import-error +with SysPath(SIX_PATH): + import six + _ANDROID_BUILD_TOOLS = {'aapt', 'dexdump', 'split-select'} _DEVIL_DEFAULT_CONFIG = os.path.abspath( @@ -53,7 +57,7 @@ def EmptyConfig(): def LocalConfigItem(dependency_name, dependency_platform, dependency_path): - if isinstance(dependency_path, basestring): + if isinstance(dependency_path, six.string_types): dependency_path = [dependency_path] return { dependency_name: { @@ -69,7 +73,7 @@ def LocalConfigItem(dependency_name, dependency_platform, dependency_path): def _GetEnvironmentVariableConfig(): env_config = EmptyConfig() path_config = ((os.environ.get(k), v) - for k, v in _LEGACY_ENVIRONMENT_VARIABLES.iteritems()) + for k, v in six.iteritems(_LEGACY_ENVIRONMENT_VARIABLES)) path_config = ((p, c) for p, c in path_config if p) for p, c in path_config: env_config['dependencies'].update( @@ -115,7 +119,8 @@ class _Environment(object): # TODO(jbudorick): Remove this recursion if/when dependency_manager # supports loading configurations directly from a dict. if configs: - with tempfile.NamedTemporaryFile(delete=False) as next_config_file: + with tempfile.NamedTemporaryFile(mode='w', + delete=False) as next_config_file: try: next_config_file.write(json.dumps(configs[0])) next_config_file.close() @@ -182,7 +187,10 @@ class _Environment(object): def GetPlatform(arch=None, device=None): if arch or device: return 'android_%s' % (arch or device.product_cpu_abi) - return '%s_%s' % (sys.platform, platform.machine()) + # use 'linux2' for linux as this is already used in json file + return '%s_%s' % ( + sys.platform if not sys.platform.startswith('linux') else 'linux2', + platform.machine()) config = _Environment() diff --git a/catapult/devil/devil/devil_env_test.py b/catapult/devil/devil/devil_env_test.py index ee7cd8fd..65fd7047 100755 --- a/catapult/devil/devil/devil_env_test.py +++ b/catapult/devil/devil/devil_env_test.py @@ -10,6 +10,7 @@ import sys import unittest from devil import devil_env +from devil.android.ndk import abis _sys_path_before = list(sys.path) with devil_env.SysPath(devil_env.PYMOCK_PATH): @@ -18,6 +19,11 @@ with devil_env.SysPath(devil_env.PYMOCK_PATH): _sys_path_after = list(sys.path) +class _MockDeviceUtils(object): + def __init__(self): + self.product_cpu_abi = abis.ARM_64 + + class DevilEnvTest(unittest.TestCase): def testSysPath(self): self.assertEquals(_sys_path_before, _sys_path_after) @@ -52,6 +58,21 @@ class DevilEnvTest(unittest.TestCase): }, }, env_config.get('dependencies')) + def testGetPlatform(self): + with mock.patch('platform.machine', mock.Mock(return_value='x86_64')): + with mock.patch('sys.platform', mock.Mock(return_value='linux2')): + platform = devil_env.GetPlatform() + self.assertEquals(platform, 'linux2_x86_64') + with mock.patch('sys.platform', mock.Mock(return_value='linux')): + platform = devil_env.GetPlatform() + self.assertEquals(platform, 'linux2_x86_64') + + platform = devil_env.GetPlatform(arch='arm64-v8a') + self.assertEquals(platform, 'android_arm64-v8a') + + device = _MockDeviceUtils() + platform = devil_env.GetPlatform(device=device) + self.assertEquals(platform, 'android_arm64-v8a') if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) diff --git a/catapult/devil/devil/utils/cmd_helper.py b/catapult/devil/devil/utils/cmd_helper.py index 634c9716..3a959450 100644 --- a/catapult/devil/devil/utils/cmd_helper.py +++ b/catapult/devil/devil/utils/cmd_helper.py @@ -10,11 +10,19 @@ import pipes import select import signal import string -import StringIO import subprocess import sys import time +CATAPULT_ROOT_PATH = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..')) +SIX_PATH = os.path.join(CATAPULT_ROOT_PATH, 'third_party', 'six') + +if SIX_PATH not in sys.path: + sys.path.append(SIX_PATH) + +import six + from devil import base_error logger = logging.getLogger(__name__) @@ -23,7 +31,8 @@ _SafeShellChars = frozenset(string.ascii_letters + string.digits + '@%_-+=:,./') # Cache the string-escape codec to ensure subprocess can find it # later. Return value doesn't matter. -codecs.lookup('string-escape') +if six.PY2: + codecs.lookup('string-escape') def SingleQuote(s): @@ -102,6 +111,7 @@ def Popen(args, cwd=None, env=None): # preexec_fn isn't supported on windows. + # pylint: disable=unexpected-keyword-arg if sys.platform == 'win32': close_fds = (stdin is None and stdout is None and stderr is None) preexec_fn = None @@ -109,7 +119,8 @@ def Popen(args, close_fds = True preexec_fn = lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL) - return subprocess.Popen( + if six.PY2: + return subprocess.Popen( args=args, cwd=cwd, stdin=stdin, @@ -118,8 +129,29 @@ def Popen(args, shell=shell, close_fds=close_fds, env=env, - preexec_fn=preexec_fn) - + preexec_fn=preexec_fn + ) + else: + # opens stdout in text mode, so that caller side always get 'str', + # and there will be no type mismatch error. + # Ignore any decoding error, so that caller will not crash due to + # uncaught exception. Decoding errors are unavoidable, as we + # do not know the encoding of the output, and in some output there + # will be multiple encodings (e.g. adb logcat) + return subprocess.Popen( + args=args, + cwd=cwd, + stdin=stdin, + stdout=stdout, + stderr=stderr, + shell=shell, + close_fds=close_fds, + env=env, + preexec_fn=preexec_fn, + universal_newlines=True, + encoding='utf-8', + errors='ignore' + ) def Call(args, stdout=None, stderr=None, shell=None, cwd=None, env=None): pipe = Popen( @@ -165,7 +197,7 @@ def GetCmdOutput(args, cwd=None, shell=False, env=None): def _ValidateAndLogCommand(args, cwd, shell): - if isinstance(args, basestring): + if isinstance(args, six.string_types): if not shell: raise Exception('string args must be run with shell=True') else: @@ -283,6 +315,12 @@ class TimeoutError(base_error.BaseError): return self._output +def _read_and_decode(fd, buffer_size): + data = os.read(fd, buffer_size) + if data and six.PY3: + data = data.decode('utf-8', errors='ignore') + return data + def _IterProcessStdoutFcntl(process, iter_timeout=None, timeout=None, @@ -316,7 +354,7 @@ def _IterProcessStdoutFcntl(process, read_fds, _, _ = select.select([child_fd], [], [], iter_aware_poll_interval) if child_fd in read_fds: - data = os.read(child_fd, buffer_size) + data = _read_and_decode(child_fd, buffer_size) if not data: break yield data @@ -328,7 +366,7 @@ def _IterProcessStdoutFcntl(process, read_fds, _, _ = select.select([child_fd], [], [], iter_aware_poll_interval) if child_fd in read_fds: - data = os.read(child_fd, buffer_size) + data = _read_and_decode(child_fd, buffer_size) if data: yield data continue @@ -365,7 +403,7 @@ def _IterProcessStdoutQueue(process, # TODO(jbudorick): Pick an appropriate read size here. while True: try: - output_chunk = os.read(process.stdout.fileno(), buffer_size) + output_chunk = _read_and_decode(process.stdout.fileno(), buffer_size) except IOError: break stdout_queue.put(output_chunk, True) @@ -452,7 +490,7 @@ def GetCmdStatusAndOutputWithTimeout(args, TimeoutError on timeout. """ _ValidateAndLogCommand(args, cwd, shell) - output = StringIO.StringIO() + output = six.StringIO() process = Popen( args, cwd=cwd, diff --git a/catapult/devil/devil/utils/cmd_helper_test.py b/catapult/devil/devil/utils/cmd_helper_test.py index 57abceb4..0eeefe16 100755 --- a/catapult/devil/devil/utils/cmd_helper_test.py +++ b/catapult/devil/devil/utils/cmd_helper_test.py @@ -33,6 +33,17 @@ class CmdHelperSingleQuoteTest(unittest.TestCase): self.assertEquals(test_string, cmd_helper.GetCmdOutput(cmd, shell=True).rstrip()) +class CmdHelperGetCmdStatusAndOutputTest(unittest.TestCase): + def testGetCmdStatusAndOutput_success(self): + cmd = 'echo "Hello World"' + status, output = cmd_helper.GetCmdStatusAndOutput(cmd, shell=True) + self.assertEqual(status, 0) + self.assertEqual(output.rstrip(), "Hello World") + + def testGetCmdStatusAndOutput_unicode(self): + # pylint: disable=no-self-use + cmd = 'echo "\x80\x31Hello World\n"' + cmd_helper.GetCmdStatusAndOutput(cmd, shell=True) class CmdHelperDoubleQuoteTest(unittest.TestCase): def testDoubleQuote_basic(self): @@ -195,7 +206,7 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): # pylint: disable=protected-access _SIMPLE_OUTPUT_SEQUENCE = [ - _ProcessOutputEvent(read_contents='1\n2\n'), + _ProcessOutputEvent(read_contents=b'1\n2\n'), ] def testIterCmdOutputLines_success(self): @@ -205,6 +216,14 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): cmd_helper._IterCmdOutputLines(mock_proc, 'mock_proc'), 1): self.assertEquals(num, int(line)) + def testIterCmdOutputLines_unicode(self): + output_sequence = [ + _ProcessOutputEvent(read_contents=b'\x80\x31\nHello\n\xE2\x98\xA0') + ] + with _MockProcess(output_sequence=output_sequence) as mock_proc: + lines = list(cmd_helper._IterCmdOutputLines(mock_proc, 'mock_proc')) + self.assertEquals(lines[1], "Hello") + def testIterCmdOutputLines_exitStatusFail(self): with self.assertRaises(subprocess.CalledProcessError): with _MockProcess( @@ -238,9 +257,9 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): def testIterCmdOutputLines_delay(self): output_sequence = [ - _ProcessOutputEvent(read_contents='1\n2\n', ts=1), + _ProcessOutputEvent(read_contents=b'1\n2\n', ts=1), _ProcessOutputEvent(read_contents=None, ts=2), - _ProcessOutputEvent(read_contents='Awake', ts=10), + _ProcessOutputEvent(read_contents=b'Awake', ts=10), ] with _MockProcess(output_sequence=output_sequence) as mock_proc: for num, line in enumerate( diff --git a/catapult/devil/devil/utils/decorators.py b/catapult/devil/devil/utils/decorators.py new file mode 100644 index 00000000..5d286107 --- /dev/null +++ b/catapult/devil/devil/utils/decorators.py @@ -0,0 +1,17 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import functools + + +def Memoize(f): + """Decorator to cache return values of function.""" + memoize_dict = {} + @functools.wraps(f) + def wrapper(*args, **kwargs): + key = repr((args, kwargs)) + if key not in memoize_dict: + memoize_dict[key] = f(*args, **kwargs) + return memoize_dict[key] + return wrapper diff --git a/catapult/devil/devil/utils/decorators_test.py b/catapult/devil/devil/utils/decorators_test.py new file mode 100755 index 00000000..f81974ac --- /dev/null +++ b/catapult/devil/devil/utils/decorators_test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unit tests for decorators.py.""" + +import unittest + +from devil.utils import decorators + + +class MemoizeDecoratorTest(unittest.TestCase): + + def testFunctionExceptionNotMemoized(self): + """Tests that |Memoize| decorator does not cache exception results.""" + + class ExceptionType1(Exception): + pass + + class ExceptionType2(Exception): + pass + + @decorators.Memoize + def raiseExceptions(): + if raiseExceptions.count == 0: + raiseExceptions.count += 1 + raise ExceptionType1() + + if raiseExceptions.count == 1: + raise ExceptionType2() + raiseExceptions.count = 0 + + with self.assertRaises(ExceptionType1): + raiseExceptions() + with self.assertRaises(ExceptionType2): + raiseExceptions() + + def testFunctionResultMemoized(self): + """Tests that |Memoize| decorator caches results.""" + + @decorators.Memoize + def memoized(): + memoized.count += 1 + return memoized.count + memoized.count = 0 + + def notMemoized(): + notMemoized.count += 1 + return notMemoized.count + notMemoized.count = 0 + + self.assertEquals(memoized(), 1) + self.assertEquals(memoized(), 1) + self.assertEquals(memoized(), 1) + + self.assertEquals(notMemoized(), 1) + self.assertEquals(notMemoized(), 2) + self.assertEquals(notMemoized(), 3) + + def testFunctionMemoizedBasedOnArgs(self): + """Tests that |Memoize| caches results based on args and kwargs.""" + + @decorators.Memoize + def returnValueBasedOnArgsKwargs(a, k=0): + return a + k + + self.assertEquals(returnValueBasedOnArgsKwargs(1, 1), 2) + self.assertEquals(returnValueBasedOnArgsKwargs(1, 2), 3) + self.assertEquals(returnValueBasedOnArgsKwargs(2, 1), 3) + self.assertEquals(returnValueBasedOnArgsKwargs(3, 3), 6) + + +if __name__ == '__main__': + unittest.main() diff --git a/catapult/devil/devil/utils/markdown_test.py b/catapult/devil/devil/utils/markdown_test.py index 621d56ba..11cd46ea 100755 --- a/catapult/devil/devil/utils/markdown_test.py +++ b/catapult/devil/devil/utils/markdown_test.py @@ -103,10 +103,9 @@ class MarkdownTest(unittest.TestCase): def testLink(self): link_text = 'Devil home' link_target = ( - 'https://github.com/catapult-project/catapult/tree/master/devil') - expected = ( - '[Devil home]' - '(https://github.com/catapult-project/catapult/tree/master/devil)') + 'https://chromium.googlesource.com/catapult.git/+/HEAD/devil') + expected = ('[Devil home]' + '(https://chromium.googlesource.com/catapult.git/+/HEAD/devil)') self.assertEquals(expected, markdown.md_link(link_text, link_target)) def testLinkTextContainsBracket(self): diff --git a/catapult/devil/devil/utils/mock_calls.py b/catapult/devil/devil/utils/mock_calls.py index 2b359385..ba96658a 100644 --- a/catapult/devil/devil/utils/mock_calls.py +++ b/catapult/devil/devil/utils/mock_calls.py @@ -51,7 +51,7 @@ class TestCase(unittest.TestCase): (call.parent.name, call.parent) for call, _ in self._expected_calls) self._patched = [ test_case.patch_call(call, side_effect=do_check(call)) - for call in watched.itervalues() + for call in watched.values() ] def __enter__(self): diff --git a/catapult/devil/devil/utils/parallelizer_test.py b/catapult/devil/devil/utils/parallelizer_test.py index 7c86148c..2d8f72aa 100644 --- a/catapult/devil/devil/utils/parallelizer_test.py +++ b/catapult/devil/devil/utils/parallelizer_test.py @@ -19,6 +19,7 @@ if __name__ == '__main__': os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) from devil.utils import parallelizer +from devil.base_error import BaseError class ParallelizerTestObject(object): @@ -58,7 +59,7 @@ class ParallelizerTestObject(object): def _write_completion_file(self): if self._completion_file_name and len(self._completion_file_name): with open(self._completion_file_name, 'w+b') as completion_file: - completion_file.write('complete') + completion_file.write(b'complete') def __getitem__(self, index): return self._thing[index] @@ -87,13 +88,13 @@ class ParallelizerTest(unittest.TestCase): self.assertEquals(expected, r) def testMutate(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] self.assertTrue(all(d.doReturnTheThing() for d in devices)) ParallelizerTestObject.parallel(devices).doSetTheThing(False).pFinish(1) self.assertTrue(not any(d.doReturnTheThing() for d in devices)) def testAllReturn(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] results = ParallelizerTestObject.parallel(devices).doReturnTheThing().pGet( 1) self.assertTrue(isinstance(results, list)) @@ -103,7 +104,7 @@ class ParallelizerTest(unittest.TestCase): def testAllRaise(self): devices = [ ParallelizerTestObject(Exception('thing %d' % i)) - for i in xrange(0, 10) + for i in range(0, 10) ] p = ParallelizerTestObject.parallel(devices).doRaiseTheThing() with self.assertRaises(Exception): @@ -117,21 +118,21 @@ class ParallelizerTest(unittest.TestCase): try: completion_files = [ tempfile.NamedTemporaryFile(delete=False) - for _ in xrange(0, parallel_device_count) + for _ in range(0, parallel_device_count) ] devices = [ ParallelizerTestObject( - i if i != exception_index else Exception(exception_msg), + i if i != exception_index else BaseError(exception_msg), completion_files[i].name) - for i in xrange(0, parallel_device_count) + for i in range(0, parallel_device_count) ] for f in completion_files: f.close() p = ParallelizerTestObject.parallel(devices) - with self.assertRaises(Exception) as e: + with self.assertRaises(BaseError) as e: p.doRaiseIfExceptionElseSleepFor(2).pGet(3) self.assertTrue(exception_msg in str(e.exception)) - for i in xrange(0, parallel_device_count): + for i in range(0, parallel_device_count): with open(completion_files[i].name) as f: if i == exception_index: self.assertEquals('', f.read()) @@ -142,7 +143,7 @@ class ParallelizerTest(unittest.TestCase): os.remove(f.name) def testReusable(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] p = ParallelizerTestObject.parallel(devices) results = p.doReturn(True).pGet(1) self.assertTrue(all(results)) @@ -152,23 +153,23 @@ class ParallelizerTest(unittest.TestCase): results = p.doRaise(Exception('reusableTest')).pGet(1) def testContained(self): - devices = [ParallelizerTestObject(i) for i in xrange(0, 10)] + devices = [ParallelizerTestObject(i) for i in range(0, 10)] results = (ParallelizerTestObject.parallel(devices).helper. doReturnStringThing().pGet(1)) self.assertTrue(isinstance(results, list)) self.assertEquals(10, len(results)) - for i in xrange(0, 10): + for i in range(0, 10): self.assertEquals(str(i), results[i]) def testGetItem(self): - devices = [ParallelizerTestObject(range(i, i + 10)) for i in xrange(0, 10)] + devices = [ParallelizerTestObject(range(i, i + 10)) for i in range(0, 10)] results = ParallelizerTestObject.parallel(devices)[9].pGet(1) - self.assertEquals(range(9, 19), results) + self.assertEquals(list(range(9, 19)), results) class SyncParallelizerTest(unittest.TestCase): def testContextManager(self): - in_context = [False for i in xrange(10)] + in_context = [False for i in range(10)] @contextlib.contextmanager def enter_into_context(i): @@ -179,7 +180,7 @@ class SyncParallelizerTest(unittest.TestCase): in_context[i] = False parallelized_context = parallelizer.SyncParallelizer( - [enter_into_context(i) for i in xrange(10)]) + [enter_into_context(i) for i in range(10)]) with parallelized_context: self.assertTrue(all(in_context)) diff --git a/catapult/devil/devil/utils/reraiser_thread_unittest.py b/catapult/devil/devil/utils/reraiser_thread_unittest.py index eb37456c..f1fabd0f 100644 --- a/catapult/devil/devil/utils/reraiser_thread_unittest.py +++ b/catapult/devil/devil/utils/reraiser_thread_unittest.py @@ -64,7 +64,7 @@ class TestReraiserThreadGroup(unittest.TestCase): ran[i] = True group = reraiser_thread.ReraiserThreadGroup() - for i in xrange(5): + for i in range(5): group.Add(reraiser_thread.ReraiserThread(f, args=[i])) group.StartAll() group.JoinAll() @@ -76,7 +76,7 @@ class TestReraiserThreadGroup(unittest.TestCase): raise TestException group = reraiser_thread.ReraiserThreadGroup( - [reraiser_thread.ReraiserThread(f) for _ in xrange(5)]) + [reraiser_thread.ReraiserThread(f) for _ in range(5)]) group.StartAll() with self.assertRaises(TestException): group.JoinAll() diff --git a/catapult/devil/devil/utils/usb_hubs.py b/catapult/devil/devil/utils/usb_hubs.py index 313cf3f6..b83fb610 100644 --- a/catapult/devil/devil/utils/usb_hubs.py +++ b/catapult/devil/devil/utils/usb_hubs.py @@ -73,7 +73,7 @@ class HubType(object): A series of (int, USBNode) tuples giving a physical port and the Node connected to it. """ - for (virtual, physical) in mapping.iteritems(): + for (virtual, physical) in mapping.items(): if node.HasPort(virtual): if isinstance(physical, dict): for res in self._GppHelper(node.PortToDevice(virtual), physical): diff --git a/catapult/devil/docs/adb_wrapper.md b/catapult/devil/docs/adb_wrapper.md index 3ca8aa68..16a666fa 100644 --- a/catapult/devil/docs/adb_wrapper.md +++ b/catapult/devil/docs/adb_wrapper.md @@ -1,4 +1,4 @@ -# [devil.android.sdk.adb_wrapper](https://github.com/catapult-project/catapult/blob/master/devil/devil/android/sdk/adb_wrapper.py) +# [devil.android.sdk.adb_wrapper](https://chromium.googlesource.com/catapult.git/+/HEAD/devil/devil/android/sdk/adb_wrapper.py) *This page was autogenerated. Run `devil/bin/generate_md_docs` to update* diff --git a/catapult/devil/docs/device_utils.md b/catapult/devil/docs/device_utils.md index 960ca894..ad37d8eb 100644 --- a/catapult/devil/docs/device_utils.md +++ b/catapult/devil/docs/device_utils.md @@ -1,4 +1,4 @@ -# [devil.android.device_utils](https://github.com/catapult-project/catapult/blob/master/devil/devil/android/device_utils.py) +# [devil.android.device_utils](https://chromium.googlesource.com/catapult.git/+/HEAD/devil/devil/android/device_utils.py) *This page was autogenerated. Run `devil/bin/generate_md_docs` to update* @@ -467,11 +467,12 @@ Get the WebView update command sysdump on the device. WebViewPackages: Dict of installed WebView providers, mapping "package name" to "reason it's valid/invalid." - It may return an empty dictionary if device does not - support the "dumpsys webviewupdate" command. + The returned dictionary may not include all of the above keys: this depends + on the support of the platform's underlying WebViewUpdateService. This may + return an empty dictionary on OS versions which do not support querying the + WebViewUpdateService. Raises: - CommandFailedError on failure. CommandTimeoutError on timeout. DeviceUnreachableError on missing device. ``` @@ -593,6 +594,8 @@ Determines whether a particular package is installed on the device. ``` Args: package: Name of the package. + version_code: The version of the package to check for as an int, if + applicable. Only used for static shared libraries, otherwise ignored. Returns: True if the application is installed, False otherwise. @@ -866,6 +869,10 @@ Reads the contents of a file from the device. Reboot the device. ``` + Note if the device has the root privilege, it will likely lose it after the + reboot. When |block| is True, it will try to restore the root status if + applicable. + Args: block: A boolean indicating if we should wait for the reboot to complete. wifi: A boolean indicating if we should wait for wifi to be enabled after diff --git a/catapult/devil/docs/markdown.md b/catapult/devil/docs/markdown.md index b2fec500..605d4a93 100644 --- a/catapult/devil/docs/markdown.md +++ b/catapult/devil/docs/markdown.md @@ -1,4 +1,4 @@ -# [devil.utils.markdown](https://github.com/catapult-project/catapult/blob/master/devil/devil/utils/markdown.py) +# [devil.utils.markdown](https://chromium.googlesource.com/catapult.git/+/HEAD/devil/devil/utils/markdown.py) *This page was autogenerated. Run `devil/bin/generate_md_docs` to update* diff --git a/catapult/devil/docs/persistent_device_list.md b/catapult/devil/docs/persistent_device_list.md index 626a8788..513f640f 100644 --- a/catapult/devil/docs/persistent_device_list.md +++ b/catapult/devil/docs/persistent_device_list.md @@ -29,7 +29,7 @@ bots that upload data to the perf dashboard. ## Where it is used The persistent device list is used in the -[chromium_android](https://cs.chromium.org/chromium/build/scripts/slave/recipe_modules/chromium_android/api.py?q=known_devices_file) +[chromium_android](https://source.chromium.org/chromium/chromium/tools/build/+/HEAD:recipes/recipe_modules/chromium_android/api.py;l=50;drc=fd928820620dff8989e853accc54b1d61657f236) recipe module, and consumed by the -[device_status.py](https://cs.chromium.org/chromium/src/third_party/catapult/devil/devil/android/tools/device_status.py?q=\-\-known%5C-devices%5C-file) +[device_status.py](https://source.chromium.org/chromium/chromium/src/+/HEAD:third_party/catapult/devil/devil/android/tools/device_status.py;l=230;drc=1e5bef4469199e4daba5d8fd885966112f8a45d5) script among others. diff --git a/catapult/systrace/atrace_helper/jni/Android.mk b/catapult/systrace/atrace_helper/jni/Android.mk index b8f6acbb..9d5cc605 100644 --- a/catapult/systrace/atrace_helper/jni/Android.mk +++ b/catapult/systrace/atrace_helper/jni/Android.mk @@ -7,9 +7,6 @@ LOCAL_PATH := $(call my-dir) include $(CLEAR_VARS) LOCAL_MODULE := atrace_helper -LOCAL_LICENSE_KINDS := SPDX-license-identifier-BSD -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := $(LOCAL_PATH)/../../../../NOTICE LOCAL_CPPFLAGS := -std=c++11 -Wall -Wextra -Werror -O2 LOCAL_CPPFLAGS += -fPIE -fno-rtti -fno-exceptions -fstack-protector LOCAL_CPP_EXTENSION := .cc diff --git a/catapult/systrace/systrace/systrace_trace_viewer.html b/catapult/systrace/systrace/systrace_trace_viewer.html index 473261d8..11efa176 100644 --- a/catapult/systrace/systrace/systrace_trace_viewer.html +++ b/catapult/systrace/systrace/systrace_trace_viewer.html @@ -5855,7 +5855,7 @@ platformSpecificTotals[rawTotalName]=parseInt(rawTotalValue,16);} if(totals.peakResidentBytes===undefined&&totals.arePeakResidentBytesResettable!==undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Optional field peak_resident_set_bytes found'+' but is_peak_rss_resetable not found in'+' process memory dump for PID='+pid+' and dump ID='+dumpId+'.'});} if(totals.arePeakResidentBytesResettable!==undefined&&totals.peakResidentBytes===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Optional field is_peak_rss_resetable found'+' but peak_resident_set_bytes not found in'+' process memory dump for PID='+pid+' and dump ID='+dumpId+'.'});} processMemoryDump.totals=totals;},parseMemoryDumpVmRegions_(processMemoryDump,dumps,pid,dumpId){const rawProcessMmaps=dumps.process_mmaps;if(rawProcessMmaps===undefined)return;const rawVmRegions=rawProcessMmaps.vm_regions;if(rawVmRegions===undefined)return;if(processMemoryDump.vmRegions!==undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'VM regions provided multiple times for'+' process memory dump for PID='+pid+' and dump ID='+dumpId+'.'});return;} -const vmRegions=new Array(rawVmRegions.length);for(let i=0;i<rawVmRegions.length;i++){const rawVmRegion=rawVmRegions[i];const byteStats={};const rawByteStats=rawVmRegion.bs;for(const rawByteStatName in rawByteStats){const rawByteStatValue=rawByteStats[rawByteStatName];if(rawByteStatValue===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Byte stat \''+rawByteStatName+'\' of VM region '+ +if(rawVmRegions===null)return;const vmRegions=new Array(rawVmRegions.length);for(let i=0;i<rawVmRegions.length;i++){const rawVmRegion=rawVmRegions[i];const byteStats={};const rawByteStats=rawVmRegion.bs;for(const rawByteStatName in rawByteStats){const rawByteStatValue=rawByteStats[rawByteStatName];if(rawByteStatValue===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Byte stat \''+rawByteStatName+'\' of VM region '+ i+' ('+rawVmRegion.mf+') in process memory dump for '+'PID='+pid+' and dump ID='+dumpId+' does not have a value.'});continue;} const byteStatName=BYTE_STAT_NAME_MAP[rawByteStatName];if(byteStatName===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Unknown byte stat name \''+rawByteStatName+'\' ('+ rawByteStatValue+') of VM region '+i+' ('+ @@ -6495,7 +6495,9 @@ for(let i=0;i<importers.length;i++){const subtraces=importers[i].extractSubtrace if(traces.length&&!this.hasEventDataDecoder_(importers)){throw new Error('Could not find an importer for the provided eventData.');} importers.sort(function(x,y){return x.importPriority-y.importPriority;});});addStageForEachImporter('Importing clock sync markers',importer=>importer.importClockSyncMarkers());addStageForEachImporter('Importing',importer=>importer.importEvents());if(this.importOptions_.customizeModelCallback){addImportStage('Customizing',()=>{this.importOptions_.customizeModelCallback(this.model_);});} addStageForEachImporter('Importing sample data',importer=>importer.importSampleData());addImportStage('Autoclosing open slices...',()=>{this.model_.autoCloseOpenSlices();this.model_.createSubSlices();});addStageForEachImporter('Finalizing import',importer=>importer.finalizeImport());addImportStage('Initializing objects (step 1/2)...',()=>this.model_.preInitializeObjects());if(this.importOptions_.pruneEmptyContainers){addImportStage('Pruning empty containers...',()=>this.model_.pruneEmptyContainers());} -addImportStage('Merging kernel with userland...',()=>this.model_.mergeKernelWithUserland());let auditors=[];addImportStage('Adding arbitrary data to model...',()=>{auditors=this.importOptions_.auditorConstructors.map(auditorConstructor=>new auditorConstructor(this.model_));auditors.forEach((auditor)=>{auditor.runAnnotate();auditor.installUserFriendlyCategoryDriverIfNeeded();});});addImportStage('Computing final world bounds...',()=>{this.model_.computeWorldBounds(this.importOptions_.shiftWorldToZero);});addImportStage('Building flow event map...',()=>this.model_.buildFlowEventIntervalTree());addImportStage('Joining object refs...',()=>this.model_.joinRefs());addImportStage('Cleaning up undeleted objects...',()=>this.model_.cleanupUndeletedObjects());addImportStage('Sorting memory dumps...',()=>this.model_.sortMemoryDumps());addImportStage('Finalizing memory dump graphs...',()=>this.model_.finalizeMemoryGraphs());addImportStage('Initializing objects (step 2/2)...',()=>this.model_.initializeObjects());addImportStage('Building event indices...',()=>this.model_.buildEventIndices());addImportStage('Building UserModel...',()=>{const userModelBuilder=new tr.importer.UserModelBuilder(this.model_);userModelBuilder.buildUserModel();});addImportStage('Sorting user expectations...',()=>this.model_.userModel.sortExpectations());addImportStage('Running auditors...',()=>{auditors.forEach(auditor=>auditor.runAudit());});addImportStage('Updating alerts...',()=>this.model_.sortAlerts());addImportStage('Update bounds...',()=>this.model_.updateBounds());addImportStage('Looking for warnings...',()=>{if(!this.model_.isTimeHighResolution){this.model_.importWarning({type:'low_resolution_timer',message:'Trace time is low resolution, trace may be unusable.',showToUser:true});}});lastTask.after(()=>{this.importing_=false;this.model_.stats.traceImportDurationMs=tr.b.Timing.getCurrentTimeMs()-importStartTimeMs;});return importTask;},createImporter_(eventData){const importerConstructor=tr.importer.Importer.findImporterFor(eventData);if(!importerConstructor){throw new Error('Couldn\'t create an importer for the provided '+'eventData.');} +addImportStage('Merging kernel with userland...',()=>this.model_.mergeKernelWithUserland());let auditors=[];addImportStage('Adding arbitrary data to model...',()=>{for(const auditorConstructor of +this.importOptions_.auditorConstructors){try{auditors.push(new auditorConstructor(this.model_));}catch(e){console.error('Failed to construct an auditor:');console.error(e);}} +auditors.forEach((auditor)=>{try{auditor.runAnnotate();auditor.installUserFriendlyCategoryDriverIfNeeded();}catch(e){console.error('Failed to run an auditor:');console.error(e);}});});addImportStage('Computing final world bounds...',()=>{this.model_.computeWorldBounds(this.importOptions_.shiftWorldToZero);});addImportStage('Building flow event map...',()=>this.model_.buildFlowEventIntervalTree());addImportStage('Joining object refs...',()=>this.model_.joinRefs());addImportStage('Cleaning up undeleted objects...',()=>this.model_.cleanupUndeletedObjects());addImportStage('Sorting memory dumps...',()=>this.model_.sortMemoryDumps());addImportStage('Finalizing memory dump graphs...',()=>this.model_.finalizeMemoryGraphs());addImportStage('Initializing objects (step 2/2)...',()=>this.model_.initializeObjects());addImportStage('Building event indices...',()=>this.model_.buildEventIndices());addImportStage('Building UserModel...',()=>{try{const userModelBuilder=new tr.importer.UserModelBuilder(this.model_);userModelBuilder.buildUserModel();}catch(e){console.error('Failed to build user model');console.error(e);}});addImportStage('Sorting user expectations...',()=>this.model_.userModel.sortExpectations());addImportStage('Running auditors...',()=>{auditors.forEach(auditor=>auditor.runAudit());});addImportStage('Updating alerts...',()=>this.model_.sortAlerts());addImportStage('Update bounds...',()=>this.model_.updateBounds());addImportStage('Looking for warnings...',()=>{if(!this.model_.isTimeHighResolution){this.model_.importWarning({type:'low_resolution_timer',message:'Trace time is low resolution, trace may be unusable.',showToUser:true});}});lastTask.after(()=>{this.importing_=false;this.model_.stats.traceImportDurationMs=tr.b.Timing.getCurrentTimeMs()-importStartTimeMs;});return importTask;},createImporter_(eventData){const importerConstructor=tr.importer.Importer.findImporterFor(eventData);if(!importerConstructor){throw new Error('Couldn\'t create an importer for the provided '+'eventData.');} return new importerConstructor(this.model_,eventData);},hasEventDataDecoder_(importers){for(let i=0;i<importers.length;++i){if(!importers[i].isTraceDataContainer())return true;} return false;}};return{ImportOptions,Import,};});'use strict';tr.exportTo('tr.e.v8',function(){const ThreadSlice=tr.model.ThreadSlice;function V8GCStatsThreadSlice(){ThreadSlice.apply(this,arguments);this.liveObjects_=JSON.parse(this.args.live);delete this.args.live;this.deadObjects_=JSON.parse(this.args.dead);delete this.args.dead;} V8GCStatsThreadSlice.prototype={__proto__:ThreadSlice.prototype,get liveObjects(){return this.liveObjects_;},get deadObjects(){return this.deadObjects_;}};ThreadSlice.subTypes.register(V8GCStatsThreadSlice,{categoryParts:['disabled-by-default-v8.gc_stats'],name:'v8 gc stats slice',pluralName:'v8 gc stats slices'});return{V8GCStatsThreadSlice,};});'use strict';tr.exportTo('tr.e.v8',function(){const ThreadSlice=tr.model.ThreadSlice;function V8ICStatsThreadSlice(){ThreadSlice.apply(this,arguments);this.icStats_=undefined;if(this.args['ic-stats']){this.icStats_=this.args['ic-stats'].data;delete this.args['ic-stats'];}} @@ -8221,12 +8223,21 @@ const firstCounter=countBlinkObjects(dumps[0],pid);const lastCounter=countBlinkO return diffCounter;} function countBlinkObjects(dump,pid){const counter=new Map();const processesMemoryDumps=dump.processMemoryDumps;if(processesMemoryDumps[pid]===undefined)return counter;const blinkObjectsDump=processesMemoryDumps[pid].memoryAllocatorDumps.find(dump=>dump.fullName==='blink_objects');for(const v of blinkObjectsDump.children){counter.set(v.name,v.numerics.object_count.value);} return counter;} -return{leakDetectionMetric,};});'use strict';tr.exportTo('tr.metrics.console',function(){const COUNT_BOUNDARIES=tr.v.HistogramBinBoundaries.createExponential(1,1e4,30);const SUMMARY_OPTIONS=tr.v.Histogram.AVERAGE_ONLY_SUMMARY_OPTIONS;const SOURCES=['all','js','network'];function consoleErrorMetric(histograms,model){const counts={};for(const source of SOURCES){counts[source]={count:0,details:[]};} +return{leakDetectionMetric,};});'use strict';tr.exportTo('tr.metrics',function(){function blinkResourceMetric(histograms,model,opt_options){const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!chromeHelper){return;} +const CATEGORY='blink';const NAME='ResourceFetcher::requestResource';let count=0;for(const helper of Object.values(chromeHelper.rendererHelpers)){if(helper.isChromeTracingUI)continue;const events=tr.e.chrome.EventFinderUtils.getMainThreadEvents(helper,NAME,CATEGORY);for(const event of events){count++;}} +histograms.createHistogram('blinkRequestResourceCount',tr.b.Unit.byName.count,count);} +tr.metrics.MetricRegistry.register(blinkResourceMetric,{supportsRangeOfInterest:false,});return{blinkResourceMetric,};});'use strict';tr.exportTo('tr.metrics.console',function(){const COUNT_BOUNDARIES=tr.v.HistogramBinBoundaries.createExponential(1,1e4,30);const SUMMARY_OPTIONS=tr.v.Histogram.AVERAGE_ONLY_SUMMARY_OPTIONS;const SOURCES=['all','js','network'];function consoleErrorMetric(histograms,model){const counts={};for(const source of SOURCES){counts[source]={count:0,details:[]};} for(const slice of model.getDescendantEvents()){if(slice.category==='blink.console'&&slice.title==='ConsoleMessage::Error'){const source=slice.args.source.toLowerCase();counts.all.count++;if(slice.args.message){counts.all.details.push({pid:slice.getProcess().pid,...slice.args.message});} if(source in counts){counts[source].count++;if(slice.args.message){counts[source].details.push({pid:slice.getProcess().pid,...slice.args.message});}}} if(slice.category==='v8.console'&&(slice.title==='V8ConsoleMessage::Exception'||slice.title==='V8ConsoleMessage::Error'||slice.title==='V8ConsoleMessage::Assert')){counts.all.count++;counts.js.count++;}} for(const source of SOURCES){const hist=histograms.createHistogram(`console:error:${source}`,tr.b.Unit.byName.count_smallerIsBetter,{value:counts[source].count,diagnostics:{details:new tr.v.d.GenericSet(counts[source].details)}},{description:`Number of ${source} console error messages`,summaryOptions:SUMMARY_OPTIONS,});}} -tr.metrics.MetricRegistry.register(consoleErrorMetric);return{consoleErrorMetric,};});'use strict';tr.exportTo('tr.metrics.sh',function(){function getCpuSnapshotsFromModel(model){const snapshots=[];for(const pid in model.processes){const snapshotInstances=model.processes[pid].objects.getAllInstancesNamed('CPUSnapshots');if(!snapshotInstances)continue;for(const object of snapshotInstances[0].snapshots){snapshots.push(object.args.processes);}} +tr.metrics.MetricRegistry.register(consoleErrorMetric);return{consoleErrorMetric,};});'use strict';tr.exportTo('tr.metrics',function(){function countSumMetric(histograms,model,opt_options){const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!chromeHelper){return;} +const CATEGORY='benchmark';const NAME='count_sum';const counts=new Map();const sums=new Map();for(const pid in chromeHelper.rendererHelpers){const helper=chromeHelper.rendererHelpers[pid];if(helper.isChromeTracingUI)continue;const events=tr.e.chrome.EventFinderUtils.getMainThreadEvents(helper,NAME,CATEGORY);for(const event of events){const c=event.args.counter;if(!c){continue;} +if(!counts.get(c)){counts.set(c,0);} +counts.set(c,counts.get(c)+1);if(event.args.value){if(!sums.get(c)){sums.set(c,0);} +sums.set(c,sums.get(c)+event.args.value);}}} +counts.forEach((value,key)=>{histograms.createHistogram('count_'+key,tr.b.Unit.byName.count,value);});sums.forEach((value,key)=>{histograms.createHistogram('sum_'+key,tr.b.Unit.byName.unitlessNumber,value);});} +tr.metrics.MetricRegistry.register(countSumMetric,{supportsRangeOfInterest:false,});return{countSumMetric,};});'use strict';tr.exportTo('tr.metrics.sh',function(){function getCpuSnapshotsFromModel(model){const snapshots=[];for(const pid in model.processes){const snapshotInstances=model.processes[pid].objects.getAllInstancesNamed('CPUSnapshots');if(!snapshotInstances)continue;for(const object of snapshotInstances[0].snapshots){snapshots.push(object.args.processes);}} return snapshots;} function getProcessSumsFromSnapshot(snapshot){const processSums=new Map();for(const processData of snapshot){const processName=processData.name;if(!(processSums.has(processName))){processSums.set(processName,{sum:0.0,paths:new Set()});} processSums.get(processName).sum+=parseFloat(processData.pCpu);if(processData.path){processSums.get(processName).paths.add(processData.path);}} @@ -8291,11 +8302,10 @@ processVideoFreezing(freezing){if(this.freezing_===undefined||this.freezing_>fre addMetricToHistograms(histograms){this.addSample_(histograms,'time_to_video_play',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.timeToVideoPlay);this.addSample_(histograms,'time_to_audio_play',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.timeToAudioPlay);this.addSample_(histograms,'dropped_frame_count',tr.b.Unit.byName.count_smallerIsBetter,this.droppedFrameCount);for(const[key,value]of this.seekTimes.entries()){const keyString=key.toString().replace('.','_');this.addSample_(histograms,'pipeline_seek_time_'+keyString,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,value.pipelineSeekTime);this.addSample_(histograms,'seek_time_'+keyString,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,value.seekTime);} this.addSample_(histograms,'buffering_time',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.bufferingTime);this.addSample_(histograms,'roughness',tr.b.Unit.byName.count_smallerIsBetter,this.roughness);this.addSample_(histograms,'freezing',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.freezing);} addSample_(histograms,name,unit,sample){if(sample===undefined)return;const histogram=histograms.getHistogramNamed(name);if(histogram===undefined){histograms.createHistogram(name,unit,sample);}else{histogram.addSample(sample);}}} -tr.metrics.MetricRegistry.register(mediaMetric);return{mediaMetric,};});'use strict';tr.exportTo('tr.metrics',function(){function memoryAblationMetric(histograms,model){const modelHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!modelHelper.gpuHelper)return;const gpuProcess=modelHelper.gpuHelper.process;const events=[...gpuProcess.findTopmostSlicesNamed('Memory.GPU.PeakMemoryUsage.AblationTimes')];const allocHistogram=histograms.createHistogram('Ablation Alloc',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,[],{binBoundaries:tr.v.HistogramBinBoundaries.createLinear(0,10000,20),description:'The amount of time spent allocating the ablation '+'memory',summaryOptions:tr.metrics.rendering.SUMMARY_OPTIONS,});const deallocHistogram=histograms.createHistogram('Ablation Dealloc',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,[],{binBoundaries:tr.v.HistogramBinBoundaries.createLinear(0,10000,20),description:'The amount of time spent deallocating the ablation '+'memory',summaryOptions:tr.metrics.rendering.SUMMARY_OPTIONS,});for(let i=0;i<events.length;i++){allocHistogram.addSample(events[i].args.alloc);deallocHistogram.addSample(events[i].args.dealloc);}} -tr.metrics.MetricRegistry.register(memoryAblationMetric,{requiredCategories:['gpu.memory'],});return{memoryAblationMetric,};});'use strict';tr.exportTo('tr.metrics.pa',function(){function pcscanMetric(histograms,model){function createNumericForProcess(name,processName,desc){function createNumericForEventTime(name,desc){const n=new tr.v.Histogram(name,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter);n.description=desc;n.customizeSummaryOptions({avg:true,count:true,max:true,min:true,std:true,sum:true});return n;} -const scheme=['pa','pcscan',processName];if(name)scheme.push(name);return createNumericForEventTime(scheme.join(':'),desc);} -function createHistsForProcess(processName){return{scan:createNumericForProcess('scan',processName,'Time for scanning heap for quarantine pointers'),sweep:createNumericForProcess('sweep',processName,'Time for sweeping quarantine'),clear:createNumericForProcess('clear',processName,'Time for clearing quarantine entries'),total:createNumericForProcess('',processName,'Total time for PCScan execution')};} -function addSample(hists,slice){if(!(slice instanceof tr.model.ThreadSlice))return;if(slice.category!=='partition_alloc')return;if(slice.title==='PCScan.Scan'){hists.scan.addSample(slice.duration);}else if(slice.title==='PCScan.Sweep'){hists.sweep.addSample(slice.duration);}else if(slice.title==='PCScan.Clear'){hists.clear.addSample(slice.duration);}else if(slice.title==='PCScan'){hists.total.addSample(slice.duration);}} +tr.metrics.MetricRegistry.register(mediaMetric);return{mediaMetric,};});'use strict';tr.exportTo('tr.metrics.pa',function(){function pcscanMetric(histograms,model){function createNumericForProcess(name,processName,context,desc){function createNumericForEventTime(name,desc){const n=new tr.v.Histogram(name,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter);n.description=desc;n.customizeSummaryOptions({avg:true,count:true,max:true,min:true,std:true,sum:true});return n;} +const scheme=['pa','pcscan',processName,context];if(name)scheme.push(name);return createNumericForEventTime(scheme.join(':'),desc);} +function createHistsForProcess(processName){return{scanner_scan:createNumericForProcess('scan',processName,'scanner','Time for scanning heap for quarantine pointers on concurrent threads'),scanner_sweep:createNumericForProcess('sweep',processName,'scanner','Time for sweeping quarantine'),scanner_clear:createNumericForProcess('clear',processName,'scanner','Time for clearing quarantine entries'),scanner_total:createNumericForProcess('',processName,'scanner','Total time for PCScan execution on concurrent threads'),mutator_scan:createNumericForProcess('scan',processName,'mutator','Time for scanning heap for quarantine pointers on mutator threads'),mutator_clear:createNumericForProcess('clear',processName,'mutator','Time for clearing heap quarantine entries on mutator threads'),mutator_total:createNumericForProcess('',processName,'mutator','Total time for PCScan execution on mutator threads (inside safepoints)'),};} +function addSample(hists,slice){if(!(slice instanceof tr.model.ThreadSlice))return;if(slice.category!=='partition_alloc')return;if(slice.title==='PCScan.Scanner.Scan'){hists.scanner_scan.addSample(slice.duration);}else if(slice.title==='PCScan.Scanner.Sweep'){hists.scanner_sweep.addSample(slice.duration);}else if(slice.title==='PCScan.Scanner.Clear'){hists.scanner_clear.addSample(slice.duration);}else if(slice.title==='PCScan.Scanner'){hists.scanner_total.addSample(slice.duration);}else if(slice.title==='PCScan.Mutator.Scan'){hists.mutator_scan.addSample(slice.duration);}else if(slice.title==='PCScan.Mutator.Clear'){hists.mutator_clear.addSample(slice.duration);}else if(slice.title==='PCScan.Mutator'){hists.mutator_total.addSample(slice.duration);}} function addHistsForProcess(processHists,processHelpers){for(const helper of Object.values(processHelpers)){const processName=tr.e.chrome.chrome_processes.canonicalizeProcessName(helper.process.name);if(!processHists.has(processName)){processHists.set(processName,createHistsForProcess(processName));} for(const slice of helper.process.getDescendantEvents()){addSample(processHists.get(processName),slice);}}} const helper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);const processHists=new Map();addHistsForProcess(processHists,helper.browserHelpers);addHistsForProcess(processHists,helper.rendererHelpers);for(const hists of processHists.values()){for(const hist of Object.values(hists)){histograms.addHistogram(hist);}}} @@ -8477,7 +8487,9 @@ function collectTimeToEvent(rendererHelper,timeToXEntries){const samples=[];for( return samples;} function collectTimeToEventInCpuTime(rendererHelper,timeToXEntries){const samples=[];for(const{targetEvent,navigationStartEvent,url}of timeToXEntries){const navStartToEventRange=tr.b.math.Range.fromExplicitRange(navigationStartEvent.start,targetEvent.start);const mainThreadCpuTime=rendererHelper.mainThread.getCpuTimeForRange(navStartToEventRange);const breakdownTree=tr.metrics.sh.generateCpuTimeBreakdownTree(rendererHelper.mainThread,navStartToEventRange);samples.push({value:mainThreadCpuTime,breakdownTree,diagnostics:{breakdown:createBreakdownDiagnostic(breakdownTree),start:new RelatedEventSet(navigationStartEvent),end:new RelatedEventSet(targetEvent),infos:new tr.v.d.GenericSet([{pid:rendererHelper.pid,start:navigationStartEvent.start,event:targetEvent.start,}]),}});} return samples;} -function findLayoutShiftSamples(rendererHelper){let sample;EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'LayoutShift','loading').forEach((events)=>{const evData=events.pop().args.data;if(evData.is_main_frame){sample={value:evData.cumulative_score};}});return sample?[sample]:[];} +function findMainFrameLayoutShiftSamples(rendererHelper){let sample;EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'LayoutShift','loading').forEach((events)=>{const evData=events.pop().args.data;if(evData.is_main_frame){sample={value:evData.cumulative_score};}});return sample?[sample]:[];} +function findAllLayoutShiftSamples(chromeHelper){let total=0;let foundMainFrame=false;for(const pid in chromeHelper.rendererHelpers){const rendererHelper=chromeHelper.rendererHelpers[pid];if(rendererHelper.isChromeTracingUI)continue;tr.e.chrome.EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'LayoutShift','loading').forEach((events)=>{for(const event of events){const evData=event.args.data;if(evData.is_main_frame){total+=evData.score;foundMainFrame=true;}else{total+=evData.weighted_score_delta;}}});} +return foundMainFrame?[{value:total}]:[];} function addFirstMeaningfulPaintSample(samples,rendererHelper,navigationStart,fmpMarkerEvent,url){const navStartToFMPRange=tr.b.math.Range.fromExplicitRange(navigationStart.start,fmpMarkerEvent.start);const networkEvents=EventFinderUtils.getNetworkEventsInRange(rendererHelper.process,navStartToFMPRange);const timeToFirstMeaningfulPaint=navStartToFMPRange.duration;const breakdownTree=tr.metrics.sh.generateWallClockTimeBreakdownTree(rendererHelper.mainThread,networkEvents,navStartToFMPRange);samples.push({value:timeToFirstMeaningfulPaint,breakdownTree,diagnostics:{breakdown:createBreakdownDiagnostic(breakdownTree),start:new RelatedEventSet(navigationStart),end:new RelatedEventSet(fmpMarkerEvent),infos:new tr.v.d.GenericSet([{url,pid:rendererHelper.pid,start:navigationStart.start,fmp:fmpMarkerEvent.start,}]),}});} function addFirstMeaningfulPaintCpuTimeSample(samples,rendererHelper,navigationStart,fmpMarkerEvent,url){const navStartToFMPRange=tr.b.math.Range.fromExplicitRange(navigationStart.start,fmpMarkerEvent.start);const mainThreadCpuTime=rendererHelper.mainThread.getCpuTimeForRange(navStartToFMPRange);const breakdownTree=tr.metrics.sh.generateCpuTimeBreakdownTree(rendererHelper.mainThread,navStartToFMPRange);samples.push({value:mainThreadCpuTime,breakdownTree,diagnostics:{breakdown:createBreakdownDiagnostic(breakdownTree),start:new RelatedEventSet(navigationStart),end:new RelatedEventSet(fmpMarkerEvent),infos:new tr.v.d.GenericSet([{url,pid:rendererHelper.pid,start:navigationStart.start,fmp:fmpMarkerEvent.start,}]),}});} function decorateInteractivitySampleWithDiagnostics_(rendererHelper,eventTimestamp,navigationStartEvent,firstContentfulPaintTime,domContentLoadedEndTime,url){if(eventTimestamp===undefined)return undefined;const navigationStartTime=navigationStartEvent.start;const navStartToEventTimeRange=tr.b.math.Range.fromExplicitRange(navigationStartTime,eventTimestamp);const networkEvents=EventFinderUtils.getNetworkEventsInRange(rendererHelper.process,navStartToEventTimeRange);const breakdownTree=tr.metrics.sh.generateWallClockTimeBreakdownTree(rendererHelper.mainThread,networkEvents,navStartToEventTimeRange);const breakdownDiagnostic=createBreakdownDiagnostic(breakdownTree);return{value:navStartToEventTimeRange.duration,diagnostics:tr.v.d.DiagnosticMap.fromObject({'Start':new RelatedEventSet(navigationStartEvent),'Navigation infos':new tr.v.d.GenericSet([{url,pid:rendererHelper.pid,navigationStartTime,firstContentfulPaintTime,domContentLoadedEndTime,eventTimestamp,}]),'Breakdown of [navStart, eventTimestamp]':breakdownDiagnostic,}),};} @@ -8490,7 +8502,7 @@ return lastCandidates;} function findLargestTextPaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents){const timeToPaintEntries=findTimeToXEntries('loading','LargestTextPaint::Candidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const timeToPaintBlockingEntries=findTimeToXEntries('loading','LargestTextPaint::NoCandidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const lastCandidateEvents=findLastCandidateForEachNavigation(timeToPaintEntries.concat(timeToPaintBlockingEntries)).filter(event=>event.targetEvent.title!=='LargestTextPaint::NoCandidate');return collectTimeToEvent(rendererHelper,lastCandidateEvents);} function findLargestImagePaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents){const timeToPaintEntries=findTimeToXEntries('loading','LargestImagePaint::Candidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const timeToPaintBlockingEntries=findTimeToXEntries('loading','LargestImagePaint::NoCandidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const lastCandidateEvents=findLastCandidateForEachNavigation(timeToPaintEntries.concat(timeToPaintBlockingEntries)).filter(event=>event.targetEvent.title!=='LargestImagePaint::NoCandidate');return collectTimeToEvent(rendererHelper,lastCandidateEvents);} function findLargestContentfulPaintHistogramSamples(allBrowserEvents){const lcp=new tr.e.chrome.LargestContentfulPaint(allBrowserEvents);const lcpSamples=lcp.findCandidates().map(candidate=>{const{durationInMilliseconds,size,type,inMainFrame,mainFrameTreeNodeId}=candidate;return{value:durationInMilliseconds,diagnostics:{size:new tr.v.d.GenericSet([size]),type:new tr.v.d.GenericSet([type]),inMainFrame:new tr.v.d.GenericSet([inMainFrame]),mainFrameTreeNodeId:new tr.v.d.GenericSet([mainFrameTreeNodeId]),},};});return lcpSamples;} -function collectLoadingMetricsForRenderer(rendererHelper){const frameToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'navigationStart','blink.user_timing');const navIdToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByNavId(rendererHelper,'navigationStart','blink.user_timing');const firstPaintSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('loading','firstPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const timeToFCPEntries=findTimeToXEntries('loading','firstContentfulPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const firstContentfulPaintSamples=collectTimeToEvent(rendererHelper,timeToFCPEntries);const firstContentfulPaintCpuTimeSamples=collectTimeToEventInCpuTime(rendererHelper,timeToFCPEntries);const onLoadSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('blink.user_timing','loadEventStart',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const aboveTheFoldLoadedToVisibleSamples=getAboveTheFoldLoadedToVisibleSamples(rendererHelper);const firstViewportReadySamples=getFirstViewportReadySamples(rendererHelper,navIdToNavStartEvents);const largestImagePaintSamples=findLargestImagePaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const largestTextPaintSamples=findLargestTextPaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const layoutShiftSamples=findLayoutShiftSamples(rendererHelper);const navigationStartSamples=timeToFCPEntries.map(entry=>{return{value:entry.navigationStartEvent.start};});return{frameToNavStartEvents,firstPaintSamples,firstContentfulPaintSamples,firstContentfulPaintCpuTimeSamples,onLoadSamples,aboveTheFoldLoadedToVisibleSamples,firstViewportReadySamples,largestImagePaintSamples,largestTextPaintSamples,layoutShiftSamples,navigationStartSamples,};} +function collectLoadingMetricsForRenderer(rendererHelper){const frameToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'navigationStart','blink.user_timing');const navIdToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByNavId(rendererHelper,'navigationStart','blink.user_timing');const firstPaintSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('loading','firstPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const timeToFCPEntries=findTimeToXEntries('loading','firstContentfulPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const firstContentfulPaintSamples=collectTimeToEvent(rendererHelper,timeToFCPEntries);const firstContentfulPaintCpuTimeSamples=collectTimeToEventInCpuTime(rendererHelper,timeToFCPEntries);const onLoadSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('blink.user_timing','loadEventStart',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const aboveTheFoldLoadedToVisibleSamples=getAboveTheFoldLoadedToVisibleSamples(rendererHelper);const firstViewportReadySamples=getFirstViewportReadySamples(rendererHelper,navIdToNavStartEvents);const largestImagePaintSamples=findLargestImagePaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const largestTextPaintSamples=findLargestTextPaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const mainFrameLayoutShiftSamples=findMainFrameLayoutShiftSamples(rendererHelper);const navigationStartSamples=timeToFCPEntries.map(entry=>{return{value:entry.navigationStartEvent.start};});return{frameToNavStartEvents,firstPaintSamples,firstContentfulPaintSamples,firstContentfulPaintCpuTimeSamples,onLoadSamples,aboveTheFoldLoadedToVisibleSamples,firstViewportReadySamples,largestImagePaintSamples,largestTextPaintSamples,mainFrameLayoutShiftSamples,navigationStartSamples,};} function collectMetricsFromLoadExpectations(model,chromeHelper){const interactiveSamples=[];const firstCpuIdleSamples=[];const firstMeaningfulPaintSamples=[];const firstMeaningfulPaintCpuTimeSamples=[];const totalBlockingTimeSamples=[];for(const expectation of model.userModel.expectations){if(!(expectation instanceof tr.model.um.LoadExpectation))continue;if(tr.e.chrome.CHROME_INTERNAL_URLS.includes(expectation.url)){continue;} const rendererHelper=chromeHelper.rendererHelpers[expectation.renderProcess.pid];if(expectation.fmpEvent!==undefined){addFirstMeaningfulPaintSample(firstMeaningfulPaintSamples,rendererHelper,expectation.navigationStart,expectation.fmpEvent,expectation.url);addFirstMeaningfulPaintCpuTimeSample(firstMeaningfulPaintCpuTimeSamples,rendererHelper,expectation.navigationStart,expectation.fmpEvent,expectation.url);} if(expectation.firstCpuIdleTime!==undefined){firstCpuIdleSamples.push(decorateInteractivitySampleWithDiagnostics_(rendererHelper,expectation.firstCpuIdleTime,expectation.navigationStart,expectation.fcpEvent.start,expectation.domContentLoadedEndEvent.start,expectation.url));} @@ -8500,7 +8512,7 @@ return{firstMeaningfulPaintSamples,firstMeaningfulPaintCpuTimeSamples,firstCpuId function addSamplesToHistogram(samples,histogram,histograms){for(const sample of samples){histogram.addSample(sample.value,sample.diagnostics);if(histogram.name!=='timeToFirstContentfulPaint')continue;if(!sample.breakdownTree)continue;for(const[category,breakdown]of Object.entries(sample.breakdownTree)){const relatedName=`${histogram.name}:${category}`;let relatedHist=histograms.getHistogramsNamed(relatedName)[0];if(!relatedHist){relatedHist=histograms.createHistogram(relatedName,histogram.unit,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,summaryOptions:{count:false,max:false,min:false,sum:false,},});let relatedNames=histogram.diagnostics.get('breakdown');if(!relatedNames){relatedNames=new tr.v.d.RelatedNameMap();histogram.diagnostics.set('breakdown',relatedNames);} relatedNames.set(category,relatedName);} relatedHist.addSample(breakdown.total,{breakdown:tr.v.d.Breakdown.fromEntries(Object.entries(breakdown.events)),});}}} -function loadingMetric(histograms,model){const firstPaintHistogram=histograms.createHistogram('timeToFirstPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintHistogram=histograms.createHistogram('timeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const onLoadHistogram=histograms.createHistogram('timeToOnload',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to onload. '+'This is temporary metric used for PCv1/v2 sanity checking',summaryOptions:SUMMARY_OPTIONS,});const firstMeaningfulPaintHistogram=histograms.createHistogram('timeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstMeaningfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const timeToInteractiveHistogram=histograms.createHistogram('timeToInteractive',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to Interactive',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const totalBlockingTimeHistogram=histograms.createHistogram('totalBlockingTime',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Total Blocking Time',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const timeToFirstCpuIdleHistogram=histograms.createHistogram('timeToFirstCpuIdle',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to First CPU Idle',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const aboveTheFoldLoadedToVisibleHistogram=histograms.createHistogram('aboveTheFoldLoadedToVisible',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from first visible to load for AMP pages only.',summaryOptions:SUMMARY_OPTIONS,});const firstViewportReadyHistogram=histograms.createHistogram('timeToFirstViewportReady',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from navigation to load for AMP pages only. ',summaryOptions:SUMMARY_OPTIONS,});const largestImagePaintHistogram=histograms.createHistogram('largestImagePaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Image Paint',summaryOptions:SUMMARY_OPTIONS,});const largestTextPaintHistogram=histograms.createHistogram('largestTextPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Text Paint',summaryOptions:SUMMARY_OPTIONS,});const largestContentfulPaintHistogram=histograms.createHistogram('largestContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Contentful Paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const layoutShiftHistogram=histograms.createHistogram('mainFrameCumulativeLayoutShift',unitlessNumber_smallerIsBetter,[],{binBoundaries:LAYOUT_SHIFT_SCORE_BOUNDARIES,description:'Main Frame Document Cumulative Layout Shift Score',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_LAYOUT],});const navigationStartHistogram=histograms.createHistogram('navigationStart',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'navigationStart',summaryOptions:SUMMARY_OPTIONS,});tr.metrics.sh.rectsBasedSpeedIndexMetric(histograms,model);const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);for(const pid in chromeHelper.rendererHelpers){const rendererHelper=chromeHelper.rendererHelpers[pid];if(rendererHelper.isChromeTracingUI)continue;const samplesSet=collectLoadingMetricsForRenderer(rendererHelper);const lcpSamples=findLargestContentfulPaintHistogramSamples(chromeHelper.browserHelper.mainThread.sliceGroup.slices);addSamplesToHistogram(lcpSamples,largestContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstPaintSamples,firstPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintSamples,firstContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintCpuTimeSamples,firstContentfulPaintCpuTimeHistogram,histograms);addSamplesToHistogram(samplesSet.onLoadSamples,onLoadHistogram,histograms);addSamplesToHistogram(samplesSet.aboveTheFoldLoadedToVisibleSamples,aboveTheFoldLoadedToVisibleHistogram,histograms);addSamplesToHistogram(samplesSet.firstViewportReadySamples,firstViewportReadyHistogram,histograms);addSamplesToHistogram(samplesSet.largestImagePaintSamples,largestImagePaintHistogram,histograms);addSamplesToHistogram(samplesSet.largestTextPaintSamples,largestTextPaintHistogram,histograms);addSamplesToHistogram(samplesSet.layoutShiftSamples,layoutShiftHistogram,histograms);addSamplesToHistogram(samplesSet.navigationStartSamples,navigationStartHistogram,histograms);} +function loadingMetric(histograms,model){const firstPaintHistogram=histograms.createHistogram('timeToFirstPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintHistogram=histograms.createHistogram('timeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const onLoadHistogram=histograms.createHistogram('timeToOnload',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to onload. '+'This is temporary metric used for PCv1/v2 sanity checking',summaryOptions:SUMMARY_OPTIONS,});const firstMeaningfulPaintHistogram=histograms.createHistogram('timeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstMeaningfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const timeToInteractiveHistogram=histograms.createHistogram('timeToInteractive',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to Interactive',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const totalBlockingTimeHistogram=histograms.createHistogram('totalBlockingTime',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Total Blocking Time',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const timeToFirstCpuIdleHistogram=histograms.createHistogram('timeToFirstCpuIdle',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to First CPU Idle',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const aboveTheFoldLoadedToVisibleHistogram=histograms.createHistogram('aboveTheFoldLoadedToVisible',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from first visible to load for AMP pages only.',summaryOptions:SUMMARY_OPTIONS,});const firstViewportReadyHistogram=histograms.createHistogram('timeToFirstViewportReady',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from navigation to load for AMP pages only. ',summaryOptions:SUMMARY_OPTIONS,});const largestImagePaintHistogram=histograms.createHistogram('largestImagePaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Image Paint',summaryOptions:SUMMARY_OPTIONS,});const largestTextPaintHistogram=histograms.createHistogram('largestTextPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Text Paint',summaryOptions:SUMMARY_OPTIONS,});const largestContentfulPaintHistogram=histograms.createHistogram('largestContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Contentful Paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const mainFrameLayoutShiftHistogram=histograms.createHistogram('mainFrameCumulativeLayoutShift',unitlessNumber_smallerIsBetter,[],{binBoundaries:LAYOUT_SHIFT_SCORE_BOUNDARIES,description:'Main Frame Document Cumulative Layout Shift Score',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_LAYOUT],});const allLayoutShiftHistogram=histograms.createHistogram('overallCumulativeLayoutShift',unitlessNumber_smallerIsBetter,[],{binBoundaries:LAYOUT_SHIFT_SCORE_BOUNDARIES,description:'Document Cumulative Layout Shift Score with iframes',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_LAYOUT],});const navigationStartHistogram=histograms.createHistogram('navigationStart',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'navigationStart',summaryOptions:SUMMARY_OPTIONS,});tr.metrics.sh.rectsBasedSpeedIndexMetric(histograms,model);const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);const allLayoutShiftSamples=findAllLayoutShiftSamples(chromeHelper);addSamplesToHistogram(allLayoutShiftSamples,allLayoutShiftHistogram,histograms);for(const pid in chromeHelper.rendererHelpers){const rendererHelper=chromeHelper.rendererHelpers[pid];if(rendererHelper.isChromeTracingUI)continue;const samplesSet=collectLoadingMetricsForRenderer(rendererHelper);const lcpSamples=findLargestContentfulPaintHistogramSamples(chromeHelper.browserHelper.mainThread.sliceGroup.slices);addSamplesToHistogram(lcpSamples,largestContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstPaintSamples,firstPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintSamples,firstContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintCpuTimeSamples,firstContentfulPaintCpuTimeHistogram,histograms);addSamplesToHistogram(samplesSet.onLoadSamples,onLoadHistogram,histograms);addSamplesToHistogram(samplesSet.aboveTheFoldLoadedToVisibleSamples,aboveTheFoldLoadedToVisibleHistogram,histograms);addSamplesToHistogram(samplesSet.firstViewportReadySamples,firstViewportReadyHistogram,histograms);addSamplesToHistogram(samplesSet.largestImagePaintSamples,largestImagePaintHistogram,histograms);addSamplesToHistogram(samplesSet.largestTextPaintSamples,largestTextPaintHistogram,histograms);addSamplesToHistogram(samplesSet.mainFrameLayoutShiftSamples,mainFrameLayoutShiftHistogram,histograms);addSamplesToHistogram(samplesSet.navigationStartSamples,navigationStartHistogram,histograms);} const samplesSet=collectMetricsFromLoadExpectations(model,chromeHelper);addSamplesToHistogram(samplesSet.firstMeaningfulPaintSamples,firstMeaningfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstMeaningfulPaintCpuTimeSamples,firstMeaningfulPaintCpuTimeHistogram,histograms);addSamplesToHistogram(samplesSet.interactiveSamples,timeToInteractiveHistogram,histograms);addSamplesToHistogram(samplesSet.firstCpuIdleSamples,timeToFirstCpuIdleHistogram,histograms);addSamplesToHistogram(samplesSet.totalBlockingTimeSamples,totalBlockingTimeHistogram,histograms);} tr.metrics.MetricRegistry.register(loadingMetric);return{loadingMetric,createBreakdownDiagnostic};});'use strict';tr.exportTo('tr.metrics',function(){const SPA_NAVIGATION_START_TO_FIRST_PAINT_DURATION_BIN_BOUNDARY=tr.v.HistogramBinBoundaries.createExponential(1,1000,50);function spaNavigationMetric(histograms,model){const histogram=new tr.v.Histogram('spaNavigationStartToFpDuration',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,SPA_NAVIGATION_START_TO_FIRST_PAINT_DURATION_BIN_BOUNDARY);histogram.description='Latency between the input event causing'+' a SPA navigation and the first paint event after it';histogram.customizeSummaryOptions({count:false,sum:false,});const modelHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!modelHelper){return;} const rendererHelpers=modelHelper.rendererHelpers;if(!rendererHelpers){return;} @@ -8987,6 +8999,7 @@ return tr.v.HistogramBinBoundaries.createExponential(1e-3,1e3,50);} function umaMetric(histograms,model){const histogramValues=new Map();const nameCounts=new Map();for(const process of model.getAllProcesses()){const histogramEvents=new Map();for(const event of process.instantEvents){if(event.title!=='UMAHistogramSamples')continue;const name=event.args.name;const events=histogramEvents.get(name)||[];if(!histogramEvents.has(name))histogramEvents.set(name,events);events.push(event);} let processName=tr.e.chrome.chrome_processes.canonicalizeProcessName(process.name);nameCounts.set(processName,(nameCounts.get(processName)||0)+1);processName=`${processName}_${nameCounts.get(processName)}`;for(const[name,events]of histogramEvents){const values=histogramValues.get(name)||{sum:0,bins:[]};if(!histogramValues.has(name))histogramValues.set(name,values);const endValues=parseBuckets_(events[events.length-1],processName);if(events.length===1){mergeBins_(values,endValues,name);}else{throw new Error('There should be at most one snapshot of UMA '+`histogram for ${name} in each process.`);}}} for(const[name,values]of histogramValues){const histogram=new tr.v.Histogram(name,getHistogramUnit_(name),getHistogramBoundaries_(name));const isLinear=getIsHistogramBinsLinear_(name);let sumOfMiddles=0;let sumOfBinLengths=0;for(const bin of values.bins){sumOfMiddles+=bin.count*(bin.min+bin.max)/2;sumOfBinLengths+=bin.count*(bin.max-bin.min);} +if(name.startsWith('CompositorLatency.Type')){let histogramBoundaries=tr.v.HistogramBinBoundaries.createLinear(0,100,101);let histogramUnit=getHistogramUnit_(name);let presentedCount=values.bins[0]?values.bins[0].count:0;let delayedCount=values.bins[1]?values.bins[1].count:0;let droppedCount=values.bins[2]?values.bins[2].count:0;let inTimeCount=presentedCount-delayedCount;let totalCount=presentedCount+droppedCount;const inTimeHistogram=new tr.v.Histogram(name+'.Percentage_of_in_time_frames',histogramUnit,histogramBoundaries);inTimeHistogram.addSample(100.0*inTimeCount/totalCount);histograms.addHistogram(inTimeHistogram);const delayedHistogram=new tr.v.Histogram(name+'.Percentage_of_delayed_frames',histogramUnit,histogramBoundaries);delayedHistogram.addSample(100.0*delayedCount/totalCount);histograms.addHistogram(delayedHistogram);const droppedHistogram=new tr.v.Histogram(name+'.Percentage_of_dropped_frames',histogramUnit,histogramBoundaries);droppedHistogram.addSample(100.0*droppedCount/totalCount);histograms.addHistogram(droppedHistogram);} const shift=(values.sum-sumOfMiddles)/sumOfBinLengths;if(isLinear&&Math.abs(shift)>0.5){throw new Error(`Samples sum is wrong for ${name}.`);} for(const bin of values.bins){if(bin.count===0)continue;const shiftedValue=(bin.min+bin.max)/2+shift*(bin.max-bin.min);for(const[processName,count]of bin.processes){bin.processes.set(processName,shiftedValue*count/bin.count);} for(let i=0;i<bin.count;i++){histogram.addSample(shiftedValue,{processes:bin.processes,events:bin.events});}} @@ -9008,7 +9021,7 @@ histograms.addHistogram(cpuTotalOptimizeCode);histograms.addHistogram(wallTotalO function computeDeoptimizeCodeMetrics(histograms,model){const cpuTotalDeoptimizeCode=new tr.v.Histogram('v8_deoptimize_code_cpu_total',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);cpuTotalDeoptimizeCode.description='cpu total time spent in code deoptimization';const wallTotalDeoptimizeCode=new tr.v.Histogram('v8_deoptimize_code_wall_total',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);wallTotalDeoptimizeCode.description='wall total time spent in code deoptimization';for(const e of model.findTopmostSlicesNamed('V8.DeoptimizeCode')){cpuTotalDeoptimizeCode.addSample(e.cpuDuration);wallTotalDeoptimizeCode.addSample(e.duration);} histograms.addHistogram(cpuTotalDeoptimizeCode);histograms.addHistogram(wallTotalDeoptimizeCode);} function executionMetric(histograms,model){computeExecuteMetrics(histograms,model);computeParseLazyMetrics(histograms,model);computeCompileIgnitionMetrics(histograms,model);computeCompileFullCodeMetrics(histograms,model);computeRecompileMetrics(histograms,model);computeOptimizeCodeMetrics(histograms,model);computeDeoptimizeCodeMetrics(histograms,model);} -tr.metrics.MetricRegistry.register(executionMetric);return{executionMetric,};});'use strict';tr.exportTo('tr.metrics.v8',function(){const TARGET_FPS=60;const MS_PER_SECOND=1000;const WINDOW_SIZE_MS=MS_PER_SECOND/TARGET_FPS;const EPSILON=1e-6;const METRICS=['v8:gc:cycle:full','v8:gc:cycle:full:cpp','v8:gc:cycle:full:mark','v8:gc:cycle:full:mark:cpp','v8:gc:cycle:full:weak','v8:gc:cycle:full:weak:cpp','v8:gc:cycle:full:sweep','v8:gc:cycle:full:sweep:cpp','v8:gc:cycle:full:compact','v8:gc:cycle:full:compact:cpp','v8:gc:cycle:main_thread:full','v8:gc:cycle:main_thread:full:cpp','v8:gc:cycle:main_thread:full:mark','v8:gc:cycle:main_thread:full:mark:cpp','v8:gc:cycle:main_thread:full:weak','v8:gc:cycle:main_thread:full:weak:cpp','v8:gc:cycle:main_thread:full:sweep','v8:gc:cycle:main_thread:full:sweep:cpp','v8:gc:cycle:main_thread:full:compact','v8:gc:cycle:main_thread:full:compact:cpp','v8:gc:event:main_thread:full:atomic','v8:gc:event:main_thread:full:atomic:cpp','v8:gc:event:main_thread:full:atomic:mark','v8:gc:event:main_thread:full:atomic:mark:cpp','v8:gc:event:main_thread:full:atomic:weak','v8:gc:event:main_thread:full:atomic:weak:cpp','v8:gc:event:main_thread:full:atomic:sweep','v8:gc:event:main_thread:full:atomic:sweep:cpp','v8:gc:event:main_thread:full:atomic:compact','v8:gc:event:main_thread:full:atomic:compact:cpp','v8:gc:event:main_thread:full:incremental','v8:gc:event:main_thread:full:incremental:cpp','v8:gc:event:main_thread:full:incremental:mark','v8:gc:event:main_thread:full:incremental:mark:cpp','v8:gc:event:main_thread:full:incremental:sweep','v8:gc:event:main_thread:full:incremental:sweep:cpp','v8:gc:cycle:young','v8:gc:cycle:main_thread:young',];const V8_FULL_ATOMIC_EVENTS=['V8.GCCompactor','V8.GCFinalizeMC','V8.GCFinalizeMCReduceMemory',];const V8_FULL_MARK_EVENTS=['V8.GC_MC_BACKGROUND_MARKING','V8.GC_MC_MARK','V8.GCIncrementalMarking','V8.GCIncrementalMarkingFinalize','V8.GCIncrementalMarkingStart',];const V8_FULL_COMPACT_EVENTS=['V8.GC_MC_BACKGROUND_EVACUATE_COPY','V8.GC_MC_BACKGROUND_EVACUATE_UPDATE_POINTERS','V8.GC_MC_EVACUATE',];const V8_FULL_SWEEP_EVENTS=['V8.GC_MC_BACKGROUND_SWEEPING','V8.GC_MC_SWEEP',];const V8_FULL_WEAK_EVENTS=['V8.GC_MC_CLEAR',];const V8_YOUNG_EVENTS=['V8.GC_SCAVENGER_BACKGROUND_SCAVENGE_PARALLEL','V8.GCScavenger',];const CPP_GC_FULL_MARK_EVENTS=['BlinkGC.AtomicPauseMarkEpilogue','BlinkGC.AtomicPauseMarkPrologue','BlinkGC.AtomicPauseMarkRoots','BlinkGC.AtomicPauseMarkTransitiveClosure','BlinkGC.ConcurrentMarkingStep','BlinkGC.IncrementalMarkingStartMarking','BlinkGC.IncrementalMarkingStep','BlinkGC.MarkBailOutObjects','BlinkGC.MarkFlushEphemeronPairs','BlinkGC.MarkFlushV8References','BlinkGC.UnifiedMarkingStep','CppGC.AtomicMark','CppGC.IncrementalMark','CppGC.ConcurrentMark',];const CPP_GC_FULL_COMPACT_EVENTS=['BlinkGC.AtomicPauseSweepAndCompact','CppGC.AtomicCompact',];const CPP_GC_FULL_SWEEP_EVENTS=['BlinkGC.CompleteSweep','BlinkGC.ConcurrentSweepingStep','BlinkGC.LazySweepInIdle','BlinkGC.LazySweepOnAllocation','CppGC.AtomicSweep','CppGC.IncrementalSweep','CppGC.ConcurrentSweep',];const CPP_GC_FULL_WEAK_EVENTS=['BlinkGC.MarkWeakProcessing','CppGC.AtomicWeak',];const RULES=[{events:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic',},{events:V8_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark',},{events:CPP_GC_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark:cpp',},{events:V8_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark',},{events:CPP_GC_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark:cpp',},{events:V8_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact',},{events:CPP_GC_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact:cpp',},{events:V8_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep:cpp',},{events:V8_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak',},{events:CPP_GC_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak:cpp',},{events:V8_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep:cpp',},{events:V8_YOUNG_EVENTS,contribute_to:'young:atomic',},];const Granularity={CYCLE:'cycle',EVENT:'event',};const ThreadType={MAIN:'main',BACKGROUND:'background',ALL_THREADS:'all_threads',};class Metric{constructor(name){const parts=name.split(':');this.granularity_=parts[2];assert(this.granularity_===Granularity.CYCLE||this.granularity_===Granularity.EVENT);this.thread_=ThreadType.ALL_THREADS;let phasesIndex=3;if(parts[3]==='main_thread'){this.thread_=ThreadType.MAIN;phasesIndex=4;} +tr.metrics.MetricRegistry.register(executionMetric);return{executionMetric,};});'use strict';tr.exportTo('tr.metrics.v8',function(){const TARGET_FPS=60;const MS_PER_SECOND=1000;const WINDOW_SIZE_MS=MS_PER_SECOND/TARGET_FPS;const EPSILON=1e-6;const METRICS=['v8:gc:cycle:full','v8:gc:cycle:full:cpp','v8:gc:cycle:full:mark','v8:gc:cycle:full:mark:cpp','v8:gc:cycle:full:weak','v8:gc:cycle:full:weak:cpp','v8:gc:cycle:full:sweep','v8:gc:cycle:full:sweep:cpp','v8:gc:cycle:full:compact','v8:gc:cycle:full:compact:cpp','v8:gc:cycle:main_thread:full','v8:gc:cycle:main_thread:full:cpp','v8:gc:cycle:main_thread:full:mark','v8:gc:cycle:main_thread:full:mark:cpp','v8:gc:cycle:main_thread:full:weak','v8:gc:cycle:main_thread:full:weak:cpp','v8:gc:cycle:main_thread:full:sweep','v8:gc:cycle:main_thread:full:sweep:cpp','v8:gc:cycle:main_thread:full:compact','v8:gc:cycle:main_thread:full:compact:cpp','v8:gc:cycle:main_thread:full:atomic','v8:gc:cycle:main_thread:full:atomic:cpp','v8:gc:cycle:main_thread:full:atomic:mark','v8:gc:cycle:main_thread:full:atomic:mark:cpp','v8:gc:cycle:main_thread:full:atomic:weak','v8:gc:cycle:main_thread:full:atomic:weak:cpp','v8:gc:cycle:main_thread:full:atomic:sweep','v8:gc:cycle:main_thread:full:atomic:sweep:cpp','v8:gc:cycle:main_thread:full:atomic:compact','v8:gc:cycle:main_thread:full:atomic:compact:cpp','v8:gc:cycle:main_thread:full:incremental','v8:gc:cycle:main_thread:full:incremental:cpp','v8:gc:cycle:main_thread:full:incremental:mark','v8:gc:cycle:main_thread:full:incremental:mark:cpp','v8:gc:cycle:main_thread:full:incremental:sweep','v8:gc:cycle:main_thread:full:incremental:sweep:cpp','v8:gc:event:main_thread:full:atomic','v8:gc:event:main_thread:full:atomic:cpp','v8:gc:event:main_thread:full:atomic:mark','v8:gc:event:main_thread:full:atomic:mark:cpp','v8:gc:event:main_thread:full:atomic:weak','v8:gc:event:main_thread:full:atomic:weak:cpp','v8:gc:event:main_thread:full:atomic:sweep','v8:gc:event:main_thread:full:atomic:sweep:cpp','v8:gc:event:main_thread:full:atomic:compact','v8:gc:event:main_thread:full:atomic:compact:cpp','v8:gc:event:main_thread:full:incremental','v8:gc:event:main_thread:full:incremental:cpp','v8:gc:event:main_thread:full:incremental:mark','v8:gc:event:main_thread:full:incremental:mark:cpp','v8:gc:event:main_thread:full:incremental:sweep','v8:gc:event:main_thread:full:incremental:sweep:cpp','v8:gc:cycle:young','v8:gc:cycle:main_thread:young',];const V8_FULL_ATOMIC_EVENTS=['V8.GC_MARK_COMPACTOR'];const V8_FULL_MARK_EVENTS=['V8.GC_MC_BACKGROUND_MARKING','V8.GC_MC_MARK','V8.GC_MC_INCREMENTAL','V8.GCIncrementalMarkingFinalize','V8.GCIncrementalMarkingStart',];const V8_FULL_COMPACT_EVENTS=['V8.GC_MC_BACKGROUND_EVACUATE_COPY','V8.GC_MC_BACKGROUND_EVACUATE_UPDATE_POINTERS','V8.GC_MC_EVACUATE',];const V8_FULL_SWEEP_EVENTS=['V8.GC_MC_BACKGROUND_SWEEPING','V8.GC_MC_SWEEP',];const V8_FULL_WEAK_EVENTS=['V8.GC_MC_CLEAR',];const V8_YOUNG_EVENTS=['V8.GC_SCAVENGER_BACKGROUND_SCAVENGE_PARALLEL','V8.GC_SCAVENGER',];const CPP_GC_FULL_MARK_EVENTS=['BlinkGC.AtomicPauseMarkEpilogue','BlinkGC.AtomicPauseMarkPrologue','BlinkGC.AtomicPauseMarkRoots','BlinkGC.AtomicPauseMarkTransitiveClosure','BlinkGC.ConcurrentMarkingStep','BlinkGC.IncrementalMarkingStartMarking','BlinkGC.IncrementalMarkingStep','BlinkGC.MarkBailOutObjects','BlinkGC.MarkFlushEphemeronPairs','BlinkGC.MarkFlushV8References','BlinkGC.UnifiedMarkingStep','CppGC.AtomicMark','CppGC.IncrementalMark','CppGC.ConcurrentMark',];const CPP_GC_FULL_COMPACT_EVENTS=['BlinkGC.AtomicPauseSweepAndCompact','CppGC.AtomicCompact',];const CPP_GC_FULL_SWEEP_EVENTS=['BlinkGC.CompleteSweep','BlinkGC.ConcurrentSweepingStep','BlinkGC.LazySweepInIdle','BlinkGC.LazySweepOnAllocation','CppGC.AtomicSweep','CppGC.IncrementalSweep','CppGC.ConcurrentSweep',];const CPP_GC_FULL_WEAK_EVENTS=['BlinkGC.MarkWeakProcessing','CppGC.AtomicWeak',];const RULES=[{events:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic',},{events:V8_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark',},{events:CPP_GC_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark:cpp',},{events:V8_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark',},{events:CPP_GC_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark:cpp',},{events:V8_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact',},{events:CPP_GC_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact:cpp',},{events:V8_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep:cpp',},{events:V8_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak',},{events:CPP_GC_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak:cpp',},{events:V8_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep:cpp',},{events:V8_YOUNG_EVENTS,contribute_to:'young:atomic',},];const Granularity={CYCLE:'cycle',EVENT:'event',};const ThreadType={MAIN:'main',BACKGROUND:'background',ALL_THREADS:'all_threads',};class Metric{constructor(name){const parts=name.split(':');this.granularity_=parts[2];assert(this.granularity_===Granularity.CYCLE||this.granularity_===Granularity.EVENT);this.thread_=ThreadType.ALL_THREADS;let phasesIndex=3;if(parts[3]==='main_thread'){this.thread_=ThreadType.MAIN;phasesIndex=4;} if(parts[3]==='background_threads'){this.thread_=ThreadType.BACKGROUND;phasesIndex=4;} this.phases_=parts.slice(phasesIndex);const maxValue=this.isPerCycleMetric()?10000:1000;const boundaries=tr.v.HistogramBinBoundaries.createExponential(0.1,maxValue,100);this.histogram=new tr.v.Histogram(name,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,boundaries);this.histogram.customizeSummaryOptions({avg:true,count:true,max:true,min:false,std:false,sum:this.isPerCycleMetric(),});} isPerCycleMetric(){return this.granularity_===Granularity.CYCLE;} @@ -9045,16 +9058,19 @@ function getEpoch(event){function checkEpochConsistency(epoch,event){if(epoch=== const result={v8:null,cpp:null};while(event){if('epoch'in event.args){if(isV8Event(event)){checkEpochConsistency(result.v8,event);result.v8=event.args.epoch;}else{checkEpochConsistency(result.cpp,event);result.cpp=event.args.epoch;}} event=event.parentSlice;} return result;} -const cppToV8=new Map();for(const event of events){const epoch=getEpoch(event);if(epoch.cpp!==null&&epoch.v8!==null){assert(!cppToV8.has(epoch.cpp)||cppToV8.get(epoch.cpp)===epoch.v8,`CppGC epoch ${epoch.cpp} corresponds to two v8 epochs `+`${cppToV8.get(epoch.cpp)} and ${epoch.v8}. `+`Detected at ${event.userFriendlyName}.`);cppToV8.set(epoch.cpp,epoch.v8);}} +function GlobalEpochFromV8(v8Epoch){return 2*v8Epoch;} +function GlobalEpochFromCpp(cppEpoch){return 2*cppEpoch+1;} +const cppToV8=new Map();for(const event of events){const epoch=getEpoch(event);if(epoch.cpp!==null&&epoch.v8!==null){if(!cppToV8.has(epoch.cpp)||cppToV8.get(epoch.cpp)>epoch.v8){cppToV8.set(epoch.cpp,epoch.v8);}}} const result=new Map();for(const event of events){const epoch=getEpoch(event);if(epoch.cpp===null&&epoch.v8===null){continue;} -assert(epoch.cpp===null||cppToV8.has(epoch.cpp),`CppGC epoch ${epoch.cpp} doesn't have the corresponding V8 epoch. `+`Detected at ${event.userFriendlyName}`);const key=epoch.v8===null?cppToV8.get(epoch.cpp):epoch.v8;if(result.has(key)){result.get(key).push(event);}else{result.set(key,[event]);}} +let globalEpoch;if(epoch.v8!==null){globalEpoch=GlobalEpochFromV8(epoch.v8);}else if(cppToV8.has(epoch.cpp)){globalEpoch=GlobalEpochFromV8(cppToV8.get(epoch.cpp));}else{globalEpoch=GlobalEpochFromCpp(epoch.cpp);} +if(result.has(globalEpoch)){result.get(globalEpoch).push(event);}else{result.set(globalEpoch,[event]);}} return result;} function addGarbageCollectionMetrics(metricNames,histograms,model){const metrics=metricNames.map(name=>new Metric(name));const gcEventNames=new Set(eventsMentionedIn(RULES));const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);for(const rendererHelper of Object.values(chromeHelper.rendererHelpers)){if(rendererHelper.isChromeTracingUI)continue;const[threads,threadTypes]=jsExecutionThreadsWithTypes(rendererHelper);const events=[];for(const thread of threads){for(const event of thread.sliceGroup.childEvents()){if(gcEventNames.has(event.title)){events.push(event);}}} for(const cycleEvents of groupByEpoch(events).values()){if(cycleEvents.some(tr.metrics.v8.utils.isForcedGarbageCollectionEvent)){continue;} for(const metric of metrics){metric.apply(RULES,cycleEvents,threadTypes);}}} for(const metric of metrics){histograms.addHistogram(metric.histogram);}} function gcMetric(histograms,model,options){options=options||{};addDurationOfTopEvents(histograms,model);addTotalDurationOfTopEvents(histograms,model);if(options.include_sub_events){addDurationOfSubEvents(histograms,model);} -addPercentageInV8ExecuteOfTopEvents(histograms,model);addTotalPercentageInV8Execute(histograms,model);addMarkCompactorMutatorUtilization(histograms,model);addTotalMarkCompactorTime(histograms,model);addTotalMarkCompactorMarkingTime(histograms,model);addScavengerSurvivedFromStackEvents(histograms,model);} +addPercentageInV8ExecuteOfTopEvents(histograms,model);addTotalPercentageInV8Execute(histograms,model);addMarkCompactorMutatorUtilization(histograms,model);addTotalMarkCompactorTime(histograms,model);addTotalMarkCompactorMarkingTime(histograms,model);addScavengerSurvivedFromStackEvents(histograms,model);addGarbageCollectionMetrics(METRICS,histograms,model);} tr.metrics.MetricRegistry.register(gcMetric);const timeDurationInMs_smallerIsBetter=tr.b.Unit.byName.timeDurationInMs_smallerIsBetter;const percentage_biggerIsBetter=tr.b.Unit.byName.normalizedPercentage_biggerIsBetter;const percentage_smallerIsBetter=tr.b.Unit.byName.normalizedPercentage_smallerIsBetter;const bytes_smallerIsBetter=tr.b.Unit.byName.sizeInBytes_smallerIsBetter;const CUSTOM_BOUNDARIES=tr.v.HistogramBinBoundaries.createLinear(0,20,200).addExponentialBins(200,100);function createNumericForTopEventTime(name){const n=new tr.v.Histogram(name,timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);n.customizeSummaryOptions({avg:true,count:true,max:true,min:false,std:true,sum:true,percentile:[0.90]});return n;} function createNumericForSubEventTime(name){const n=new tr.v.Histogram(name,timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);n.customizeSummaryOptions({avg:true,count:false,max:true,min:false,std:false,sum:false,percentile:[0.90]});return n;} function createNumericForIdleTime(name){const n=new tr.v.Histogram(name,timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);n.customizeSummaryOptions({avg:true,count:false,max:true,min:false,std:false,sum:true,percentile:[]});return n;} @@ -9228,9 +9244,9 @@ for(const tab of this.$.tabs.tabs){tab.nodes=undefined;} this.$.tabs.clearSubViews();if(this.displayedNode_===undefined){this.$.tabs.label='No heap node provided.';return;} for(const[dimension,children]of this.displayedNode_.childNodes){if(!this.dimensionToTab_.has(dimension)){this.dimensionToTab_.set(dimension,document.createElement('tr-ui-a-memory-dump-heap-details-breakdown-view-tab'));} const tab=this.dimensionToTab_.get(dimension);tab.aggregationMode=this.aggregationMode_;tab.dimension=dimension;tab.nodes=children;this.$.tabs.addSubView(tab);tab.rebuild();if(dimension===previouslySelectedDimension){this.$.tabs.selectedSubView=tab;if(previouslySelectedTabFocused){tab.focus();}}} -if(this.$.tabs.tabs.length>0){this.$.tabs.label='Break selected node further by:';}else{this.$.tabs.label='Selected node cannot be broken down any further.';}},onKeyDown_(keyEvent){if(!this.displayedNode_)return;let keyHandled=false;switch(keyEvent.keyCode){case 8:{if(!this.displayedNode_.parentNode)break;const viewEvent=new tr.b.Event('enter-node');viewEvent.node=this.displayedNode_.parentNode;this.dispatchEvent(viewEvent);keyHandled=true;break;} +if(this.$.tabs.tabs.length>0){this.$.tabs.label='Break selected node further by:';}else{this.$.tabs.label='Selected node cannot be broken down any further.';}},onKeyDown_(keyEvent){if(!this.displayedNode_)return;let keyHandled=false;switch(keyEvent.keyCode){case 8:{if(!this.displayedNode_.parentNode)break;const viewEvent=new tr.b.Event('enter-node',true);viewEvent.node=this.displayedNode_.parentNode;this.dispatchEvent(viewEvent);keyHandled=true;break;} case 37:case 39:{const wasFocused=this.$.tabs.selectedSubView.isFocused;keyHandled=keyEvent.keyCode===37?this.$.tabs.selectPreviousTabIfPossible():this.$.tabs.selectNextTabIfPossible();if(wasFocused&&keyHandled){this.$.tabs.selectedSubView.focus();}}} -if(!keyHandled)return;keyEvent.stopPropagation();keyEvent.preventDefault();}});Polymer({is:'tr-ui-a-memory-dump-heap-details-breakdown-view-tab',behaviors:[tr.ui.analysis.RebuildableBehavior],created(){this.dimension_=undefined;this.nodes_=undefined;this.aggregationMode_=undefined;this.displayLongTail_=false;},ready(){this.$.table.addEventListener('step-into',function(tableEvent){const viewEvent=new tr.b.Event('enter-node');viewEvent.node=tableEvent.tableRow;this.dispatchEvent(viewEvent);}.bind(this));},get displayLongTail(){return this.displayLongTail_;},set displayLongTail(newValue){if(this.displayLongTail===newValue)return;this.displayLongTail_=newValue;this.scheduleRebuild_();},get dimension(){return this.dimension_;},set dimension(dimension){this.dimension_=dimension;this.scheduleRebuild_();},get nodes(){return this.nodes_;},set nodes(nodes){this.nodes_=nodes;this.scheduleRebuild_();},get nodes(){return this.nodes_||[];},get dimensionLabel_(){if(this.dimension_===undefined)return'(undefined)';return this.dimension_.label;},get tabLabel(){let nodeCount=0;if(this.nodes_){nodeCount=this.nodes_.length;} +if(!keyHandled)return;keyEvent.stopPropagation();keyEvent.preventDefault();}});Polymer({is:'tr-ui-a-memory-dump-heap-details-breakdown-view-tab',behaviors:[tr.ui.analysis.RebuildableBehavior],created(){this.dimension_=undefined;this.nodes_=undefined;this.aggregationMode_=undefined;this.displayLongTail_=false;},ready(){this.$.table.addEventListener('step-into',function(tableEvent){const viewEvent=new tr.b.Event('enter-node',true);viewEvent.node=tableEvent.tableRow;this.dispatchEvent(viewEvent);}.bind(this));},get displayLongTail(){return this.displayLongTail_;},set displayLongTail(newValue){if(this.displayLongTail===newValue)return;this.displayLongTail_=newValue;this.scheduleRebuild_();},get dimension(){return this.dimension_;},set dimension(dimension){this.dimension_=dimension;this.scheduleRebuild_();},get nodes(){return this.nodes_;},set nodes(nodes){this.nodes_=nodes;this.scheduleRebuild_();},get nodes(){return this.nodes_||[];},get dimensionLabel_(){if(this.dimension_===undefined)return'(undefined)';return this.dimension_.label;},get tabLabel(){let nodeCount=0;if(this.nodes_){nodeCount=this.nodes_.length;} return this.dimensionLabel_+' ('+nodeCount+')';},get tabIcon(){if(this.dimension_===undefined||this.dimension_===tr.ui.analysis.HeapDetailsRowDimension.ROOT){return undefined;} return{text:this.dimension_.symbol,style:'color: '+tr.b.ColorScheme.getColorForReservedNameAsString(this.dimension_.color)+';'};},get aggregationMode(){return this.aggregationMode_;},set aggregationMode(aggregationMode){this.aggregationMode_=aggregationMode;this.scheduleRebuild_();},focus(){this.$.table.focus();},blur(){this.$.table.blur();},get isFocused(){return this.$.table.isFocused;},onRebuild_(){this.$.table.selectionMode=tr.ui.b.TableFormat.SelectionMode.ROW;this.$.table.emptyValue='Cannot break down by '+ this.dimensionLabel_.toLowerCase()+' any further.';const[state,rows]=this.getRows_();const total=this.nodes.length;const displayed=rows.length;const hidden=total-displayed;this.updateInfoBar_(state,[total,displayed,hidden]);this.$.table.tableRows=rows;this.$.table.tableColumns=this.createColumns_(rows);if(this.$.table.sortColumnIndex===undefined){this.$.table.sortColumnIndex=0;this.$.table.sortDescending=false;} diff --git a/catapult/systrace/systrace/util.py b/catapult/systrace/systrace/util.py index 063f9ed9..797d6756 100644 --- a/catapult/systrace/systrace/util.py +++ b/catapult/systrace/systrace/util.py @@ -69,8 +69,8 @@ def get_tracing_path(device_serial=None): options, _ = parser.parse_args() device_serial = options.device_serial - adb_output, adb_return_code = run_adb_shell(mount_info_args, device_serial) - if adb_return_code == 0 and 'debugfs' not in adb_output: + adb_output, adb_return_code = run_adb_shell(mount_info_args, device_serial, ) + if adb_return_code == 0 and 'tracefs on /sys/kernel/tracing' in adb_output: return '/sys/kernel/tracing' return '/sys/kernel/debug/tracing' diff --git a/catapult/third_party/zipfile/LICENSE b/catapult/third_party/zipfile/LICENSE deleted file mode 100644 index 84a3337c..00000000 --- a/catapult/third_party/zipfile/LICENSE +++ /dev/null @@ -1,255 +0,0 @@ -A. HISTORY OF THE SOFTWARE -========================== - -Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands -as a successor of a language called ABC. Guido remains Python's -principal author, although it includes many contributions from others. - -In 1995, Guido continued his work on Python at the Corporation for -National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) -in Reston, Virginia where he released several versions of the -software. - -In May 2000, Guido and the Python core development team moved to -BeOpen.com to form the BeOpen PythonLabs team. In October of the same -year, the PythonLabs team moved to Digital Creations (now Zope -Corporation, see http://www.zope.com). In 2001, the Python Software -Foundation (PSF, see http://www.python.org/psf/) was formed, a -non-profit organization created specifically to own Python-related -Intellectual Property. Zope Corporation is a sponsoring member of -the PSF. - -All Python releases are Open Source (see http://www.opensource.org for -the Open Source Definition). Historically, most, but not all, Python -releases have also been GPL-compatible; the table below summarizes -the various releases. - - Release Derived Year Owner GPL- - from compatible? (1) - - 0.9.0 thru 1.2 1991-1995 CWI yes - 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes - 1.6 1.5.2 2000 CNRI no - 2.0 1.6 2000 BeOpen.com no - 1.6.1 1.6 2001 CNRI yes (2) - 2.1 2.0+1.6.1 2001 PSF no - 2.0.1 2.0+1.6.1 2001 PSF yes - 2.1.1 2.1+2.0.1 2001 PSF yes - 2.1.2 2.1.1 2002 PSF yes - 2.1.3 2.1.2 2002 PSF yes - 2.2 and above 2.1.1 2001-now PSF yes - -Footnotes: - -(1) GPL-compatible doesn't mean that we're distributing Python under - the GPL. All Python licenses, unlike the GPL, let you distribute - a modified version without making your changes open source. The - GPL-compatible licenses make it possible to combine Python with - other software that is released under the GPL; the others don't. - -(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, - because its license has a choice of law clause. According to - CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 - is "not incompatible" with the GPL. - -Thanks to the many outside volunteers who have worked under Guido's -direction to make these releases possible. - - -B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON -=============================================================== - -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF hereby -grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -analyze, test, perform and/or display publicly, prepare derivative works, -distribute, and otherwise use Python alone or in any derivative version, -provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016 Python Software Foundation; All Rights -Reserved" are retained in Python alone or in any derivative version prepared by -Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an -office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the -Individual or Organization ("Licensee") accessing and otherwise using -this software in source or binary form and its associated -documentation ("the Software"). - -2. Subject to the terms and conditions of this BeOpen Python License -Agreement, BeOpen hereby grants Licensee a non-exclusive, -royalty-free, world-wide license to reproduce, analyze, test, perform -and/or display publicly, prepare derivative works, distribute, and -otherwise use the Software alone or in any derivative version, -provided, however, that the BeOpen Python License is retained in the -Software, alone or in any derivative version prepared by Licensee. - -3. BeOpen is making the Software available to Licensee on an "AS IS" -basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE -SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS -AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY -DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -5. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -6. This License Agreement shall be governed by and interpreted in all -respects by the law of the State of California, excluding conflict of -law provisions. Nothing in this License Agreement shall be deemed to -create any relationship of agency, partnership, or joint venture -between BeOpen and Licensee. This License Agreement does not grant -permission to use BeOpen trademarks or trade names in a trademark -sense to endorse or promote products or services of Licensee, or any -third party. As an exception, the "BeOpen Python" logos available at -http://www.pythonlabs.com/logos.html may be used according to the -permissions granted on that web page. - -7. By copying, installing or otherwise using the software, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- - -1. This LICENSE AGREEMENT is between the Corporation for National -Research Initiatives, having an office at 1895 Preston White Drive, -Reston, VA 20191 ("CNRI"), and the Individual or Organization -("Licensee") accessing and otherwise using Python 1.6.1 software in -source or binary form and its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, CNRI -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python 1.6.1 -alone or in any derivative version, provided, however, that CNRI's -License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) -1995-2001 Corporation for National Research Initiatives; All Rights -Reserved" are retained in Python 1.6.1 alone or in any derivative -version prepared by Licensee. Alternately, in lieu of CNRI's License -Agreement, Licensee may substitute the following text (omitting the -quotes): "Python 1.6.1 is made available subject to the terms and -conditions in CNRI's License Agreement. This Agreement together with -Python 1.6.1 may be located on the Internet using the following -unique, persistent identifier (known as a handle): 1895.22/1013. This -Agreement may also be obtained from a proxy server on the Internet -using the following URL: http://hdl.handle.net/1895.22/1013". - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python 1.6.1 or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python 1.6.1. - -4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" -basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. This License Agreement shall be governed by the federal -intellectual property law of the United States, including without -limitation the federal copyright law, and, to the extent such -U.S. federal law does not apply, by the law of the Commonwealth of -Virginia, excluding Virginia's conflict of law provisions. -Notwithstanding the foregoing, with regard to derivative works based -on Python 1.6.1 that incorporate non-separable material that was -previously distributed under the GNU General Public License (GPL), the -law of the Commonwealth of Virginia shall govern this License -Agreement only as to issues arising under or with respect to -Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this -License Agreement shall be deemed to create any relationship of -agency, partnership, or joint venture between CNRI and Licensee. This -License Agreement does not grant permission to use CNRI trademarks or -trade name in a trademark sense to endorse or promote products or -services of Licensee, or any third party. - -8. By clicking on the "ACCEPT" button where indicated, or by copying, -installing or otherwise using Python 1.6.1, Licensee agrees to be -bound by the terms and conditions of this License Agreement. - - ACCEPT - - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, -The Netherlands. All rights reserved. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the name of Stichting Mathematisch -Centrum or CWI not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO -THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE -FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/catapult/third_party/zipfile/README.chromium b/catapult/third_party/zipfile/README.chromium deleted file mode 100644 index f45d3cff..00000000 --- a/catapult/third_party/zipfile/README.chromium +++ /dev/null @@ -1,16 +0,0 @@ -Name: Python zipfile module -Short Name: zipfile -URL: https://github.com/python/cpython/blob/master/Lib/zipfile.py -Version: 2.7.13 -License: Python -License File: NOT_SHIPPED -Security Critical: no - -Description: -This is a copy of the zipfile module from Python 2.7.13. This snapshot was -taken to workaround a bug in older Python versions such as 2.7.3. See -<https://bugs.python.org/issue6972> and specifically the fix at -<https://hg.python.org/cpython/rev/0c5fa35c9f12#l3.11>. - -Local Modifications: -Renamed zipfile.py to zipfile_2_7_13.py to avoid conflicting with system copy. diff --git a/catapult/third_party/zipfile/zipfile_2_7_13.py b/catapult/third_party/zipfile/zipfile_2_7_13.py deleted file mode 100644 index 1d106508..00000000 --- a/catapult/third_party/zipfile/zipfile_2_7_13.py +++ /dev/null @@ -1,1543 +0,0 @@ -""" -Read and write ZIP files. -""" -import struct, os, time, sys, shutil -import binascii, cStringIO, stat -import io -import re -import string - -try: - import zlib # We may need its compression method - crc32 = zlib.crc32 -except ImportError: - zlib = None - crc32 = binascii.crc32 - -__all__ = ["BadZipfile", "error", "ZIP_STORED", "ZIP_DEFLATED", "is_zipfile", - "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile" ] - -class BadZipfile(Exception): - pass - - -class LargeZipFile(Exception): - """ - Raised when writing a zipfile, the zipfile requires ZIP64 extensions - and those extensions are disabled. - """ - -error = BadZipfile # The exception raised by this module - -ZIP64_LIMIT = (1 << 31) - 1 -ZIP_FILECOUNT_LIMIT = (1 << 16) - 1 -ZIP_MAX_COMMENT = (1 << 16) - 1 - -# constants for Zip file compression methods -ZIP_STORED = 0 -ZIP_DEFLATED = 8 -# Other ZIP compression methods not supported - -# Below are some formats and associated data for reading/writing headers using -# the struct module. The names and structures of headers/records are those used -# in the PKWARE description of the ZIP file format: -# http://www.pkware.com/documents/casestudies/APPNOTE.TXT -# (URL valid as of January 2008) - -# The "end of central directory" structure, magic number, size, and indices -# (section V.I in the format document) -structEndArchive = "<4s4H2LH" -stringEndArchive = "PK\005\006" -sizeEndCentDir = struct.calcsize(structEndArchive) - -_ECD_SIGNATURE = 0 -_ECD_DISK_NUMBER = 1 -_ECD_DISK_START = 2 -_ECD_ENTRIES_THIS_DISK = 3 -_ECD_ENTRIES_TOTAL = 4 -_ECD_SIZE = 5 -_ECD_OFFSET = 6 -_ECD_COMMENT_SIZE = 7 -# These last two indices are not part of the structure as defined in the -# spec, but they are used internally by this module as a convenience -_ECD_COMMENT = 8 -_ECD_LOCATION = 9 - -# The "central directory" structure, magic number, size, and indices -# of entries in the structure (section V.F in the format document) -structCentralDir = "<4s4B4HL2L5H2L" -stringCentralDir = "PK\001\002" -sizeCentralDir = struct.calcsize(structCentralDir) - -# indexes of entries in the central directory structure -_CD_SIGNATURE = 0 -_CD_CREATE_VERSION = 1 -_CD_CREATE_SYSTEM = 2 -_CD_EXTRACT_VERSION = 3 -_CD_EXTRACT_SYSTEM = 4 -_CD_FLAG_BITS = 5 -_CD_COMPRESS_TYPE = 6 -_CD_TIME = 7 -_CD_DATE = 8 -_CD_CRC = 9 -_CD_COMPRESSED_SIZE = 10 -_CD_UNCOMPRESSED_SIZE = 11 -_CD_FILENAME_LENGTH = 12 -_CD_EXTRA_FIELD_LENGTH = 13 -_CD_COMMENT_LENGTH = 14 -_CD_DISK_NUMBER_START = 15 -_CD_INTERNAL_FILE_ATTRIBUTES = 16 -_CD_EXTERNAL_FILE_ATTRIBUTES = 17 -_CD_LOCAL_HEADER_OFFSET = 18 - -# The "local file header" structure, magic number, size, and indices -# (section V.A in the format document) -structFileHeader = "<4s2B4HL2L2H" -stringFileHeader = "PK\003\004" -sizeFileHeader = struct.calcsize(structFileHeader) - -_FH_SIGNATURE = 0 -_FH_EXTRACT_VERSION = 1 -_FH_EXTRACT_SYSTEM = 2 -_FH_GENERAL_PURPOSE_FLAG_BITS = 3 -_FH_COMPRESSION_METHOD = 4 -_FH_LAST_MOD_TIME = 5 -_FH_LAST_MOD_DATE = 6 -_FH_CRC = 7 -_FH_COMPRESSED_SIZE = 8 -_FH_UNCOMPRESSED_SIZE = 9 -_FH_FILENAME_LENGTH = 10 -_FH_EXTRA_FIELD_LENGTH = 11 - -# The "Zip64 end of central directory locator" structure, magic number, and size -structEndArchive64Locator = "<4sLQL" -stringEndArchive64Locator = "PK\x06\x07" -sizeEndCentDir64Locator = struct.calcsize(structEndArchive64Locator) - -# The "Zip64 end of central directory" record, magic number, size, and indices -# (section V.G in the format document) -structEndArchive64 = "<4sQ2H2L4Q" -stringEndArchive64 = "PK\x06\x06" -sizeEndCentDir64 = struct.calcsize(structEndArchive64) - -_CD64_SIGNATURE = 0 -_CD64_DIRECTORY_RECSIZE = 1 -_CD64_CREATE_VERSION = 2 -_CD64_EXTRACT_VERSION = 3 -_CD64_DISK_NUMBER = 4 -_CD64_DISK_NUMBER_START = 5 -_CD64_NUMBER_ENTRIES_THIS_DISK = 6 -_CD64_NUMBER_ENTRIES_TOTAL = 7 -_CD64_DIRECTORY_SIZE = 8 -_CD64_OFFSET_START_CENTDIR = 9 - -def _check_zipfile(fp): - try: - if _EndRecData(fp): - return True # file has correct magic number - except IOError: - pass - return False - -def is_zipfile(filename): - """Quickly see if a file is a ZIP file by checking the magic number. - - The filename argument may be a file or file-like object too. - """ - result = False - try: - if hasattr(filename, "read"): - result = _check_zipfile(fp=filename) - else: - with open(filename, "rb") as fp: - result = _check_zipfile(fp) - except IOError: - pass - return result - -def _EndRecData64(fpin, offset, endrec): - """ - Read the ZIP64 end-of-archive records and use that to update endrec - """ - try: - fpin.seek(offset - sizeEndCentDir64Locator, 2) - except IOError: - # If the seek fails, the file is not large enough to contain a ZIP64 - # end-of-archive record, so just return the end record we were given. - return endrec - - data = fpin.read(sizeEndCentDir64Locator) - if len(data) != sizeEndCentDir64Locator: - return endrec - sig, diskno, reloff, disks = struct.unpack(structEndArchive64Locator, data) - if sig != stringEndArchive64Locator: - return endrec - - if diskno != 0 or disks != 1: - raise BadZipfile("zipfiles that span multiple disks are not supported") - - # Assume no 'zip64 extensible data' - fpin.seek(offset - sizeEndCentDir64Locator - sizeEndCentDir64, 2) - data = fpin.read(sizeEndCentDir64) - if len(data) != sizeEndCentDir64: - return endrec - sig, sz, create_version, read_version, disk_num, disk_dir, \ - dircount, dircount2, dirsize, diroffset = \ - struct.unpack(structEndArchive64, data) - if sig != stringEndArchive64: - return endrec - - # Update the original endrec using data from the ZIP64 record - endrec[_ECD_SIGNATURE] = sig - endrec[_ECD_DISK_NUMBER] = disk_num - endrec[_ECD_DISK_START] = disk_dir - endrec[_ECD_ENTRIES_THIS_DISK] = dircount - endrec[_ECD_ENTRIES_TOTAL] = dircount2 - endrec[_ECD_SIZE] = dirsize - endrec[_ECD_OFFSET] = diroffset - return endrec - - -def _EndRecData(fpin): - """Return data from the "End of Central Directory" record, or None. - - The data is a list of the nine items in the ZIP "End of central dir" - record followed by a tenth item, the file seek offset of this record.""" - - # Determine file size - fpin.seek(0, 2) - filesize = fpin.tell() - - # Check to see if this is ZIP file with no archive comment (the - # "end of central directory" structure should be the last item in the - # file if this is the case). - try: - fpin.seek(-sizeEndCentDir, 2) - except IOError: - return None - data = fpin.read() - if (len(data) == sizeEndCentDir and - data[0:4] == stringEndArchive and - data[-2:] == b"\000\000"): - # the signature is correct and there's no comment, unpack structure - endrec = struct.unpack(structEndArchive, data) - endrec=list(endrec) - - # Append a blank comment and record start offset - endrec.append("") - endrec.append(filesize - sizeEndCentDir) - - # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, -sizeEndCentDir, endrec) - - # Either this is not a ZIP file, or it is a ZIP file with an archive - # comment. Search the end of the file for the "end of central directory" - # record signature. The comment is the last item in the ZIP file and may be - # up to 64K long. It is assumed that the "end of central directory" magic - # number does not appear in the comment. - maxCommentStart = max(filesize - (1 << 16) - sizeEndCentDir, 0) - fpin.seek(maxCommentStart, 0) - data = fpin.read() - start = data.rfind(stringEndArchive) - if start >= 0: - # found the magic number; attempt to unpack and interpret - recData = data[start:start+sizeEndCentDir] - if len(recData) != sizeEndCentDir: - # Zip file is corrupted. - return None - endrec = list(struct.unpack(structEndArchive, recData)) - commentSize = endrec[_ECD_COMMENT_SIZE] #as claimed by the zip file - comment = data[start+sizeEndCentDir:start+sizeEndCentDir+commentSize] - endrec.append(comment) - endrec.append(maxCommentStart + start) - - # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, maxCommentStart + start - filesize, - endrec) - - # Unable to find a valid end of central directory structure - return None - - -class ZipInfo (object): - """Class with attributes describing each file in the ZIP archive.""" - - __slots__ = ( - 'orig_filename', - 'filename', - 'date_time', - 'compress_type', - 'comment', - 'extra', - 'create_system', - 'create_version', - 'extract_version', - 'reserved', - 'flag_bits', - 'volume', - 'internal_attr', - 'external_attr', - 'header_offset', - 'CRC', - 'compress_size', - 'file_size', - '_raw_time', - ) - - def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): - self.orig_filename = filename # Original file name in archive - - # Terminate the file name at the first null byte. Null bytes in file - # names are used as tricks by viruses in archives. - null_byte = filename.find(chr(0)) - if null_byte >= 0: - filename = filename[0:null_byte] - # This is used to ensure paths in generated ZIP files always use - # forward slashes as the directory separator, as required by the - # ZIP format specification. - if os.sep != "/" and os.sep in filename: - filename = filename.replace(os.sep, "/") - - self.filename = filename # Normalized file name - self.date_time = date_time # year, month, day, hour, min, sec - - if date_time[0] < 1980: - raise ValueError('ZIP does not support timestamps before 1980') - - # Standard values: - self.compress_type = ZIP_STORED # Type of compression for the file - self.comment = "" # Comment for each file - self.extra = "" # ZIP extra data - if sys.platform == 'win32': - self.create_system = 0 # System which created ZIP archive - else: - # Assume everything else is unix-y - self.create_system = 3 # System which created ZIP archive - self.create_version = 20 # Version which created ZIP archive - self.extract_version = 20 # Version needed to extract archive - self.reserved = 0 # Must be zero - self.flag_bits = 0 # ZIP flag bits - self.volume = 0 # Volume number of file header - self.internal_attr = 0 # Internal attributes - self.external_attr = 0 # External file attributes - # Other attributes are set by class ZipFile: - # header_offset Byte offset to the file header - # CRC CRC-32 of the uncompressed file - # compress_size Size of the compressed file - # file_size Size of the uncompressed file - - def FileHeader(self, zip64=None): - """Return the per-file header as a string.""" - dt = self.date_time - dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - if self.flag_bits & 0x08: - # Set these to zero because we write them after the file data - CRC = compress_size = file_size = 0 - else: - CRC = self.CRC - compress_size = self.compress_size - file_size = self.file_size - - extra = self.extra - - if zip64 is None: - zip64 = file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT - if zip64: - fmt = '<HHQQ' - extra = extra + struct.pack(fmt, - 1, struct.calcsize(fmt)-4, file_size, compress_size) - if file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT: - if not zip64: - raise LargeZipFile("Filesize would require ZIP64 extensions") - # File is larger than what fits into a 4 byte integer, - # fall back to the ZIP64 extension - file_size = 0xffffffff - compress_size = 0xffffffff - self.extract_version = max(45, self.extract_version) - self.create_version = max(45, self.extract_version) - - filename, flag_bits = self._encodeFilenameFlags() - header = struct.pack(structFileHeader, stringFileHeader, - self.extract_version, self.reserved, flag_bits, - self.compress_type, dostime, dosdate, CRC, - compress_size, file_size, - len(filename), len(extra)) - return header + filename + extra - - def _encodeFilenameFlags(self): - if isinstance(self.filename, unicode): - try: - return self.filename.encode('ascii'), self.flag_bits - except UnicodeEncodeError: - return self.filename.encode('utf-8'), self.flag_bits | 0x800 - else: - return self.filename, self.flag_bits - - def _decodeFilename(self): - if self.flag_bits & 0x800: - return self.filename.decode('utf-8') - else: - return self.filename - - def _decodeExtra(self): - # Try to decode the extra field. - extra = self.extra - unpack = struct.unpack - while len(extra) >= 4: - tp, ln = unpack('<HH', extra[:4]) - if tp == 1: - if ln >= 24: - counts = unpack('<QQQ', extra[4:28]) - elif ln == 16: - counts = unpack('<QQ', extra[4:20]) - elif ln == 8: - counts = unpack('<Q', extra[4:12]) - elif ln == 0: - counts = () - else: - raise RuntimeError, "Corrupt extra field %s"%(ln,) - - idx = 0 - - # ZIP64 extension (large files and/or large archives) - if self.file_size in (0xffffffffffffffffL, 0xffffffffL): - self.file_size = counts[idx] - idx += 1 - - if self.compress_size == 0xFFFFFFFFL: - self.compress_size = counts[idx] - idx += 1 - - if self.header_offset == 0xffffffffL: - old = self.header_offset - self.header_offset = counts[idx] - idx+=1 - - extra = extra[ln+4:] - - -class _ZipDecrypter: - """Class to handle decryption of files stored within a ZIP archive. - - ZIP supports a password-based form of encryption. Even though known - plaintext attacks have been found against it, it is still useful - to be able to get data out of such a file. - - Usage: - zd = _ZipDecrypter(mypwd) - plain_char = zd(cypher_char) - plain_text = map(zd, cypher_text) - """ - - def _GenerateCRCTable(): - """Generate a CRC-32 table. - - ZIP encryption uses the CRC32 one-byte primitive for scrambling some - internal keys. We noticed that a direct implementation is faster than - relying on binascii.crc32(). - """ - poly = 0xedb88320 - table = [0] * 256 - for i in range(256): - crc = i - for j in range(8): - if crc & 1: - crc = ((crc >> 1) & 0x7FFFFFFF) ^ poly - else: - crc = ((crc >> 1) & 0x7FFFFFFF) - table[i] = crc - return table - crctable = _GenerateCRCTable() - - def _crc32(self, ch, crc): - """Compute the CRC32 primitive on one byte.""" - return ((crc >> 8) & 0xffffff) ^ self.crctable[(crc ^ ord(ch)) & 0xff] - - def __init__(self, pwd): - self.key0 = 305419896 - self.key1 = 591751049 - self.key2 = 878082192 - for p in pwd: - self._UpdateKeys(p) - - def _UpdateKeys(self, c): - self.key0 = self._crc32(c, self.key0) - self.key1 = (self.key1 + (self.key0 & 255)) & 4294967295 - self.key1 = (self.key1 * 134775813 + 1) & 4294967295 - self.key2 = self._crc32(chr((self.key1 >> 24) & 255), self.key2) - - def __call__(self, c): - """Decrypt a single character.""" - c = ord(c) - k = self.key2 | 2 - c = c ^ (((k * (k^1)) >> 8) & 255) - c = chr(c) - self._UpdateKeys(c) - return c - - -compressor_names = { - 0: 'store', - 1: 'shrink', - 2: 'reduce', - 3: 'reduce', - 4: 'reduce', - 5: 'reduce', - 6: 'implode', - 7: 'tokenize', - 8: 'deflate', - 9: 'deflate64', - 10: 'implode', - 12: 'bzip2', - 14: 'lzma', - 18: 'terse', - 19: 'lz77', - 97: 'wavpack', - 98: 'ppmd', -} - - -class ZipExtFile(io.BufferedIOBase): - """File-like object for reading an archive member. - Is returned by ZipFile.open(). - """ - - # Max size supported by decompressor. - MAX_N = 1 << 31 - 1 - - # Read from compressed files in 4k blocks. - MIN_READ_SIZE = 4096 - - # Search for universal newlines or line chunks. - PATTERN = re.compile(r'^(?P<chunk>[^\r\n]+)|(?P<newline>\n|\r\n?)') - - def __init__(self, fileobj, mode, zipinfo, decrypter=None, - close_fileobj=False): - self._fileobj = fileobj - self._decrypter = decrypter - self._close_fileobj = close_fileobj - - self._compress_type = zipinfo.compress_type - self._compress_size = zipinfo.compress_size - self._compress_left = zipinfo.compress_size - - if self._compress_type == ZIP_DEFLATED: - self._decompressor = zlib.decompressobj(-15) - elif self._compress_type != ZIP_STORED: - descr = compressor_names.get(self._compress_type) - if descr: - raise NotImplementedError("compression type %d (%s)" % (self._compress_type, descr)) - else: - raise NotImplementedError("compression type %d" % (self._compress_type,)) - self._unconsumed = '' - - self._readbuffer = '' - self._offset = 0 - - self._universal = 'U' in mode - self.newlines = None - - # Adjust read size for encrypted files since the first 12 bytes - # are for the encryption/password information. - if self._decrypter is not None: - self._compress_left -= 12 - - self.mode = mode - self.name = zipinfo.filename - - if hasattr(zipinfo, 'CRC'): - self._expected_crc = zipinfo.CRC - self._running_crc = crc32(b'') & 0xffffffff - else: - self._expected_crc = None - - def readline(self, limit=-1): - """Read and return a line from the stream. - - If limit is specified, at most limit bytes will be read. - """ - - if not self._universal and limit < 0: - # Shortcut common case - newline found in buffer. - i = self._readbuffer.find('\n', self._offset) + 1 - if i > 0: - line = self._readbuffer[self._offset: i] - self._offset = i - return line - - if not self._universal: - return io.BufferedIOBase.readline(self, limit) - - line = '' - while limit < 0 or len(line) < limit: - readahead = self.peek(2) - if readahead == '': - return line - - # - # Search for universal newlines or line chunks. - # - # The pattern returns either a line chunk or a newline, but not - # both. Combined with peek(2), we are assured that the sequence - # '\r\n' is always retrieved completely and never split into - # separate newlines - '\r', '\n' due to coincidental readaheads. - # - match = self.PATTERN.search(readahead) - newline = match.group('newline') - if newline is not None: - if self.newlines is None: - self.newlines = [] - if newline not in self.newlines: - self.newlines.append(newline) - self._offset += len(newline) - return line + '\n' - - chunk = match.group('chunk') - if limit >= 0: - chunk = chunk[: limit - len(line)] - - self._offset += len(chunk) - line += chunk - - return line - - def peek(self, n=1): - """Returns buffered bytes without advancing the position.""" - if n > len(self._readbuffer) - self._offset: - chunk = self.read(n) - if len(chunk) > self._offset: - self._readbuffer = chunk + self._readbuffer[self._offset:] - self._offset = 0 - else: - self._offset -= len(chunk) - - # Return up to 512 bytes to reduce allocation overhead for tight loops. - return self._readbuffer[self._offset: self._offset + 512] - - def readable(self): - return True - - def read(self, n=-1): - """Read and return up to n bytes. - If the argument is omitted, None, or negative, data is read and returned until EOF is reached.. - """ - buf = '' - if n is None: - n = -1 - while True: - if n < 0: - data = self.read1(n) - elif n > len(buf): - data = self.read1(n - len(buf)) - else: - return buf - if len(data) == 0: - return buf - buf += data - - def _update_crc(self, newdata, eof): - # Update the CRC using the given data. - if self._expected_crc is None: - # No need to compute the CRC if we don't have a reference value - return - self._running_crc = crc32(newdata, self._running_crc) & 0xffffffff - # Check the CRC if we're at the end of the file - if eof and self._running_crc != self._expected_crc: - raise BadZipfile("Bad CRC-32 for file %r" % self.name) - - def read1(self, n): - """Read up to n bytes with at most one read() system call.""" - - # Simplify algorithm (branching) by transforming negative n to large n. - if n < 0 or n is None: - n = self.MAX_N - - # Bytes available in read buffer. - len_readbuffer = len(self._readbuffer) - self._offset - - # Read from file. - if self._compress_left > 0 and n > len_readbuffer + len(self._unconsumed): - nbytes = n - len_readbuffer - len(self._unconsumed) - nbytes = max(nbytes, self.MIN_READ_SIZE) - nbytes = min(nbytes, self._compress_left) - - data = self._fileobj.read(nbytes) - self._compress_left -= len(data) - - if data and self._decrypter is not None: - data = ''.join(map(self._decrypter, data)) - - if self._compress_type == ZIP_STORED: - self._update_crc(data, eof=(self._compress_left==0)) - self._readbuffer = self._readbuffer[self._offset:] + data - self._offset = 0 - else: - # Prepare deflated bytes for decompression. - self._unconsumed += data - - # Handle unconsumed data. - if (len(self._unconsumed) > 0 and n > len_readbuffer and - self._compress_type == ZIP_DEFLATED): - data = self._decompressor.decompress( - self._unconsumed, - max(n - len_readbuffer, self.MIN_READ_SIZE) - ) - - self._unconsumed = self._decompressor.unconsumed_tail - eof = len(self._unconsumed) == 0 and self._compress_left == 0 - if eof: - data += self._decompressor.flush() - - self._update_crc(data, eof=eof) - self._readbuffer = self._readbuffer[self._offset:] + data - self._offset = 0 - - # Read from buffer. - data = self._readbuffer[self._offset: self._offset + n] - self._offset += len(data) - return data - - def close(self): - try : - if self._close_fileobj: - self._fileobj.close() - finally: - super(ZipExtFile, self).close() - - -class ZipFile(object): - """ Class with methods to open, read, write, close, list zip files. - - z = ZipFile(file, mode="r", compression=ZIP_STORED, allowZip64=False) - - file: Either the path to the file, or a file-like object. - If it is a path, the file will be opened and closed by ZipFile. - mode: The mode can be either read "r", write "w" or append "a". - compression: ZIP_STORED (no compression) or ZIP_DEFLATED (requires zlib). - allowZip64: if True ZipFile will create files with ZIP64 extensions when - needed, otherwise it will raise an exception when this would - be necessary. - - """ - - fp = None # Set here since __del__ checks it - - def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=False): - """Open the ZIP file with mode read "r", write "w" or append "a".""" - if mode not in ("r", "w", "a"): - raise RuntimeError('ZipFile() requires mode "r", "w", or "a"') - - if compression == ZIP_STORED: - pass - elif compression == ZIP_DEFLATED: - if not zlib: - raise RuntimeError,\ - "Compression requires the (missing) zlib module" - else: - raise RuntimeError, "That compression method is not supported" - - self._allowZip64 = allowZip64 - self._didModify = False - self.debug = 0 # Level of printing: 0 through 3 - self.NameToInfo = {} # Find file info given name - self.filelist = [] # List of ZipInfo instances for archive - self.compression = compression # Method of compression - self.mode = key = mode.replace('b', '')[0] - self.pwd = None - self._comment = '' - - # Check if we were passed a file-like object - if isinstance(file, basestring): - self._filePassed = 0 - self.filename = file - modeDict = {'r' : 'rb', 'w': 'wb', 'a' : 'r+b'} - try: - self.fp = open(file, modeDict[mode]) - except IOError: - if mode == 'a': - mode = key = 'w' - self.fp = open(file, modeDict[mode]) - else: - raise - else: - self._filePassed = 1 - self.fp = file - self.filename = getattr(file, 'name', None) - - try: - if key == 'r': - self._RealGetContents() - elif key == 'w': - # set the modified flag so central directory gets written - # even if no files are added to the archive - self._didModify = True - self._start_disk = self.fp.tell() - elif key == 'a': - try: - # See if file is a zip file - self._RealGetContents() - # seek to start of directory and overwrite - self.fp.seek(self.start_dir, 0) - except BadZipfile: - # file is not a zip file, just append - self.fp.seek(0, 2) - - # set the modified flag so central directory gets written - # even if no files are added to the archive - self._didModify = True - self._start_disk = self.fp.tell() - else: - raise RuntimeError('Mode must be "r", "w" or "a"') - except: - fp = self.fp - self.fp = None - if not self._filePassed: - fp.close() - raise - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - def _RealGetContents(self): - """Read in the table of contents for the ZIP file.""" - fp = self.fp - try: - endrec = _EndRecData(fp) - except IOError: - raise BadZipfile("File is not a zip file") - if not endrec: - raise BadZipfile, "File is not a zip file" - if self.debug > 1: - print endrec - size_cd = endrec[_ECD_SIZE] # bytes in central directory - offset_cd = endrec[_ECD_OFFSET] # offset of central directory - self._comment = endrec[_ECD_COMMENT] # archive comment - - # self._start_disk: Position of the start of ZIP archive - # It is zero, unless ZIP was concatenated to another file - self._start_disk = endrec[_ECD_LOCATION] - size_cd - offset_cd - if endrec[_ECD_SIGNATURE] == stringEndArchive64: - # If Zip64 extension structures are present, account for them - self._start_disk -= (sizeEndCentDir64 + sizeEndCentDir64Locator) - - if self.debug > 2: - inferred = self._start_disk + offset_cd - print "given, inferred, offset", offset_cd, inferred, self._start_disk - # self.start_dir: Position of start of central directory - self.start_dir = offset_cd + self._start_disk - fp.seek(self.start_dir, 0) - data = fp.read(size_cd) - fp = cStringIO.StringIO(data) - total = 0 - while total < size_cd: - centdir = fp.read(sizeCentralDir) - if len(centdir) != sizeCentralDir: - raise BadZipfile("Truncated central directory") - centdir = struct.unpack(structCentralDir, centdir) - if centdir[_CD_SIGNATURE] != stringCentralDir: - raise BadZipfile("Bad magic number for central directory") - if self.debug > 2: - print centdir - filename = fp.read(centdir[_CD_FILENAME_LENGTH]) - # Create ZipInfo instance to store file information - x = ZipInfo(filename) - x.extra = fp.read(centdir[_CD_EXTRA_FIELD_LENGTH]) - x.comment = fp.read(centdir[_CD_COMMENT_LENGTH]) - x.header_offset = centdir[_CD_LOCAL_HEADER_OFFSET] - (x.create_version, x.create_system, x.extract_version, x.reserved, - x.flag_bits, x.compress_type, t, d, - x.CRC, x.compress_size, x.file_size) = centdir[1:12] - x.volume, x.internal_attr, x.external_attr = centdir[15:18] - # Convert date/time code to (year, month, day, hour, min, sec) - x._raw_time = t - x.date_time = ( (d>>9)+1980, (d>>5)&0xF, d&0x1F, - t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) - - x._decodeExtra() - x.header_offset = x.header_offset + self._start_disk - x.filename = x._decodeFilename() - self.filelist.append(x) - self.NameToInfo[x.filename] = x - - # update total bytes read from central directory - total = (total + sizeCentralDir + centdir[_CD_FILENAME_LENGTH] - + centdir[_CD_EXTRA_FIELD_LENGTH] - + centdir[_CD_COMMENT_LENGTH]) - - if self.debug > 2: - print "total", total - - - def namelist(self): - """Return a list of file names in the archive.""" - l = [] - for data in self.filelist: - l.append(data.filename) - return l - - def infolist(self): - """Return a list of class ZipInfo instances for files in the - archive.""" - return self.filelist - - def printdir(self): - """Print a table of contents for the zip file.""" - print "%-46s %19s %12s" % ("File Name", "Modified ", "Size") - for zinfo in self.filelist: - date = "%d-%02d-%02d %02d:%02d:%02d" % zinfo.date_time[:6] - print "%-46s %s %12d" % (zinfo.filename, date, zinfo.file_size) - - def testzip(self): - """Read all the files and check the CRC.""" - chunk_size = 2 ** 20 - for zinfo in self.filelist: - try: - # Read by chunks, to avoid an OverflowError or a - # MemoryError with very large embedded files. - with self.open(zinfo.filename, "r") as f: - while f.read(chunk_size): # Check CRC-32 - pass - except BadZipfile: - return zinfo.filename - - def getinfo(self, name): - """Return the instance of ZipInfo given 'name'.""" - info = self.NameToInfo.get(name) - if info is None: - raise KeyError( - 'There is no item named %r in the archive' % name) - - return info - - def setpassword(self, pwd): - """Set default password for encrypted files.""" - self.pwd = pwd - - @property - def comment(self): - """The comment text associated with the ZIP file.""" - return self._comment - - @comment.setter - def comment(self, comment): - # check for valid comment length - if len(comment) > ZIP_MAX_COMMENT: - import warnings - warnings.warn('Archive comment is too long; truncating to %d bytes' - % ZIP_MAX_COMMENT, stacklevel=2) - comment = comment[:ZIP_MAX_COMMENT] - self._comment = comment - self._didModify = True - - def read(self, name, pwd=None): - """Return file bytes (as a string) for name.""" - return self.open(name, "r", pwd).read() - - def open(self, name, mode="r", pwd=None): - """Return file-like object for 'name'.""" - if mode not in ("r", "U", "rU"): - raise RuntimeError, 'open() requires mode "r", "U", or "rU"' - if not self.fp: - raise RuntimeError, \ - "Attempt to read ZIP archive that was already closed" - - # Only open a new file for instances where we were not - # given a file object in the constructor - if self._filePassed: - zef_file = self.fp - should_close = False - else: - zef_file = open(self.filename, 'rb') - should_close = True - - try: - # Make sure we have an info object - if isinstance(name, ZipInfo): - # 'name' is already an info object - zinfo = name - else: - # Get info object for name - zinfo = self.getinfo(name) - - zef_file.seek(zinfo.header_offset, 0) - - # Skip the file header: - fheader = zef_file.read(sizeFileHeader) - if len(fheader) != sizeFileHeader: - raise BadZipfile("Truncated file header") - fheader = struct.unpack(structFileHeader, fheader) - if fheader[_FH_SIGNATURE] != stringFileHeader: - raise BadZipfile("Bad magic number for file header") - - fname = zef_file.read(fheader[_FH_FILENAME_LENGTH]) - if fheader[_FH_EXTRA_FIELD_LENGTH]: - zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH]) - - if fname != zinfo.orig_filename: - raise BadZipfile, \ - 'File name in directory "%s" and header "%s" differ.' % ( - zinfo.orig_filename, fname) - - # check for encrypted flag & handle password - is_encrypted = zinfo.flag_bits & 0x1 - zd = None - if is_encrypted: - if not pwd: - pwd = self.pwd - if not pwd: - raise RuntimeError, "File %s is encrypted, " \ - "password required for extraction" % name - - zd = _ZipDecrypter(pwd) - # The first 12 bytes in the cypher stream is an encryption header - # used to strengthen the algorithm. The first 11 bytes are - # completely random, while the 12th contains the MSB of the CRC, - # or the MSB of the file time depending on the header type - # and is used to check the correctness of the password. - bytes = zef_file.read(12) - h = map(zd, bytes[0:12]) - if zinfo.flag_bits & 0x8: - # compare against the file type from extended local headers - check_byte = (zinfo._raw_time >> 8) & 0xff - else: - # compare against the CRC otherwise - check_byte = (zinfo.CRC >> 24) & 0xff - if ord(h[11]) != check_byte: - raise RuntimeError("Bad password for file", name) - - return ZipExtFile(zef_file, mode, zinfo, zd, - close_fileobj=should_close) - except: - if should_close: - zef_file.close() - raise - - def extract(self, member, path=None, pwd=None): - """Extract a member from the archive to the current working directory, - using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a ZipInfo object. You can - specify a different directory using `path'. - """ - if not isinstance(member, ZipInfo): - member = self.getinfo(member) - - if path is None: - path = os.getcwd() - - return self._extract_member(member, path, pwd) - - def extractall(self, path=None, members=None, pwd=None): - """Extract all members from the archive to the current working - directory. `path' specifies a different directory to extract to. - `members' is optional and must be a subset of the list returned - by namelist(). - """ - if members is None: - members = self.namelist() - - for zipinfo in members: - self.extract(zipinfo, path, pwd) - - def _extract_member(self, member, targetpath, pwd): - """Extract the ZipInfo object 'member' to a physical - file on the path targetpath. - """ - # build the destination pathname, replacing - # forward slashes to platform specific separators. - arcname = member.filename.replace('/', os.path.sep) - - if os.path.altsep: - arcname = arcname.replace(os.path.altsep, os.path.sep) - # interpret absolute pathname as relative, remove drive letter or - # UNC path, redundant separators, "." and ".." components. - arcname = os.path.splitdrive(arcname)[1] - arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) - if x not in ('', os.path.curdir, os.path.pardir)) - if os.path.sep == '\\': - # filter illegal characters on Windows - illegal = ':<>|"?*' - if isinstance(arcname, unicode): - table = {ord(c): ord('_') for c in illegal} - else: - table = string.maketrans(illegal, '_' * len(illegal)) - arcname = arcname.translate(table) - # remove trailing dots - arcname = (x.rstrip('.') for x in arcname.split(os.path.sep)) - arcname = os.path.sep.join(x for x in arcname if x) - - targetpath = os.path.join(targetpath, arcname) - targetpath = os.path.normpath(targetpath) - - # Create all upper directories if necessary. - upperdirs = os.path.dirname(targetpath) - if upperdirs and not os.path.exists(upperdirs): - os.makedirs(upperdirs) - - if member.filename[-1] == '/': - if not os.path.isdir(targetpath): - os.mkdir(targetpath) - return targetpath - - with self.open(member, pwd=pwd) as source, \ - file(targetpath, "wb") as target: - shutil.copyfileobj(source, target) - - return targetpath - - def _writecheck(self, zinfo): - """Check for errors before writing a file to the archive.""" - if zinfo.filename in self.NameToInfo: - import warnings - warnings.warn('Duplicate name: %r' % zinfo.filename, stacklevel=3) - if self.mode not in ("w", "a"): - raise RuntimeError, 'write() requires mode "w" or "a"' - if not self.fp: - raise RuntimeError, \ - "Attempt to write ZIP archive that was already closed" - if zinfo.compress_type == ZIP_DEFLATED and not zlib: - raise RuntimeError, \ - "Compression requires the (missing) zlib module" - if zinfo.compress_type not in (ZIP_STORED, ZIP_DEFLATED): - raise RuntimeError, \ - "That compression method is not supported" - if not self._allowZip64: - requires_zip64 = None - if len(self.filelist) >= ZIP_FILECOUNT_LIMIT: - requires_zip64 = "Files count" - elif zinfo.file_size > ZIP64_LIMIT: - requires_zip64 = "Filesize" - elif zinfo.header_offset > ZIP64_LIMIT: - requires_zip64 = "Zipfile size" - if requires_zip64: - raise LargeZipFile(requires_zip64 + - " would require ZIP64 extensions") - - def write(self, filename, arcname=None, compress_type=None): - """Put the bytes from filename into the archive under the name - arcname.""" - if not self.fp: - raise RuntimeError( - "Attempt to write to ZIP archive that was already closed") - - st = os.stat(filename) - isdir = stat.S_ISDIR(st.st_mode) - mtime = time.localtime(st.st_mtime) - date_time = mtime[0:6] - # Create ZipInfo instance to store file information - if arcname is None: - arcname = filename - arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) - while arcname[0] in (os.sep, os.altsep): - arcname = arcname[1:] - if isdir: - arcname += '/' - zinfo = ZipInfo(arcname, date_time) - zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes - if isdir: - zinfo.compress_type = ZIP_STORED - elif compress_type is None: - zinfo.compress_type = self.compression - else: - zinfo.compress_type = compress_type - - zinfo.file_size = st.st_size - zinfo.flag_bits = 0x00 - zinfo.header_offset = self.fp.tell() # Start of header bytes - - self._writecheck(zinfo) - self._didModify = True - - if isdir: - zinfo.file_size = 0 - zinfo.compress_size = 0 - zinfo.CRC = 0 - zinfo.external_attr |= 0x10 # MS-DOS directory flag - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo - self.fp.write(zinfo.FileHeader(False)) - return - - with open(filename, "rb") as fp: - # Must overwrite CRC and sizes with correct data later - zinfo.CRC = CRC = 0 - zinfo.compress_size = compress_size = 0 - # Compressed size can be larger than uncompressed size - zip64 = self._allowZip64 and \ - zinfo.file_size * 1.05 > ZIP64_LIMIT - self.fp.write(zinfo.FileHeader(zip64)) - if zinfo.compress_type == ZIP_DEFLATED: - cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, - zlib.DEFLATED, -15) - else: - cmpr = None - file_size = 0 - while 1: - buf = fp.read(1024 * 8) - if not buf: - break - file_size = file_size + len(buf) - CRC = crc32(buf, CRC) & 0xffffffff - if cmpr: - buf = cmpr.compress(buf) - compress_size = compress_size + len(buf) - self.fp.write(buf) - if cmpr: - buf = cmpr.flush() - compress_size = compress_size + len(buf) - self.fp.write(buf) - zinfo.compress_size = compress_size - else: - zinfo.compress_size = file_size - zinfo.CRC = CRC - zinfo.file_size = file_size - if not zip64 and self._allowZip64: - if file_size > ZIP64_LIMIT: - raise RuntimeError('File size has increased during compressing') - if compress_size > ZIP64_LIMIT: - raise RuntimeError('Compressed size larger than uncompressed size') - # Seek backwards and write file header (which will now include - # correct CRC and file sizes) - position = self.fp.tell() # Preserve current position in file - self.fp.seek(zinfo.header_offset, 0) - self.fp.write(zinfo.FileHeader(zip64)) - self.fp.seek(position, 0) - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo - - def writestr(self, zinfo_or_arcname, bytes, compress_type=None): - """Write a file into the archive. The contents is the string - 'bytes'. 'zinfo_or_arcname' is either a ZipInfo instance or - the name of the file in the archive.""" - if not isinstance(zinfo_or_arcname, ZipInfo): - zinfo = ZipInfo(filename=zinfo_or_arcname, - date_time=time.localtime(time.time())[:6]) - - zinfo.compress_type = self.compression - if zinfo.filename[-1] == '/': - zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x - zinfo.external_attr |= 0x10 # MS-DOS directory flag - else: - zinfo.external_attr = 0o600 << 16 # ?rw------- - else: - zinfo = zinfo_or_arcname - - if not self.fp: - raise RuntimeError( - "Attempt to write to ZIP archive that was already closed") - - if compress_type is not None: - zinfo.compress_type = compress_type - - zinfo.file_size = len(bytes) # Uncompressed size - zinfo.header_offset = self.fp.tell() # Start of header bytes - self._writecheck(zinfo) - self._didModify = True - zinfo.CRC = crc32(bytes) & 0xffffffff # CRC-32 checksum - if zinfo.compress_type == ZIP_DEFLATED: - co = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, - zlib.DEFLATED, -15) - bytes = co.compress(bytes) + co.flush() - zinfo.compress_size = len(bytes) # Compressed size - else: - zinfo.compress_size = zinfo.file_size - zip64 = zinfo.file_size > ZIP64_LIMIT or \ - zinfo.compress_size > ZIP64_LIMIT - if zip64 and not self._allowZip64: - raise LargeZipFile("Filesize would require ZIP64 extensions") - self.fp.write(zinfo.FileHeader(zip64)) - self.fp.write(bytes) - if zinfo.flag_bits & 0x08: - # Write CRC and file sizes after the file data - fmt = '<LQQ' if zip64 else '<LLL' - self.fp.write(struct.pack(fmt, zinfo.CRC, zinfo.compress_size, - zinfo.file_size)) - self.fp.flush() - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo - - def __del__(self): - """Call the "close()" method in case the user forgot.""" - self.close() - - def close(self): - """Close the file, and for mode "w" and "a" write the ending - records.""" - if self.fp is None: - return - - try: - if self.mode in ("w", "a") and self._didModify: # write ending records - pos1 = self.fp.tell() - for zinfo in self.filelist: # write central directory - dt = zinfo.date_time - dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - extra = [] - if zinfo.file_size > ZIP64_LIMIT \ - or zinfo.compress_size > ZIP64_LIMIT: - extra.append(zinfo.file_size) - extra.append(zinfo.compress_size) - file_size = 0xffffffff - compress_size = 0xffffffff - else: - file_size = zinfo.file_size - compress_size = zinfo.compress_size - - header_offset = zinfo.header_offset - self._start_disk - if header_offset > ZIP64_LIMIT: - extra.append(header_offset) - header_offset = 0xffffffffL - - extra_data = zinfo.extra - if extra: - # Append a ZIP64 field to the extra's - extra_data = struct.pack( - '<HH' + 'Q'*len(extra), - 1, 8*len(extra), *extra) + extra_data - - extract_version = max(45, zinfo.extract_version) - create_version = max(45, zinfo.create_version) - else: - extract_version = zinfo.extract_version - create_version = zinfo.create_version - - try: - filename, flag_bits = zinfo._encodeFilenameFlags() - centdir = struct.pack(structCentralDir, - stringCentralDir, create_version, - zinfo.create_system, extract_version, zinfo.reserved, - flag_bits, zinfo.compress_type, dostime, dosdate, - zinfo.CRC, compress_size, file_size, - len(filename), len(extra_data), len(zinfo.comment), - 0, zinfo.internal_attr, zinfo.external_attr, - header_offset) - except DeprecationWarning: - print >>sys.stderr, (structCentralDir, - stringCentralDir, create_version, - zinfo.create_system, extract_version, zinfo.reserved, - zinfo.flag_bits, zinfo.compress_type, dostime, dosdate, - zinfo.CRC, compress_size, file_size, - len(zinfo.filename), len(extra_data), len(zinfo.comment), - 0, zinfo.internal_attr, zinfo.external_attr, - header_offset) - raise - self.fp.write(centdir) - self.fp.write(filename) - self.fp.write(extra_data) - self.fp.write(zinfo.comment) - - pos2 = self.fp.tell() - # Write end-of-zip-archive record - centDirCount = len(self.filelist) - centDirSize = pos2 - pos1 - centDirOffset = pos1 - self._start_disk - requires_zip64 = None - if centDirCount > ZIP_FILECOUNT_LIMIT: - requires_zip64 = "Files count" - elif centDirOffset > ZIP64_LIMIT: - requires_zip64 = "Central directory offset" - elif centDirSize > ZIP64_LIMIT: - requires_zip64 = "Central directory size" - if requires_zip64: - # Need to write the ZIP64 end-of-archive records - if not self._allowZip64: - raise LargeZipFile(requires_zip64 + - " would require ZIP64 extensions") - zip64endrec = struct.pack( - structEndArchive64, stringEndArchive64, - 44, 45, 45, 0, 0, centDirCount, centDirCount, - centDirSize, centDirOffset) - self.fp.write(zip64endrec) - - zip64locrec = struct.pack( - structEndArchive64Locator, - stringEndArchive64Locator, 0, pos2, 1) - self.fp.write(zip64locrec) - centDirCount = min(centDirCount, 0xFFFF) - centDirSize = min(centDirSize, 0xFFFFFFFF) - centDirOffset = min(centDirOffset, 0xFFFFFFFF) - - endrec = struct.pack(structEndArchive, stringEndArchive, - 0, 0, centDirCount, centDirCount, - centDirSize, centDirOffset, len(self._comment)) - self.fp.write(endrec) - self.fp.write(self._comment) - self.fp.flush() - finally: - fp = self.fp - self.fp = None - if not self._filePassed: - fp.close() - - -class PyZipFile(ZipFile): - """Class to create ZIP archives with Python library files and packages.""" - - def writepy(self, pathname, basename = ""): - """Add all files from "pathname" to the ZIP archive. - - If pathname is a package directory, search the directory and - all package subdirectories recursively for all *.py and enter - the modules into the archive. If pathname is a plain - directory, listdir *.py and enter all modules. Else, pathname - must be a Python *.py file and the module will be put into the - archive. Added modules are always module.pyo or module.pyc. - This method will compile the module.py into module.pyc if - necessary. - """ - dir, name = os.path.split(pathname) - if os.path.isdir(pathname): - initname = os.path.join(pathname, "__init__.py") - if os.path.isfile(initname): - # This is a package directory, add it - if basename: - basename = "%s/%s" % (basename, name) - else: - basename = name - if self.debug: - print "Adding package in", pathname, "as", basename - fname, arcname = self._get_codename(initname[0:-3], basename) - if self.debug: - print "Adding", arcname - self.write(fname, arcname) - dirlist = os.listdir(pathname) - dirlist.remove("__init__.py") - # Add all *.py files and package subdirectories - for filename in dirlist: - path = os.path.join(pathname, filename) - root, ext = os.path.splitext(filename) - if os.path.isdir(path): - if os.path.isfile(os.path.join(path, "__init__.py")): - # This is a package directory, add it - self.writepy(path, basename) # Recursive call - elif ext == ".py": - fname, arcname = self._get_codename(path[0:-3], - basename) - if self.debug: - print "Adding", arcname - self.write(fname, arcname) - else: - # This is NOT a package directory, add its files at top level - if self.debug: - print "Adding files from directory", pathname - for filename in os.listdir(pathname): - path = os.path.join(pathname, filename) - root, ext = os.path.splitext(filename) - if ext == ".py": - fname, arcname = self._get_codename(path[0:-3], - basename) - if self.debug: - print "Adding", arcname - self.write(fname, arcname) - else: - if pathname[-3:] != ".py": - raise RuntimeError, \ - 'Files added with writepy() must end with ".py"' - fname, arcname = self._get_codename(pathname[0:-3], basename) - if self.debug: - print "Adding file", arcname - self.write(fname, arcname) - - def _get_codename(self, pathname, basename): - """Return (filename, archivename) for the path. - - Given a module name path, return the correct file path and - archive name, compiling if necessary. For example, given - /python/lib/string, return (/python/lib/string.pyc, string). - """ - file_py = pathname + ".py" - file_pyc = pathname + ".pyc" - file_pyo = pathname + ".pyo" - if os.path.isfile(file_pyo) and \ - os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime: - fname = file_pyo # Use .pyo file - elif not os.path.isfile(file_pyc) or \ - os.stat(file_pyc).st_mtime < os.stat(file_py).st_mtime: - import py_compile - if self.debug: - print "Compiling", file_py - try: - py_compile.compile(file_py, file_pyc, None, True) - except py_compile.PyCompileError,err: - print err.msg - fname = file_pyc - else: - fname = file_pyc - archivename = os.path.split(fname)[1] - if basename: - archivename = "%s/%s" % (basename, archivename) - return (fname, archivename) - - -def main(args = None): - import textwrap - USAGE=textwrap.dedent("""\ - Usage: - zipfile.py -l zipfile.zip # Show listing of a zipfile - zipfile.py -t zipfile.zip # Test if a zipfile is valid - zipfile.py -e zipfile.zip target # Extract zipfile into target dir - zipfile.py -c zipfile.zip src ... # Create zipfile from sources - """) - if args is None: - args = sys.argv[1:] - - if not args or args[0] not in ('-l', '-c', '-e', '-t'): - print USAGE - sys.exit(1) - - if args[0] == '-l': - if len(args) != 2: - print USAGE - sys.exit(1) - with ZipFile(args[1], 'r') as zf: - zf.printdir() - - elif args[0] == '-t': - if len(args) != 2: - print USAGE - sys.exit(1) - with ZipFile(args[1], 'r') as zf: - badfile = zf.testzip() - if badfile: - print("The following enclosed file is corrupted: {!r}".format(badfile)) - print "Done testing" - - elif args[0] == '-e': - if len(args) != 3: - print USAGE - sys.exit(1) - - with ZipFile(args[1], 'r') as zf: - zf.extractall(args[2]) - - elif args[0] == '-c': - if len(args) < 3: - print USAGE - sys.exit(1) - - def addToZip(zf, path, zippath): - if os.path.isfile(path): - zf.write(path, zippath, ZIP_DEFLATED) - elif os.path.isdir(path): - if zippath: - zf.write(path, zippath) - for nm in os.listdir(path): - addToZip(zf, - os.path.join(path, nm), os.path.join(zippath, nm)) - # else: ignore - - with ZipFile(args[1], 'w', allowZip64=True) as zf: - for path in args[2:]: - zippath = os.path.basename(path) - if not zippath: - zippath = os.path.basename(os.path.dirname(path)) - if zippath in ('', os.curdir, os.pardir): - zippath = '' - addToZip(zf, path, zippath) - -if __name__ == "__main__": - main() |