diff options
| author | Hridya Valsaraju <hridya@google.com> | 2021-04-15 15:57:00 -0700 |
|---|---|---|
| committer | Hridya Valsaraju <hridya@google.com> | 2021-04-15 15:57:00 -0700 |
| commit | 4537bffda5f89155f2f27afc7c80e4c0354c1c20 (patch) | |
| tree | b9a6b60023b853553ae095370bf251a2f1f7640e | |
| parent | 25821f7b195e31b5b957ddc720a6bc12700362a0 (diff) | |
| download | chromium-trace-4537bffda5f89155f2f27afc7c80e4c0354c1c20.tar.gz | |
Update catapult to latest version(ab9d330fe2)
git log --oneline --no-merges 91735e2e6..ab9d330fe2a3 tracing systrace
ab9d330fe systrace: check if tracefs in mounted in /sys/kernel/tracing
361ae742a systrace: fix tracefs device path detection
22a35ea6a [tracing] Avoid very large vulcanized strings in histogram_viewer.html
494a8eb59 [tracing] Add blink-resource metric
a297a0839 Reland "Remove memoryAblationMetric"
b0fb5822d Used undefined bins as bins of count zero
d229e77e8 Check the bins to be defined before using them in UMA metrics calculations
de74236cf PCScan: Add :clear and :overall metrics for mutator
e46359da4 Move metadata in OWNERS files to DIR_METADATA files
999f35f30 Revert "Remove memoryAblationMetric"
7caeed04d Remove memoryAblationMetric
b965b659c Added metrics for percentage of inTime/delayed/dropped frames
e5299ca5b Fix crash while processing memory dumps
4f4de5e16 PCScan: Update metric to reflect scope contexts
2528f75cb Fix drill down in heap dump pane
1f5037810 trace-viewer: Don't let optional features block import
d2aa56920 Reland "Enable the new GC metrics""
e6e7c9314 Revert "Enable the new GC metrics"
9e0992ffc Enable the new GC metrics
79a90eb5e Include missing header
138dc4ba2 Calculate Overall CLS including all renderer processes.
c8f9f36bb Add a generic count/sum metric.
73a0597ed Add overallCumulativeLayoutShift metric that includes weighted scores for iframes.
Test: systrace
Bug: 185400108
Change-Id: I195c394b78094c4385b308ab6abf1cf57009569b
62 files changed, 756 insertions, 2050 deletions
diff --git a/UPSTREAM_REVISION b/UPSTREAM_REVISION index ab641bd2..7988ceb2 100644 --- a/UPSTREAM_REVISION +++ b/UPSTREAM_REVISION @@ -1 +1 @@ -91735e2e6775c098eb32840a8903e5a9111fad77 +ab9d330fe2a32f84b4b5fe141958c0a0a857c5c9 diff --git a/catapult/common/py_utils/py_utils/ts_proxy_server.py b/catapult/common/py_utils/py_utils/ts_proxy_server.py index 0e168a53..ffed090f 100644 --- a/catapult/common/py_utils/py_utils/ts_proxy_server.py +++ b/catapult/common/py_utils/py_utils/ts_proxy_server.py @@ -194,6 +194,7 @@ class TsProxyServer(object): if not self._proc: return try: + self._IssueCommand('exit', timeout=10) py_utils.WaitFor(lambda: self._proc.poll() is not None, 10) except py_utils.TimeoutException: # signal.SIGINT is not supported on Windows. diff --git a/catapult/dependency_manager/dependency_manager/__init__.py b/catapult/dependency_manager/dependency_manager/__init__.py index 3b18f06a..4b595c5b 100644 --- a/catapult/dependency_manager/dependency_manager/__init__.py +++ b/catapult/dependency_manager/dependency_manager/__init__.py @@ -20,8 +20,8 @@ def _AddDirToPythonPath(*path_parts): _AddDirToPythonPath(CATAPULT_PATH, 'common', 'py_utils') _AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'mock') +_AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'six') _AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'pyfakefs') -_AddDirToPythonPath(CATAPULT_THIRD_PARTY_PATH, 'zipfile') _AddDirToPythonPath(DEPENDENCY_MANAGER_PATH) diff --git a/catapult/dependency_manager/dependency_manager/dependency_manager_util.py b/catapult/dependency_manager/dependency_manager/dependency_manager_util.py index ca0174e0..a8e21b81 100644 --- a/catapult/dependency_manager/dependency_manager/dependency_manager_util.py +++ b/catapult/dependency_manager/dependency_manager/dependency_manager_util.py @@ -7,7 +7,9 @@ import shutil import stat import subprocess import sys -import zipfile_2_7_13 as zipfile +import zipfile + +import six from dependency_manager import exceptions @@ -17,7 +19,7 @@ def _WinReadOnlyHandler(func, path, execinfo): os.chmod(path, stat.S_IWRITE) func(path) else: - raise execinfo[0], execinfo[1], execinfo[2] + six.reraise(*execinfo) def RemoveDir(dir_path): diff --git a/catapult/dependency_manager/dependency_manager/local_path_info.py b/catapult/dependency_manager/dependency_manager/local_path_info.py index 8ac0152f..56009662 100644 --- a/catapult/dependency_manager/dependency_manager/local_path_info.py +++ b/catapult/dependency_manager/dependency_manager/local_path_info.py @@ -4,6 +4,7 @@ import os +import six class LocalPathInfo(object): @@ -66,4 +67,4 @@ class LocalPathInfo(object): def _ParseLocalPaths(local_paths): if not local_paths: return [] - return [[e] if isinstance(e, basestring) else e for e in local_paths] + return [[e] if isinstance(e, six.string_types) else e for e in local_paths] diff --git a/catapult/devil/DIR_METADATA b/catapult/devil/DIR_METADATA new file mode 100644 index 00000000..7608b2f5 --- /dev/null +++ b/catapult/devil/DIR_METADATA @@ -0,0 +1,3 @@ +monorail { + component: "Test>Devil" +} diff --git a/catapult/devil/PRESUBMIT.py b/catapult/devil/PRESUBMIT.py index 0b49eb80..ec48dddf 100644 --- a/catapult/devil/PRESUBMIT.py +++ b/catapult/devil/PRESUBMIT.py @@ -30,14 +30,23 @@ def _RunUnitTests(input_api, output_api): output_api.PresubmitPromptWarning) return input_api.RunTests([ - input_api.Command( - name='devil/bin/run_py_tests', - cmd=[ - input_api.os_path.join(input_api.PresubmitLocalPath(), 'bin', - 'run_py_tests') - ], - kwargs={'env': test_env}, - message=message_type) + input_api.Command(name='devil/bin/run_py_tests', + cmd=[ + input_api.os_path.join( + input_api.PresubmitLocalPath(), 'bin', + 'run_py_tests') + ], + kwargs={'env': test_env}, + message=message_type), + input_api.Command(name='devil/bin/run_py3_tests', + cmd=[ + input_api.os_path.join( + input_api.PresubmitLocalPath(), 'bin', + 'run_py3_tests') + ], + kwargs={'env': test_env}, + message=message_type, + python3=True), ]) diff --git a/catapult/devil/bin/generate_md_docs b/catapult/devil/bin/generate_md_docs index d1dbf06f..4c6f0f91 100755 --- a/catapult/devil/bin/generate_md_docs +++ b/catapult/devil/bin/generate_md_docs @@ -8,7 +8,7 @@ import os import sys _DEVIL_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -_DEVIL_URL = ('https://github.com/catapult-project/catapult/blob/master/devil/') +_DEVIL_URL = ('https://chromium.googlesource.com/catapult.git/+/HEAD/devil/') sys.path.append(_DEVIL_PATH) from devil.utils import cmd_helper diff --git a/catapult/devil/bin/run_py3_tests b/catapult/devil/bin/run_py3_tests new file mode 100755 index 00000000..3250ff15 --- /dev/null +++ b/catapult/devil/bin/run_py3_tests @@ -0,0 +1,113 @@ +#!/usr/bin/env vpython3 +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Runs all python3-compatible tests in devil.""" + +import os +import sys +import unittest + +_DEVIL_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +_SIX_PATH = os.path.join(_DEVIL_PATH, '..', 'third_party', 'six') + +sys.path.append(_DEVIL_PATH) +sys.path.append(_SIX_PATH) +# Import compatible tests here by full module path +import devil.android.apk_helper_test +import devil.android.app_ui_test +import devil.android.battery_utils_test +import devil.android.cpu_temperature_test +import devil.android.decorators_test +import devil.android.device_denylist_test +import devil.android.device_errors_test +import devil.android.device_utils_test +import devil.android.fastboot_utils_test +import devil.android.flag_changer_test +import devil.android.logcat_monitor_test +import devil.android.md5sum_test +import devil.android.perf.perf_control_test +import devil.android.perf.surface_stats_collector_test +import devil.android.sdk.adb_wrapper_test +import devil.android.sdk.shared_prefs_test +import devil.android.tools.device_monitor_test +import devil.android.tools.script_common_test +import devil.android.tools.system_app_test +import devil.devil_env_test +import devil.utils.cmd_helper_test +import devil.utils.decorators_test +import devil.utils.find_usb_devices_test +import devil.utils.geometry_test +import devil.utils.lazy.weak_constant_test +import devil.utils.lsusb_test +import devil.utils.markdown_test +import devil.utils.mock_calls_test +import devil.utils.parallelizer_test +import devil.utils.reraiser_thread_unittest +import devil.utils.timeout_retry_unittest +import devil.utils.zip_utils_test + +PY3_COMPATIBLE_TESTS = [ + # Add full test module path here + devil.android.apk_helper_test, + devil.android.app_ui_test, + devil.android.battery_utils_test, + devil.android.cpu_temperature_test, + devil.android.decorators_test, + devil.android.device_denylist_test, + devil.android.device_errors_test, + devil.android.device_utils_test, + devil.android.fastboot_utils_test, + devil.android.flag_changer_test, + devil.android.logcat_monitor_test, + devil.android.md5sum_test, + devil.android.perf.perf_control_test, + devil.android.perf.surface_stats_collector_test, + devil.android.sdk.adb_wrapper_test, + devil.android.sdk.shared_prefs_test, + devil.android.tools.device_monitor_test, + devil.android.tools.script_common_test, + devil.android.tools.system_app_test, + devil.devil_env_test, + devil.utils.cmd_helper_test, + devil.utils.decorators_test, + devil.utils.find_usb_devices_test, + devil.utils.geometry_test, + devil.utils.lazy.weak_constant_test, + devil.utils.lsusb_test, + devil.utils.markdown_test, + devil.utils.mock_calls_test, + devil.utils.parallelizer_test, + devil.utils.reraiser_thread_unittest, + devil.utils.timeout_retry_unittest, + devil.utils.zip_utils_test, +] + + +def main(): + # TODO(crbug.com/1007101): Use six.PY2 directly once we're using six via + # vpython. + if sys.version_info[0] == 2: + print('Somehow running under python2.') + return 1 + + # Tests mock out internal details of methods, and the ANDROID_SERIAL can + # change which internal methods are called. Since tests don't actually use + # devices, it should be fine to delete the variable. + if 'ANDROID_SERIAL' in os.environ: + del os.environ['ANDROID_SERIAL'] + + # This does not use typ for now, as typ has vpython dependencies that haven't + # yet been updated for python3. + result = unittest.TextTestRunner().run(unittest.TestSuite( + unittest.defaultTestLoader.loadTestsFromModule(test_module) + for test_module in PY3_COMPATIBLE_TESTS + )) + + return 0 if result.wasSuccessful() else 1 + + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/catapult/devil/devil/android/apk_helper.py b/catapult/devil/devil/android/apk_helper.py index fdece072..4d723a56 100644 --- a/catapult/devil/devil/android/apk_helper.py +++ b/catapult/devil/devil/android/apk_helper.py @@ -11,6 +11,8 @@ import shutil import tempfile import zipfile +import six + from devil import base_error from devil.android.ndk import abis from devil.android.sdk import aapt @@ -69,7 +71,7 @@ def GetInstrumentationName(apk_path): def ToHelper(path_or_helper): """Creates an ApkHelper unless one is already given.""" - if not isinstance(path_or_helper, basestring): + if not isinstance(path_or_helper, six.string_types): return path_or_helper elif path_or_helper.endswith('.apk'): return ApkHelper(path_or_helper) @@ -86,7 +88,7 @@ def ToSplitHelper(path_or_helper, split_apks): if sorted(path_or_helper.split_apk_paths) != sorted(split_apks): raise ApkHelperError('Helper has different split APKs') return path_or_helper - elif (isinstance(path_or_helper, basestring) + elif (isinstance(path_or_helper, six.string_types) and path_or_helper.endswith('.apk')): return SplitApkHelper(path_or_helper, split_apks) diff --git a/catapult/devil/devil/android/app_ui.py b/catapult/devil/devil/android/app_ui.py index 399c2ee3..4f7af1d7 100644 --- a/catapult/devil/devil/android/app_ui.py +++ b/catapult/devil/devil/android/app_ui.py @@ -51,7 +51,7 @@ class _UiNode(object): A geometry.Rectangle instance. """ d = _RE_BOUNDS.match(self._GetAttribute('bounds')).groupdict() - return geometry.Rectangle.FromDict({k: int(v) for k, v in d.iteritems()}) + return geometry.Rectangle.FromDict({k: int(v) for k, v in d.items()}) def Tap(self, point=None, dp_units=False): """Send a tap event to the UI node. @@ -150,7 +150,7 @@ class _UiNode(object): and ':id/' not in resource_id): kwargs['resource_id'] = '%s:id/%s' % (self._package, resource_id) - criteria = [(k.replace('_', '-'), v) for k, v in kwargs.iteritems() + criteria = [(k.replace('_', '-'), v) for k, v in kwargs.items() if v is not None] if not criteria: raise TypeError('At least one search criteria should be specified') @@ -193,8 +193,12 @@ class AppUi(object): A UI node instance pointing to the root of the xml screenshot. """ with device_temp_file.DeviceTempFile(self._device.adb) as dtemp: - self._device.RunShellCommand(['uiautomator', 'dump', dtemp.name], - check_return=True) + output = self._device.RunShellCommand( + ['uiautomator', 'dump', dtemp.name], single_line=True, + check_return=True) + if output.startswith('ERROR:'): + raise RuntimeError( + 'uiautomator dump command returned error: {}'.format(output)) xml_node = element_tree.fromstring( self._device.ReadFile(dtemp.name, force_pull=True)) return _UiNode(self._device, xml_node, package=self._package) diff --git a/catapult/devil/devil/android/app_ui_test.py b/catapult/devil/devil/android/app_ui_test.py index 938fd408..50f00ca1 100644 --- a/catapult/devil/devil/android/app_ui_test.py +++ b/catapult/devil/devil/android/app_ui_test.py @@ -80,7 +80,7 @@ class UiAppTest(unittest.TestCase): def assertNodeHasAttribs(self, node, attr): # pylint: disable=protected-access - for key, value in attr.iteritems(): + for key, value in attr.items(): self.assertEquals(node._GetAttribute(key), value) def assertTappedOnceAt(self, x, y): diff --git a/catapult/devil/devil/android/battery_utils.py b/catapult/devil/devil/android/battery_utils.py index e8134d2b..d680f03f 100644 --- a/catapult/devil/devil/android/battery_utils.py +++ b/catapult/devil/devil/android/battery_utils.py @@ -300,7 +300,7 @@ class BatteryUtils(object): 'uid': uid, 'data': pwi_entries[uid] } - for p, uid in self._cache['uids'].iteritems() + for p, uid in self._cache['uids'].items() } return {'system_total': system_total, 'per_package': per_package} diff --git a/catapult/devil/devil/android/cpu_temperature_test.py b/catapult/devil/devil/android/cpu_temperature_test.py index 8d082bb9..47cc28a6 100644 --- a/catapult/devil/devil/android/cpu_temperature_test.py +++ b/catapult/devil/devil/android/cpu_temperature_test.py @@ -69,9 +69,11 @@ class CpuTemperatureGetThermalDeviceInformationTest(CpuTemperatureTest): 'cpu6': '/sys/class/thermal/thermal_zone17/temp', 'cpu7': '/sys/class/thermal/thermal_zone18/temp' } - self.assertEqual( - cmp(correct_information, - self.cpu_temp.GetDeviceInfoForTesting().get('cpu_temps')), 0) + + self.assertDictEqual( + correct_information, + self.cpu_temp.GetDeviceInfoForTesting().get('cpu_temps') + ) class CpuTemperatureIsSupportedTest(CpuTemperatureTest): diff --git a/catapult/devil/devil/android/decorators.py b/catapult/devil/devil/android/decorators.py index 0b3778aa..11d2494b 100644 --- a/catapult/devil/devil/android/decorators.py +++ b/catapult/devil/devil/android/decorators.py @@ -9,6 +9,8 @@ import functools import itertools import sys +import six + from devil.android import device_errors from devil.utils import cmd_helper from devil.utils import reraiser_thread @@ -56,14 +58,19 @@ def _TimeoutRetryWrapper(f, desc = '%s(%s)' % (f.__name__, ', '.join( itertools.chain( (str(a) for a in args), - ('%s=%s' % (k, str(v)) for k, v in kwargs.iteritems())))) + ('%s=%s' % (k, str(v)) for k, v in six.iteritems(kwargs))))) return timeout_retry.Run( impl, timeout, retries, desc=desc, retry_if_func=retry_if_func) except reraiser_thread.TimeoutError as e: - raise device_errors.CommandTimeoutError(str(e)), None, (sys.exc_info()[2]) + six.reraise( + device_errors.CommandTimeoutError, + device_errors.CommandTimeoutError(str(e)), + sys.exc_info()[2]) except cmd_helper.TimeoutError as e: - raise device_errors.CommandTimeoutError( - str(e), output=e.output), None, (sys.exc_info()[2]) + six.reraise( + device_errors.CommandTimeoutError, + device_errors.CommandTimeoutError(str(e), output=e.output), + sys.exc_info()[2]) return timeout_retry_wrapper diff --git a/catapult/devil/devil/android/device_errors.py b/catapult/devil/devil/android/device_errors.py index 6e710876..75bf7e3f 100644 --- a/catapult/devil/devil/android/device_errors.py +++ b/catapult/devil/devil/android/device_errors.py @@ -23,6 +23,8 @@ The class hierarchy for device exceptions is: """ +import six + from devil import base_error from devil.utils import cmd_helper from devil.utils import parallelizer @@ -158,7 +160,7 @@ class AdbShellCommandFailedError(AdbCommandFailedError): segments.append(' exit status: %s\n' % status) if output: segments.append(' output:\n') - if isinstance(output, basestring): + if isinstance(output, six.string_types): output_lines = output.splitlines() else: output_lines = output diff --git a/catapult/devil/devil/android/device_list.py b/catapult/devil/devil/android/device_list.py index cd631dbd..5fb586f6 100644 --- a/catapult/devil/devil/android/device_list.py +++ b/catapult/devil/devil/android/device_list.py @@ -7,6 +7,8 @@ import json import logging import os +import six + logger = logging.getLogger(__name__) @@ -26,7 +28,7 @@ def GetPersistentDeviceList(file_name): with open(file_name) as f: devices = json.load(f) if not isinstance(devices, list) or not all( - isinstance(d, basestring) for d in devices): + isinstance(d, six.string_types) for d in devices): logger.warning('Unrecognized device file format: %s', devices) return [] return [d for d in devices if d != '(error)'] diff --git a/catapult/devil/devil/android/device_utils.py b/catapult/devil/devil/android/device_utils.py index 7b7dad24..093bfc71 100644 --- a/catapult/devil/devil/android/device_utils.py +++ b/catapult/devil/devil/android/device_utils.py @@ -24,6 +24,8 @@ import time import threading import uuid +import six + from devil import base_error from devil import devil_env from devil.utils import cmd_helper @@ -105,6 +107,13 @@ _RESTART_ADBD_SCRIPT = """ restart & """ +_UNZIP_AND_CHMOD_SCRIPT = """ + {bin_dir}/unzip {zip_file} && (for dir in {dirs} + do + chmod -R 777 "$dir" || exit 1 + done) +""" + # Not all permissions can be set. _PERMISSIONS_DENYLIST_RE = re.compile('|'.join( fnmatch.translate(p) for p in [ @@ -131,6 +140,7 @@ _PERMISSIONS_DENYLIST_RE = re.compile('|'.join( 'android.permission.INTERNET', 'android.permission.KILL_BACKGROUND_PROCESSES', 'android.permission.MANAGE_ACCOUNTS', + 'android.permission.MANAGE_EXTERNAL_STORAGE', 'android.permission.MODIFY_AUDIO_SETTINGS', 'android.permission.NFC', 'android.permission.QUERY_ALL_PACKAGES', @@ -314,30 +324,6 @@ def GetAVDs(): return avds -@decorators.WithExplicitTimeoutAndRetries(_DEFAULT_TIMEOUT, _DEFAULT_RETRIES) -def RestartServer(): - """Restarts the adb server. - - Raises: - CommandFailedError if we fail to kill or restart the server. - """ - - def adb_killed(): - return not adb_wrapper.AdbWrapper.IsServerOnline() - - def adb_started(): - return adb_wrapper.AdbWrapper.IsServerOnline() - - adb_wrapper.AdbWrapper.KillServer() - if not timeout_retry.WaitFor(adb_killed, wait_period=1, max_tries=5): - # TODO(crbug.com/442319): Switch this to raise an exception if we - # figure out why sometimes not all adb servers on bots get killed. - logger.warning('Failed to kill adb server') - adb_wrapper.AdbWrapper.StartServer() - if not timeout_retry.WaitFor(adb_started, wait_period=1, max_tries=5): - raise device_errors.CommandFailedError('Failed to start adb server') - - def _ParseModeString(mode_str): """Parse a mode string, e.g. 'drwxrwxrwx', into a st_mode value. @@ -376,7 +362,7 @@ def _CreateAdbWrapper(device): def _FormatPartialOutputError(output): - lines = output.splitlines() if isinstance(output, basestring) else output + lines = output.splitlines() if isinstance(output, six.string_types) else output message = ['Partial output found:'] if len(lines) > 11: message.extend('- %s' % line for line in lines[:5]) @@ -468,7 +454,7 @@ class DeviceUtils(object): operation should be retried on failure if no explicit value is provided. """ self.adb = None - if isinstance(device, basestring): + if isinstance(device, six.string_types): self.adb = _CreateAdbWrapper(device) elif isinstance(device, adb_wrapper.AdbWrapper): self.adb = device @@ -1533,7 +1519,7 @@ class DeviceUtils(object): else: raise - if isinstance(cmd, basestring): + if isinstance(cmd, six.string_types): if not shell: # TODO(crbug.com/1029769): Make this an error instead. logger.warning( @@ -1543,7 +1529,7 @@ class DeviceUtils(object): else: cmd = ' '.join(cmd_helper.SingleQuote(s) for s in cmd) if env: - env = ' '.join(env_quote(k, v) for k, v in env.iteritems()) + env = ' '.join(env_quote(k, v) for k, v in env.items()) cmd = '%s %s' % (env, cmd) if cwd: cmd = 'cd %s && %s' % (cmd_helper.SingleQuote(cwd), cmd) @@ -1740,7 +1726,7 @@ class DeviceUtils(object): cmd.append('-w') if raw: cmd.append('-r') - for k, v in extras.iteritems(): + for k, v in extras.items(): cmd.extend(['-e', str(k), str(v)]) cmd.append(component) @@ -2223,13 +2209,19 @@ class DeviceUtils(object): self.adb, suffix='.zip') as device_temp: self.adb.Push(zip_path, device_temp.name) - quoted_dirs = ' '.join(cmd_helper.SingleQuote(d) for d in dirs) - self.RunShellCommand( - 'unzip %s&&chmod -R 777 %s' % (device_temp.name, quoted_dirs), - shell=True, - as_root=True, - env={'PATH': '%s:$PATH' % install_commands.BIN_DIR}, - check_return=True) + with device_temp_file.DeviceTempFile(self.adb, suffix='.sh') as script: + # Read dirs from temp file to avoid potential errors like + # "Argument list too long" (crbug.com/1174331) when the list + # is too long. + self.WriteFile( + script.name, + _UNZIP_AND_CHMOD_SCRIPT.format(bin_dir=install_commands.BIN_DIR, + zip_file=device_temp.name, + dirs=' '.join(dirs))) + + self.RunShellCommand(['source', script.name], + check_return=True, + as_root=True) return True @@ -2263,7 +2255,7 @@ class DeviceUtils(object): DeviceUnreachableError on missing device. """ paths = device_paths - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): paths = (paths, ) if not paths: return True @@ -2322,7 +2314,7 @@ class DeviceUtils(object): args.append('-f') if recursive: args.append('-r') - if isinstance(device_path, basestring): + if isinstance(device_path, six.string_types): args.append(device_path if not rename else _RenamePath(device_path)) else: args.extend( @@ -2596,7 +2588,7 @@ class DeviceUtils(object): """ entries = self._ParseLongLsOutput(device_path, as_root=as_root, **kwargs) for d in entries: - for key, value in d.items(): + for key, value in list(d.items()): if value is None: del d[key] # Remove missing fields. d['st_mode'] = _ParseModeString(d['st_mode']) @@ -2960,7 +2952,7 @@ class DeviceUtils(object): """ assert isinstance( property_name, - basestring), ("property_name is not a string: %r" % property_name) + six.string_types), ("property_name is not a string: %r" % property_name) if cache: # It takes ~120ms to query a single property, and ~130ms to query all @@ -3004,8 +2996,8 @@ class DeviceUtils(object): """ assert isinstance( property_name, - basestring), ("property_name is not a string: %r" % property_name) - assert isinstance(value, basestring), "value is not a string: %r" % value + six.string_types), ("property_name is not a string: %r" % property_name) + assert isinstance(value, six.string_types), "value is not a string: %r" % value self.RunShellCommand(['setprop', property_name, value], check_return=True) prop_cache = self._cache['getprop'] @@ -3084,18 +3076,19 @@ class DeviceUtils(object): Returns: A list of ProcessInfo tuples with |name|, |pid|, and |ppid| fields. """ + # pylint: disable=broad-except process_name = process_name or '' processes = [] for line in self._GetPsOutput(process_name): row = line.split() try: - row = {k: row[i] for k, i in _PS_COLUMNS.iteritems()} + row = {k: row[i] for k, i in _PS_COLUMNS.items()} if row['pid'] == 'PID' or process_name not in row['name']: # Skip over header and non-matching processes. continue row['pid'] = int(row['pid']) row['ppid'] = int(row['ppid']) - except StandardError: # e.g. IndexError, TypeError, ValueError. + except Exception: # e.g. IndexError, TypeError, ValueError. logging.warning('failed to parse ps line: %r', line) continue processes.append(ProcessInfo(**row)) @@ -3550,10 +3543,10 @@ class DeviceUtils(object): # When using a cache across script invokations, verify that apps have # not been uninstalled. self._cache['package_apk_paths_to_verify'] = set( - self._cache['package_apk_paths'].iterkeys()) + self._cache['package_apk_paths']) package_apk_checksums = obj.get('package_apk_checksums', {}) - for k, v in package_apk_checksums.iteritems(): + for k, v in package_apk_checksums.items(): package_apk_checksums[k] = set(v) self._cache['package_apk_checksums'] = package_apk_checksums device_path_checksums = obj.get('device_path_checksums', {}) @@ -3577,27 +3570,27 @@ class DeviceUtils(object): obj['package_apk_paths'] = self._cache['package_apk_paths'] obj['package_apk_checksums'] = self._cache['package_apk_checksums'] # JSON can't handle sets. - for k, v in obj['package_apk_checksums'].iteritems(): + for k, v in obj['package_apk_checksums'].items(): obj['package_apk_checksums'][k] = list(v) obj['device_path_checksums'] = self._cache['device_path_checksums'] return json.dumps(obj, separators=(',', ':')) @classmethod - def parallel(cls, devices, async=False): + def parallel(cls, devices, asyn=False): """Creates a Parallelizer to operate over the provided list of devices. Args: devices: A list of either DeviceUtils instances or objects from from which DeviceUtils instances can be constructed. If None, all attached devices will be used. - async: If true, returns a Parallelizer that runs operations + asyn: If true, returns a Parallelizer that runs operations asynchronously. Returns: A Parallelizer operating over |devices|. """ devices = [d if isinstance(d, cls) else cls(d) for d in devices] - if async: + if asyn: return parallelizer.Parallelizer(devices) else: return parallelizer.SyncParallelizer(devices) @@ -3710,7 +3703,7 @@ class DeviceUtils(object): else: reset_usb.reset_all_android_devices() - for attempt in xrange(retries + 1): + for attempt in range(retries + 1): try: return _get_devices() except device_errors.NoDevicesError: @@ -3728,7 +3721,7 @@ class DeviceUtils(object): 'No devices found. Will try again after restarting adb server ' 'and a short nap of %d s.', sleep_s) time.sleep(sleep_s) - RestartServer() + adb_wrapper.RestartServer() @decorators.WithTimeoutAndRetriesFromInstance() def RestartAdbd(self, timeout=None, retries=None): @@ -3745,6 +3738,17 @@ class DeviceUtils(object): if not permissions: return + # For Andorid-11(R), enable MANAGE_EXTERNAL_STORAGE for testing. + # See https://bit.ly/2MBjBIM for details. + if ('android.permission.MANAGE_EXTERNAL_STORAGE' in permissions + and self.build_version_sdk >= version_codes.R): + script_manage_ext_storage = [ + 'appops set {package} MANAGE_EXTERNAL_STORAGE allow', + 'echo "{sep}MANAGE_EXTERNAL_STORAGE{sep}$?{sep}"', + ] + else: + script_manage_ext_storage = [] + permissions = set(p for p in permissions if not _PERMISSIONS_DENYLIST_RE.match(p)) @@ -3752,10 +3756,15 @@ class DeviceUtils(object): and 'android.permission.READ_EXTERNAL_STORAGE' not in permissions): permissions.add('android.permission.READ_EXTERNAL_STORAGE') - script = ';'.join([ - 'p={package}', 'for q in {permissions}', 'do pm grant "$p" "$q"', - 'echo "{sep}$q{sep}$?{sep}"', 'done' - ]).format( + script_raw = [ + 'p={package}', + 'for q in {permissions}', + 'do pm grant "$p" "$q"', + 'echo "{sep}$q{sep}$?{sep}"', + 'done', + ] + script_manage_ext_storage + + script = ';'.join(script_raw).format( package=cmd_helper.SingleQuote(package), permissions=' '.join( cmd_helper.SingleQuote(p) for p in sorted(permissions)), diff --git a/catapult/devil/devil/android/device_utils_test.py b/catapult/devil/devil/android/device_utils_test.py index 62313c5b..8e583f01 100755 --- a/catapult/devil/devil/android/device_utils_test.py +++ b/catapult/devil/devil/android/device_utils_test.py @@ -11,6 +11,7 @@ Unit tests for the contents of device_utils.py (mostly DeviceUtils). import collections import contextlib +import io import json import logging import os @@ -19,6 +20,8 @@ import stat import sys import unittest +import six + from devil import devil_env from devil.android import device_errors from devil.android import device_signal @@ -117,9 +120,10 @@ class DeviceUtilsInitTest(unittest.TestCase): self.assertEqual(serial_as_str, d.adb.GetDeviceSerial()) def testInitWithUnicode(self): - serial_as_unicode = unicode('fedcba9876543210') - d = device_utils.DeviceUtils(serial_as_unicode) - self.assertEqual(serial_as_unicode, d.adb.GetDeviceSerial()) + if six.PY2: + serial_as_unicode = unicode('fedcba9876543210') + d = device_utils.DeviceUtils(serial_as_unicode) + self.assertEqual(serial_as_unicode, d.adb.GetDeviceSerial()) def testInitWithAdbWrapper(self): serial = '123456789abcdef0' @@ -162,12 +166,12 @@ class DeviceUtilsRestartServerTest(mock_calls.TestCase): ['pgrep', 'adb']), (1, '')), (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutput( ['pgrep', 'adb']), (0, '123\n'))): - device_utils.RestartServer() + adb_wrapper.RestartServer() class MockTempFile(object): def __init__(self, name='/tmp/some/file'): - self.file = mock.MagicMock(spec=file) + self.file = mock.MagicMock(spec=io.BufferedIOBase) self.file.name = name self.file.name_quoted = cmd_helper.SingleQuote(name) @@ -217,6 +221,12 @@ class DeviceUtilsTest(mock_calls.TestCase): self.adb, default_timeout=10, default_retries=0) self.watchMethodCalls(self.call.adb, ignore=['GetDeviceSerial']) + def safeAssertItemsEqual(self, expected, actual): + if six.PY2: + self.assertItemsEqual(expected, actual) + else: + self.assertCountEqual(expected, actual) # pylint: disable=no-member + def AdbCommandError(self, args=None, output=None, status=None, msg=None): if args is None: args = ['[unspecified]'] @@ -2194,11 +2204,16 @@ class DeviceUtilsPushChangedFilesZippedTest(DeviceUtilsTest): self.assertFalse( self.device._PushChangedFilesZipped(test_files, ['/test/dir'])) - def _testPushChangedFilesZipped_spec(self, test_files): + def _testPushChangedFilesZipped_spec(self, test_files, test_dirs): @contextlib.contextmanager def mock_zip_temp_dir(): yield '/test/temp/dir' + expected_cmd = ''.join([ + '\n /data/local/tmp/bin/unzip %s &&', + ' (for dir in %s\n do\n chmod -R 777 "$dir" || exit 1\n', + ' done)\n' + ]) % ('/sdcard/foo123.zip', ' '.join(test_dirs)) with self.assertCalls( (self.call.device._MaybeInstallCommands(), True), (mock.call.py_utils.tempfile_ext.NamedTemporaryDirectory(), @@ -2207,25 +2222,27 @@ class DeviceUtilsPushChangedFilesZippedTest(DeviceUtilsTest): (mock.call.os.path.getsize('/test/temp/dir/tmp.zip'), 123), (self.call.device.NeedsSU(), True), (mock.call.devil.android.device_temp_file.DeviceTempFile( - self.adb, suffix='.zip'), MockTempFile('/test/sdcard/foo123.zip')), - self.call.adb.Push('/test/temp/dir/tmp.zip', '/test/sdcard/foo123.zip'), - self.call.device.RunShellCommand( - 'unzip /test/sdcard/foo123.zip&&chmod -R 777 /test/dir', - shell=True, - as_root=True, - env={'PATH': '/data/local/tmp/bin:$PATH'}, - check_return=True)): + self.adb, suffix='.zip'), MockTempFile('/sdcard/foo123.zip')), + self.call.adb.Push('/test/temp/dir/tmp.zip', '/sdcard/foo123.zip'), + (mock.call.devil.android.device_temp_file.DeviceTempFile( + self.adb, suffix='.sh'), MockTempFile('/sdcard/temp-123.sh')), + self.call.device.WriteFile('/sdcard/temp-123.sh', expected_cmd), + (self.call.device.RunShellCommand(['source', '/sdcard/temp-123.sh'], + check_return=True, + as_root=True))): self.assertTrue( - self.device._PushChangedFilesZipped(test_files, ['/test/dir'])) + self.device._PushChangedFilesZipped(test_files, test_dirs)) def testPushChangedFilesZipped_single(self): - self._testPushChangedFilesZipped_spec([('/test/host/path/file1', - '/test/device/path/file1')]) + self._testPushChangedFilesZipped_spec( + [('/test/host/path/file1', '/test/device/path/file1')], + ['/test/dir1']) def testPushChangedFilesZipped_multiple(self): self._testPushChangedFilesZipped_spec( [('/test/host/path/file1', '/test/device/path/file1'), - ('/test/host/path/file2', '/test/device/path/file2')]) + ('/test/host/path/file2', '/test/device/path/file2')], + ['/test/dir1', '/test/dir2']) class DeviceUtilsPathExistsTest(DeviceUtilsTest): @@ -2374,7 +2391,8 @@ class DeviceUtilsReadFileTest(DeviceUtilsTest): with self.assertCalls( (mock.call.tempfile.mkdtemp(), tmp_host_dir), (self.call.adb.Pull('/path/to/device/file', mock.ANY)), - (mock.call.__builtin__.open(mock.ANY, 'r'), tmp_host), + (mock.call.__builtin__.open(mock.ANY, 'r'), tmp_host) if six.PY2 else \ + (mock.call.builtins.open(mock.ANY, 'r'), tmp_host), (mock.call.os.path.exists(tmp_host_dir), True), (mock.call.shutil.rmtree(tmp_host_dir), None)): self.assertEquals('some interesting contents', @@ -2587,8 +2605,8 @@ class DeviceUtilsStatDirectoryTest(DeviceUtilsTest): self.getStatEntries(path_given='/foo/bar', path_listed='/foo/bar/') def testStatDirectory_fileList(self): - self.assertItemsEqual(self.getStatEntries().keys(), self.FILENAMES) - self.assertItemsEqual(self.getListEntries(), self.FILENAMES) + self.safeAssertItemsEqual(self.getStatEntries().keys(), self.FILENAMES) + self.safeAssertItemsEqual(self.getListEntries(), self.FILENAMES) def testStatDirectory_fileModes(self): expected_modes = ( @@ -2656,7 +2674,7 @@ class DeviceUtilsStatDirectoryTest(DeviceUtilsTest): def testStatDirectory_symbolicLinks(self): entries = self.getStatEntries() self.assertEqual(entries['lnk']['symbolic_link_to'], '/a/path') - for d in entries.itervalues(): + for d in entries.values(): self.assertEqual('symbolic_link_to' in d, stat.S_ISLNK(d['st_mode'])) @@ -3463,7 +3481,7 @@ class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): device_utils.DeviceUtils.HealthyDevices(device_arg=[], retries=0) @mock.patch('time.sleep') - @mock.patch('devil.android.device_utils.RestartServer') + @mock.patch('devil.android.sdk.adb_wrapper.RestartServer') def testHealthyDevices_EmptyListDeviceArg_no_attached_with_retry( self, mock_restart, mock_sleep): with self.assertCalls( @@ -3481,7 +3499,7 @@ class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): mock.call(8), mock.call(16)]) @mock.patch('time.sleep') - @mock.patch('devil.android.device_utils.RestartServer') + @mock.patch('devil.android.sdk.adb_wrapper.RestartServer') def testHealthyDevices_EmptyListDeviceArg_no_attached_with_resets( self, mock_restart, mock_sleep): # The reset_usb import fails on windows. Mock the full import here so it can @@ -3557,7 +3575,7 @@ class DeviceUtilsGrantPermissionsTest(DeviceUtilsTest): def _PmGrantShellCall(self, package, permissions): fragment = 'p=%s;for q in %s;' % (package, ' '.join(sorted(permissions))) results = [] - for permission, result in sorted(permissions.iteritems()): + for permission, result in sorted(permissions.items()): if result: output, status = result + '\n', 1 else: @@ -3610,6 +3628,23 @@ class DeviceUtilsGrantPermissionsTest(DeviceUtilsTest): self.device.GrantPermissions('package', [WRITE]) self.assertEqual(logger.warnings, []) + def testGrantPermissions_ManageExtrnalStorage(self): + with PatchLogger() as logger: + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.R): + with self.assertCalls( + (self.call.device.RunShellCommand( + AnyStringWith('appops set pkg MANAGE_EXTERNAL_STORAGE allow'), + shell=True, + raw_output=True, + large_output=True, + check_return=True), + '{sep}MANAGE_EXTERNAL_STORAGE{sep}0{sep}\n'.format( + sep=device_utils._SHELL_OUTPUT_SEPARATOR))): + self.device.GrantPermissions( + 'pkg', ['android.permission.MANAGE_EXTERNAL_STORAGE']) + self.assertEqual(logger.warnings, []) + def testGrantPermissions_DenyList(self): with PatchLogger() as logger: with self.patch_call( @@ -3866,6 +3901,12 @@ class IterPushableComponentsTest(unittest.TestCase): yield Layout(layout_root, basic_file, symlink, symlink_dir, dir1, dir2) + def safeAssertItemsEqual(self, expected, actual): + if six.PY2: + self.assertItemsEqual(expected, actual) + else: + self.assertCountEqual(expected, actual) # pylint: disable=no-member + def testFile(self): with self.sampleLayout() as layout: device_path = '/sdcard/basic_file' @@ -3873,7 +3914,7 @@ class IterPushableComponentsTest(unittest.TestCase): expected = [(layout.basic_file, device_path, True)] actual = list( device_utils._IterPushableComponents(layout.basic_file, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testSymlinkFile(self): with self.sampleLayout() as layout: @@ -3883,7 +3924,7 @@ class IterPushableComponentsTest(unittest.TestCase): actual = list( device_utils._IterPushableComponents(layout.symlink_file, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testDirectoryWithNoSymlink(self): with self.sampleLayout() as layout: @@ -3893,7 +3934,7 @@ class IterPushableComponentsTest(unittest.TestCase): actual = list( device_utils._IterPushableComponents(layout.dir_without_symlinks, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testDirectoryWithSymlink(self): with self.sampleLayout() as layout: @@ -3910,7 +3951,7 @@ class IterPushableComponentsTest(unittest.TestCase): actual = list( device_utils._IterPushableComponents(layout.dir_with_symlinks, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testSymlinkDirectory(self): with self.sampleLayout() as layout: @@ -3919,7 +3960,7 @@ class IterPushableComponentsTest(unittest.TestCase): expected = [(os.path.realpath(layout.symlink_dir), device_path, False)] actual = list( device_utils._IterPushableComponents(layout.symlink_dir, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) def testDirectoryWithNestedSymlink(self): with self.sampleLayout() as layout: @@ -3947,7 +3988,7 @@ class IterPushableComponentsTest(unittest.TestCase): ] actual = list( device_utils._IterPushableComponents(layout.root, device_path)) - self.assertItemsEqual(expected, actual) + self.safeAssertItemsEqual(expected, actual) class DeviceUtilsGetTracingPathTest(DeviceUtilsTest): diff --git a/catapult/devil/devil/android/fastboot_utils_test.py b/catapult/devil/devil/android/fastboot_utils_test.py index 1ad73190..ed891393 100755 --- a/catapult/devil/devil/android/fastboot_utils_test.py +++ b/catapult/devil/devil/android/fastboot_utils_test.py @@ -13,6 +13,8 @@ import io import logging import unittest +import six + from devil import devil_env from devil.android import device_errors from devil.android import device_utils @@ -347,16 +349,22 @@ class FastbootUtilsFastbootMode(FastbootUtilsTest): pass +if six.PY2: + _BUILTIN_OPEN = '__builtin__.open' +else: + _BUILTIN_OPEN = 'builtins.open' + + class FastbootUtilsVerifyBoard(FastbootUtilsTest): def testVerifyBoard_bothValid(self): mock_file = io.StringIO(u'require board=%s\n' % _BOARD) - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_VALID_FILES): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_BothNotValid(self): mock_file = io.StringIO(u'abc') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_INVALID_FILES): self.assertFalse(self.assertFalse(self.fastboot._VerifyBoard('test'))) @@ -366,31 +374,31 @@ class FastbootUtilsVerifyBoard(FastbootUtilsTest): def testVerifyBoard_ZipNotFoundFileValid(self): mock_file = io.StringIO(u'require board=%s\n' % _BOARD) - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=['android-info.txt']): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_zipNotValidFileIs(self): mock_file = io.StringIO(u'require board=%s\n' % _BOARD) - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_INVALID_FILES): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_fileNotValidZipIs(self): mock_file = io.StringIO(u'require board=WrongBoard') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_VALID_FILES): self.assertFalse(self.fastboot._VerifyBoard('test')) def testVerifyBoard_noBoardInFileValidZip(self): mock_file = io.StringIO(u'Regex wont match') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_VALID_FILES): self.assertTrue(self.fastboot._VerifyBoard('test')) def testVerifyBoard_noBoardInFileInvalidZip(self): mock_file = io.StringIO(u'Regex wont match') - with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch(_BUILTIN_OPEN, return_value=mock_file, create=True): with mock.patch('os.listdir', return_value=_INVALID_FILES): self.assertFalse(self.fastboot._VerifyBoard('test')) @@ -419,7 +427,7 @@ class FastbootUtilsFindAndVerifyPartitionsAndImages(FastbootUtilsTest): ] with mock.patch('os.listdir', return_value=files): imgs = self.fastboot._FindAndVerifyPartitionsAndImages(PARTITIONS, 'test') - parts = imgs.keys() + parts = list(imgs.keys()) self.assertDictEqual(imgs, img_check) self.assertListEqual(parts, parts_check) @@ -451,7 +459,7 @@ class FastbootUtilsFindAndVerifyPartitionsAndImages(FastbootUtilsTest): with self.patch_call(self.call.fastboot.supports_ab, return_value=False): imgs = self.fastboot._FindAndVerifyPartitionsAndImages( PARTITIONS, 'test') - parts = imgs.keys() + parts = list(imgs.keys()) self.assertDictEqual(imgs, img_check) self.assertListEqual(parts, parts_check) diff --git a/catapult/devil/devil/android/flag_changer_test.py b/catapult/devil/devil/android/flag_changer_test.py index 564ead6e..9c155f13 100755 --- a/catapult/devil/devil/android/flag_changer_test.py +++ b/catapult/devil/devil/android/flag_changer_test.py @@ -6,6 +6,8 @@ import posixpath import unittest +import six + from devil.android import flag_changer _CMDLINE_FILE = 'chrome-command-line' @@ -109,15 +111,22 @@ class ParseSerializeFlagsTest(unittest.TestCase): def _testParseCmdLine(self, command_line, expected_flags): # Start with a command line, check that flags are parsed as expected. # pylint: disable=protected-access + # pylint: disable=no-member flags = flag_changer._ParseFlags(command_line) - self.assertItemsEqual(flags, expected_flags) + if six.PY2: + self.assertItemsEqual(flags, expected_flags) + else: + self.assertCountEqual(flags, expected_flags) # Check that flags survive a round-trip. # Note: Although new_command_line and command_line may not match, they # should describe the same set of flags. new_command_line = flag_changer._SerializeFlags(flags) new_flags = flag_changer._ParseFlags(new_command_line) - self.assertItemsEqual(new_flags, expected_flags) + if six.PY2: + self.assertItemsEqual(new_flags, expected_flags) + else: + self.assertCountEqual(new_flags, expected_flags) def testParseCmdLine_simple(self): self._testParseCmdLine('chrome --foo --bar="a b" --baz=true --fine="ok"', diff --git a/catapult/devil/devil/android/logcat_monitor.py b/catapult/devil/devil/android/logcat_monitor.py index bec74440..df306b0a 100644 --- a/catapult/devil/devil/android/logcat_monitor.py +++ b/catapult/devil/devil/android/logcat_monitor.py @@ -13,6 +13,8 @@ import tempfile import threading import time +import six + from devil.android import decorators from devil.android import device_errors from devil.android.sdk import adb_wrapper @@ -102,9 +104,9 @@ class LogcatMonitor(object): raise LogcatMonitorCommandError( 'Must be recording logcat when calling |WaitFor|', device_serial=str(self._adb)) - if isinstance(success_regex, basestring): + if isinstance(success_regex, six.string_types): success_regex = re.compile(success_regex) - if isinstance(failure_regex, basestring): + if isinstance(failure_regex, six.string_types): failure_regex = re.compile(failure_regex) logger.debug('Waiting %d seconds for "%s"', timeout, success_regex.pattern) @@ -220,10 +222,14 @@ class LogcatMonitor(object): Clears the logcat if |clear| was set in |__init__|. """ + # pylint: disable=unexpected-keyword-arg if self._clear: self._adb.Logcat(clear=True) if not self._record_file: - self._record_file = tempfile.NamedTemporaryFile(mode='a', bufsize=1) + if six.PY2: + self._record_file = tempfile.NamedTemporaryFile(mode='a', bufsize=1) + else: + self._record_file = tempfile.NamedTemporaryFile(mode='a', buffering=1) self._StartRecording() def Stop(self): diff --git a/catapult/devil/devil/android/logcat_monitor_test.py b/catapult/devil/devil/android/logcat_monitor_test.py index 7f2f10a6..356fe041 100755 --- a/catapult/devil/devil/android/logcat_monitor_test.py +++ b/catapult/devil/devil/android/logcat_monitor_test.py @@ -9,6 +9,8 @@ import itertools import threading import unittest +import six + from devil import devil_env from devil.android import logcat_monitor from devil.android.sdk import adb_wrapper @@ -24,6 +26,13 @@ def _CreateTestLog(raw_logcat=None): return test_log +def zip_longest(expected, actual): + # pylint: disable=no-member + if six.PY2: + return itertools.izip_longest(expected, actual) + else: + return itertools.zip_longest(expected, actual) + class LogcatMonitorTest(unittest.TestCase): _TEST_THREADTIME_LOGCAT_DATA = [ @@ -44,7 +53,7 @@ class LogcatMonitorTest(unittest.TestCase): ] def assertIterEqual(self, expected_iter, actual_iter): - for expected, actual in itertools.izip_longest(expected_iter, actual_iter): + for expected, actual in zip_longest(expected_iter, actual_iter): self.assertIsNotNone( expected, msg='actual has unexpected elements starting with %s' % str(actual)) diff --git a/catapult/devil/devil/android/md5sum.py b/catapult/devil/devil/android/md5sum.py index 8adf4ef7..e67f3f60 100644 --- a/catapult/devil/devil/android/md5sum.py +++ b/catapult/devil/devil/android/md5sum.py @@ -3,10 +3,12 @@ # found in the LICENSE file. import base64 +import io import gzip import os import re -import StringIO + +import six from devil import devil_env from devil.android import device_errors @@ -38,7 +40,7 @@ def CalculateHostMd5Sums(paths): Returns: A dict mapping file paths to their respective md5sum checksums. """ - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): paths = [paths] paths = list(paths) @@ -47,13 +49,17 @@ def CalculateHostMd5Sums(paths): raise IOError('File not built: %s' % md5sum_bin_host_path) out = "" for i in range(0, len(paths), _MAX_PATHS_PER_INVOCATION): - mem_file = StringIO.StringIO() + mem_file = io.BytesIO() compressed = gzip.GzipFile(fileobj=mem_file, mode="wb") - compressed.write(";".join( - [os.path.realpath(p) for p in paths[i:i+_MAX_PATHS_PER_INVOCATION]])) + data = ";".join( + [os.path.realpath(p) for p in paths[i:i+_MAX_PATHS_PER_INVOCATION]]) + if six.PY3: + data = data.encode('utf-8') + compressed.write(data) compressed.close() compressed_paths = base64.b64encode(mem_file.getvalue()) - out += cmd_helper.GetCmdOutput([md5sum_bin_host_path, "-gz", compressed_paths]) + out += cmd_helper.GetCmdOutput( + [md5sum_bin_host_path, "-gz", compressed_paths]) return dict(zip(paths, out.splitlines())) @@ -72,7 +78,7 @@ def CalculateDeviceMd5Sums(paths, device): if not paths: return {} - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): paths = [paths] paths = list(paths) @@ -97,9 +103,12 @@ def CalculateDeviceMd5Sums(paths, device): # Make sure it can find libbase.so md5sum_script += 'export LD_LIBRARY_PATH=%s;' % MD5SUM_DEVICE_LIB_PATH for i in range(0, len(paths), _MAX_PATHS_PER_INVOCATION): - mem_file = StringIO.StringIO() + mem_file = io.BytesIO() compressed = gzip.GzipFile(fileobj=mem_file, mode="wb") - compressed.write(";".join(paths[i:i+_MAX_PATHS_PER_INVOCATION])) + data = ";".join(paths[i:i+_MAX_PATHS_PER_INVOCATION]) + if six.PY3: + data = data.encode('utf-8') + compressed.write(data) compressed.close() compressed_paths = base64.b64encode(mem_file.getvalue()) md5sum_script += '$a -gz %s;' % compressed_paths diff --git a/catapult/devil/devil/android/md5sum_test.py b/catapult/devil/devil/android/md5sum_test.py index 548b2d02..9a51313e 100755 --- a/catapult/devil/devil/android/md5sum_test.py +++ b/catapult/devil/devil/android/md5sum_test.py @@ -112,7 +112,7 @@ class Md5SumTest(unittest.TestCase): def testCalculateDeviceMd5Sums_generator(self): test_path = ('/storage/emulated/legacy/test/file%d.dat' % n - for n in xrange(0, 2)) + for n in range(0, 2)) device = mock.NonCallableMock() device_md5sum_output = [ diff --git a/catapult/devil/devil/android/perf/perf_control.py b/catapult/devil/devil/android/perf/perf_control.py index 59485e0e..398b27fa 100644 --- a/catapult/devil/devil/android/perf/perf_control.py +++ b/catapult/devil/devil/android/perf/perf_control.py @@ -203,7 +203,7 @@ class PerfControl(object): if not isinstance(cpu_max_freq, dict): self._SetScalingMaxFreqForCpus(cpu_max_freq, self._cpu_file_list) else: - for key, max_frequency in cpu_max_freq.iteritems(): + for key, max_frequency in cpu_max_freq.items(): # Convert 'X' to 'cpuX' and 'X..Y' to 'cpuX cpu<X+1> .. cpuY'. if '..' in key: range_min, range_max = key.split('..') @@ -211,7 +211,7 @@ class PerfControl(object): else: range_min = range_max = int(key) cpu_files = [ - 'cpu%d' % number for number in xrange(range_min, range_max + 1) + 'cpu%d' % number for number in range(range_min, range_max + 1) ] # Set the |max_frequency| on requested subset of the cores. self._SetScalingMaxFreqForCpus(max_frequency, ' '.join(cpu_files)) diff --git a/catapult/devil/devil/android/perf/perf_control_test.py b/catapult/devil/devil/android/perf/perf_control_test.py index bde54d5a..a841a0e2 100644 --- a/catapult/devil/devil/android/perf/perf_control_test.py +++ b/catapult/devil/devil/android/perf/perf_control_test.py @@ -47,7 +47,7 @@ class PerfControlTest(unittest.TestCase): # pylint: disable=no-self-use def testNexus5HighPerfMode(self): # Mock out the device state for PerfControl. - cpu_list = ['cpu%d' % cpu for cpu in xrange(4)] + cpu_list = ['cpu%d' % cpu for cpu in range(4)] mock_device = mock.Mock(spec=device_utils.DeviceUtils) mock_device.product_model = 'Nexus 5' mock_device.adb = mock.Mock(spec=adb_wrapper.AdbWrapper) @@ -70,7 +70,7 @@ class PerfControlTest(unittest.TestCase): def testNexus5XHighPerfMode(self): # Mock out the device state for PerfControl. - cpu_list = ['cpu%d' % cpu for cpu in xrange(6)] + cpu_list = ['cpu%d' % cpu for cpu in range(6)] mock_device = mock.Mock(spec=device_utils.DeviceUtils) mock_device.product_model = 'Nexus 5X' mock_device.adb = mock.Mock(spec=adb_wrapper.AdbWrapper) @@ -93,7 +93,7 @@ class PerfControlTest(unittest.TestCase): def testNexus5XDefaultPerfMode(self): # Mock out the device state for PerfControl. - cpu_list = ['cpu%d' % cpu for cpu in xrange(6)] + cpu_list = ['cpu%d' % cpu for cpu in range(6)] mock_device = mock.Mock(spec=device_utils.DeviceUtils) mock_device.product_model = 'Nexus 5X' mock_device.adb = mock.Mock(spec=adb_wrapper.AdbWrapper) diff --git a/catapult/devil/devil/android/perf/surface_stats_collector.py b/catapult/devil/devil/android/perf/surface_stats_collector.py index 6240624f..4ddc6f5d 100644 --- a/catapult/devil/devil/android/perf/surface_stats_collector.py +++ b/catapult/devil/devil/android/perf/surface_stats_collector.py @@ -3,10 +3,16 @@ # found in the LICENSE file. import logging -import Queue import re import threading +import six + +if six.PY3: + import queue # pylint: disable=wrong-import-order +else: + import Queue as queue # pylint: disable=wrong-import-order + # Log marker containing SurfaceTexture timestamps. _SURFACE_TEXTURE_TIMESTAMPS_MESSAGE = 'SurfaceTexture update timestamps' _SURFACE_TEXTURE_TIMESTAMP_RE = r'\d+' @@ -37,7 +43,7 @@ class SurfaceStatsCollector(object): if self._ClearSurfaceFlingerLatencyData(): self._get_data_event = threading.Event() self._stop_event = threading.Event() - self._data_queue = Queue.Queue() + self._data_queue = queue.Queue() self._collector_thread = threading.Thread(target=self._CollectorThread) self._collector_thread.start() else: @@ -148,6 +154,10 @@ class SurfaceStatsCollector(object): return ParseFrameData(output, parse_timestamps=bool(window_name)) +def to_long_int(val): + """Cast val to a long int type.""" + return long(val) if six.PY2 else int(val) + def ParseFrameData(lines, parse_timestamps): # adb shell dumpsys SurfaceFlinger --latency <window name> # prints some information about the last 128 frames displayed in @@ -174,6 +184,7 @@ def ParseFrameData(lines, parse_timestamps): # (each time the number above changes, we have a "jank"). # If this happens a lot during an animation, the animation appears # janky, even if it runs at 60 fps in average. + # pylint: disable=redefined-variable-type results = [] for line in lines: # Skip over lines with anything other than digits and whitespace. @@ -186,7 +197,8 @@ def ParseFrameData(lines, parse_timestamps): timestamps = [] nanoseconds_per_millisecond = 1e6 - refresh_period = long(results[0]) / nanoseconds_per_millisecond + refresh_period = to_long_int(results[0]) / nanoseconds_per_millisecond + if not parse_timestamps: return refresh_period, timestamps @@ -201,7 +213,8 @@ def ParseFrameData(lines, parse_timestamps): if len(fields) != 3: logging.warning('Unexpected line: %s', line) continue - timestamp = long(fields[1]) + timestamp = to_long_int(fields[1]) + if timestamp == pending_fence_timestamp: continue timestamp /= nanoseconds_per_millisecond diff --git a/catapult/devil/devil/android/ports.py b/catapult/devil/devil/android/ports.py index f25c8bfd..4a7c2945 100644 --- a/catapult/devil/devil/android/ports.py +++ b/catapult/devil/devil/android/ports.py @@ -159,7 +159,7 @@ def IsHttpServerConnectable(host, message the server returns when connect status is false. """ assert tries >= 1 - for i in xrange(0, tries): + for i in range(0, tries): client_error = None try: with contextlib.closing( diff --git a/catapult/devil/devil/android/sdk/aapt.py b/catapult/devil/devil/android/sdk/aapt.py index 76c7ef68..fd354079 100644 --- a/catapult/devil/devil/android/sdk/aapt.py +++ b/catapult/devil/devil/android/sdk/aapt.py @@ -3,6 +3,8 @@ # found in the LICENSE file. """This module wraps the Android Asset Packaging Tool.""" +import six + from devil.android.sdk import build_tools from devil.utils import cmd_helper from devil.utils import lazy @@ -36,6 +38,6 @@ def Dump(what, apk, assets=None): assets: List of assets in apk you want to dump information for. """ assets = assets or [] - if isinstance(assets, basestring): + if isinstance(assets, six.string_types): assets = [assets] return _RunAaptCmd(['dump', what, apk] + assets).splitlines() diff --git a/catapult/devil/devil/android/sdk/adb_wrapper.py b/catapult/devil/devil/android/sdk/adb_wrapper.py index 71928d58..d8992242 100644 --- a/catapult/devil/devil/android/sdk/adb_wrapper.py +++ b/catapult/devil/devil/android/sdk/adb_wrapper.py @@ -18,6 +18,8 @@ import posixpath import re import subprocess +import six + from devil import base_error from devil import devil_env from devil.android import decorators @@ -127,6 +129,30 @@ def _IsExtraneousLine(line, send_cmd): return send_cmd.rstrip() in line +@decorators.WithExplicitTimeoutAndRetries(timeout=60, retries=3) +def RestartServer(): + """Restarts the adb server. + + Raises: + CommandFailedError if we fail to kill or restart the server. + """ + + def adb_killed(): + return not AdbWrapper.IsServerOnline() + + def adb_started(): + return AdbWrapper.IsServerOnline() + + AdbWrapper.KillServer() + if not timeout_retry.WaitFor(adb_killed, wait_period=1, max_tries=5): + # TODO(crbug.com/442319): Switch this to raise an exception if we + # figure out why sometimes not all adb servers on bots get killed. + logger.warning('Failed to kill adb server') + AdbWrapper.StartServer() + if not timeout_retry.WaitFor(adb_started, wait_period=1, max_tries=5): + raise device_errors.CommandFailedError('Failed to start adb server') + + class AdbWrapper(object): """A wrapper around a local Android Debug Bridge executable.""" @@ -1105,7 +1131,7 @@ class AdbWrapper(object): Returns: The output of the emulator console command. """ - if isinstance(cmd, basestring): + if isinstance(cmd, six.string_types): cmd = [cmd] return self._RunDeviceAdbCmd(['emu'] + cmd, timeout, retries) diff --git a/catapult/devil/devil/android/sdk/dexdump.py b/catapult/devil/devil/android/sdk/dexdump.py index 2a59e9bf..c71442ce 100644 --- a/catapult/devil/devil/android/sdk/dexdump.py +++ b/catapult/devil/devil/android/sdk/dexdump.py @@ -2,6 +2,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import six + from devil.android.sdk import build_tools from devil.utils import cmd_helper from devil.utils import lazy @@ -20,7 +22,7 @@ def DexDump(dexfiles, file_summary=False): An iterable over the output lines. """ # TODO(jbudorick): Add support for more options as necessary. - if isinstance(dexfiles, basestring): + if isinstance(dexfiles, six.string_types): dexfiles = [dexfiles] args = [_dexdump_path.read()] + dexfiles if file_summary: diff --git a/catapult/devil/devil/android/sdk/intent.py b/catapult/devil/devil/android/sdk/intent.py index 69c7d900..2ea38c33 100644 --- a/catapult/devil/devil/android/sdk/intent.py +++ b/catapult/devil/devil/android/sdk/intent.py @@ -116,7 +116,7 @@ class Intent(object): if self.flags: args.extend(['-f', self.flags]) if self.extras: - for key, value in self.extras.iteritems(): + for key, value in self.extras.items(): if value is None: args.extend(['--esn', key]) elif isinstance(value, str): diff --git a/catapult/devil/devil/android/sdk/shared_prefs.py b/catapult/devil/devil/android/sdk/shared_prefs.py index 7b12bf54..32b5bc4d 100644 --- a/catapult/devil/devil/android/sdk/shared_prefs.py +++ b/catapult/devil/devil/android/sdk/shared_prefs.py @@ -11,6 +11,8 @@ import logging import posixpath from xml.etree import ElementTree +import six + from devil.android import device_errors from devil.android.sdk import version_codes @@ -43,7 +45,10 @@ class BasePref(object): def __str__(self): """Get the underlying xml element as a string.""" - return ElementTree.tostring(self._elem) + if six.PY2: + return ElementTree.tostring(self._elem) + else: + return ElementTree.tostring(self._elem, encoding="unicode") def get(self): """Get the value of this preference.""" @@ -231,7 +236,11 @@ class SharedPrefs(object): def __str__(self): """Get the underlying xml document as a string.""" - return _XML_DECLARATION + ElementTree.tostring(self.xml) + if six.PY2: + return _XML_DECLARATION + ElementTree.tostring(self.xml) + else: + return _XML_DECLARATION + \ + ElementTree.tostring(self.xml, encoding="unicode") @property def package(self): diff --git a/catapult/devil/devil/android/sdk/shared_prefs_test.py b/catapult/devil/devil/android/sdk/shared_prefs_test.py index 7374c892..b50d9e0b 100755 --- a/catapult/devil/devil/android/sdk/shared_prefs_test.py +++ b/catapult/devil/devil/android/sdk/shared_prefs_test.py @@ -132,6 +132,8 @@ class SharedPrefsTest(unittest.TestCase): }) # data survived roundtrip def testForceCommit(self): + type(self.device).build_version_sdk = mock.PropertyMock( + return_value=version_codes.LOLLIPOP_MR1) prefs = shared_prefs.SharedPrefs(self.device, 'com.some.package', 'prefs.xml') prefs.Load() diff --git a/catapult/devil/devil/android/sdk/version_codes.py b/catapult/devil/devil/android/sdk/version_codes.py index 943b9d3f..564f30da 100644 --- a/catapult/devil/devil/android/sdk/version_codes.py +++ b/catapult/devil/devil/android/sdk/version_codes.py @@ -20,3 +20,4 @@ OREO = 26 OREO_MR1 = 27 PIE = 28 Q = 29 +R = 30 diff --git a/catapult/devil/devil/android/tools/device_monitor.py b/catapult/devil/devil/android/tools/device_monitor.py index 26e89a24..730df141 100755 --- a/catapult/devil/devil/android/tools/device_monitor.py +++ b/catapult/devil/devil/android/tools/device_monitor.py @@ -184,7 +184,7 @@ def get_all_status(denylist): } if denylist: - for device, reason in denylist.Read().iteritems(): + for device, reason in denylist.Read().items(): status_dict['devices'][device] = { 'state': reason.get('reason', 'denylisted') } diff --git a/catapult/devil/devil/android/tools/device_monitor_test.py b/catapult/devil/devil/android/tools/device_monitor_test.py index 8082d26e..1bb5680a 100755 --- a/catapult/devil/devil/android/tools/device_monitor_test.py +++ b/catapult/devil/devil/android/tools/device_monitor_test.py @@ -7,6 +7,8 @@ import os import sys import unittest +import six + if __name__ == '__main__': sys.path.append( os.path.abspath( @@ -53,7 +55,7 @@ class DeviceMonitorTest(unittest.TestCase): } def mock_run_shell(cmd, **_kwargs): - args = cmd.split() if isinstance(cmd, basestring) else cmd + args = cmd.split() if isinstance(cmd, six.string_types) else cmd try: return self.cmd_outputs[args[0]] except KeyError: diff --git a/catapult/devil/devil/android/tools/system_app.py b/catapult/devil/devil/android/tools/system_app.py index 62bf5a59..50d85595 100755 --- a/catapult/devil/devil/android/tools/system_app.py +++ b/catapult/devil/devil/android/tools/system_app.py @@ -22,6 +22,7 @@ from devil.android import decorators from devil.android import device_errors from devil.android import device_temp_file from devil.android.sdk import version_codes +from devil.android.sdk import adb_wrapper from devil.android.tools import script_common from devil.utils import cmd_helper from devil.utils import parallelizer @@ -128,7 +129,15 @@ _ENABLE_MODIFICATION_PROP = 'devil.modify_sys_apps' def _ShouldRetryModification(exc): - return not isinstance(exc, device_errors.CommandTimeoutError) + try: + if isinstance(exc, device_errors.CommandTimeoutError): + logger.info('Restarting the adb server') + adb_wrapper.RestartServer() + return True + except Exception: # pylint: disable=broad-except + logger.exception(('Caught an exception when deciding' + ' to retry system modification')) + return False # timeout and retries are both required by the decorator, but neither @@ -172,7 +181,7 @@ def _SetUpSystemAppModification(device, timeout=None, retries=None): # Point the user to documentation, since there's a good chance they can # workaround this on an emulator. docs_url = ('https://chromium.googlesource.com/chromium/src/+/' - 'master/docs/android_emulator.md#writable-system-partition') + 'HEAD/docs/android_emulator.md#writable-system-partition') logger.error( 'Did you start the emulator with "-writable-system?"\n' 'See %s\n', docs_url) @@ -187,6 +196,22 @@ def _TearDownSystemAppModification(device, timeout=None, retries=None): try: + # The function may be re-entered after the the device loses root + # privilege. For instance if the adb server is restarted before + # re-entering the function then the device may lose root privilege. + # Therefore we need to do a sanity check for root privilege + # on the device and then re-enable root privilege if the device + # does not have it. + if not device.HasRoot(): + logger.warning('Need to re-enable root.') + device.EnableRoot() + + if not device.HasRoot(): + raise device_errors.CommandFailedError( + ('Failed to tear down modification of ' + 'system apps on non-rooted device.'), + str(device)) + device.SetProp(_ENABLE_MODIFICATION_PROP, '0') device.Reboot() device.WaitUntilFullyBooted() diff --git a/catapult/devil/devil/android/tools/video_recorder.py b/catapult/devil/devil/android/tools/video_recorder.py index 984931f3..4004c467 100755 --- a/catapult/devil/devil/android/tools/video_recorder.py +++ b/catapult/devil/devil/android/tools/video_recorder.py @@ -164,7 +164,7 @@ def main(): parallel_devices = device_utils.DeviceUtils.parallel(script_common.GetDevices( args.devices, args.denylist_file), - async=True) + asyn=True) stop_recording = threading.Event() running_recording = parallel_devices.pMap(record_video, stop_recording) print 'Recording. Press Enter to stop.', diff --git a/catapult/devil/devil/base_error.py b/catapult/devil/devil/base_error.py index b7b33ccf..dd9ed496 100644 --- a/catapult/devil/devil/base_error.py +++ b/catapult/devil/devil/base_error.py @@ -9,6 +9,7 @@ class BaseError(Exception): def __init__(self, message, is_infra_error=False): super(BaseError, self).__init__(message) self._is_infra_error = is_infra_error + self.message = message def __eq__(self, other): return (self.message == other.message diff --git a/catapult/devil/devil/devil_env.py b/catapult/devil/devil/devil_env.py index b61d1b0e..2d576885 100644 --- a/catapult/devil/devil/devil_env.py +++ b/catapult/devil/devil/devil_env.py @@ -16,6 +16,7 @@ CATAPULT_ROOT_PATH = os.path.abspath( DEPENDENCY_MANAGER_PATH = os.path.join(CATAPULT_ROOT_PATH, 'dependency_manager') PYMOCK_PATH = os.path.join(CATAPULT_ROOT_PATH, 'third_party', 'mock') PY_UTILS_PATH = os.path.join(CATAPULT_ROOT_PATH, 'common', 'py_utils') +SIX_PATH = os.path.join(CATAPULT_ROOT_PATH, 'third_party', 'six') @contextlib.contextmanager @@ -31,6 +32,9 @@ def SysPath(path): with SysPath(DEPENDENCY_MANAGER_PATH): import dependency_manager # pylint: disable=import-error +with SysPath(SIX_PATH): + import six + _ANDROID_BUILD_TOOLS = {'aapt', 'dexdump', 'split-select'} _DEVIL_DEFAULT_CONFIG = os.path.abspath( @@ -53,7 +57,7 @@ def EmptyConfig(): def LocalConfigItem(dependency_name, dependency_platform, dependency_path): - if isinstance(dependency_path, basestring): + if isinstance(dependency_path, six.string_types): dependency_path = [dependency_path] return { dependency_name: { @@ -69,7 +73,7 @@ def LocalConfigItem(dependency_name, dependency_platform, dependency_path): def _GetEnvironmentVariableConfig(): env_config = EmptyConfig() path_config = ((os.environ.get(k), v) - for k, v in _LEGACY_ENVIRONMENT_VARIABLES.iteritems()) + for k, v in six.iteritems(_LEGACY_ENVIRONMENT_VARIABLES)) path_config = ((p, c) for p, c in path_config if p) for p, c in path_config: env_config['dependencies'].update( @@ -115,7 +119,8 @@ class _Environment(object): # TODO(jbudorick): Remove this recursion if/when dependency_manager # supports loading configurations directly from a dict. if configs: - with tempfile.NamedTemporaryFile(delete=False) as next_config_file: + with tempfile.NamedTemporaryFile(mode='w', + delete=False) as next_config_file: try: next_config_file.write(json.dumps(configs[0])) next_config_file.close() @@ -182,7 +187,10 @@ class _Environment(object): def GetPlatform(arch=None, device=None): if arch or device: return 'android_%s' % (arch or device.product_cpu_abi) - return '%s_%s' % (sys.platform, platform.machine()) + # use 'linux2' for linux as this is already used in json file + return '%s_%s' % ( + sys.platform if not sys.platform.startswith('linux') else 'linux2', + platform.machine()) config = _Environment() diff --git a/catapult/devil/devil/devil_env_test.py b/catapult/devil/devil/devil_env_test.py index ee7cd8fd..65fd7047 100755 --- a/catapult/devil/devil/devil_env_test.py +++ b/catapult/devil/devil/devil_env_test.py @@ -10,6 +10,7 @@ import sys import unittest from devil import devil_env +from devil.android.ndk import abis _sys_path_before = list(sys.path) with devil_env.SysPath(devil_env.PYMOCK_PATH): @@ -18,6 +19,11 @@ with devil_env.SysPath(devil_env.PYMOCK_PATH): _sys_path_after = list(sys.path) +class _MockDeviceUtils(object): + def __init__(self): + self.product_cpu_abi = abis.ARM_64 + + class DevilEnvTest(unittest.TestCase): def testSysPath(self): self.assertEquals(_sys_path_before, _sys_path_after) @@ -52,6 +58,21 @@ class DevilEnvTest(unittest.TestCase): }, }, env_config.get('dependencies')) + def testGetPlatform(self): + with mock.patch('platform.machine', mock.Mock(return_value='x86_64')): + with mock.patch('sys.platform', mock.Mock(return_value='linux2')): + platform = devil_env.GetPlatform() + self.assertEquals(platform, 'linux2_x86_64') + with mock.patch('sys.platform', mock.Mock(return_value='linux')): + platform = devil_env.GetPlatform() + self.assertEquals(platform, 'linux2_x86_64') + + platform = devil_env.GetPlatform(arch='arm64-v8a') + self.assertEquals(platform, 'android_arm64-v8a') + + device = _MockDeviceUtils() + platform = devil_env.GetPlatform(device=device) + self.assertEquals(platform, 'android_arm64-v8a') if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) diff --git a/catapult/devil/devil/utils/cmd_helper.py b/catapult/devil/devil/utils/cmd_helper.py index 634c9716..3a959450 100644 --- a/catapult/devil/devil/utils/cmd_helper.py +++ b/catapult/devil/devil/utils/cmd_helper.py @@ -10,11 +10,19 @@ import pipes import select import signal import string -import StringIO import subprocess import sys import time +CATAPULT_ROOT_PATH = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..')) +SIX_PATH = os.path.join(CATAPULT_ROOT_PATH, 'third_party', 'six') + +if SIX_PATH not in sys.path: + sys.path.append(SIX_PATH) + +import six + from devil import base_error logger = logging.getLogger(__name__) @@ -23,7 +31,8 @@ _SafeShellChars = frozenset(string.ascii_letters + string.digits + '@%_-+=:,./') # Cache the string-escape codec to ensure subprocess can find it # later. Return value doesn't matter. -codecs.lookup('string-escape') +if six.PY2: + codecs.lookup('string-escape') def SingleQuote(s): @@ -102,6 +111,7 @@ def Popen(args, cwd=None, env=None): # preexec_fn isn't supported on windows. + # pylint: disable=unexpected-keyword-arg if sys.platform == 'win32': close_fds = (stdin is None and stdout is None and stderr is None) preexec_fn = None @@ -109,7 +119,8 @@ def Popen(args, close_fds = True preexec_fn = lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL) - return subprocess.Popen( + if six.PY2: + return subprocess.Popen( args=args, cwd=cwd, stdin=stdin, @@ -118,8 +129,29 @@ def Popen(args, shell=shell, close_fds=close_fds, env=env, - preexec_fn=preexec_fn) - + preexec_fn=preexec_fn + ) + else: + # opens stdout in text mode, so that caller side always get 'str', + # and there will be no type mismatch error. + # Ignore any decoding error, so that caller will not crash due to + # uncaught exception. Decoding errors are unavoidable, as we + # do not know the encoding of the output, and in some output there + # will be multiple encodings (e.g. adb logcat) + return subprocess.Popen( + args=args, + cwd=cwd, + stdin=stdin, + stdout=stdout, + stderr=stderr, + shell=shell, + close_fds=close_fds, + env=env, + preexec_fn=preexec_fn, + universal_newlines=True, + encoding='utf-8', + errors='ignore' + ) def Call(args, stdout=None, stderr=None, shell=None, cwd=None, env=None): pipe = Popen( @@ -165,7 +197,7 @@ def GetCmdOutput(args, cwd=None, shell=False, env=None): def _ValidateAndLogCommand(args, cwd, shell): - if isinstance(args, basestring): + if isinstance(args, six.string_types): if not shell: raise Exception('string args must be run with shell=True') else: @@ -283,6 +315,12 @@ class TimeoutError(base_error.BaseError): return self._output +def _read_and_decode(fd, buffer_size): + data = os.read(fd, buffer_size) + if data and six.PY3: + data = data.decode('utf-8', errors='ignore') + return data + def _IterProcessStdoutFcntl(process, iter_timeout=None, timeout=None, @@ -316,7 +354,7 @@ def _IterProcessStdoutFcntl(process, read_fds, _, _ = select.select([child_fd], [], [], iter_aware_poll_interval) if child_fd in read_fds: - data = os.read(child_fd, buffer_size) + data = _read_and_decode(child_fd, buffer_size) if not data: break yield data @@ -328,7 +366,7 @@ def _IterProcessStdoutFcntl(process, read_fds, _, _ = select.select([child_fd], [], [], iter_aware_poll_interval) if child_fd in read_fds: - data = os.read(child_fd, buffer_size) + data = _read_and_decode(child_fd, buffer_size) if data: yield data continue @@ -365,7 +403,7 @@ def _IterProcessStdoutQueue(process, # TODO(jbudorick): Pick an appropriate read size here. while True: try: - output_chunk = os.read(process.stdout.fileno(), buffer_size) + output_chunk = _read_and_decode(process.stdout.fileno(), buffer_size) except IOError: break stdout_queue.put(output_chunk, True) @@ -452,7 +490,7 @@ def GetCmdStatusAndOutputWithTimeout(args, TimeoutError on timeout. """ _ValidateAndLogCommand(args, cwd, shell) - output = StringIO.StringIO() + output = six.StringIO() process = Popen( args, cwd=cwd, diff --git a/catapult/devil/devil/utils/cmd_helper_test.py b/catapult/devil/devil/utils/cmd_helper_test.py index 57abceb4..0eeefe16 100755 --- a/catapult/devil/devil/utils/cmd_helper_test.py +++ b/catapult/devil/devil/utils/cmd_helper_test.py @@ -33,6 +33,17 @@ class CmdHelperSingleQuoteTest(unittest.TestCase): self.assertEquals(test_string, cmd_helper.GetCmdOutput(cmd, shell=True).rstrip()) +class CmdHelperGetCmdStatusAndOutputTest(unittest.TestCase): + def testGetCmdStatusAndOutput_success(self): + cmd = 'echo "Hello World"' + status, output = cmd_helper.GetCmdStatusAndOutput(cmd, shell=True) + self.assertEqual(status, 0) + self.assertEqual(output.rstrip(), "Hello World") + + def testGetCmdStatusAndOutput_unicode(self): + # pylint: disable=no-self-use + cmd = 'echo "\x80\x31Hello World\n"' + cmd_helper.GetCmdStatusAndOutput(cmd, shell=True) class CmdHelperDoubleQuoteTest(unittest.TestCase): def testDoubleQuote_basic(self): @@ -195,7 +206,7 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): # pylint: disable=protected-access _SIMPLE_OUTPUT_SEQUENCE = [ - _ProcessOutputEvent(read_contents='1\n2\n'), + _ProcessOutputEvent(read_contents=b'1\n2\n'), ] def testIterCmdOutputLines_success(self): @@ -205,6 +216,14 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): cmd_helper._IterCmdOutputLines(mock_proc, 'mock_proc'), 1): self.assertEquals(num, int(line)) + def testIterCmdOutputLines_unicode(self): + output_sequence = [ + _ProcessOutputEvent(read_contents=b'\x80\x31\nHello\n\xE2\x98\xA0') + ] + with _MockProcess(output_sequence=output_sequence) as mock_proc: + lines = list(cmd_helper._IterCmdOutputLines(mock_proc, 'mock_proc')) + self.assertEquals(lines[1], "Hello") + def testIterCmdOutputLines_exitStatusFail(self): with self.assertRaises(subprocess.CalledProcessError): with _MockProcess( @@ -238,9 +257,9 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): def testIterCmdOutputLines_delay(self): output_sequence = [ - _ProcessOutputEvent(read_contents='1\n2\n', ts=1), + _ProcessOutputEvent(read_contents=b'1\n2\n', ts=1), _ProcessOutputEvent(read_contents=None, ts=2), - _ProcessOutputEvent(read_contents='Awake', ts=10), + _ProcessOutputEvent(read_contents=b'Awake', ts=10), ] with _MockProcess(output_sequence=output_sequence) as mock_proc: for num, line in enumerate( diff --git a/catapult/devil/devil/utils/decorators.py b/catapult/devil/devil/utils/decorators.py new file mode 100644 index 00000000..5d286107 --- /dev/null +++ b/catapult/devil/devil/utils/decorators.py @@ -0,0 +1,17 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import functools + + +def Memoize(f): + """Decorator to cache return values of function.""" + memoize_dict = {} + @functools.wraps(f) + def wrapper(*args, **kwargs): + key = repr((args, kwargs)) + if key not in memoize_dict: + memoize_dict[key] = f(*args, **kwargs) + return memoize_dict[key] + return wrapper diff --git a/catapult/devil/devil/utils/decorators_test.py b/catapult/devil/devil/utils/decorators_test.py new file mode 100755 index 00000000..f81974ac --- /dev/null +++ b/catapult/devil/devil/utils/decorators_test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unit tests for decorators.py.""" + +import unittest + +from devil.utils import decorators + + +class MemoizeDecoratorTest(unittest.TestCase): + + def testFunctionExceptionNotMemoized(self): + """Tests that |Memoize| decorator does not cache exception results.""" + + class ExceptionType1(Exception): + pass + + class ExceptionType2(Exception): + pass + + @decorators.Memoize + def raiseExceptions(): + if raiseExceptions.count == 0: + raiseExceptions.count += 1 + raise ExceptionType1() + + if raiseExceptions.count == 1: + raise ExceptionType2() + raiseExceptions.count = 0 + + with self.assertRaises(ExceptionType1): + raiseExceptions() + with self.assertRaises(ExceptionType2): + raiseExceptions() + + def testFunctionResultMemoized(self): + """Tests that |Memoize| decorator caches results.""" + + @decorators.Memoize + def memoized(): + memoized.count += 1 + return memoized.count + memoized.count = 0 + + def notMemoized(): + notMemoized.count += 1 + return notMemoized.count + notMemoized.count = 0 + + self.assertEquals(memoized(), 1) + self.assertEquals(memoized(), 1) + self.assertEquals(memoized(), 1) + + self.assertEquals(notMemoized(), 1) + self.assertEquals(notMemoized(), 2) + self.assertEquals(notMemoized(), 3) + + def testFunctionMemoizedBasedOnArgs(self): + """Tests that |Memoize| caches results based on args and kwargs.""" + + @decorators.Memoize + def returnValueBasedOnArgsKwargs(a, k=0): + return a + k + + self.assertEquals(returnValueBasedOnArgsKwargs(1, 1), 2) + self.assertEquals(returnValueBasedOnArgsKwargs(1, 2), 3) + self.assertEquals(returnValueBasedOnArgsKwargs(2, 1), 3) + self.assertEquals(returnValueBasedOnArgsKwargs(3, 3), 6) + + +if __name__ == '__main__': + unittest.main() diff --git a/catapult/devil/devil/utils/markdown_test.py b/catapult/devil/devil/utils/markdown_test.py index 621d56ba..11cd46ea 100755 --- a/catapult/devil/devil/utils/markdown_test.py +++ b/catapult/devil/devil/utils/markdown_test.py @@ -103,10 +103,9 @@ class MarkdownTest(unittest.TestCase): def testLink(self): link_text = 'Devil home' link_target = ( - 'https://github.com/catapult-project/catapult/tree/master/devil') - expected = ( - '[Devil home]' - '(https://github.com/catapult-project/catapult/tree/master/devil)') + 'https://chromium.googlesource.com/catapult.git/+/HEAD/devil') + expected = ('[Devil home]' + '(https://chromium.googlesource.com/catapult.git/+/HEAD/devil)') self.assertEquals(expected, markdown.md_link(link_text, link_target)) def testLinkTextContainsBracket(self): diff --git a/catapult/devil/devil/utils/mock_calls.py b/catapult/devil/devil/utils/mock_calls.py index 2b359385..ba96658a 100644 --- a/catapult/devil/devil/utils/mock_calls.py +++ b/catapult/devil/devil/utils/mock_calls.py @@ -51,7 +51,7 @@ class TestCase(unittest.TestCase): (call.parent.name, call.parent) for call, _ in self._expected_calls) self._patched = [ test_case.patch_call(call, side_effect=do_check(call)) - for call in watched.itervalues() + for call in watched.values() ] def __enter__(self): diff --git a/catapult/devil/devil/utils/parallelizer_test.py b/catapult/devil/devil/utils/parallelizer_test.py index 7c86148c..2d8f72aa 100644 --- a/catapult/devil/devil/utils/parallelizer_test.py +++ b/catapult/devil/devil/utils/parallelizer_test.py @@ -19,6 +19,7 @@ if __name__ == '__main__': os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) from devil.utils import parallelizer +from devil.base_error import BaseError class ParallelizerTestObject(object): @@ -58,7 +59,7 @@ class ParallelizerTestObject(object): def _write_completion_file(self): if self._completion_file_name and len(self._completion_file_name): with open(self._completion_file_name, 'w+b') as completion_file: - completion_file.write('complete') + completion_file.write(b'complete') def __getitem__(self, index): return self._thing[index] @@ -87,13 +88,13 @@ class ParallelizerTest(unittest.TestCase): self.assertEquals(expected, r) def testMutate(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] self.assertTrue(all(d.doReturnTheThing() for d in devices)) ParallelizerTestObject.parallel(devices).doSetTheThing(False).pFinish(1) self.assertTrue(not any(d.doReturnTheThing() for d in devices)) def testAllReturn(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] results = ParallelizerTestObject.parallel(devices).doReturnTheThing().pGet( 1) self.assertTrue(isinstance(results, list)) @@ -103,7 +104,7 @@ class ParallelizerTest(unittest.TestCase): def testAllRaise(self): devices = [ ParallelizerTestObject(Exception('thing %d' % i)) - for i in xrange(0, 10) + for i in range(0, 10) ] p = ParallelizerTestObject.parallel(devices).doRaiseTheThing() with self.assertRaises(Exception): @@ -117,21 +118,21 @@ class ParallelizerTest(unittest.TestCase): try: completion_files = [ tempfile.NamedTemporaryFile(delete=False) - for _ in xrange(0, parallel_device_count) + for _ in range(0, parallel_device_count) ] devices = [ ParallelizerTestObject( - i if i != exception_index else Exception(exception_msg), + i if i != exception_index else BaseError(exception_msg), completion_files[i].name) - for i in xrange(0, parallel_device_count) + for i in range(0, parallel_device_count) ] for f in completion_files: f.close() p = ParallelizerTestObject.parallel(devices) - with self.assertRaises(Exception) as e: + with self.assertRaises(BaseError) as e: p.doRaiseIfExceptionElseSleepFor(2).pGet(3) self.assertTrue(exception_msg in str(e.exception)) - for i in xrange(0, parallel_device_count): + for i in range(0, parallel_device_count): with open(completion_files[i].name) as f: if i == exception_index: self.assertEquals('', f.read()) @@ -142,7 +143,7 @@ class ParallelizerTest(unittest.TestCase): os.remove(f.name) def testReusable(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] p = ParallelizerTestObject.parallel(devices) results = p.doReturn(True).pGet(1) self.assertTrue(all(results)) @@ -152,23 +153,23 @@ class ParallelizerTest(unittest.TestCase): results = p.doRaise(Exception('reusableTest')).pGet(1) def testContained(self): - devices = [ParallelizerTestObject(i) for i in xrange(0, 10)] + devices = [ParallelizerTestObject(i) for i in range(0, 10)] results = (ParallelizerTestObject.parallel(devices).helper. doReturnStringThing().pGet(1)) self.assertTrue(isinstance(results, list)) self.assertEquals(10, len(results)) - for i in xrange(0, 10): + for i in range(0, 10): self.assertEquals(str(i), results[i]) def testGetItem(self): - devices = [ParallelizerTestObject(range(i, i + 10)) for i in xrange(0, 10)] + devices = [ParallelizerTestObject(range(i, i + 10)) for i in range(0, 10)] results = ParallelizerTestObject.parallel(devices)[9].pGet(1) - self.assertEquals(range(9, 19), results) + self.assertEquals(list(range(9, 19)), results) class SyncParallelizerTest(unittest.TestCase): def testContextManager(self): - in_context = [False for i in xrange(10)] + in_context = [False for i in range(10)] @contextlib.contextmanager def enter_into_context(i): @@ -179,7 +180,7 @@ class SyncParallelizerTest(unittest.TestCase): in_context[i] = False parallelized_context = parallelizer.SyncParallelizer( - [enter_into_context(i) for i in xrange(10)]) + [enter_into_context(i) for i in range(10)]) with parallelized_context: self.assertTrue(all(in_context)) diff --git a/catapult/devil/devil/utils/reraiser_thread_unittest.py b/catapult/devil/devil/utils/reraiser_thread_unittest.py index eb37456c..f1fabd0f 100644 --- a/catapult/devil/devil/utils/reraiser_thread_unittest.py +++ b/catapult/devil/devil/utils/reraiser_thread_unittest.py @@ -64,7 +64,7 @@ class TestReraiserThreadGroup(unittest.TestCase): ran[i] = True group = reraiser_thread.ReraiserThreadGroup() - for i in xrange(5): + for i in range(5): group.Add(reraiser_thread.ReraiserThread(f, args=[i])) group.StartAll() group.JoinAll() @@ -76,7 +76,7 @@ class TestReraiserThreadGroup(unittest.TestCase): raise TestException group = reraiser_thread.ReraiserThreadGroup( - [reraiser_thread.ReraiserThread(f) for _ in xrange(5)]) + [reraiser_thread.ReraiserThread(f) for _ in range(5)]) group.StartAll() with self.assertRaises(TestException): group.JoinAll() diff --git a/catapult/devil/devil/utils/usb_hubs.py b/catapult/devil/devil/utils/usb_hubs.py index 313cf3f6..b83fb610 100644 --- a/catapult/devil/devil/utils/usb_hubs.py +++ b/catapult/devil/devil/utils/usb_hubs.py @@ -73,7 +73,7 @@ class HubType(object): A series of (int, USBNode) tuples giving a physical port and the Node connected to it. """ - for (virtual, physical) in mapping.iteritems(): + for (virtual, physical) in mapping.items(): if node.HasPort(virtual): if isinstance(physical, dict): for res in self._GppHelper(node.PortToDevice(virtual), physical): diff --git a/catapult/devil/docs/adb_wrapper.md b/catapult/devil/docs/adb_wrapper.md index 3ca8aa68..16a666fa 100644 --- a/catapult/devil/docs/adb_wrapper.md +++ b/catapult/devil/docs/adb_wrapper.md @@ -1,4 +1,4 @@ -# [devil.android.sdk.adb_wrapper](https://github.com/catapult-project/catapult/blob/master/devil/devil/android/sdk/adb_wrapper.py) +# [devil.android.sdk.adb_wrapper](https://chromium.googlesource.com/catapult.git/+/HEAD/devil/devil/android/sdk/adb_wrapper.py) *This page was autogenerated. Run `devil/bin/generate_md_docs` to update* diff --git a/catapult/devil/docs/device_utils.md b/catapult/devil/docs/device_utils.md index 960ca894..ad37d8eb 100644 --- a/catapult/devil/docs/device_utils.md +++ b/catapult/devil/docs/device_utils.md @@ -1,4 +1,4 @@ -# [devil.android.device_utils](https://github.com/catapult-project/catapult/blob/master/devil/devil/android/device_utils.py) +# [devil.android.device_utils](https://chromium.googlesource.com/catapult.git/+/HEAD/devil/devil/android/device_utils.py) *This page was autogenerated. Run `devil/bin/generate_md_docs` to update* @@ -467,11 +467,12 @@ Get the WebView update command sysdump on the device. WebViewPackages: Dict of installed WebView providers, mapping "package name" to "reason it's valid/invalid." - It may return an empty dictionary if device does not - support the "dumpsys webviewupdate" command. + The returned dictionary may not include all of the above keys: this depends + on the support of the platform's underlying WebViewUpdateService. This may + return an empty dictionary on OS versions which do not support querying the + WebViewUpdateService. Raises: - CommandFailedError on failure. CommandTimeoutError on timeout. DeviceUnreachableError on missing device. ``` @@ -593,6 +594,8 @@ Determines whether a particular package is installed on the device. ``` Args: package: Name of the package. + version_code: The version of the package to check for as an int, if + applicable. Only used for static shared libraries, otherwise ignored. Returns: True if the application is installed, False otherwise. @@ -866,6 +869,10 @@ Reads the contents of a file from the device. Reboot the device. ``` + Note if the device has the root privilege, it will likely lose it after the + reboot. When |block| is True, it will try to restore the root status if + applicable. + Args: block: A boolean indicating if we should wait for the reboot to complete. wifi: A boolean indicating if we should wait for wifi to be enabled after diff --git a/catapult/devil/docs/markdown.md b/catapult/devil/docs/markdown.md index b2fec500..605d4a93 100644 --- a/catapult/devil/docs/markdown.md +++ b/catapult/devil/docs/markdown.md @@ -1,4 +1,4 @@ -# [devil.utils.markdown](https://github.com/catapult-project/catapult/blob/master/devil/devil/utils/markdown.py) +# [devil.utils.markdown](https://chromium.googlesource.com/catapult.git/+/HEAD/devil/devil/utils/markdown.py) *This page was autogenerated. Run `devil/bin/generate_md_docs` to update* diff --git a/catapult/devil/docs/persistent_device_list.md b/catapult/devil/docs/persistent_device_list.md index 626a8788..513f640f 100644 --- a/catapult/devil/docs/persistent_device_list.md +++ b/catapult/devil/docs/persistent_device_list.md @@ -29,7 +29,7 @@ bots that upload data to the perf dashboard. ## Where it is used The persistent device list is used in the -[chromium_android](https://cs.chromium.org/chromium/build/scripts/slave/recipe_modules/chromium_android/api.py?q=known_devices_file) +[chromium_android](https://source.chromium.org/chromium/chromium/tools/build/+/HEAD:recipes/recipe_modules/chromium_android/api.py;l=50;drc=fd928820620dff8989e853accc54b1d61657f236) recipe module, and consumed by the -[device_status.py](https://cs.chromium.org/chromium/src/third_party/catapult/devil/devil/android/tools/device_status.py?q=\-\-known%5C-devices%5C-file) +[device_status.py](https://source.chromium.org/chromium/chromium/src/+/HEAD:third_party/catapult/devil/devil/android/tools/device_status.py;l=230;drc=1e5bef4469199e4daba5d8fd885966112f8a45d5) script among others. diff --git a/catapult/systrace/atrace_helper/jni/Android.mk b/catapult/systrace/atrace_helper/jni/Android.mk index b8f6acbb..9d5cc605 100644 --- a/catapult/systrace/atrace_helper/jni/Android.mk +++ b/catapult/systrace/atrace_helper/jni/Android.mk @@ -7,9 +7,6 @@ LOCAL_PATH := $(call my-dir) include $(CLEAR_VARS) LOCAL_MODULE := atrace_helper -LOCAL_LICENSE_KINDS := SPDX-license-identifier-BSD -LOCAL_LICENSE_CONDITIONS := notice -LOCAL_NOTICE_FILE := $(LOCAL_PATH)/../../../../NOTICE LOCAL_CPPFLAGS := -std=c++11 -Wall -Wextra -Werror -O2 LOCAL_CPPFLAGS += -fPIE -fno-rtti -fno-exceptions -fstack-protector LOCAL_CPP_EXTENSION := .cc diff --git a/catapult/systrace/systrace/systrace_trace_viewer.html b/catapult/systrace/systrace/systrace_trace_viewer.html index 473261d8..11efa176 100644 --- a/catapult/systrace/systrace/systrace_trace_viewer.html +++ b/catapult/systrace/systrace/systrace_trace_viewer.html @@ -5855,7 +5855,7 @@ platformSpecificTotals[rawTotalName]=parseInt(rawTotalValue,16);} if(totals.peakResidentBytes===undefined&&totals.arePeakResidentBytesResettable!==undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Optional field peak_resident_set_bytes found'+' but is_peak_rss_resetable not found in'+' process memory dump for PID='+pid+' and dump ID='+dumpId+'.'});} if(totals.arePeakResidentBytesResettable!==undefined&&totals.peakResidentBytes===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Optional field is_peak_rss_resetable found'+' but peak_resident_set_bytes not found in'+' process memory dump for PID='+pid+' and dump ID='+dumpId+'.'});} processMemoryDump.totals=totals;},parseMemoryDumpVmRegions_(processMemoryDump,dumps,pid,dumpId){const rawProcessMmaps=dumps.process_mmaps;if(rawProcessMmaps===undefined)return;const rawVmRegions=rawProcessMmaps.vm_regions;if(rawVmRegions===undefined)return;if(processMemoryDump.vmRegions!==undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'VM regions provided multiple times for'+' process memory dump for PID='+pid+' and dump ID='+dumpId+'.'});return;} -const vmRegions=new Array(rawVmRegions.length);for(let i=0;i<rawVmRegions.length;i++){const rawVmRegion=rawVmRegions[i];const byteStats={};const rawByteStats=rawVmRegion.bs;for(const rawByteStatName in rawByteStats){const rawByteStatValue=rawByteStats[rawByteStatName];if(rawByteStatValue===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Byte stat \''+rawByteStatName+'\' of VM region '+ +if(rawVmRegions===null)return;const vmRegions=new Array(rawVmRegions.length);for(let i=0;i<rawVmRegions.length;i++){const rawVmRegion=rawVmRegions[i];const byteStats={};const rawByteStats=rawVmRegion.bs;for(const rawByteStatName in rawByteStats){const rawByteStatValue=rawByteStats[rawByteStatName];if(rawByteStatValue===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Byte stat \''+rawByteStatName+'\' of VM region '+ i+' ('+rawVmRegion.mf+') in process memory dump for '+'PID='+pid+' and dump ID='+dumpId+' does not have a value.'});continue;} const byteStatName=BYTE_STAT_NAME_MAP[rawByteStatName];if(byteStatName===undefined){this.model_.importWarning({type:'memory_dump_parse_error',message:'Unknown byte stat name \''+rawByteStatName+'\' ('+ rawByteStatValue+') of VM region '+i+' ('+ @@ -6495,7 +6495,9 @@ for(let i=0;i<importers.length;i++){const subtraces=importers[i].extractSubtrace if(traces.length&&!this.hasEventDataDecoder_(importers)){throw new Error('Could not find an importer for the provided eventData.');} importers.sort(function(x,y){return x.importPriority-y.importPriority;});});addStageForEachImporter('Importing clock sync markers',importer=>importer.importClockSyncMarkers());addStageForEachImporter('Importing',importer=>importer.importEvents());if(this.importOptions_.customizeModelCallback){addImportStage('Customizing',()=>{this.importOptions_.customizeModelCallback(this.model_);});} addStageForEachImporter('Importing sample data',importer=>importer.importSampleData());addImportStage('Autoclosing open slices...',()=>{this.model_.autoCloseOpenSlices();this.model_.createSubSlices();});addStageForEachImporter('Finalizing import',importer=>importer.finalizeImport());addImportStage('Initializing objects (step 1/2)...',()=>this.model_.preInitializeObjects());if(this.importOptions_.pruneEmptyContainers){addImportStage('Pruning empty containers...',()=>this.model_.pruneEmptyContainers());} -addImportStage('Merging kernel with userland...',()=>this.model_.mergeKernelWithUserland());let auditors=[];addImportStage('Adding arbitrary data to model...',()=>{auditors=this.importOptions_.auditorConstructors.map(auditorConstructor=>new auditorConstructor(this.model_));auditors.forEach((auditor)=>{auditor.runAnnotate();auditor.installUserFriendlyCategoryDriverIfNeeded();});});addImportStage('Computing final world bounds...',()=>{this.model_.computeWorldBounds(this.importOptions_.shiftWorldToZero);});addImportStage('Building flow event map...',()=>this.model_.buildFlowEventIntervalTree());addImportStage('Joining object refs...',()=>this.model_.joinRefs());addImportStage('Cleaning up undeleted objects...',()=>this.model_.cleanupUndeletedObjects());addImportStage('Sorting memory dumps...',()=>this.model_.sortMemoryDumps());addImportStage('Finalizing memory dump graphs...',()=>this.model_.finalizeMemoryGraphs());addImportStage('Initializing objects (step 2/2)...',()=>this.model_.initializeObjects());addImportStage('Building event indices...',()=>this.model_.buildEventIndices());addImportStage('Building UserModel...',()=>{const userModelBuilder=new tr.importer.UserModelBuilder(this.model_);userModelBuilder.buildUserModel();});addImportStage('Sorting user expectations...',()=>this.model_.userModel.sortExpectations());addImportStage('Running auditors...',()=>{auditors.forEach(auditor=>auditor.runAudit());});addImportStage('Updating alerts...',()=>this.model_.sortAlerts());addImportStage('Update bounds...',()=>this.model_.updateBounds());addImportStage('Looking for warnings...',()=>{if(!this.model_.isTimeHighResolution){this.model_.importWarning({type:'low_resolution_timer',message:'Trace time is low resolution, trace may be unusable.',showToUser:true});}});lastTask.after(()=>{this.importing_=false;this.model_.stats.traceImportDurationMs=tr.b.Timing.getCurrentTimeMs()-importStartTimeMs;});return importTask;},createImporter_(eventData){const importerConstructor=tr.importer.Importer.findImporterFor(eventData);if(!importerConstructor){throw new Error('Couldn\'t create an importer for the provided '+'eventData.');} +addImportStage('Merging kernel with userland...',()=>this.model_.mergeKernelWithUserland());let auditors=[];addImportStage('Adding arbitrary data to model...',()=>{for(const auditorConstructor of +this.importOptions_.auditorConstructors){try{auditors.push(new auditorConstructor(this.model_));}catch(e){console.error('Failed to construct an auditor:');console.error(e);}} +auditors.forEach((auditor)=>{try{auditor.runAnnotate();auditor.installUserFriendlyCategoryDriverIfNeeded();}catch(e){console.error('Failed to run an auditor:');console.error(e);}});});addImportStage('Computing final world bounds...',()=>{this.model_.computeWorldBounds(this.importOptions_.shiftWorldToZero);});addImportStage('Building flow event map...',()=>this.model_.buildFlowEventIntervalTree());addImportStage('Joining object refs...',()=>this.model_.joinRefs());addImportStage('Cleaning up undeleted objects...',()=>this.model_.cleanupUndeletedObjects());addImportStage('Sorting memory dumps...',()=>this.model_.sortMemoryDumps());addImportStage('Finalizing memory dump graphs...',()=>this.model_.finalizeMemoryGraphs());addImportStage('Initializing objects (step 2/2)...',()=>this.model_.initializeObjects());addImportStage('Building event indices...',()=>this.model_.buildEventIndices());addImportStage('Building UserModel...',()=>{try{const userModelBuilder=new tr.importer.UserModelBuilder(this.model_);userModelBuilder.buildUserModel();}catch(e){console.error('Failed to build user model');console.error(e);}});addImportStage('Sorting user expectations...',()=>this.model_.userModel.sortExpectations());addImportStage('Running auditors...',()=>{auditors.forEach(auditor=>auditor.runAudit());});addImportStage('Updating alerts...',()=>this.model_.sortAlerts());addImportStage('Update bounds...',()=>this.model_.updateBounds());addImportStage('Looking for warnings...',()=>{if(!this.model_.isTimeHighResolution){this.model_.importWarning({type:'low_resolution_timer',message:'Trace time is low resolution, trace may be unusable.',showToUser:true});}});lastTask.after(()=>{this.importing_=false;this.model_.stats.traceImportDurationMs=tr.b.Timing.getCurrentTimeMs()-importStartTimeMs;});return importTask;},createImporter_(eventData){const importerConstructor=tr.importer.Importer.findImporterFor(eventData);if(!importerConstructor){throw new Error('Couldn\'t create an importer for the provided '+'eventData.');} return new importerConstructor(this.model_,eventData);},hasEventDataDecoder_(importers){for(let i=0;i<importers.length;++i){if(!importers[i].isTraceDataContainer())return true;} return false;}};return{ImportOptions,Import,};});'use strict';tr.exportTo('tr.e.v8',function(){const ThreadSlice=tr.model.ThreadSlice;function V8GCStatsThreadSlice(){ThreadSlice.apply(this,arguments);this.liveObjects_=JSON.parse(this.args.live);delete this.args.live;this.deadObjects_=JSON.parse(this.args.dead);delete this.args.dead;} V8GCStatsThreadSlice.prototype={__proto__:ThreadSlice.prototype,get liveObjects(){return this.liveObjects_;},get deadObjects(){return this.deadObjects_;}};ThreadSlice.subTypes.register(V8GCStatsThreadSlice,{categoryParts:['disabled-by-default-v8.gc_stats'],name:'v8 gc stats slice',pluralName:'v8 gc stats slices'});return{V8GCStatsThreadSlice,};});'use strict';tr.exportTo('tr.e.v8',function(){const ThreadSlice=tr.model.ThreadSlice;function V8ICStatsThreadSlice(){ThreadSlice.apply(this,arguments);this.icStats_=undefined;if(this.args['ic-stats']){this.icStats_=this.args['ic-stats'].data;delete this.args['ic-stats'];}} @@ -8221,12 +8223,21 @@ const firstCounter=countBlinkObjects(dumps[0],pid);const lastCounter=countBlinkO return diffCounter;} function countBlinkObjects(dump,pid){const counter=new Map();const processesMemoryDumps=dump.processMemoryDumps;if(processesMemoryDumps[pid]===undefined)return counter;const blinkObjectsDump=processesMemoryDumps[pid].memoryAllocatorDumps.find(dump=>dump.fullName==='blink_objects');for(const v of blinkObjectsDump.children){counter.set(v.name,v.numerics.object_count.value);} return counter;} -return{leakDetectionMetric,};});'use strict';tr.exportTo('tr.metrics.console',function(){const COUNT_BOUNDARIES=tr.v.HistogramBinBoundaries.createExponential(1,1e4,30);const SUMMARY_OPTIONS=tr.v.Histogram.AVERAGE_ONLY_SUMMARY_OPTIONS;const SOURCES=['all','js','network'];function consoleErrorMetric(histograms,model){const counts={};for(const source of SOURCES){counts[source]={count:0,details:[]};} +return{leakDetectionMetric,};});'use strict';tr.exportTo('tr.metrics',function(){function blinkResourceMetric(histograms,model,opt_options){const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!chromeHelper){return;} +const CATEGORY='blink';const NAME='ResourceFetcher::requestResource';let count=0;for(const helper of Object.values(chromeHelper.rendererHelpers)){if(helper.isChromeTracingUI)continue;const events=tr.e.chrome.EventFinderUtils.getMainThreadEvents(helper,NAME,CATEGORY);for(const event of events){count++;}} +histograms.createHistogram('blinkRequestResourceCount',tr.b.Unit.byName.count,count);} +tr.metrics.MetricRegistry.register(blinkResourceMetric,{supportsRangeOfInterest:false,});return{blinkResourceMetric,};});'use strict';tr.exportTo('tr.metrics.console',function(){const COUNT_BOUNDARIES=tr.v.HistogramBinBoundaries.createExponential(1,1e4,30);const SUMMARY_OPTIONS=tr.v.Histogram.AVERAGE_ONLY_SUMMARY_OPTIONS;const SOURCES=['all','js','network'];function consoleErrorMetric(histograms,model){const counts={};for(const source of SOURCES){counts[source]={count:0,details:[]};} for(const slice of model.getDescendantEvents()){if(slice.category==='blink.console'&&slice.title==='ConsoleMessage::Error'){const source=slice.args.source.toLowerCase();counts.all.count++;if(slice.args.message){counts.all.details.push({pid:slice.getProcess().pid,...slice.args.message});} if(source in counts){counts[source].count++;if(slice.args.message){counts[source].details.push({pid:slice.getProcess().pid,...slice.args.message});}}} if(slice.category==='v8.console'&&(slice.title==='V8ConsoleMessage::Exception'||slice.title==='V8ConsoleMessage::Error'||slice.title==='V8ConsoleMessage::Assert')){counts.all.count++;counts.js.count++;}} for(const source of SOURCES){const hist=histograms.createHistogram(`console:error:${source}`,tr.b.Unit.byName.count_smallerIsBetter,{value:counts[source].count,diagnostics:{details:new tr.v.d.GenericSet(counts[source].details)}},{description:`Number of ${source} console error messages`,summaryOptions:SUMMARY_OPTIONS,});}} -tr.metrics.MetricRegistry.register(consoleErrorMetric);return{consoleErrorMetric,};});'use strict';tr.exportTo('tr.metrics.sh',function(){function getCpuSnapshotsFromModel(model){const snapshots=[];for(const pid in model.processes){const snapshotInstances=model.processes[pid].objects.getAllInstancesNamed('CPUSnapshots');if(!snapshotInstances)continue;for(const object of snapshotInstances[0].snapshots){snapshots.push(object.args.processes);}} +tr.metrics.MetricRegistry.register(consoleErrorMetric);return{consoleErrorMetric,};});'use strict';tr.exportTo('tr.metrics',function(){function countSumMetric(histograms,model,opt_options){const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!chromeHelper){return;} +const CATEGORY='benchmark';const NAME='count_sum';const counts=new Map();const sums=new Map();for(const pid in chromeHelper.rendererHelpers){const helper=chromeHelper.rendererHelpers[pid];if(helper.isChromeTracingUI)continue;const events=tr.e.chrome.EventFinderUtils.getMainThreadEvents(helper,NAME,CATEGORY);for(const event of events){const c=event.args.counter;if(!c){continue;} +if(!counts.get(c)){counts.set(c,0);} +counts.set(c,counts.get(c)+1);if(event.args.value){if(!sums.get(c)){sums.set(c,0);} +sums.set(c,sums.get(c)+event.args.value);}}} +counts.forEach((value,key)=>{histograms.createHistogram('count_'+key,tr.b.Unit.byName.count,value);});sums.forEach((value,key)=>{histograms.createHistogram('sum_'+key,tr.b.Unit.byName.unitlessNumber,value);});} +tr.metrics.MetricRegistry.register(countSumMetric,{supportsRangeOfInterest:false,});return{countSumMetric,};});'use strict';tr.exportTo('tr.metrics.sh',function(){function getCpuSnapshotsFromModel(model){const snapshots=[];for(const pid in model.processes){const snapshotInstances=model.processes[pid].objects.getAllInstancesNamed('CPUSnapshots');if(!snapshotInstances)continue;for(const object of snapshotInstances[0].snapshots){snapshots.push(object.args.processes);}} return snapshots;} function getProcessSumsFromSnapshot(snapshot){const processSums=new Map();for(const processData of snapshot){const processName=processData.name;if(!(processSums.has(processName))){processSums.set(processName,{sum:0.0,paths:new Set()});} processSums.get(processName).sum+=parseFloat(processData.pCpu);if(processData.path){processSums.get(processName).paths.add(processData.path);}} @@ -8291,11 +8302,10 @@ processVideoFreezing(freezing){if(this.freezing_===undefined||this.freezing_>fre addMetricToHistograms(histograms){this.addSample_(histograms,'time_to_video_play',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.timeToVideoPlay);this.addSample_(histograms,'time_to_audio_play',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.timeToAudioPlay);this.addSample_(histograms,'dropped_frame_count',tr.b.Unit.byName.count_smallerIsBetter,this.droppedFrameCount);for(const[key,value]of this.seekTimes.entries()){const keyString=key.toString().replace('.','_');this.addSample_(histograms,'pipeline_seek_time_'+keyString,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,value.pipelineSeekTime);this.addSample_(histograms,'seek_time_'+keyString,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,value.seekTime);} this.addSample_(histograms,'buffering_time',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.bufferingTime);this.addSample_(histograms,'roughness',tr.b.Unit.byName.count_smallerIsBetter,this.roughness);this.addSample_(histograms,'freezing',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,this.freezing);} addSample_(histograms,name,unit,sample){if(sample===undefined)return;const histogram=histograms.getHistogramNamed(name);if(histogram===undefined){histograms.createHistogram(name,unit,sample);}else{histogram.addSample(sample);}}} -tr.metrics.MetricRegistry.register(mediaMetric);return{mediaMetric,};});'use strict';tr.exportTo('tr.metrics',function(){function memoryAblationMetric(histograms,model){const modelHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!modelHelper.gpuHelper)return;const gpuProcess=modelHelper.gpuHelper.process;const events=[...gpuProcess.findTopmostSlicesNamed('Memory.GPU.PeakMemoryUsage.AblationTimes')];const allocHistogram=histograms.createHistogram('Ablation Alloc',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,[],{binBoundaries:tr.v.HistogramBinBoundaries.createLinear(0,10000,20),description:'The amount of time spent allocating the ablation '+'memory',summaryOptions:tr.metrics.rendering.SUMMARY_OPTIONS,});const deallocHistogram=histograms.createHistogram('Ablation Dealloc',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,[],{binBoundaries:tr.v.HistogramBinBoundaries.createLinear(0,10000,20),description:'The amount of time spent deallocating the ablation '+'memory',summaryOptions:tr.metrics.rendering.SUMMARY_OPTIONS,});for(let i=0;i<events.length;i++){allocHistogram.addSample(events[i].args.alloc);deallocHistogram.addSample(events[i].args.dealloc);}} -tr.metrics.MetricRegistry.register(memoryAblationMetric,{requiredCategories:['gpu.memory'],});return{memoryAblationMetric,};});'use strict';tr.exportTo('tr.metrics.pa',function(){function pcscanMetric(histograms,model){function createNumericForProcess(name,processName,desc){function createNumericForEventTime(name,desc){const n=new tr.v.Histogram(name,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter);n.description=desc;n.customizeSummaryOptions({avg:true,count:true,max:true,min:true,std:true,sum:true});return n;} -const scheme=['pa','pcscan',processName];if(name)scheme.push(name);return createNumericForEventTime(scheme.join(':'),desc);} -function createHistsForProcess(processName){return{scan:createNumericForProcess('scan',processName,'Time for scanning heap for quarantine pointers'),sweep:createNumericForProcess('sweep',processName,'Time for sweeping quarantine'),clear:createNumericForProcess('clear',processName,'Time for clearing quarantine entries'),total:createNumericForProcess('',processName,'Total time for PCScan execution')};} -function addSample(hists,slice){if(!(slice instanceof tr.model.ThreadSlice))return;if(slice.category!=='partition_alloc')return;if(slice.title==='PCScan.Scan'){hists.scan.addSample(slice.duration);}else if(slice.title==='PCScan.Sweep'){hists.sweep.addSample(slice.duration);}else if(slice.title==='PCScan.Clear'){hists.clear.addSample(slice.duration);}else if(slice.title==='PCScan'){hists.total.addSample(slice.duration);}} +tr.metrics.MetricRegistry.register(mediaMetric);return{mediaMetric,};});'use strict';tr.exportTo('tr.metrics.pa',function(){function pcscanMetric(histograms,model){function createNumericForProcess(name,processName,context,desc){function createNumericForEventTime(name,desc){const n=new tr.v.Histogram(name,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter);n.description=desc;n.customizeSummaryOptions({avg:true,count:true,max:true,min:true,std:true,sum:true});return n;} +const scheme=['pa','pcscan',processName,context];if(name)scheme.push(name);return createNumericForEventTime(scheme.join(':'),desc);} +function createHistsForProcess(processName){return{scanner_scan:createNumericForProcess('scan',processName,'scanner','Time for scanning heap for quarantine pointers on concurrent threads'),scanner_sweep:createNumericForProcess('sweep',processName,'scanner','Time for sweeping quarantine'),scanner_clear:createNumericForProcess('clear',processName,'scanner','Time for clearing quarantine entries'),scanner_total:createNumericForProcess('',processName,'scanner','Total time for PCScan execution on concurrent threads'),mutator_scan:createNumericForProcess('scan',processName,'mutator','Time for scanning heap for quarantine pointers on mutator threads'),mutator_clear:createNumericForProcess('clear',processName,'mutator','Time for clearing heap quarantine entries on mutator threads'),mutator_total:createNumericForProcess('',processName,'mutator','Total time for PCScan execution on mutator threads (inside safepoints)'),};} +function addSample(hists,slice){if(!(slice instanceof tr.model.ThreadSlice))return;if(slice.category!=='partition_alloc')return;if(slice.title==='PCScan.Scanner.Scan'){hists.scanner_scan.addSample(slice.duration);}else if(slice.title==='PCScan.Scanner.Sweep'){hists.scanner_sweep.addSample(slice.duration);}else if(slice.title==='PCScan.Scanner.Clear'){hists.scanner_clear.addSample(slice.duration);}else if(slice.title==='PCScan.Scanner'){hists.scanner_total.addSample(slice.duration);}else if(slice.title==='PCScan.Mutator.Scan'){hists.mutator_scan.addSample(slice.duration);}else if(slice.title==='PCScan.Mutator.Clear'){hists.mutator_clear.addSample(slice.duration);}else if(slice.title==='PCScan.Mutator'){hists.mutator_total.addSample(slice.duration);}} function addHistsForProcess(processHists,processHelpers){for(const helper of Object.values(processHelpers)){const processName=tr.e.chrome.chrome_processes.canonicalizeProcessName(helper.process.name);if(!processHists.has(processName)){processHists.set(processName,createHistsForProcess(processName));} for(const slice of helper.process.getDescendantEvents()){addSample(processHists.get(processName),slice);}}} const helper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);const processHists=new Map();addHistsForProcess(processHists,helper.browserHelpers);addHistsForProcess(processHists,helper.rendererHelpers);for(const hists of processHists.values()){for(const hist of Object.values(hists)){histograms.addHistogram(hist);}}} @@ -8477,7 +8487,9 @@ function collectTimeToEvent(rendererHelper,timeToXEntries){const samples=[];for( return samples;} function collectTimeToEventInCpuTime(rendererHelper,timeToXEntries){const samples=[];for(const{targetEvent,navigationStartEvent,url}of timeToXEntries){const navStartToEventRange=tr.b.math.Range.fromExplicitRange(navigationStartEvent.start,targetEvent.start);const mainThreadCpuTime=rendererHelper.mainThread.getCpuTimeForRange(navStartToEventRange);const breakdownTree=tr.metrics.sh.generateCpuTimeBreakdownTree(rendererHelper.mainThread,navStartToEventRange);samples.push({value:mainThreadCpuTime,breakdownTree,diagnostics:{breakdown:createBreakdownDiagnostic(breakdownTree),start:new RelatedEventSet(navigationStartEvent),end:new RelatedEventSet(targetEvent),infos:new tr.v.d.GenericSet([{pid:rendererHelper.pid,start:navigationStartEvent.start,event:targetEvent.start,}]),}});} return samples;} -function findLayoutShiftSamples(rendererHelper){let sample;EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'LayoutShift','loading').forEach((events)=>{const evData=events.pop().args.data;if(evData.is_main_frame){sample={value:evData.cumulative_score};}});return sample?[sample]:[];} +function findMainFrameLayoutShiftSamples(rendererHelper){let sample;EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'LayoutShift','loading').forEach((events)=>{const evData=events.pop().args.data;if(evData.is_main_frame){sample={value:evData.cumulative_score};}});return sample?[sample]:[];} +function findAllLayoutShiftSamples(chromeHelper){let total=0;let foundMainFrame=false;for(const pid in chromeHelper.rendererHelpers){const rendererHelper=chromeHelper.rendererHelpers[pid];if(rendererHelper.isChromeTracingUI)continue;tr.e.chrome.EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'LayoutShift','loading').forEach((events)=>{for(const event of events){const evData=event.args.data;if(evData.is_main_frame){total+=evData.score;foundMainFrame=true;}else{total+=evData.weighted_score_delta;}}});} +return foundMainFrame?[{value:total}]:[];} function addFirstMeaningfulPaintSample(samples,rendererHelper,navigationStart,fmpMarkerEvent,url){const navStartToFMPRange=tr.b.math.Range.fromExplicitRange(navigationStart.start,fmpMarkerEvent.start);const networkEvents=EventFinderUtils.getNetworkEventsInRange(rendererHelper.process,navStartToFMPRange);const timeToFirstMeaningfulPaint=navStartToFMPRange.duration;const breakdownTree=tr.metrics.sh.generateWallClockTimeBreakdownTree(rendererHelper.mainThread,networkEvents,navStartToFMPRange);samples.push({value:timeToFirstMeaningfulPaint,breakdownTree,diagnostics:{breakdown:createBreakdownDiagnostic(breakdownTree),start:new RelatedEventSet(navigationStart),end:new RelatedEventSet(fmpMarkerEvent),infos:new tr.v.d.GenericSet([{url,pid:rendererHelper.pid,start:navigationStart.start,fmp:fmpMarkerEvent.start,}]),}});} function addFirstMeaningfulPaintCpuTimeSample(samples,rendererHelper,navigationStart,fmpMarkerEvent,url){const navStartToFMPRange=tr.b.math.Range.fromExplicitRange(navigationStart.start,fmpMarkerEvent.start);const mainThreadCpuTime=rendererHelper.mainThread.getCpuTimeForRange(navStartToFMPRange);const breakdownTree=tr.metrics.sh.generateCpuTimeBreakdownTree(rendererHelper.mainThread,navStartToFMPRange);samples.push({value:mainThreadCpuTime,breakdownTree,diagnostics:{breakdown:createBreakdownDiagnostic(breakdownTree),start:new RelatedEventSet(navigationStart),end:new RelatedEventSet(fmpMarkerEvent),infos:new tr.v.d.GenericSet([{url,pid:rendererHelper.pid,start:navigationStart.start,fmp:fmpMarkerEvent.start,}]),}});} function decorateInteractivitySampleWithDiagnostics_(rendererHelper,eventTimestamp,navigationStartEvent,firstContentfulPaintTime,domContentLoadedEndTime,url){if(eventTimestamp===undefined)return undefined;const navigationStartTime=navigationStartEvent.start;const navStartToEventTimeRange=tr.b.math.Range.fromExplicitRange(navigationStartTime,eventTimestamp);const networkEvents=EventFinderUtils.getNetworkEventsInRange(rendererHelper.process,navStartToEventTimeRange);const breakdownTree=tr.metrics.sh.generateWallClockTimeBreakdownTree(rendererHelper.mainThread,networkEvents,navStartToEventTimeRange);const breakdownDiagnostic=createBreakdownDiagnostic(breakdownTree);return{value:navStartToEventTimeRange.duration,diagnostics:tr.v.d.DiagnosticMap.fromObject({'Start':new RelatedEventSet(navigationStartEvent),'Navigation infos':new tr.v.d.GenericSet([{url,pid:rendererHelper.pid,navigationStartTime,firstContentfulPaintTime,domContentLoadedEndTime,eventTimestamp,}]),'Breakdown of [navStart, eventTimestamp]':breakdownDiagnostic,}),};} @@ -8490,7 +8502,7 @@ return lastCandidates;} function findLargestTextPaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents){const timeToPaintEntries=findTimeToXEntries('loading','LargestTextPaint::Candidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const timeToPaintBlockingEntries=findTimeToXEntries('loading','LargestTextPaint::NoCandidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const lastCandidateEvents=findLastCandidateForEachNavigation(timeToPaintEntries.concat(timeToPaintBlockingEntries)).filter(event=>event.targetEvent.title!=='LargestTextPaint::NoCandidate');return collectTimeToEvent(rendererHelper,lastCandidateEvents);} function findLargestImagePaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents){const timeToPaintEntries=findTimeToXEntries('loading','LargestImagePaint::Candidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const timeToPaintBlockingEntries=findTimeToXEntries('loading','LargestImagePaint::NoCandidate',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const lastCandidateEvents=findLastCandidateForEachNavigation(timeToPaintEntries.concat(timeToPaintBlockingEntries)).filter(event=>event.targetEvent.title!=='LargestImagePaint::NoCandidate');return collectTimeToEvent(rendererHelper,lastCandidateEvents);} function findLargestContentfulPaintHistogramSamples(allBrowserEvents){const lcp=new tr.e.chrome.LargestContentfulPaint(allBrowserEvents);const lcpSamples=lcp.findCandidates().map(candidate=>{const{durationInMilliseconds,size,type,inMainFrame,mainFrameTreeNodeId}=candidate;return{value:durationInMilliseconds,diagnostics:{size:new tr.v.d.GenericSet([size]),type:new tr.v.d.GenericSet([type]),inMainFrame:new tr.v.d.GenericSet([inMainFrame]),mainFrameTreeNodeId:new tr.v.d.GenericSet([mainFrameTreeNodeId]),},};});return lcpSamples;} -function collectLoadingMetricsForRenderer(rendererHelper){const frameToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'navigationStart','blink.user_timing');const navIdToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByNavId(rendererHelper,'navigationStart','blink.user_timing');const firstPaintSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('loading','firstPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const timeToFCPEntries=findTimeToXEntries('loading','firstContentfulPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const firstContentfulPaintSamples=collectTimeToEvent(rendererHelper,timeToFCPEntries);const firstContentfulPaintCpuTimeSamples=collectTimeToEventInCpuTime(rendererHelper,timeToFCPEntries);const onLoadSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('blink.user_timing','loadEventStart',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const aboveTheFoldLoadedToVisibleSamples=getAboveTheFoldLoadedToVisibleSamples(rendererHelper);const firstViewportReadySamples=getFirstViewportReadySamples(rendererHelper,navIdToNavStartEvents);const largestImagePaintSamples=findLargestImagePaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const largestTextPaintSamples=findLargestTextPaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const layoutShiftSamples=findLayoutShiftSamples(rendererHelper);const navigationStartSamples=timeToFCPEntries.map(entry=>{return{value:entry.navigationStartEvent.start};});return{frameToNavStartEvents,firstPaintSamples,firstContentfulPaintSamples,firstContentfulPaintCpuTimeSamples,onLoadSamples,aboveTheFoldLoadedToVisibleSamples,firstViewportReadySamples,largestImagePaintSamples,largestTextPaintSamples,layoutShiftSamples,navigationStartSamples,};} +function collectLoadingMetricsForRenderer(rendererHelper){const frameToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByFrame(rendererHelper,'navigationStart','blink.user_timing');const navIdToNavStartEvents=EventFinderUtils.getSortedMainThreadEventsByNavId(rendererHelper,'navigationStart','blink.user_timing');const firstPaintSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('loading','firstPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const timeToFCPEntries=findTimeToXEntries('loading','firstContentfulPaint',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const firstContentfulPaintSamples=collectTimeToEvent(rendererHelper,timeToFCPEntries);const firstContentfulPaintCpuTimeSamples=collectTimeToEventInCpuTime(rendererHelper,timeToFCPEntries);const onLoadSamples=collectTimeToEvent(rendererHelper,findTimeToXEntries('blink.user_timing','loadEventStart',rendererHelper,frameToNavStartEvents,navIdToNavStartEvents));const aboveTheFoldLoadedToVisibleSamples=getAboveTheFoldLoadedToVisibleSamples(rendererHelper);const firstViewportReadySamples=getFirstViewportReadySamples(rendererHelper,navIdToNavStartEvents);const largestImagePaintSamples=findLargestImagePaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const largestTextPaintSamples=findLargestTextPaintSamples(rendererHelper,frameToNavStartEvents,navIdToNavStartEvents);const mainFrameLayoutShiftSamples=findMainFrameLayoutShiftSamples(rendererHelper);const navigationStartSamples=timeToFCPEntries.map(entry=>{return{value:entry.navigationStartEvent.start};});return{frameToNavStartEvents,firstPaintSamples,firstContentfulPaintSamples,firstContentfulPaintCpuTimeSamples,onLoadSamples,aboveTheFoldLoadedToVisibleSamples,firstViewportReadySamples,largestImagePaintSamples,largestTextPaintSamples,mainFrameLayoutShiftSamples,navigationStartSamples,};} function collectMetricsFromLoadExpectations(model,chromeHelper){const interactiveSamples=[];const firstCpuIdleSamples=[];const firstMeaningfulPaintSamples=[];const firstMeaningfulPaintCpuTimeSamples=[];const totalBlockingTimeSamples=[];for(const expectation of model.userModel.expectations){if(!(expectation instanceof tr.model.um.LoadExpectation))continue;if(tr.e.chrome.CHROME_INTERNAL_URLS.includes(expectation.url)){continue;} const rendererHelper=chromeHelper.rendererHelpers[expectation.renderProcess.pid];if(expectation.fmpEvent!==undefined){addFirstMeaningfulPaintSample(firstMeaningfulPaintSamples,rendererHelper,expectation.navigationStart,expectation.fmpEvent,expectation.url);addFirstMeaningfulPaintCpuTimeSample(firstMeaningfulPaintCpuTimeSamples,rendererHelper,expectation.navigationStart,expectation.fmpEvent,expectation.url);} if(expectation.firstCpuIdleTime!==undefined){firstCpuIdleSamples.push(decorateInteractivitySampleWithDiagnostics_(rendererHelper,expectation.firstCpuIdleTime,expectation.navigationStart,expectation.fcpEvent.start,expectation.domContentLoadedEndEvent.start,expectation.url));} @@ -8500,7 +8512,7 @@ return{firstMeaningfulPaintSamples,firstMeaningfulPaintCpuTimeSamples,firstCpuId function addSamplesToHistogram(samples,histogram,histograms){for(const sample of samples){histogram.addSample(sample.value,sample.diagnostics);if(histogram.name!=='timeToFirstContentfulPaint')continue;if(!sample.breakdownTree)continue;for(const[category,breakdown]of Object.entries(sample.breakdownTree)){const relatedName=`${histogram.name}:${category}`;let relatedHist=histograms.getHistogramsNamed(relatedName)[0];if(!relatedHist){relatedHist=histograms.createHistogram(relatedName,histogram.unit,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,summaryOptions:{count:false,max:false,min:false,sum:false,},});let relatedNames=histogram.diagnostics.get('breakdown');if(!relatedNames){relatedNames=new tr.v.d.RelatedNameMap();histogram.diagnostics.set('breakdown',relatedNames);} relatedNames.set(category,relatedName);} relatedHist.addSample(breakdown.total,{breakdown:tr.v.d.Breakdown.fromEntries(Object.entries(breakdown.events)),});}}} -function loadingMetric(histograms,model){const firstPaintHistogram=histograms.createHistogram('timeToFirstPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintHistogram=histograms.createHistogram('timeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const onLoadHistogram=histograms.createHistogram('timeToOnload',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to onload. '+'This is temporary metric used for PCv1/v2 sanity checking',summaryOptions:SUMMARY_OPTIONS,});const firstMeaningfulPaintHistogram=histograms.createHistogram('timeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstMeaningfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const timeToInteractiveHistogram=histograms.createHistogram('timeToInteractive',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to Interactive',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const totalBlockingTimeHistogram=histograms.createHistogram('totalBlockingTime',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Total Blocking Time',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const timeToFirstCpuIdleHistogram=histograms.createHistogram('timeToFirstCpuIdle',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to First CPU Idle',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const aboveTheFoldLoadedToVisibleHistogram=histograms.createHistogram('aboveTheFoldLoadedToVisible',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from first visible to load for AMP pages only.',summaryOptions:SUMMARY_OPTIONS,});const firstViewportReadyHistogram=histograms.createHistogram('timeToFirstViewportReady',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from navigation to load for AMP pages only. ',summaryOptions:SUMMARY_OPTIONS,});const largestImagePaintHistogram=histograms.createHistogram('largestImagePaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Image Paint',summaryOptions:SUMMARY_OPTIONS,});const largestTextPaintHistogram=histograms.createHistogram('largestTextPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Text Paint',summaryOptions:SUMMARY_OPTIONS,});const largestContentfulPaintHistogram=histograms.createHistogram('largestContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Contentful Paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const layoutShiftHistogram=histograms.createHistogram('mainFrameCumulativeLayoutShift',unitlessNumber_smallerIsBetter,[],{binBoundaries:LAYOUT_SHIFT_SCORE_BOUNDARIES,description:'Main Frame Document Cumulative Layout Shift Score',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_LAYOUT],});const navigationStartHistogram=histograms.createHistogram('navigationStart',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'navigationStart',summaryOptions:SUMMARY_OPTIONS,});tr.metrics.sh.rectsBasedSpeedIndexMetric(histograms,model);const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);for(const pid in chromeHelper.rendererHelpers){const rendererHelper=chromeHelper.rendererHelpers[pid];if(rendererHelper.isChromeTracingUI)continue;const samplesSet=collectLoadingMetricsForRenderer(rendererHelper);const lcpSamples=findLargestContentfulPaintHistogramSamples(chromeHelper.browserHelper.mainThread.sliceGroup.slices);addSamplesToHistogram(lcpSamples,largestContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstPaintSamples,firstPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintSamples,firstContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintCpuTimeSamples,firstContentfulPaintCpuTimeHistogram,histograms);addSamplesToHistogram(samplesSet.onLoadSamples,onLoadHistogram,histograms);addSamplesToHistogram(samplesSet.aboveTheFoldLoadedToVisibleSamples,aboveTheFoldLoadedToVisibleHistogram,histograms);addSamplesToHistogram(samplesSet.firstViewportReadySamples,firstViewportReadyHistogram,histograms);addSamplesToHistogram(samplesSet.largestImagePaintSamples,largestImagePaintHistogram,histograms);addSamplesToHistogram(samplesSet.largestTextPaintSamples,largestTextPaintHistogram,histograms);addSamplesToHistogram(samplesSet.layoutShiftSamples,layoutShiftHistogram,histograms);addSamplesToHistogram(samplesSet.navigationStartSamples,navigationStartHistogram,histograms);} +function loadingMetric(histograms,model){const firstPaintHistogram=histograms.createHistogram('timeToFirstPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintHistogram=histograms.createHistogram('timeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstContentfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first contentful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const onLoadHistogram=histograms.createHistogram('timeToOnload',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to onload. '+'This is temporary metric used for PCv1/v2 sanity checking',summaryOptions:SUMMARY_OPTIONS,});const firstMeaningfulPaintHistogram=histograms.createHistogram('timeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const firstMeaningfulPaintCpuTimeHistogram=histograms.createHistogram('cpuTimeToFirstMeaningfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'CPU time to first meaningful paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const timeToInteractiveHistogram=histograms.createHistogram('timeToInteractive',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to Interactive',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const totalBlockingTimeHistogram=histograms.createHistogram('totalBlockingTime',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Total Blocking Time',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const timeToFirstCpuIdleHistogram=histograms.createHistogram('timeToFirstCpuIdle',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time to First CPU Idle',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_INTERACTIVITY],});const aboveTheFoldLoadedToVisibleHistogram=histograms.createHistogram('aboveTheFoldLoadedToVisible',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from first visible to load for AMP pages only.',summaryOptions:SUMMARY_OPTIONS,});const firstViewportReadyHistogram=histograms.createHistogram('timeToFirstViewportReady',timeDurationInMs_smallerIsBetter,[],{binBoundaries:TIME_TO_INTERACTIVE_BOUNDARIES,description:'Time from navigation to load for AMP pages only. ',summaryOptions:SUMMARY_OPTIONS,});const largestImagePaintHistogram=histograms.createHistogram('largestImagePaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Image Paint',summaryOptions:SUMMARY_OPTIONS,});const largestTextPaintHistogram=histograms.createHistogram('largestTextPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Text Paint',summaryOptions:SUMMARY_OPTIONS,});const largestContentfulPaintHistogram=histograms.createHistogram('largestContentfulPaint',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'Time to Largest Contentful Paint',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_PAINT],});const mainFrameLayoutShiftHistogram=histograms.createHistogram('mainFrameCumulativeLayoutShift',unitlessNumber_smallerIsBetter,[],{binBoundaries:LAYOUT_SHIFT_SCORE_BOUNDARIES,description:'Main Frame Document Cumulative Layout Shift Score',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_LAYOUT],});const allLayoutShiftHistogram=histograms.createHistogram('overallCumulativeLayoutShift',unitlessNumber_smallerIsBetter,[],{binBoundaries:LAYOUT_SHIFT_SCORE_BOUNDARIES,description:'Document Cumulative Layout Shift Score with iframes',summaryOptions:SUMMARY_OPTIONS,alertGrouping:[tr.v.d.ALERT_GROUPS.LOADING_LAYOUT],});const navigationStartHistogram=histograms.createHistogram('navigationStart',timeDurationInMs_smallerIsBetter,[],{binBoundaries:LOADING_METRIC_BOUNDARIES,description:'navigationStart',summaryOptions:SUMMARY_OPTIONS,});tr.metrics.sh.rectsBasedSpeedIndexMetric(histograms,model);const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);const allLayoutShiftSamples=findAllLayoutShiftSamples(chromeHelper);addSamplesToHistogram(allLayoutShiftSamples,allLayoutShiftHistogram,histograms);for(const pid in chromeHelper.rendererHelpers){const rendererHelper=chromeHelper.rendererHelpers[pid];if(rendererHelper.isChromeTracingUI)continue;const samplesSet=collectLoadingMetricsForRenderer(rendererHelper);const lcpSamples=findLargestContentfulPaintHistogramSamples(chromeHelper.browserHelper.mainThread.sliceGroup.slices);addSamplesToHistogram(lcpSamples,largestContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstPaintSamples,firstPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintSamples,firstContentfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstContentfulPaintCpuTimeSamples,firstContentfulPaintCpuTimeHistogram,histograms);addSamplesToHistogram(samplesSet.onLoadSamples,onLoadHistogram,histograms);addSamplesToHistogram(samplesSet.aboveTheFoldLoadedToVisibleSamples,aboveTheFoldLoadedToVisibleHistogram,histograms);addSamplesToHistogram(samplesSet.firstViewportReadySamples,firstViewportReadyHistogram,histograms);addSamplesToHistogram(samplesSet.largestImagePaintSamples,largestImagePaintHistogram,histograms);addSamplesToHistogram(samplesSet.largestTextPaintSamples,largestTextPaintHistogram,histograms);addSamplesToHistogram(samplesSet.mainFrameLayoutShiftSamples,mainFrameLayoutShiftHistogram,histograms);addSamplesToHistogram(samplesSet.navigationStartSamples,navigationStartHistogram,histograms);} const samplesSet=collectMetricsFromLoadExpectations(model,chromeHelper);addSamplesToHistogram(samplesSet.firstMeaningfulPaintSamples,firstMeaningfulPaintHistogram,histograms);addSamplesToHistogram(samplesSet.firstMeaningfulPaintCpuTimeSamples,firstMeaningfulPaintCpuTimeHistogram,histograms);addSamplesToHistogram(samplesSet.interactiveSamples,timeToInteractiveHistogram,histograms);addSamplesToHistogram(samplesSet.firstCpuIdleSamples,timeToFirstCpuIdleHistogram,histograms);addSamplesToHistogram(samplesSet.totalBlockingTimeSamples,totalBlockingTimeHistogram,histograms);} tr.metrics.MetricRegistry.register(loadingMetric);return{loadingMetric,createBreakdownDiagnostic};});'use strict';tr.exportTo('tr.metrics',function(){const SPA_NAVIGATION_START_TO_FIRST_PAINT_DURATION_BIN_BOUNDARY=tr.v.HistogramBinBoundaries.createExponential(1,1000,50);function spaNavigationMetric(histograms,model){const histogram=new tr.v.Histogram('spaNavigationStartToFpDuration',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,SPA_NAVIGATION_START_TO_FIRST_PAINT_DURATION_BIN_BOUNDARY);histogram.description='Latency between the input event causing'+' a SPA navigation and the first paint event after it';histogram.customizeSummaryOptions({count:false,sum:false,});const modelHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);if(!modelHelper){return;} const rendererHelpers=modelHelper.rendererHelpers;if(!rendererHelpers){return;} @@ -8987,6 +8999,7 @@ return tr.v.HistogramBinBoundaries.createExponential(1e-3,1e3,50);} function umaMetric(histograms,model){const histogramValues=new Map();const nameCounts=new Map();for(const process of model.getAllProcesses()){const histogramEvents=new Map();for(const event of process.instantEvents){if(event.title!=='UMAHistogramSamples')continue;const name=event.args.name;const events=histogramEvents.get(name)||[];if(!histogramEvents.has(name))histogramEvents.set(name,events);events.push(event);} let processName=tr.e.chrome.chrome_processes.canonicalizeProcessName(process.name);nameCounts.set(processName,(nameCounts.get(processName)||0)+1);processName=`${processName}_${nameCounts.get(processName)}`;for(const[name,events]of histogramEvents){const values=histogramValues.get(name)||{sum:0,bins:[]};if(!histogramValues.has(name))histogramValues.set(name,values);const endValues=parseBuckets_(events[events.length-1],processName);if(events.length===1){mergeBins_(values,endValues,name);}else{throw new Error('There should be at most one snapshot of UMA '+`histogram for ${name} in each process.`);}}} for(const[name,values]of histogramValues){const histogram=new tr.v.Histogram(name,getHistogramUnit_(name),getHistogramBoundaries_(name));const isLinear=getIsHistogramBinsLinear_(name);let sumOfMiddles=0;let sumOfBinLengths=0;for(const bin of values.bins){sumOfMiddles+=bin.count*(bin.min+bin.max)/2;sumOfBinLengths+=bin.count*(bin.max-bin.min);} +if(name.startsWith('CompositorLatency.Type')){let histogramBoundaries=tr.v.HistogramBinBoundaries.createLinear(0,100,101);let histogramUnit=getHistogramUnit_(name);let presentedCount=values.bins[0]?values.bins[0].count:0;let delayedCount=values.bins[1]?values.bins[1].count:0;let droppedCount=values.bins[2]?values.bins[2].count:0;let inTimeCount=presentedCount-delayedCount;let totalCount=presentedCount+droppedCount;const inTimeHistogram=new tr.v.Histogram(name+'.Percentage_of_in_time_frames',histogramUnit,histogramBoundaries);inTimeHistogram.addSample(100.0*inTimeCount/totalCount);histograms.addHistogram(inTimeHistogram);const delayedHistogram=new tr.v.Histogram(name+'.Percentage_of_delayed_frames',histogramUnit,histogramBoundaries);delayedHistogram.addSample(100.0*delayedCount/totalCount);histograms.addHistogram(delayedHistogram);const droppedHistogram=new tr.v.Histogram(name+'.Percentage_of_dropped_frames',histogramUnit,histogramBoundaries);droppedHistogram.addSample(100.0*droppedCount/totalCount);histograms.addHistogram(droppedHistogram);} const shift=(values.sum-sumOfMiddles)/sumOfBinLengths;if(isLinear&&Math.abs(shift)>0.5){throw new Error(`Samples sum is wrong for ${name}.`);} for(const bin of values.bins){if(bin.count===0)continue;const shiftedValue=(bin.min+bin.max)/2+shift*(bin.max-bin.min);for(const[processName,count]of bin.processes){bin.processes.set(processName,shiftedValue*count/bin.count);} for(let i=0;i<bin.count;i++){histogram.addSample(shiftedValue,{processes:bin.processes,events:bin.events});}} @@ -9008,7 +9021,7 @@ histograms.addHistogram(cpuTotalOptimizeCode);histograms.addHistogram(wallTotalO function computeDeoptimizeCodeMetrics(histograms,model){const cpuTotalDeoptimizeCode=new tr.v.Histogram('v8_deoptimize_code_cpu_total',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);cpuTotalDeoptimizeCode.description='cpu total time spent in code deoptimization';const wallTotalDeoptimizeCode=new tr.v.Histogram('v8_deoptimize_code_wall_total',tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);wallTotalDeoptimizeCode.description='wall total time spent in code deoptimization';for(const e of model.findTopmostSlicesNamed('V8.DeoptimizeCode')){cpuTotalDeoptimizeCode.addSample(e.cpuDuration);wallTotalDeoptimizeCode.addSample(e.duration);} histograms.addHistogram(cpuTotalDeoptimizeCode);histograms.addHistogram(wallTotalDeoptimizeCode);} function executionMetric(histograms,model){computeExecuteMetrics(histograms,model);computeParseLazyMetrics(histograms,model);computeCompileIgnitionMetrics(histograms,model);computeCompileFullCodeMetrics(histograms,model);computeRecompileMetrics(histograms,model);computeOptimizeCodeMetrics(histograms,model);computeDeoptimizeCodeMetrics(histograms,model);} -tr.metrics.MetricRegistry.register(executionMetric);return{executionMetric,};});'use strict';tr.exportTo('tr.metrics.v8',function(){const TARGET_FPS=60;const MS_PER_SECOND=1000;const WINDOW_SIZE_MS=MS_PER_SECOND/TARGET_FPS;const EPSILON=1e-6;const METRICS=['v8:gc:cycle:full','v8:gc:cycle:full:cpp','v8:gc:cycle:full:mark','v8:gc:cycle:full:mark:cpp','v8:gc:cycle:full:weak','v8:gc:cycle:full:weak:cpp','v8:gc:cycle:full:sweep','v8:gc:cycle:full:sweep:cpp','v8:gc:cycle:full:compact','v8:gc:cycle:full:compact:cpp','v8:gc:cycle:main_thread:full','v8:gc:cycle:main_thread:full:cpp','v8:gc:cycle:main_thread:full:mark','v8:gc:cycle:main_thread:full:mark:cpp','v8:gc:cycle:main_thread:full:weak','v8:gc:cycle:main_thread:full:weak:cpp','v8:gc:cycle:main_thread:full:sweep','v8:gc:cycle:main_thread:full:sweep:cpp','v8:gc:cycle:main_thread:full:compact','v8:gc:cycle:main_thread:full:compact:cpp','v8:gc:event:main_thread:full:atomic','v8:gc:event:main_thread:full:atomic:cpp','v8:gc:event:main_thread:full:atomic:mark','v8:gc:event:main_thread:full:atomic:mark:cpp','v8:gc:event:main_thread:full:atomic:weak','v8:gc:event:main_thread:full:atomic:weak:cpp','v8:gc:event:main_thread:full:atomic:sweep','v8:gc:event:main_thread:full:atomic:sweep:cpp','v8:gc:event:main_thread:full:atomic:compact','v8:gc:event:main_thread:full:atomic:compact:cpp','v8:gc:event:main_thread:full:incremental','v8:gc:event:main_thread:full:incremental:cpp','v8:gc:event:main_thread:full:incremental:mark','v8:gc:event:main_thread:full:incremental:mark:cpp','v8:gc:event:main_thread:full:incremental:sweep','v8:gc:event:main_thread:full:incremental:sweep:cpp','v8:gc:cycle:young','v8:gc:cycle:main_thread:young',];const V8_FULL_ATOMIC_EVENTS=['V8.GCCompactor','V8.GCFinalizeMC','V8.GCFinalizeMCReduceMemory',];const V8_FULL_MARK_EVENTS=['V8.GC_MC_BACKGROUND_MARKING','V8.GC_MC_MARK','V8.GCIncrementalMarking','V8.GCIncrementalMarkingFinalize','V8.GCIncrementalMarkingStart',];const V8_FULL_COMPACT_EVENTS=['V8.GC_MC_BACKGROUND_EVACUATE_COPY','V8.GC_MC_BACKGROUND_EVACUATE_UPDATE_POINTERS','V8.GC_MC_EVACUATE',];const V8_FULL_SWEEP_EVENTS=['V8.GC_MC_BACKGROUND_SWEEPING','V8.GC_MC_SWEEP',];const V8_FULL_WEAK_EVENTS=['V8.GC_MC_CLEAR',];const V8_YOUNG_EVENTS=['V8.GC_SCAVENGER_BACKGROUND_SCAVENGE_PARALLEL','V8.GCScavenger',];const CPP_GC_FULL_MARK_EVENTS=['BlinkGC.AtomicPauseMarkEpilogue','BlinkGC.AtomicPauseMarkPrologue','BlinkGC.AtomicPauseMarkRoots','BlinkGC.AtomicPauseMarkTransitiveClosure','BlinkGC.ConcurrentMarkingStep','BlinkGC.IncrementalMarkingStartMarking','BlinkGC.IncrementalMarkingStep','BlinkGC.MarkBailOutObjects','BlinkGC.MarkFlushEphemeronPairs','BlinkGC.MarkFlushV8References','BlinkGC.UnifiedMarkingStep','CppGC.AtomicMark','CppGC.IncrementalMark','CppGC.ConcurrentMark',];const CPP_GC_FULL_COMPACT_EVENTS=['BlinkGC.AtomicPauseSweepAndCompact','CppGC.AtomicCompact',];const CPP_GC_FULL_SWEEP_EVENTS=['BlinkGC.CompleteSweep','BlinkGC.ConcurrentSweepingStep','BlinkGC.LazySweepInIdle','BlinkGC.LazySweepOnAllocation','CppGC.AtomicSweep','CppGC.IncrementalSweep','CppGC.ConcurrentSweep',];const CPP_GC_FULL_WEAK_EVENTS=['BlinkGC.MarkWeakProcessing','CppGC.AtomicWeak',];const RULES=[{events:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic',},{events:V8_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark',},{events:CPP_GC_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark:cpp',},{events:V8_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark',},{events:CPP_GC_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark:cpp',},{events:V8_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact',},{events:CPP_GC_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact:cpp',},{events:V8_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep:cpp',},{events:V8_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak',},{events:CPP_GC_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak:cpp',},{events:V8_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep:cpp',},{events:V8_YOUNG_EVENTS,contribute_to:'young:atomic',},];const Granularity={CYCLE:'cycle',EVENT:'event',};const ThreadType={MAIN:'main',BACKGROUND:'background',ALL_THREADS:'all_threads',};class Metric{constructor(name){const parts=name.split(':');this.granularity_=parts[2];assert(this.granularity_===Granularity.CYCLE||this.granularity_===Granularity.EVENT);this.thread_=ThreadType.ALL_THREADS;let phasesIndex=3;if(parts[3]==='main_thread'){this.thread_=ThreadType.MAIN;phasesIndex=4;} +tr.metrics.MetricRegistry.register(executionMetric);return{executionMetric,};});'use strict';tr.exportTo('tr.metrics.v8',function(){const TARGET_FPS=60;const MS_PER_SECOND=1000;const WINDOW_SIZE_MS=MS_PER_SECOND/TARGET_FPS;const EPSILON=1e-6;const METRICS=['v8:gc:cycle:full','v8:gc:cycle:full:cpp','v8:gc:cycle:full:mark','v8:gc:cycle:full:mark:cpp','v8:gc:cycle:full:weak','v8:gc:cycle:full:weak:cpp','v8:gc:cycle:full:sweep','v8:gc:cycle:full:sweep:cpp','v8:gc:cycle:full:compact','v8:gc:cycle:full:compact:cpp','v8:gc:cycle:main_thread:full','v8:gc:cycle:main_thread:full:cpp','v8:gc:cycle:main_thread:full:mark','v8:gc:cycle:main_thread:full:mark:cpp','v8:gc:cycle:main_thread:full:weak','v8:gc:cycle:main_thread:full:weak:cpp','v8:gc:cycle:main_thread:full:sweep','v8:gc:cycle:main_thread:full:sweep:cpp','v8:gc:cycle:main_thread:full:compact','v8:gc:cycle:main_thread:full:compact:cpp','v8:gc:cycle:main_thread:full:atomic','v8:gc:cycle:main_thread:full:atomic:cpp','v8:gc:cycle:main_thread:full:atomic:mark','v8:gc:cycle:main_thread:full:atomic:mark:cpp','v8:gc:cycle:main_thread:full:atomic:weak','v8:gc:cycle:main_thread:full:atomic:weak:cpp','v8:gc:cycle:main_thread:full:atomic:sweep','v8:gc:cycle:main_thread:full:atomic:sweep:cpp','v8:gc:cycle:main_thread:full:atomic:compact','v8:gc:cycle:main_thread:full:atomic:compact:cpp','v8:gc:cycle:main_thread:full:incremental','v8:gc:cycle:main_thread:full:incremental:cpp','v8:gc:cycle:main_thread:full:incremental:mark','v8:gc:cycle:main_thread:full:incremental:mark:cpp','v8:gc:cycle:main_thread:full:incremental:sweep','v8:gc:cycle:main_thread:full:incremental:sweep:cpp','v8:gc:event:main_thread:full:atomic','v8:gc:event:main_thread:full:atomic:cpp','v8:gc:event:main_thread:full:atomic:mark','v8:gc:event:main_thread:full:atomic:mark:cpp','v8:gc:event:main_thread:full:atomic:weak','v8:gc:event:main_thread:full:atomic:weak:cpp','v8:gc:event:main_thread:full:atomic:sweep','v8:gc:event:main_thread:full:atomic:sweep:cpp','v8:gc:event:main_thread:full:atomic:compact','v8:gc:event:main_thread:full:atomic:compact:cpp','v8:gc:event:main_thread:full:incremental','v8:gc:event:main_thread:full:incremental:cpp','v8:gc:event:main_thread:full:incremental:mark','v8:gc:event:main_thread:full:incremental:mark:cpp','v8:gc:event:main_thread:full:incremental:sweep','v8:gc:event:main_thread:full:incremental:sweep:cpp','v8:gc:cycle:young','v8:gc:cycle:main_thread:young',];const V8_FULL_ATOMIC_EVENTS=['V8.GC_MARK_COMPACTOR'];const V8_FULL_MARK_EVENTS=['V8.GC_MC_BACKGROUND_MARKING','V8.GC_MC_MARK','V8.GC_MC_INCREMENTAL','V8.GCIncrementalMarkingFinalize','V8.GCIncrementalMarkingStart',];const V8_FULL_COMPACT_EVENTS=['V8.GC_MC_BACKGROUND_EVACUATE_COPY','V8.GC_MC_BACKGROUND_EVACUATE_UPDATE_POINTERS','V8.GC_MC_EVACUATE',];const V8_FULL_SWEEP_EVENTS=['V8.GC_MC_BACKGROUND_SWEEPING','V8.GC_MC_SWEEP',];const V8_FULL_WEAK_EVENTS=['V8.GC_MC_CLEAR',];const V8_YOUNG_EVENTS=['V8.GC_SCAVENGER_BACKGROUND_SCAVENGE_PARALLEL','V8.GC_SCAVENGER',];const CPP_GC_FULL_MARK_EVENTS=['BlinkGC.AtomicPauseMarkEpilogue','BlinkGC.AtomicPauseMarkPrologue','BlinkGC.AtomicPauseMarkRoots','BlinkGC.AtomicPauseMarkTransitiveClosure','BlinkGC.ConcurrentMarkingStep','BlinkGC.IncrementalMarkingStartMarking','BlinkGC.IncrementalMarkingStep','BlinkGC.MarkBailOutObjects','BlinkGC.MarkFlushEphemeronPairs','BlinkGC.MarkFlushV8References','BlinkGC.UnifiedMarkingStep','CppGC.AtomicMark','CppGC.IncrementalMark','CppGC.ConcurrentMark',];const CPP_GC_FULL_COMPACT_EVENTS=['BlinkGC.AtomicPauseSweepAndCompact','CppGC.AtomicCompact',];const CPP_GC_FULL_SWEEP_EVENTS=['BlinkGC.CompleteSweep','BlinkGC.ConcurrentSweepingStep','BlinkGC.LazySweepInIdle','BlinkGC.LazySweepOnAllocation','CppGC.AtomicSweep','CppGC.IncrementalSweep','CppGC.ConcurrentSweep',];const CPP_GC_FULL_WEAK_EVENTS=['BlinkGC.MarkWeakProcessing','CppGC.AtomicWeak',];const RULES=[{events:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic',},{events:V8_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark',},{events:CPP_GC_FULL_MARK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:mark:cpp',},{events:V8_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark',},{events:CPP_GC_FULL_MARK_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:mark:cpp',},{events:V8_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact',},{events:CPP_GC_FULL_COMPACT_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:compact:cpp',},{events:V8_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:sweep:cpp',},{events:V8_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak',},{events:CPP_GC_FULL_WEAK_EVENTS,inside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:atomic:weak:cpp',},{events:V8_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep',},{events:CPP_GC_FULL_SWEEP_EVENTS,outside:V8_FULL_ATOMIC_EVENTS,contribute_to:'full:incremental:sweep:cpp',},{events:V8_YOUNG_EVENTS,contribute_to:'young:atomic',},];const Granularity={CYCLE:'cycle',EVENT:'event',};const ThreadType={MAIN:'main',BACKGROUND:'background',ALL_THREADS:'all_threads',};class Metric{constructor(name){const parts=name.split(':');this.granularity_=parts[2];assert(this.granularity_===Granularity.CYCLE||this.granularity_===Granularity.EVENT);this.thread_=ThreadType.ALL_THREADS;let phasesIndex=3;if(parts[3]==='main_thread'){this.thread_=ThreadType.MAIN;phasesIndex=4;} if(parts[3]==='background_threads'){this.thread_=ThreadType.BACKGROUND;phasesIndex=4;} this.phases_=parts.slice(phasesIndex);const maxValue=this.isPerCycleMetric()?10000:1000;const boundaries=tr.v.HistogramBinBoundaries.createExponential(0.1,maxValue,100);this.histogram=new tr.v.Histogram(name,tr.b.Unit.byName.timeDurationInMs_smallerIsBetter,boundaries);this.histogram.customizeSummaryOptions({avg:true,count:true,max:true,min:false,std:false,sum:this.isPerCycleMetric(),});} isPerCycleMetric(){return this.granularity_===Granularity.CYCLE;} @@ -9045,16 +9058,19 @@ function getEpoch(event){function checkEpochConsistency(epoch,event){if(epoch=== const result={v8:null,cpp:null};while(event){if('epoch'in event.args){if(isV8Event(event)){checkEpochConsistency(result.v8,event);result.v8=event.args.epoch;}else{checkEpochConsistency(result.cpp,event);result.cpp=event.args.epoch;}} event=event.parentSlice;} return result;} -const cppToV8=new Map();for(const event of events){const epoch=getEpoch(event);if(epoch.cpp!==null&&epoch.v8!==null){assert(!cppToV8.has(epoch.cpp)||cppToV8.get(epoch.cpp)===epoch.v8,`CppGC epoch ${epoch.cpp} corresponds to two v8 epochs `+`${cppToV8.get(epoch.cpp)} and ${epoch.v8}. `+`Detected at ${event.userFriendlyName}.`);cppToV8.set(epoch.cpp,epoch.v8);}} +function GlobalEpochFromV8(v8Epoch){return 2*v8Epoch;} +function GlobalEpochFromCpp(cppEpoch){return 2*cppEpoch+1;} +const cppToV8=new Map();for(const event of events){const epoch=getEpoch(event);if(epoch.cpp!==null&&epoch.v8!==null){if(!cppToV8.has(epoch.cpp)||cppToV8.get(epoch.cpp)>epoch.v8){cppToV8.set(epoch.cpp,epoch.v8);}}} const result=new Map();for(const event of events){const epoch=getEpoch(event);if(epoch.cpp===null&&epoch.v8===null){continue;} -assert(epoch.cpp===null||cppToV8.has(epoch.cpp),`CppGC epoch ${epoch.cpp} doesn't have the corresponding V8 epoch. `+`Detected at ${event.userFriendlyName}`);const key=epoch.v8===null?cppToV8.get(epoch.cpp):epoch.v8;if(result.has(key)){result.get(key).push(event);}else{result.set(key,[event]);}} +let globalEpoch;if(epoch.v8!==null){globalEpoch=GlobalEpochFromV8(epoch.v8);}else if(cppToV8.has(epoch.cpp)){globalEpoch=GlobalEpochFromV8(cppToV8.get(epoch.cpp));}else{globalEpoch=GlobalEpochFromCpp(epoch.cpp);} +if(result.has(globalEpoch)){result.get(globalEpoch).push(event);}else{result.set(globalEpoch,[event]);}} return result;} function addGarbageCollectionMetrics(metricNames,histograms,model){const metrics=metricNames.map(name=>new Metric(name));const gcEventNames=new Set(eventsMentionedIn(RULES));const chromeHelper=model.getOrCreateHelper(tr.model.helpers.ChromeModelHelper);for(const rendererHelper of Object.values(chromeHelper.rendererHelpers)){if(rendererHelper.isChromeTracingUI)continue;const[threads,threadTypes]=jsExecutionThreadsWithTypes(rendererHelper);const events=[];for(const thread of threads){for(const event of thread.sliceGroup.childEvents()){if(gcEventNames.has(event.title)){events.push(event);}}} for(const cycleEvents of groupByEpoch(events).values()){if(cycleEvents.some(tr.metrics.v8.utils.isForcedGarbageCollectionEvent)){continue;} for(const metric of metrics){metric.apply(RULES,cycleEvents,threadTypes);}}} for(const metric of metrics){histograms.addHistogram(metric.histogram);}} function gcMetric(histograms,model,options){options=options||{};addDurationOfTopEvents(histograms,model);addTotalDurationOfTopEvents(histograms,model);if(options.include_sub_events){addDurationOfSubEvents(histograms,model);} -addPercentageInV8ExecuteOfTopEvents(histograms,model);addTotalPercentageInV8Execute(histograms,model);addMarkCompactorMutatorUtilization(histograms,model);addTotalMarkCompactorTime(histograms,model);addTotalMarkCompactorMarkingTime(histograms,model);addScavengerSurvivedFromStackEvents(histograms,model);} +addPercentageInV8ExecuteOfTopEvents(histograms,model);addTotalPercentageInV8Execute(histograms,model);addMarkCompactorMutatorUtilization(histograms,model);addTotalMarkCompactorTime(histograms,model);addTotalMarkCompactorMarkingTime(histograms,model);addScavengerSurvivedFromStackEvents(histograms,model);addGarbageCollectionMetrics(METRICS,histograms,model);} tr.metrics.MetricRegistry.register(gcMetric);const timeDurationInMs_smallerIsBetter=tr.b.Unit.byName.timeDurationInMs_smallerIsBetter;const percentage_biggerIsBetter=tr.b.Unit.byName.normalizedPercentage_biggerIsBetter;const percentage_smallerIsBetter=tr.b.Unit.byName.normalizedPercentage_smallerIsBetter;const bytes_smallerIsBetter=tr.b.Unit.byName.sizeInBytes_smallerIsBetter;const CUSTOM_BOUNDARIES=tr.v.HistogramBinBoundaries.createLinear(0,20,200).addExponentialBins(200,100);function createNumericForTopEventTime(name){const n=new tr.v.Histogram(name,timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);n.customizeSummaryOptions({avg:true,count:true,max:true,min:false,std:true,sum:true,percentile:[0.90]});return n;} function createNumericForSubEventTime(name){const n=new tr.v.Histogram(name,timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);n.customizeSummaryOptions({avg:true,count:false,max:true,min:false,std:false,sum:false,percentile:[0.90]});return n;} function createNumericForIdleTime(name){const n=new tr.v.Histogram(name,timeDurationInMs_smallerIsBetter,CUSTOM_BOUNDARIES);n.customizeSummaryOptions({avg:true,count:false,max:true,min:false,std:false,sum:true,percentile:[]});return n;} @@ -9228,9 +9244,9 @@ for(const tab of this.$.tabs.tabs){tab.nodes=undefined;} this.$.tabs.clearSubViews();if(this.displayedNode_===undefined){this.$.tabs.label='No heap node provided.';return;} for(const[dimension,children]of this.displayedNode_.childNodes){if(!this.dimensionToTab_.has(dimension)){this.dimensionToTab_.set(dimension,document.createElement('tr-ui-a-memory-dump-heap-details-breakdown-view-tab'));} const tab=this.dimensionToTab_.get(dimension);tab.aggregationMode=this.aggregationMode_;tab.dimension=dimension;tab.nodes=children;this.$.tabs.addSubView(tab);tab.rebuild();if(dimension===previouslySelectedDimension){this.$.tabs.selectedSubView=tab;if(previouslySelectedTabFocused){tab.focus();}}} -if(this.$.tabs.tabs.length>0){this.$.tabs.label='Break selected node further by:';}else{this.$.tabs.label='Selected node cannot be broken down any further.';}},onKeyDown_(keyEvent){if(!this.displayedNode_)return;let keyHandled=false;switch(keyEvent.keyCode){case 8:{if(!this.displayedNode_.parentNode)break;const viewEvent=new tr.b.Event('enter-node');viewEvent.node=this.displayedNode_.parentNode;this.dispatchEvent(viewEvent);keyHandled=true;break;} +if(this.$.tabs.tabs.length>0){this.$.tabs.label='Break selected node further by:';}else{this.$.tabs.label='Selected node cannot be broken down any further.';}},onKeyDown_(keyEvent){if(!this.displayedNode_)return;let keyHandled=false;switch(keyEvent.keyCode){case 8:{if(!this.displayedNode_.parentNode)break;const viewEvent=new tr.b.Event('enter-node',true);viewEvent.node=this.displayedNode_.parentNode;this.dispatchEvent(viewEvent);keyHandled=true;break;} case 37:case 39:{const wasFocused=this.$.tabs.selectedSubView.isFocused;keyHandled=keyEvent.keyCode===37?this.$.tabs.selectPreviousTabIfPossible():this.$.tabs.selectNextTabIfPossible();if(wasFocused&&keyHandled){this.$.tabs.selectedSubView.focus();}}} -if(!keyHandled)return;keyEvent.stopPropagation();keyEvent.preventDefault();}});Polymer({is:'tr-ui-a-memory-dump-heap-details-breakdown-view-tab',behaviors:[tr.ui.analysis.RebuildableBehavior],created(){this.dimension_=undefined;this.nodes_=undefined;this.aggregationMode_=undefined;this.displayLongTail_=false;},ready(){this.$.table.addEventListener('step-into',function(tableEvent){const viewEvent=new tr.b.Event('enter-node');viewEvent.node=tableEvent.tableRow;this.dispatchEvent(viewEvent);}.bind(this));},get displayLongTail(){return this.displayLongTail_;},set displayLongTail(newValue){if(this.displayLongTail===newValue)return;this.displayLongTail_=newValue;this.scheduleRebuild_();},get dimension(){return this.dimension_;},set dimension(dimension){this.dimension_=dimension;this.scheduleRebuild_();},get nodes(){return this.nodes_;},set nodes(nodes){this.nodes_=nodes;this.scheduleRebuild_();},get nodes(){return this.nodes_||[];},get dimensionLabel_(){if(this.dimension_===undefined)return'(undefined)';return this.dimension_.label;},get tabLabel(){let nodeCount=0;if(this.nodes_){nodeCount=this.nodes_.length;} +if(!keyHandled)return;keyEvent.stopPropagation();keyEvent.preventDefault();}});Polymer({is:'tr-ui-a-memory-dump-heap-details-breakdown-view-tab',behaviors:[tr.ui.analysis.RebuildableBehavior],created(){this.dimension_=undefined;this.nodes_=undefined;this.aggregationMode_=undefined;this.displayLongTail_=false;},ready(){this.$.table.addEventListener('step-into',function(tableEvent){const viewEvent=new tr.b.Event('enter-node',true);viewEvent.node=tableEvent.tableRow;this.dispatchEvent(viewEvent);}.bind(this));},get displayLongTail(){return this.displayLongTail_;},set displayLongTail(newValue){if(this.displayLongTail===newValue)return;this.displayLongTail_=newValue;this.scheduleRebuild_();},get dimension(){return this.dimension_;},set dimension(dimension){this.dimension_=dimension;this.scheduleRebuild_();},get nodes(){return this.nodes_;},set nodes(nodes){this.nodes_=nodes;this.scheduleRebuild_();},get nodes(){return this.nodes_||[];},get dimensionLabel_(){if(this.dimension_===undefined)return'(undefined)';return this.dimension_.label;},get tabLabel(){let nodeCount=0;if(this.nodes_){nodeCount=this.nodes_.length;} return this.dimensionLabel_+' ('+nodeCount+')';},get tabIcon(){if(this.dimension_===undefined||this.dimension_===tr.ui.analysis.HeapDetailsRowDimension.ROOT){return undefined;} return{text:this.dimension_.symbol,style:'color: '+tr.b.ColorScheme.getColorForReservedNameAsString(this.dimension_.color)+';'};},get aggregationMode(){return this.aggregationMode_;},set aggregationMode(aggregationMode){this.aggregationMode_=aggregationMode;this.scheduleRebuild_();},focus(){this.$.table.focus();},blur(){this.$.table.blur();},get isFocused(){return this.$.table.isFocused;},onRebuild_(){this.$.table.selectionMode=tr.ui.b.TableFormat.SelectionMode.ROW;this.$.table.emptyValue='Cannot break down by '+ this.dimensionLabel_.toLowerCase()+' any further.';const[state,rows]=this.getRows_();const total=this.nodes.length;const displayed=rows.length;const hidden=total-displayed;this.updateInfoBar_(state,[total,displayed,hidden]);this.$.table.tableRows=rows;this.$.table.tableColumns=this.createColumns_(rows);if(this.$.table.sortColumnIndex===undefined){this.$.table.sortColumnIndex=0;this.$.table.sortDescending=false;} diff --git a/catapult/systrace/systrace/util.py b/catapult/systrace/systrace/util.py index 063f9ed9..797d6756 100644 --- a/catapult/systrace/systrace/util.py +++ b/catapult/systrace/systrace/util.py @@ -69,8 +69,8 @@ def get_tracing_path(device_serial=None): options, _ = parser.parse_args() device_serial = options.device_serial - adb_output, adb_return_code = run_adb_shell(mount_info_args, device_serial) - if adb_return_code == 0 and 'debugfs' not in adb_output: + adb_output, adb_return_code = run_adb_shell(mount_info_args, device_serial, ) + if adb_return_code == 0 and 'tracefs on /sys/kernel/tracing' in adb_output: return '/sys/kernel/tracing' return '/sys/kernel/debug/tracing' diff --git a/catapult/third_party/zipfile/LICENSE b/catapult/third_party/zipfile/LICENSE deleted file mode 100644 index 84a3337c..00000000 --- a/catapult/third_party/zipfile/LICENSE +++ /dev/null @@ -1,255 +0,0 @@ -A. HISTORY OF THE SOFTWARE -========================== - -Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands -as a successor of a language called ABC. Guido remains Python's -principal author, although it includes many contributions from others. - -In 1995, Guido continued his work on Python at the Corporation for -National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) -in Reston, Virginia where he released several versions of the -software. - -In May 2000, Guido and the Python core development team moved to -BeOpen.com to form the BeOpen PythonLabs team. In October of the same -year, the PythonLabs team moved to Digital Creations (now Zope -Corporation, see http://www.zope.com). In 2001, the Python Software -Foundation (PSF, see http://www.python.org/psf/) was formed, a -non-profit organization created specifically to own Python-related -Intellectual Property. Zope Corporation is a sponsoring member of -the PSF. - -All Python releases are Open Source (see http://www.opensource.org for -the Open Source Definition). Historically, most, but not all, Python -releases have also been GPL-compatible; the table below summarizes -the various releases. - - Release Derived Year Owner GPL- - from compatible? (1) - - 0.9.0 thru 1.2 1991-1995 CWI yes - 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes - 1.6 1.5.2 2000 CNRI no - 2.0 1.6 2000 BeOpen.com no - 1.6.1 1.6 2001 CNRI yes (2) - 2.1 2.0+1.6.1 2001 PSF no - 2.0.1 2.0+1.6.1 2001 PSF yes - 2.1.1 2.1+2.0.1 2001 PSF yes - 2.1.2 2.1.1 2002 PSF yes - 2.1.3 2.1.2 2002 PSF yes - 2.2 and above 2.1.1 2001-now PSF yes - -Footnotes: - -(1) GPL-compatible doesn't mean that we're distributing Python under - the GPL. All Python licenses, unlike the GPL, let you distribute - a modified version without making your changes open source. The - GPL-compatible licenses make it possible to combine Python with - other software that is released under the GPL; the others don't. - -(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, - because its license has a choice of law clause. According to - CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 - is "not incompatible" with the GPL. - -Thanks to the many outside volunteers who have worked under Guido's -direction to make these releases possible. - - -B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON -=============================================================== - -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF hereby -grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -analyze, test, perform and/or display publicly, prepare derivative works, -distribute, and otherwise use Python alone or in any derivative version, -provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016 Python Software Foundation; All Rights -Reserved" are retained in Python alone or in any derivative version prepared by -Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an -office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the -Individual or Organization ("Licensee") accessing and otherwise using -this software in source or binary form and its associated -documentation ("the Software"). - -2. Subject to the terms and conditions of this BeOpen Python License -Agreement, BeOpen hereby grants Licensee a non-exclusive, -royalty-free, world-wide license to reproduce, analyze, test, perform -and/or display publicly, prepare derivative works, distribute, and -otherwise use the Software alone or in any derivative version, -provided, however, that the BeOpen Python License is retained in the -Software, alone or in any derivative version prepared by Licensee. - -3. BeOpen is making the Software available to Licensee on an "AS IS" -basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE -SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS -AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY -DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -5. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -6. This License Agreement shall be governed by and interpreted in all -respects by the law of the State of California, excluding conflict of -law provisions. Nothing in this License Agreement shall be deemed to -create any relationship of agency, partnership, or joint venture -between BeOpen and Licensee. This License Agreement does not grant -permission to use BeOpen trademarks or trade names in a trademark -sense to endorse or promote products or services of Licensee, or any -third party. As an exception, the "BeOpen Python" logos available at -http://www.pythonlabs.com/logos.html may be used according to the -permissions granted on that web page. - -7. By copying, installing or otherwise using the software, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- - -1. This LICENSE AGREEMENT is between the Corporation for National -Research Initiatives, having an office at 1895 Preston White Drive, -Reston, VA 20191 ("CNRI"), and the Individual or Organization -("Licensee") accessing and otherwise using Python 1.6.1 software in -source or binary form and its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, CNRI -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python 1.6.1 -alone or in any derivative version, provided, however, that CNRI's -License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) -1995-2001 Corporation for National Research Initiatives; All Rights -Reserved" are retained in Python 1.6.1 alone or in any derivative -version prepared by Licensee. Alternately, in lieu of CNRI's License -Agreement, Licensee may substitute the following text (omitting the -quotes): "Python 1.6.1 is made available subject to the terms and -conditions in CNRI's License Agreement. This Agreement together with -Python 1.6.1 may be located on the Internet using the following -unique, persistent identifier (known as a handle): 1895.22/1013. This -Agreement may also be obtained from a proxy server on the Internet -using the following URL: http://hdl.handle.net/1895.22/1013". - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python 1.6.1 or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python 1.6.1. - -4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" -basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. This License Agreement shall be governed by the federal -intellectual property law of the United States, including without -limitation the federal copyright law, and, to the extent such -U.S. federal law does not apply, by the law of the Commonwealth of -Virginia, excluding Virginia's conflict of law provisions. -Notwithstanding the foregoing, with regard to derivative works based -on Python 1.6.1 that incorporate non-separable material that was -previously distributed under the GNU General Public License (GPL), the -law of the Commonwealth of Virginia shall govern this License -Agreement only as to issues arising under or with respect to -Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this -License Agreement shall be deemed to create any relationship of -agency, partnership, or joint venture between CNRI and Licensee. This -License Agreement does not grant permission to use CNRI trademarks or -trade name in a trademark sense to endorse or promote products or -services of Licensee, or any third party. - -8. By clicking on the "ACCEPT" button where indicated, or by copying, -installing or otherwise using Python 1.6.1, Licensee agrees to be -bound by the terms and conditions of this License Agreement. - - ACCEPT - - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, -The Netherlands. All rights reserved. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the name of Stichting Mathematisch -Centrum or CWI not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO -THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE -FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/catapult/third_party/zipfile/README.chromium b/catapult/third_party/zipfile/README.chromium deleted file mode 100644 index f45d3cff..00000000 --- a/catapult/third_party/zipfile/README.chromium +++ /dev/null @@ -1,16 +0,0 @@ -Name: Python zipfile module -Short Name: zipfile -URL: https://github.com/python/cpython/blob/master/Lib/zipfile.py -Version: 2.7.13 -License: Python -License File: NOT_SHIPPED -Security Critical: no - -Description: -This is a copy of the zipfile module from Python 2.7.13. This snapshot was -taken to workaround a bug in older Python versions such as 2.7.3. See -<https://bugs.python.org/issue6972> and specifically the fix at -<https://hg.python.org/cpython/rev/0c5fa35c9f12#l3.11>. - -Local Modifications: -Renamed zipfile.py to zipfile_2_7_13.py to avoid conflicting with system copy. diff --git a/catapult/third_party/zipfile/zipfile_2_7_13.py b/catapult/third_party/zipfile/zipfile_2_7_13.py deleted file mode 100644 index 1d106508..00000000 --- a/catapult/third_party/zipfile/zipfile_2_7_13.py +++ /dev/null @@ -1,1543 +0,0 @@ -""" -Read and write ZIP files. -""" -import struct, os, time, sys, shutil -import binascii, cStringIO, stat -import io -import re -import string - -try: - import zlib # We may need its compression method - crc32 = zlib.crc32 -except ImportError: - zlib = None - crc32 = binascii.crc32 - -__all__ = ["BadZipfile", "error", "ZIP_STORED", "ZIP_DEFLATED", "is_zipfile", - "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile" ] - -class BadZipfile(Exception): - pass - - -class LargeZipFile(Exception): - """ - Raised when writing a zipfile, the zipfile requires ZIP64 extensions - and those extensions are disabled. - """ - -error = BadZipfile # The exception raised by this module - -ZIP64_LIMIT = (1 << 31) - 1 -ZIP_FILECOUNT_LIMIT = (1 << 16) - 1 -ZIP_MAX_COMMENT = (1 << 16) - 1 - -# constants for Zip file compression methods -ZIP_STORED = 0 -ZIP_DEFLATED = 8 -# Other ZIP compression methods not supported - -# Below are some formats and associated data for reading/writing headers using -# the struct module. The names and structures of headers/records are those used -# in the PKWARE description of the ZIP file format: -# http://www.pkware.com/documents/casestudies/APPNOTE.TXT -# (URL valid as of January 2008) - -# The "end of central directory" structure, magic number, size, and indices -# (section V.I in the format document) -structEndArchive = "<4s4H2LH" -stringEndArchive = "PK\005\006" -sizeEndCentDir = struct.calcsize(structEndArchive) - -_ECD_SIGNATURE = 0 -_ECD_DISK_NUMBER = 1 -_ECD_DISK_START = 2 -_ECD_ENTRIES_THIS_DISK = 3 -_ECD_ENTRIES_TOTAL = 4 -_ECD_SIZE = 5 -_ECD_OFFSET = 6 -_ECD_COMMENT_SIZE = 7 -# These last two indices are not part of the structure as defined in the -# spec, but they are used internally by this module as a convenience -_ECD_COMMENT = 8 -_ECD_LOCATION = 9 - -# The "central directory" structure, magic number, size, and indices -# of entries in the structure (section V.F in the format document) -structCentralDir = "<4s4B4HL2L5H2L" -stringCentralDir = "PK\001\002" -sizeCentralDir = struct.calcsize(structCentralDir) - -# indexes of entries in the central directory structure -_CD_SIGNATURE = 0 -_CD_CREATE_VERSION = 1 -_CD_CREATE_SYSTEM = 2 -_CD_EXTRACT_VERSION = 3 -_CD_EXTRACT_SYSTEM = 4 -_CD_FLAG_BITS = 5 -_CD_COMPRESS_TYPE = 6 -_CD_TIME = 7 -_CD_DATE = 8 -_CD_CRC = 9 -_CD_COMPRESSED_SIZE = 10 -_CD_UNCOMPRESSED_SIZE = 11 -_CD_FILENAME_LENGTH = 12 -_CD_EXTRA_FIELD_LENGTH = 13 -_CD_COMMENT_LENGTH = 14 -_CD_DISK_NUMBER_START = 15 -_CD_INTERNAL_FILE_ATTRIBUTES = 16 -_CD_EXTERNAL_FILE_ATTRIBUTES = 17 -_CD_LOCAL_HEADER_OFFSET = 18 - -# The "local file header" structure, magic number, size, and indices -# (section V.A in the format document) -structFileHeader = "<4s2B4HL2L2H" -stringFileHeader = "PK\003\004" -sizeFileHeader = struct.calcsize(structFileHeader) - -_FH_SIGNATURE = 0 -_FH_EXTRACT_VERSION = 1 -_FH_EXTRACT_SYSTEM = 2 -_FH_GENERAL_PURPOSE_FLAG_BITS = 3 -_FH_COMPRESSION_METHOD = 4 -_FH_LAST_MOD_TIME = 5 -_FH_LAST_MOD_DATE = 6 -_FH_CRC = 7 -_FH_COMPRESSED_SIZE = 8 -_FH_UNCOMPRESSED_SIZE = 9 -_FH_FILENAME_LENGTH = 10 -_FH_EXTRA_FIELD_LENGTH = 11 - -# The "Zip64 end of central directory locator" structure, magic number, and size -structEndArchive64Locator = "<4sLQL" -stringEndArchive64Locator = "PK\x06\x07" -sizeEndCentDir64Locator = struct.calcsize(structEndArchive64Locator) - -# The "Zip64 end of central directory" record, magic number, size, and indices -# (section V.G in the format document) -structEndArchive64 = "<4sQ2H2L4Q" -stringEndArchive64 = "PK\x06\x06" -sizeEndCentDir64 = struct.calcsize(structEndArchive64) - -_CD64_SIGNATURE = 0 -_CD64_DIRECTORY_RECSIZE = 1 -_CD64_CREATE_VERSION = 2 -_CD64_EXTRACT_VERSION = 3 -_CD64_DISK_NUMBER = 4 -_CD64_DISK_NUMBER_START = 5 -_CD64_NUMBER_ENTRIES_THIS_DISK = 6 -_CD64_NUMBER_ENTRIES_TOTAL = 7 -_CD64_DIRECTORY_SIZE = 8 -_CD64_OFFSET_START_CENTDIR = 9 - -def _check_zipfile(fp): - try: - if _EndRecData(fp): - return True # file has correct magic number - except IOError: - pass - return False - -def is_zipfile(filename): - """Quickly see if a file is a ZIP file by checking the magic number. - - The filename argument may be a file or file-like object too. - """ - result = False - try: - if hasattr(filename, "read"): - result = _check_zipfile(fp=filename) - else: - with open(filename, "rb") as fp: - result = _check_zipfile(fp) - except IOError: - pass - return result - -def _EndRecData64(fpin, offset, endrec): - """ - Read the ZIP64 end-of-archive records and use that to update endrec - """ - try: - fpin.seek(offset - sizeEndCentDir64Locator, 2) - except IOError: - # If the seek fails, the file is not large enough to contain a ZIP64 - # end-of-archive record, so just return the end record we were given. - return endrec - - data = fpin.read(sizeEndCentDir64Locator) - if len(data) != sizeEndCentDir64Locator: - return endrec - sig, diskno, reloff, disks = struct.unpack(structEndArchive64Locator, data) - if sig != stringEndArchive64Locator: - return endrec - - if diskno != 0 or disks != 1: - raise BadZipfile("zipfiles that span multiple disks are not supported") - - # Assume no 'zip64 extensible data' - fpin.seek(offset - sizeEndCentDir64Locator - sizeEndCentDir64, 2) - data = fpin.read(sizeEndCentDir64) - if len(data) != sizeEndCentDir64: - return endrec - sig, sz, create_version, read_version, disk_num, disk_dir, \ - dircount, dircount2, dirsize, diroffset = \ - struct.unpack(structEndArchive64, data) - if sig != stringEndArchive64: - return endrec - - # Update the original endrec using data from the ZIP64 record - endrec[_ECD_SIGNATURE] = sig - endrec[_ECD_DISK_NUMBER] = disk_num - endrec[_ECD_DISK_START] = disk_dir - endrec[_ECD_ENTRIES_THIS_DISK] = dircount - endrec[_ECD_ENTRIES_TOTAL] = dircount2 - endrec[_ECD_SIZE] = dirsize - endrec[_ECD_OFFSET] = diroffset - return endrec - - -def _EndRecData(fpin): - """Return data from the "End of Central Directory" record, or None. - - The data is a list of the nine items in the ZIP "End of central dir" - record followed by a tenth item, the file seek offset of this record.""" - - # Determine file size - fpin.seek(0, 2) - filesize = fpin.tell() - - # Check to see if this is ZIP file with no archive comment (the - # "end of central directory" structure should be the last item in the - # file if this is the case). - try: - fpin.seek(-sizeEndCentDir, 2) - except IOError: - return None - data = fpin.read() - if (len(data) == sizeEndCentDir and - data[0:4] == stringEndArchive and - data[-2:] == b"\000\000"): - # the signature is correct and there's no comment, unpack structure - endrec = struct.unpack(structEndArchive, data) - endrec=list(endrec) - - # Append a blank comment and record start offset - endrec.append("") - endrec.append(filesize - sizeEndCentDir) - - # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, -sizeEndCentDir, endrec) - - # Either this is not a ZIP file, or it is a ZIP file with an archive - # comment. Search the end of the file for the "end of central directory" - # record signature. The comment is the last item in the ZIP file and may be - # up to 64K long. It is assumed that the "end of central directory" magic - # number does not appear in the comment. - maxCommentStart = max(filesize - (1 << 16) - sizeEndCentDir, 0) - fpin.seek(maxCommentStart, 0) - data = fpin.read() - start = data.rfind(stringEndArchive) - if start >= 0: - # found the magic number; attempt to unpack and interpret - recData = data[start:start+sizeEndCentDir] - if len(recData) != sizeEndCentDir: - # Zip file is corrupted. - return None - endrec = list(struct.unpack(structEndArchive, recData)) - commentSize = endrec[_ECD_COMMENT_SIZE] #as claimed by the zip file - comment = data[start+sizeEndCentDir:start+sizeEndCentDir+commentSize] - endrec.append(comment) - endrec.append(maxCommentStart + start) - - # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, maxCommentStart + start - filesize, - endrec) - - # Unable to find a valid end of central directory structure - return None - - -class ZipInfo (object): - """Class with attributes describing each file in the ZIP archive.""" - - __slots__ = ( - 'orig_filename', - 'filename', - 'date_time', - 'compress_type', - 'comment', - 'extra', - 'create_system', - 'create_version', - 'extract_version', - 'reserved', - 'flag_bits', - 'volume', - 'internal_attr', - 'external_attr', - 'header_offset', - 'CRC', - 'compress_size', - 'file_size', - '_raw_time', - ) - - def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): - self.orig_filename = filename # Original file name in archive - - # Terminate the file name at the first null byte. Null bytes in file - # names are used as tricks by viruses in archives. - null_byte = filename.find(chr(0)) - if null_byte >= 0: - filename = filename[0:null_byte] - # This is used to ensure paths in generated ZIP files always use - # forward slashes as the directory separator, as required by the - # ZIP format specification. - if os.sep != "/" and os.sep in filename: - filename = filename.replace(os.sep, "/") - - self.filename = filename # Normalized file name - self.date_time = date_time # year, month, day, hour, min, sec - - if date_time[0] < 1980: - raise ValueError('ZIP does not support timestamps before 1980') - - # Standard values: - self.compress_type = ZIP_STORED # Type of compression for the file - self.comment = "" # Comment for each file - self.extra = "" # ZIP extra data - if sys.platform == 'win32': - self.create_system = 0 # System which created ZIP archive - else: - # Assume everything else is unix-y - self.create_system = 3 # System which created ZIP archive - self.create_version = 20 # Version which created ZIP archive - self.extract_version = 20 # Version needed to extract archive - self.reserved = 0 # Must be zero - self.flag_bits = 0 # ZIP flag bits - self.volume = 0 # Volume number of file header - self.internal_attr = 0 # Internal attributes - self.external_attr = 0 # External file attributes - # Other attributes are set by class ZipFile: - # header_offset Byte offset to the file header - # CRC CRC-32 of the uncompressed file - # compress_size Size of the compressed file - # file_size Size of the uncompressed file - - def FileHeader(self, zip64=None): - """Return the per-file header as a string.""" - dt = self.date_time - dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - if self.flag_bits & 0x08: - # Set these to zero because we write them after the file data - CRC = compress_size = file_size = 0 - else: - CRC = self.CRC - compress_size = self.compress_size - file_size = self.file_size - - extra = self.extra - - if zip64 is None: - zip64 = file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT - if zip64: - fmt = '<HHQQ' - extra = extra + struct.pack(fmt, - 1, struct.calcsize(fmt)-4, file_size, compress_size) - if file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT: - if not zip64: - raise LargeZipFile("Filesize would require ZIP64 extensions") - # File is larger than what fits into a 4 byte integer, - # fall back to the ZIP64 extension - file_size = 0xffffffff - compress_size = 0xffffffff - self.extract_version = max(45, self.extract_version) - self.create_version = max(45, self.extract_version) - - filename, flag_bits = self._encodeFilenameFlags() - header = struct.pack(structFileHeader, stringFileHeader, - self.extract_version, self.reserved, flag_bits, - self.compress_type, dostime, dosdate, CRC, - compress_size, file_size, - len(filename), len(extra)) - return header + filename + extra - - def _encodeFilenameFlags(self): - if isinstance(self.filename, unicode): - try: - return self.filename.encode('ascii'), self.flag_bits - except UnicodeEncodeError: - return self.filename.encode('utf-8'), self.flag_bits | 0x800 - else: - return self.filename, self.flag_bits - - def _decodeFilename(self): - if self.flag_bits & 0x800: - return self.filename.decode('utf-8') - else: - return self.filename - - def _decodeExtra(self): - # Try to decode the extra field. - extra = self.extra - unpack = struct.unpack - while len(extra) >= 4: - tp, ln = unpack('<HH', extra[:4]) - if tp == 1: - if ln >= 24: - counts = unpack('<QQQ', extra[4:28]) - elif ln == 16: - counts = unpack('<QQ', extra[4:20]) - elif ln == 8: - counts = unpack('<Q', extra[4:12]) - elif ln == 0: - counts = () - else: - raise RuntimeError, "Corrupt extra field %s"%(ln,) - - idx = 0 - - # ZIP64 extension (large files and/or large archives) - if self.file_size in (0xffffffffffffffffL, 0xffffffffL): - self.file_size = counts[idx] - idx += 1 - - if self.compress_size == 0xFFFFFFFFL: - self.compress_size = counts[idx] - idx += 1 - - if self.header_offset == 0xffffffffL: - old = self.header_offset - self.header_offset = counts[idx] - idx+=1 - - extra = extra[ln+4:] - - -class _ZipDecrypter: - """Class to handle decryption of files stored within a ZIP archive. - - ZIP supports a password-based form of encryption. Even though known - plaintext attacks have been found against it, it is still useful - to be able to get data out of such a file. - - Usage: - zd = _ZipDecrypter(mypwd) - plain_char = zd(cypher_char) - plain_text = map(zd, cypher_text) - """ - - def _GenerateCRCTable(): - """Generate a CRC-32 table. - - ZIP encryption uses the CRC32 one-byte primitive for scrambling some - internal keys. We noticed that a direct implementation is faster than - relying on binascii.crc32(). - """ - poly = 0xedb88320 - table = [0] * 256 - for i in range(256): - crc = i - for j in range(8): - if crc & 1: - crc = ((crc >> 1) & 0x7FFFFFFF) ^ poly - else: - crc = ((crc >> 1) & 0x7FFFFFFF) - table[i] = crc - return table - crctable = _GenerateCRCTable() - - def _crc32(self, ch, crc): - """Compute the CRC32 primitive on one byte.""" - return ((crc >> 8) & 0xffffff) ^ self.crctable[(crc ^ ord(ch)) & 0xff] - - def __init__(self, pwd): - self.key0 = 305419896 - self.key1 = 591751049 - self.key2 = 878082192 - for p in pwd: - self._UpdateKeys(p) - - def _UpdateKeys(self, c): - self.key0 = self._crc32(c, self.key0) - self.key1 = (self.key1 + (self.key0 & 255)) & 4294967295 - self.key1 = (self.key1 * 134775813 + 1) & 4294967295 - self.key2 = self._crc32(chr((self.key1 >> 24) & 255), self.key2) - - def __call__(self, c): - """Decrypt a single character.""" - c = ord(c) - k = self.key2 | 2 - c = c ^ (((k * (k^1)) >> 8) & 255) - c = chr(c) - self._UpdateKeys(c) - return c - - -compressor_names = { - 0: 'store', - 1: 'shrink', - 2: 'reduce', - 3: 'reduce', - 4: 'reduce', - 5: 'reduce', - 6: 'implode', - 7: 'tokenize', - 8: 'deflate', - 9: 'deflate64', - 10: 'implode', - 12: 'bzip2', - 14: 'lzma', - 18: 'terse', - 19: 'lz77', - 97: 'wavpack', - 98: 'ppmd', -} - - -class ZipExtFile(io.BufferedIOBase): - """File-like object for reading an archive member. - Is returned by ZipFile.open(). - """ - - # Max size supported by decompressor. - MAX_N = 1 << 31 - 1 - - # Read from compressed files in 4k blocks. - MIN_READ_SIZE = 4096 - - # Search for universal newlines or line chunks. - PATTERN = re.compile(r'^(?P<chunk>[^\r\n]+)|(?P<newline>\n|\r\n?)') - - def __init__(self, fileobj, mode, zipinfo, decrypter=None, - close_fileobj=False): - self._fileobj = fileobj - self._decrypter = decrypter - self._close_fileobj = close_fileobj - - self._compress_type = zipinfo.compress_type - self._compress_size = zipinfo.compress_size - self._compress_left = zipinfo.compress_size - - if self._compress_type == ZIP_DEFLATED: - self._decompressor = zlib.decompressobj(-15) - elif self._compress_type != ZIP_STORED: - descr = compressor_names.get(self._compress_type) - if descr: - raise NotImplementedError("compression type %d (%s)" % (self._compress_type, descr)) - else: - raise NotImplementedError("compression type %d" % (self._compress_type,)) - self._unconsumed = '' - - self._readbuffer = '' - self._offset = 0 - - self._universal = 'U' in mode - self.newlines = None - - # Adjust read size for encrypted files since the first 12 bytes - # are for the encryption/password information. - if self._decrypter is not None: - self._compress_left -= 12 - - self.mode = mode - self.name = zipinfo.filename - - if hasattr(zipinfo, 'CRC'): - self._expected_crc = zipinfo.CRC - self._running_crc = crc32(b'') & 0xffffffff - else: - self._expected_crc = None - - def readline(self, limit=-1): - """Read and return a line from the stream. - - If limit is specified, at most limit bytes will be read. - """ - - if not self._universal and limit < 0: - # Shortcut common case - newline found in buffer. - i = self._readbuffer.find('\n', self._offset) + 1 - if i > 0: - line = self._readbuffer[self._offset: i] - self._offset = i - return line - - if not self._universal: - return io.BufferedIOBase.readline(self, limit) - - line = '' - while limit < 0 or len(line) < limit: - readahead = self.peek(2) - if readahead == '': - return line - - # - # Search for universal newlines or line chunks. - # - # The pattern returns either a line chunk or a newline, but not - # both. Combined with peek(2), we are assured that the sequence - # '\r\n' is always retrieved completely and never split into - # separate newlines - '\r', '\n' due to coincidental readaheads. - # - match = self.PATTERN.search(readahead) - newline = match.group('newline') - if newline is not None: - if self.newlines is None: - self.newlines = [] - if newline not in self.newlines: - self.newlines.append(newline) - self._offset += len(newline) - return line + '\n' - - chunk = match.group('chunk') - if limit >= 0: - chunk = chunk[: limit - len(line)] - - self._offset += len(chunk) - line += chunk - - return line - - def peek(self, n=1): - """Returns buffered bytes without advancing the position.""" - if n > len(self._readbuffer) - self._offset: - chunk = self.read(n) - if len(chunk) > self._offset: - self._readbuffer = chunk + self._readbuffer[self._offset:] - self._offset = 0 - else: - self._offset -= len(chunk) - - # Return up to 512 bytes to reduce allocation overhead for tight loops. - return self._readbuffer[self._offset: self._offset + 512] - - def readable(self): - return True - - def read(self, n=-1): - """Read and return up to n bytes. - If the argument is omitted, None, or negative, data is read and returned until EOF is reached.. - """ - buf = '' - if n is None: - n = -1 - while True: - if n < 0: - data = self.read1(n) - elif n > len(buf): - data = self.read1(n - len(buf)) - else: - return buf - if len(data) == 0: - return buf - buf += data - - def _update_crc(self, newdata, eof): - # Update the CRC using the given data. - if self._expected_crc is None: - # No need to compute the CRC if we don't have a reference value - return - self._running_crc = crc32(newdata, self._running_crc) & 0xffffffff - # Check the CRC if we're at the end of the file - if eof and self._running_crc != self._expected_crc: - raise BadZipfile("Bad CRC-32 for file %r" % self.name) - - def read1(self, n): - """Read up to n bytes with at most one read() system call.""" - - # Simplify algorithm (branching) by transforming negative n to large n. - if n < 0 or n is None: - n = self.MAX_N - - # Bytes available in read buffer. - len_readbuffer = len(self._readbuffer) - self._offset - - # Read from file. - if self._compress_left > 0 and n > len_readbuffer + len(self._unconsumed): - nbytes = n - len_readbuffer - len(self._unconsumed) - nbytes = max(nbytes, self.MIN_READ_SIZE) - nbytes = min(nbytes, self._compress_left) - - data = self._fileobj.read(nbytes) - self._compress_left -= len(data) - - if data and self._decrypter is not None: - data = ''.join(map(self._decrypter, data)) - - if self._compress_type == ZIP_STORED: - self._update_crc(data, eof=(self._compress_left==0)) - self._readbuffer = self._readbuffer[self._offset:] + data - self._offset = 0 - else: - # Prepare deflated bytes for decompression. - self._unconsumed += data - - # Handle unconsumed data. - if (len(self._unconsumed) > 0 and n > len_readbuffer and - self._compress_type == ZIP_DEFLATED): - data = self._decompressor.decompress( - self._unconsumed, - max(n - len_readbuffer, self.MIN_READ_SIZE) - ) - - self._unconsumed = self._decompressor.unconsumed_tail - eof = len(self._unconsumed) == 0 and self._compress_left == 0 - if eof: - data += self._decompressor.flush() - - self._update_crc(data, eof=eof) - self._readbuffer = self._readbuffer[self._offset:] + data - self._offset = 0 - - # Read from buffer. - data = self._readbuffer[self._offset: self._offset + n] - self._offset += len(data) - return data - - def close(self): - try : - if self._close_fileobj: - self._fileobj.close() - finally: - super(ZipExtFile, self).close() - - -class ZipFile(object): - """ Class with methods to open, read, write, close, list zip files. - - z = ZipFile(file, mode="r", compression=ZIP_STORED, allowZip64=False) - - file: Either the path to the file, or a file-like object. - If it is a path, the file will be opened and closed by ZipFile. - mode: The mode can be either read "r", write "w" or append "a". - compression: ZIP_STORED (no compression) or ZIP_DEFLATED (requires zlib). - allowZip64: if True ZipFile will create files with ZIP64 extensions when - needed, otherwise it will raise an exception when this would - be necessary. - - """ - - fp = None # Set here since __del__ checks it - - def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=False): - """Open the ZIP file with mode read "r", write "w" or append "a".""" - if mode not in ("r", "w", "a"): - raise RuntimeError('ZipFile() requires mode "r", "w", or "a"') - - if compression == ZIP_STORED: - pass - elif compression == ZIP_DEFLATED: - if not zlib: - raise RuntimeError,\ - "Compression requires the (missing) zlib module" - else: - raise RuntimeError, "That compression method is not supported" - - self._allowZip64 = allowZip64 - self._didModify = False - self.debug = 0 # Level of printing: 0 through 3 - self.NameToInfo = {} # Find file info given name - self.filelist = [] # List of ZipInfo instances for archive - self.compression = compression # Method of compression - self.mode = key = mode.replace('b', '')[0] - self.pwd = None - self._comment = '' - - # Check if we were passed a file-like object - if isinstance(file, basestring): - self._filePassed = 0 - self.filename = file - modeDict = {'r' : 'rb', 'w': 'wb', 'a' : 'r+b'} - try: - self.fp = open(file, modeDict[mode]) - except IOError: - if mode == 'a': - mode = key = 'w' - self.fp = open(file, modeDict[mode]) - else: - raise - else: - self._filePassed = 1 - self.fp = file - self.filename = getattr(file, 'name', None) - - try: - if key == 'r': - self._RealGetContents() - elif key == 'w': - # set the modified flag so central directory gets written - # even if no files are added to the archive - self._didModify = True - self._start_disk = self.fp.tell() - elif key == 'a': - try: - # See if file is a zip file - self._RealGetContents() - # seek to start of directory and overwrite - self.fp.seek(self.start_dir, 0) - except BadZipfile: - # file is not a zip file, just append - self.fp.seek(0, 2) - - # set the modified flag so central directory gets written - # even if no files are added to the archive - self._didModify = True - self._start_disk = self.fp.tell() - else: - raise RuntimeError('Mode must be "r", "w" or "a"') - except: - fp = self.fp - self.fp = None - if not self._filePassed: - fp.close() - raise - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - def _RealGetContents(self): - """Read in the table of contents for the ZIP file.""" - fp = self.fp - try: - endrec = _EndRecData(fp) - except IOError: - raise BadZipfile("File is not a zip file") - if not endrec: - raise BadZipfile, "File is not a zip file" - if self.debug > 1: - print endrec - size_cd = endrec[_ECD_SIZE] # bytes in central directory - offset_cd = endrec[_ECD_OFFSET] # offset of central directory - self._comment = endrec[_ECD_COMMENT] # archive comment - - # self._start_disk: Position of the start of ZIP archive - # It is zero, unless ZIP was concatenated to another file - self._start_disk = endrec[_ECD_LOCATION] - size_cd - offset_cd - if endrec[_ECD_SIGNATURE] == stringEndArchive64: - # If Zip64 extension structures are present, account for them - self._start_disk -= (sizeEndCentDir64 + sizeEndCentDir64Locator) - - if self.debug > 2: - inferred = self._start_disk + offset_cd - print "given, inferred, offset", offset_cd, inferred, self._start_disk - # self.start_dir: Position of start of central directory - self.start_dir = offset_cd + self._start_disk - fp.seek(self.start_dir, 0) - data = fp.read(size_cd) - fp = cStringIO.StringIO(data) - total = 0 - while total < size_cd: - centdir = fp.read(sizeCentralDir) - if len(centdir) != sizeCentralDir: - raise BadZipfile("Truncated central directory") - centdir = struct.unpack(structCentralDir, centdir) - if centdir[_CD_SIGNATURE] != stringCentralDir: - raise BadZipfile("Bad magic number for central directory") - if self.debug > 2: - print centdir - filename = fp.read(centdir[_CD_FILENAME_LENGTH]) - # Create ZipInfo instance to store file information - x = ZipInfo(filename) - x.extra = fp.read(centdir[_CD_EXTRA_FIELD_LENGTH]) - x.comment = fp.read(centdir[_CD_COMMENT_LENGTH]) - x.header_offset = centdir[_CD_LOCAL_HEADER_OFFSET] - (x.create_version, x.create_system, x.extract_version, x.reserved, - x.flag_bits, x.compress_type, t, d, - x.CRC, x.compress_size, x.file_size) = centdir[1:12] - x.volume, x.internal_attr, x.external_attr = centdir[15:18] - # Convert date/time code to (year, month, day, hour, min, sec) - x._raw_time = t - x.date_time = ( (d>>9)+1980, (d>>5)&0xF, d&0x1F, - t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) - - x._decodeExtra() - x.header_offset = x.header_offset + self._start_disk - x.filename = x._decodeFilename() - self.filelist.append(x) - self.NameToInfo[x.filename] = x - - # update total bytes read from central directory - total = (total + sizeCentralDir + centdir[_CD_FILENAME_LENGTH] - + centdir[_CD_EXTRA_FIELD_LENGTH] - + centdir[_CD_COMMENT_LENGTH]) - - if self.debug > 2: - print "total", total - - - def namelist(self): - """Return a list of file names in the archive.""" - l = [] - for data in self.filelist: - l.append(data.filename) - return l - - def infolist(self): - """Return a list of class ZipInfo instances for files in the - archive.""" - return self.filelist - - def printdir(self): - """Print a table of contents for the zip file.""" - print "%-46s %19s %12s" % ("File Name", "Modified ", "Size") - for zinfo in self.filelist: - date = "%d-%02d-%02d %02d:%02d:%02d" % zinfo.date_time[:6] - print "%-46s %s %12d" % (zinfo.filename, date, zinfo.file_size) - - def testzip(self): - """Read all the files and check the CRC.""" - chunk_size = 2 ** 20 - for zinfo in self.filelist: - try: - # Read by chunks, to avoid an OverflowError or a - # MemoryError with very large embedded files. - with self.open(zinfo.filename, "r") as f: - while f.read(chunk_size): # Check CRC-32 - pass - except BadZipfile: - return zinfo.filename - - def getinfo(self, name): - """Return the instance of ZipInfo given 'name'.""" - info = self.NameToInfo.get(name) - if info is None: - raise KeyError( - 'There is no item named %r in the archive' % name) - - return info - - def setpassword(self, pwd): - """Set default password for encrypted files.""" - self.pwd = pwd - - @property - def comment(self): - """The comment text associated with the ZIP file.""" - return self._comment - - @comment.setter - def comment(self, comment): - # check for valid comment length - if len(comment) > ZIP_MAX_COMMENT: - import warnings - warnings.warn('Archive comment is too long; truncating to %d bytes' - % ZIP_MAX_COMMENT, stacklevel=2) - comment = comment[:ZIP_MAX_COMMENT] - self._comment = comment - self._didModify = True - - def read(self, name, pwd=None): - """Return file bytes (as a string) for name.""" - return self.open(name, "r", pwd).read() - - def open(self, name, mode="r", pwd=None): - """Return file-like object for 'name'.""" - if mode not in ("r", "U", "rU"): - raise RuntimeError, 'open() requires mode "r", "U", or "rU"' - if not self.fp: - raise RuntimeError, \ - "Attempt to read ZIP archive that was already closed" - - # Only open a new file for instances where we were not - # given a file object in the constructor - if self._filePassed: - zef_file = self.fp - should_close = False - else: - zef_file = open(self.filename, 'rb') - should_close = True - - try: - # Make sure we have an info object - if isinstance(name, ZipInfo): - # 'name' is already an info object - zinfo = name - else: - # Get info object for name - zinfo = self.getinfo(name) - - zef_file.seek(zinfo.header_offset, 0) - - # Skip the file header: - fheader = zef_file.read(sizeFileHeader) - if len(fheader) != sizeFileHeader: - raise BadZipfile("Truncated file header") - fheader = struct.unpack(structFileHeader, fheader) - if fheader[_FH_SIGNATURE] != stringFileHeader: - raise BadZipfile("Bad magic number for file header") - - fname = zef_file.read(fheader[_FH_FILENAME_LENGTH]) - if fheader[_FH_EXTRA_FIELD_LENGTH]: - zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH]) - - if fname != zinfo.orig_filename: - raise BadZipfile, \ - 'File name in directory "%s" and header "%s" differ.' % ( - zinfo.orig_filename, fname) - - # check for encrypted flag & handle password - is_encrypted = zinfo.flag_bits & 0x1 - zd = None - if is_encrypted: - if not pwd: - pwd = self.pwd - if not pwd: - raise RuntimeError, "File %s is encrypted, " \ - "password required for extraction" % name - - zd = _ZipDecrypter(pwd) - # The first 12 bytes in the cypher stream is an encryption header - # used to strengthen the algorithm. The first 11 bytes are - # completely random, while the 12th contains the MSB of the CRC, - # or the MSB of the file time depending on the header type - # and is used to check the correctness of the password. - bytes = zef_file.read(12) - h = map(zd, bytes[0:12]) - if zinfo.flag_bits & 0x8: - # compare against the file type from extended local headers - check_byte = (zinfo._raw_time >> 8) & 0xff - else: - # compare against the CRC otherwise - check_byte = (zinfo.CRC >> 24) & 0xff - if ord(h[11]) != check_byte: - raise RuntimeError("Bad password for file", name) - - return ZipExtFile(zef_file, mode, zinfo, zd, - close_fileobj=should_close) - except: - if should_close: - zef_file.close() - raise - - def extract(self, member, path=None, pwd=None): - """Extract a member from the archive to the current working directory, - using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a ZipInfo object. You can - specify a different directory using `path'. - """ - if not isinstance(member, ZipInfo): - member = self.getinfo(member) - - if path is None: - path = os.getcwd() - - return self._extract_member(member, path, pwd) - - def extractall(self, path=None, members=None, pwd=None): - """Extract all members from the archive to the current working - directory. `path' specifies a different directory to extract to. - `members' is optional and must be a subset of the list returned - by namelist(). - """ - if members is None: - members = self.namelist() - - for zipinfo in members: - self.extract(zipinfo, path, pwd) - - def _extract_member(self, member, targetpath, pwd): - """Extract the ZipInfo object 'member' to a physical - file on the path targetpath. - """ - # build the destination pathname, replacing - # forward slashes to platform specific separators. - arcname = member.filename.replace('/', os.path.sep) - - if os.path.altsep: - arcname = arcname.replace(os.path.altsep, os.path.sep) - # interpret absolute pathname as relative, remove drive letter or - # UNC path, redundant separators, "." and ".." components. - arcname = os.path.splitdrive(arcname)[1] - arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) - if x not in ('', os.path.curdir, os.path.pardir)) - if os.path.sep == '\\': - # filter illegal characters on Windows - illegal = ':<>|"?*' - if isinstance(arcname, unicode): - table = {ord(c): ord('_') for c in illegal} - else: - table = string.maketrans(illegal, '_' * len(illegal)) - arcname = arcname.translate(table) - # remove trailing dots - arcname = (x.rstrip('.') for x in arcname.split(os.path.sep)) - arcname = os.path.sep.join(x for x in arcname if x) - - targetpath = os.path.join(targetpath, arcname) - targetpath = os.path.normpath(targetpath) - - # Create all upper directories if necessary. - upperdirs = os.path.dirname(targetpath) - if upperdirs and not os.path.exists(upperdirs): - os.makedirs(upperdirs) - - if member.filename[-1] == '/': - if not os.path.isdir(targetpath): - os.mkdir(targetpath) - return targetpath - - with self.open(member, pwd=pwd) as source, \ - file(targetpath, "wb") as target: - shutil.copyfileobj(source, target) - - return targetpath - - def _writecheck(self, zinfo): - """Check for errors before writing a file to the archive.""" - if zinfo.filename in self.NameToInfo: - import warnings - warnings.warn('Duplicate name: %r' % zinfo.filename, stacklevel=3) - if self.mode not in ("w", "a"): - raise RuntimeError, 'write() requires mode "w" or "a"' - if not self.fp: - raise RuntimeError, \ - "Attempt to write ZIP archive that was already closed" - if zinfo.compress_type == ZIP_DEFLATED and not zlib: - raise RuntimeError, \ - "Compression requires the (missing) zlib module" - if zinfo.compress_type not in (ZIP_STORED, ZIP_DEFLATED): - raise RuntimeError, \ - "That compression method is not supported" - if not self._allowZip64: - requires_zip64 = None - if len(self.filelist) >= ZIP_FILECOUNT_LIMIT: - requires_zip64 = "Files count" - elif zinfo.file_size > ZIP64_LIMIT: - requires_zip64 = "Filesize" - elif zinfo.header_offset > ZIP64_LIMIT: - requires_zip64 = "Zipfile size" - if requires_zip64: - raise LargeZipFile(requires_zip64 + - " would require ZIP64 extensions") - - def write(self, filename, arcname=None, compress_type=None): - """Put the bytes from filename into the archive under the name - arcname.""" - if not self.fp: - raise RuntimeError( - "Attempt to write to ZIP archive that was already closed") - - st = os.stat(filename) - isdir = stat.S_ISDIR(st.st_mode) - mtime = time.localtime(st.st_mtime) - date_time = mtime[0:6] - # Create ZipInfo instance to store file information - if arcname is None: - arcname = filename - arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) - while arcname[0] in (os.sep, os.altsep): - arcname = arcname[1:] - if isdir: - arcname += '/' - zinfo = ZipInfo(arcname, date_time) - zinfo.external_attr = (st[0] & 0xFFFF) << 16L # Unix attributes - if isdir: - zinfo.compress_type = ZIP_STORED - elif compress_type is None: - zinfo.compress_type = self.compression - else: - zinfo.compress_type = compress_type - - zinfo.file_size = st.st_size - zinfo.flag_bits = 0x00 - zinfo.header_offset = self.fp.tell() # Start of header bytes - - self._writecheck(zinfo) - self._didModify = True - - if isdir: - zinfo.file_size = 0 - zinfo.compress_size = 0 - zinfo.CRC = 0 - zinfo.external_attr |= 0x10 # MS-DOS directory flag - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo - self.fp.write(zinfo.FileHeader(False)) - return - - with open(filename, "rb") as fp: - # Must overwrite CRC and sizes with correct data later - zinfo.CRC = CRC = 0 - zinfo.compress_size = compress_size = 0 - # Compressed size can be larger than uncompressed size - zip64 = self._allowZip64 and \ - zinfo.file_size * 1.05 > ZIP64_LIMIT - self.fp.write(zinfo.FileHeader(zip64)) - if zinfo.compress_type == ZIP_DEFLATED: - cmpr = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, - zlib.DEFLATED, -15) - else: - cmpr = None - file_size = 0 - while 1: - buf = fp.read(1024 * 8) - if not buf: - break - file_size = file_size + len(buf) - CRC = crc32(buf, CRC) & 0xffffffff - if cmpr: - buf = cmpr.compress(buf) - compress_size = compress_size + len(buf) - self.fp.write(buf) - if cmpr: - buf = cmpr.flush() - compress_size = compress_size + len(buf) - self.fp.write(buf) - zinfo.compress_size = compress_size - else: - zinfo.compress_size = file_size - zinfo.CRC = CRC - zinfo.file_size = file_size - if not zip64 and self._allowZip64: - if file_size > ZIP64_LIMIT: - raise RuntimeError('File size has increased during compressing') - if compress_size > ZIP64_LIMIT: - raise RuntimeError('Compressed size larger than uncompressed size') - # Seek backwards and write file header (which will now include - # correct CRC and file sizes) - position = self.fp.tell() # Preserve current position in file - self.fp.seek(zinfo.header_offset, 0) - self.fp.write(zinfo.FileHeader(zip64)) - self.fp.seek(position, 0) - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo - - def writestr(self, zinfo_or_arcname, bytes, compress_type=None): - """Write a file into the archive. The contents is the string - 'bytes'. 'zinfo_or_arcname' is either a ZipInfo instance or - the name of the file in the archive.""" - if not isinstance(zinfo_or_arcname, ZipInfo): - zinfo = ZipInfo(filename=zinfo_or_arcname, - date_time=time.localtime(time.time())[:6]) - - zinfo.compress_type = self.compression - if zinfo.filename[-1] == '/': - zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x - zinfo.external_attr |= 0x10 # MS-DOS directory flag - else: - zinfo.external_attr = 0o600 << 16 # ?rw------- - else: - zinfo = zinfo_or_arcname - - if not self.fp: - raise RuntimeError( - "Attempt to write to ZIP archive that was already closed") - - if compress_type is not None: - zinfo.compress_type = compress_type - - zinfo.file_size = len(bytes) # Uncompressed size - zinfo.header_offset = self.fp.tell() # Start of header bytes - self._writecheck(zinfo) - self._didModify = True - zinfo.CRC = crc32(bytes) & 0xffffffff # CRC-32 checksum - if zinfo.compress_type == ZIP_DEFLATED: - co = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, - zlib.DEFLATED, -15) - bytes = co.compress(bytes) + co.flush() - zinfo.compress_size = len(bytes) # Compressed size - else: - zinfo.compress_size = zinfo.file_size - zip64 = zinfo.file_size > ZIP64_LIMIT or \ - zinfo.compress_size > ZIP64_LIMIT - if zip64 and not self._allowZip64: - raise LargeZipFile("Filesize would require ZIP64 extensions") - self.fp.write(zinfo.FileHeader(zip64)) - self.fp.write(bytes) - if zinfo.flag_bits & 0x08: - # Write CRC and file sizes after the file data - fmt = '<LQQ' if zip64 else '<LLL' - self.fp.write(struct.pack(fmt, zinfo.CRC, zinfo.compress_size, - zinfo.file_size)) - self.fp.flush() - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo - - def __del__(self): - """Call the "close()" method in case the user forgot.""" - self.close() - - def close(self): - """Close the file, and for mode "w" and "a" write the ending - records.""" - if self.fp is None: - return - - try: - if self.mode in ("w", "a") and self._didModify: # write ending records - pos1 = self.fp.tell() - for zinfo in self.filelist: # write central directory - dt = zinfo.date_time - dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - extra = [] - if zinfo.file_size > ZIP64_LIMIT \ - or zinfo.compress_size > ZIP64_LIMIT: - extra.append(zinfo.file_size) - extra.append(zinfo.compress_size) - file_size = 0xffffffff - compress_size = 0xffffffff - else: - file_size = zinfo.file_size - compress_size = zinfo.compress_size - - header_offset = zinfo.header_offset - self._start_disk - if header_offset > ZIP64_LIMIT: - extra.append(header_offset) - header_offset = 0xffffffffL - - extra_data = zinfo.extra - if extra: - # Append a ZIP64 field to the extra's - extra_data = struct.pack( - '<HH' + 'Q'*len(extra), - 1, 8*len(extra), *extra) + extra_data - - extract_version = max(45, zinfo.extract_version) - create_version = max(45, zinfo.create_version) - else: - extract_version = zinfo.extract_version - create_version = zinfo.create_version - - try: - filename, flag_bits = zinfo._encodeFilenameFlags() - centdir = struct.pack(structCentralDir, - stringCentralDir, create_version, - zinfo.create_system, extract_version, zinfo.reserved, - flag_bits, zinfo.compress_type, dostime, dosdate, - zinfo.CRC, compress_size, file_size, - len(filename), len(extra_data), len(zinfo.comment), - 0, zinfo.internal_attr, zinfo.external_attr, - header_offset) - except DeprecationWarning: - print >>sys.stderr, (structCentralDir, - stringCentralDir, create_version, - zinfo.create_system, extract_version, zinfo.reserved, - zinfo.flag_bits, zinfo.compress_type, dostime, dosdate, - zinfo.CRC, compress_size, file_size, - len(zinfo.filename), len(extra_data), len(zinfo.comment), - 0, zinfo.internal_attr, zinfo.external_attr, - header_offset) - raise - self.fp.write(centdir) - self.fp.write(filename) - self.fp.write(extra_data) - self.fp.write(zinfo.comment) - - pos2 = self.fp.tell() - # Write end-of-zip-archive record - centDirCount = len(self.filelist) - centDirSize = pos2 - pos1 - centDirOffset = pos1 - self._start_disk - requires_zip64 = None - if centDirCount > ZIP_FILECOUNT_LIMIT: - requires_zip64 = "Files count" - elif centDirOffset > ZIP64_LIMIT: - requires_zip64 = "Central directory offset" - elif centDirSize > ZIP64_LIMIT: - requires_zip64 = "Central directory size" - if requires_zip64: - # Need to write the ZIP64 end-of-archive records - if not self._allowZip64: - raise LargeZipFile(requires_zip64 + - " would require ZIP64 extensions") - zip64endrec = struct.pack( - structEndArchive64, stringEndArchive64, - 44, 45, 45, 0, 0, centDirCount, centDirCount, - centDirSize, centDirOffset) - self.fp.write(zip64endrec) - - zip64locrec = struct.pack( - structEndArchive64Locator, - stringEndArchive64Locator, 0, pos2, 1) - self.fp.write(zip64locrec) - centDirCount = min(centDirCount, 0xFFFF) - centDirSize = min(centDirSize, 0xFFFFFFFF) - centDirOffset = min(centDirOffset, 0xFFFFFFFF) - - endrec = struct.pack(structEndArchive, stringEndArchive, - 0, 0, centDirCount, centDirCount, - centDirSize, centDirOffset, len(self._comment)) - self.fp.write(endrec) - self.fp.write(self._comment) - self.fp.flush() - finally: - fp = self.fp - self.fp = None - if not self._filePassed: - fp.close() - - -class PyZipFile(ZipFile): - """Class to create ZIP archives with Python library files and packages.""" - - def writepy(self, pathname, basename = ""): - """Add all files from "pathname" to the ZIP archive. - - If pathname is a package directory, search the directory and - all package subdirectories recursively for all *.py and enter - the modules into the archive. If pathname is a plain - directory, listdir *.py and enter all modules. Else, pathname - must be a Python *.py file and the module will be put into the - archive. Added modules are always module.pyo or module.pyc. - This method will compile the module.py into module.pyc if - necessary. - """ - dir, name = os.path.split(pathname) - if os.path.isdir(pathname): - initname = os.path.join(pathname, "__init__.py") - if os.path.isfile(initname): - # This is a package directory, add it - if basename: - basename = "%s/%s" % (basename, name) - else: - basename = name - if self.debug: - print "Adding package in", pathname, "as", basename - fname, arcname = self._get_codename(initname[0:-3], basename) - if self.debug: - print "Adding", arcname - self.write(fname, arcname) - dirlist = os.listdir(pathname) - dirlist.remove("__init__.py") - # Add all *.py files and package subdirectories - for filename in dirlist: - path = os.path.join(pathname, filename) - root, ext = os.path.splitext(filename) - if os.path.isdir(path): - if os.path.isfile(os.path.join(path, "__init__.py")): - # This is a package directory, add it - self.writepy(path, basename) # Recursive call - elif ext == ".py": - fname, arcname = self._get_codename(path[0:-3], - basename) - if self.debug: - print "Adding", arcname - self.write(fname, arcname) - else: - # This is NOT a package directory, add its files at top level - if self.debug: - print "Adding files from directory", pathname - for filename in os.listdir(pathname): - path = os.path.join(pathname, filename) - root, ext = os.path.splitext(filename) - if ext == ".py": - fname, arcname = self._get_codename(path[0:-3], - basename) - if self.debug: - print "Adding", arcname - self.write(fname, arcname) - else: - if pathname[-3:] != ".py": - raise RuntimeError, \ - 'Files added with writepy() must end with ".py"' - fname, arcname = self._get_codename(pathname[0:-3], basename) - if self.debug: - print "Adding file", arcname - self.write(fname, arcname) - - def _get_codename(self, pathname, basename): - """Return (filename, archivename) for the path. - - Given a module name path, return the correct file path and - archive name, compiling if necessary. For example, given - /python/lib/string, return (/python/lib/string.pyc, string). - """ - file_py = pathname + ".py" - file_pyc = pathname + ".pyc" - file_pyo = pathname + ".pyo" - if os.path.isfile(file_pyo) and \ - os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime: - fname = file_pyo # Use .pyo file - elif not os.path.isfile(file_pyc) or \ - os.stat(file_pyc).st_mtime < os.stat(file_py).st_mtime: - import py_compile - if self.debug: - print "Compiling", file_py - try: - py_compile.compile(file_py, file_pyc, None, True) - except py_compile.PyCompileError,err: - print err.msg - fname = file_pyc - else: - fname = file_pyc - archivename = os.path.split(fname)[1] - if basename: - archivename = "%s/%s" % (basename, archivename) - return (fname, archivename) - - -def main(args = None): - import textwrap - USAGE=textwrap.dedent("""\ - Usage: - zipfile.py -l zipfile.zip # Show listing of a zipfile - zipfile.py -t zipfile.zip # Test if a zipfile is valid - zipfile.py -e zipfile.zip target # Extract zipfile into target dir - zipfile.py -c zipfile.zip src ... # Create zipfile from sources - """) - if args is None: - args = sys.argv[1:] - - if not args or args[0] not in ('-l', '-c', '-e', '-t'): - print USAGE - sys.exit(1) - - if args[0] == '-l': - if len(args) != 2: - print USAGE - sys.exit(1) - with ZipFile(args[1], 'r') as zf: - zf.printdir() - - elif args[0] == '-t': - if len(args) != 2: - print USAGE - sys.exit(1) - with ZipFile(args[1], 'r') as zf: - badfile = zf.testzip() - if badfile: - print("The following enclosed file is corrupted: {!r}".format(badfile)) - print "Done testing" - - elif args[0] == '-e': - if len(args) != 3: - print USAGE - sys.exit(1) - - with ZipFile(args[1], 'r') as zf: - zf.extractall(args[2]) - - elif args[0] == '-c': - if len(args) < 3: - print USAGE - sys.exit(1) - - def addToZip(zf, path, zippath): - if os.path.isfile(path): - zf.write(path, zippath, ZIP_DEFLATED) - elif os.path.isdir(path): - if zippath: - zf.write(path, zippath) - for nm in os.listdir(path): - addToZip(zf, - os.path.join(path, nm), os.path.join(zippath, nm)) - # else: ignore - - with ZipFile(args[1], 'w', allowZip64=True) as zf: - for path in args[2:]: - zippath = os.path.basename(path) - if not zippath: - zippath = os.path.basename(os.path.dirname(path)) - if zippath in ('', os.curdir, os.pardir): - zippath = '' - addToZip(zf, path, zippath) - -if __name__ == "__main__": - main() |