diff options
Diffstat (limited to 'systrace/catapult/common/py_utils')
38 files changed, 776 insertions, 103 deletions
diff --git a/systrace/catapult/common/py_utils/bin/run_tests b/systrace/catapult/common/py_utils/bin/run_tests new file mode 100755 index 0000000..66a4b59 --- /dev/null +++ b/systrace/catapult/common/py_utils/bin/run_tests @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# Copyright (c) 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import sys + +_CATAPULT_PATH = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..')) + +_PY_UTILS_PATH = os.path.abspath( + os.path.join(_CATAPULT_PATH, 'common', 'py_utils')) + + +def _RunTestsOrDie(top_level_dir): + exit_code = run_with_typ.Run(top_level_dir, path=[_PY_UTILS_PATH]) + if exit_code: + sys.exit(exit_code) + + +def _AddToPathIfNeeded(path): + if path not in sys.path: + sys.path.insert(0, path) + + +if __name__ == '__main__': + _AddToPathIfNeeded(_CATAPULT_PATH) + + from hooks import install + if '--no-install-hooks' in sys.argv: + sys.argv.remove('--no-install-hooks') + else: + install.InstallHooks() + + from catapult_build import run_with_typ + _RunTestsOrDie(_PY_UTILS_PATH) + sys.exit(0) diff --git a/systrace/catapult/common/py_utils/py_utils/__init__.py b/systrace/catapult/common/py_utils/py_utils/__init__.py index fba0897..0d7b052 100644 --- a/systrace/catapult/common/py_utils/py_utils/__init__.py +++ b/systrace/catapult/common/py_utils/py_utils/__init__.py @@ -4,6 +4,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from __future__ import print_function + import functools import inspect import os @@ -74,8 +76,8 @@ _AddDirToPythonPath(os.path.join(GetCatapultDir(), 'third_party', 'mox3')) _AddDirToPythonPath( os.path.join(GetCatapultDir(), 'third_party', 'pyfakefs')) -from devil.utils import timeout_retry -from devil.utils import reraiser_thread +from devil.utils import timeout_retry # pylint: disable=wrong-import-position +from devil.utils import reraiser_thread # pylint: disable=wrong-import-position # Decorator that adds timeout functionality to a function. @@ -100,7 +102,7 @@ def TimeoutDeco(func, default_timeout): try: return timeout_retry.Run(func, timeout, 0, args=args) except reraiser_thread.TimeoutError: - print '%s timed out.' % func.__name__ + print('%s timed out.' % func.__name__) return False return RunWithTimeout diff --git a/systrace/catapult/common/py_utils/py_utils/binary_manager.py b/systrace/catapult/common/py_utils/py_utils/binary_manager.py index 8af08cf..2d3ac8a 100644 --- a/systrace/catapult/common/py_utils/py_utils/binary_manager.py +++ b/systrace/catapult/common/py_utils/py_utils/binary_manager.py @@ -13,7 +13,7 @@ class BinaryManager(object): """ def __init__(self, config_files): - if not config_files or type(config_files) != list: + if not config_files or not isinstance(config_files, list): raise ValueError( 'Must supply a list of config files to the BinaryManager') configs = [dependency_manager.BaseConfig(config) for config in config_files] diff --git a/systrace/catapult/common/py_utils/py_utils/camel_case.py b/systrace/catapult/common/py_utils/py_utils/camel_case.py index 9a76890..dbebb22 100644 --- a/systrace/catapult/common/py_utils/py_utils/camel_case.py +++ b/systrace/catapult/common/py_utils/py_utils/camel_case.py @@ -2,7 +2,11 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import re +import six def ToUnderscore(obj): @@ -11,7 +15,7 @@ def ToUnderscore(obj): Descends recursively into lists and dicts, converting all dict keys. Returns a newly allocated object of the same structure as the input. """ - if isinstance(obj, basestring): + if isinstance(obj, six.string_types): return re.sub('(?!^)([A-Z]+)', r'_\1', obj).lower() elif isinstance(obj, list): @@ -19,7 +23,7 @@ def ToUnderscore(obj): elif isinstance(obj, dict): output = {} - for k, v in obj.iteritems(): + for k, v in six.iteritems(obj): if isinstance(v, list) or isinstance(v, dict): output[ToUnderscore(k)] = ToUnderscore(v) else: diff --git a/systrace/catapult/common/py_utils/py_utils/chrome_binaries.json b/systrace/catapult/common/py_utils/py_utils/chrome_binaries.json index ce357c7..437cbb3 100644 --- a/systrace/catapult/common/py_utils/py_utils/chrome_binaries.json +++ b/systrace/catapult/common/py_utils/py_utils/chrome_binaries.json @@ -6,22 +6,22 @@ "cloud_storage_bucket": "chrome-telemetry", "file_info": { "mac_x86_64": { - "cloud_storage_hash": "b321a01b2c98fe62b1876655b10436c2226b1b76", + "cloud_storage_hash": "381a491e14ab523b8db4cdf3c993713678237af8", "download_path": "bin/reference_builds/chrome-mac64.zip", "path_within_archive": "chrome-mac/Google Chrome.app/Contents/MacOS/Google Chrome", - "version_in_cs": "62.0.3194.0" + "version_in_cs": "77.0.3822.0" }, "win_AMD64": { - "cloud_storage_hash": "2da1c7861745ab0e8f666f119eeb58c1410710cc", - "download_path": "bin\\reference_build\\chrome-win64-pgo.zip", - "path_within_archive": "chrome-win64-pgo\\chrome.exe", - "version_in_cs": "62.0.3194.0" + "cloud_storage_hash": "600ee522c410efe1de2f593c0efc32ae113a7d99", + "download_path": "bin\\reference_build\\chrome-win64-clang.zip", + "path_within_archive": "chrome-win64-clang\\chrome.exe", + "version_in_cs": "77.0.3822.0" }, "win_x86": { - "cloud_storage_hash": "270abd11621386be612af02b707844cba06c0dbd", - "download_path": "bin\\reference_build\\chrome-win32-pgo.zip", - "path_within_archive": "chrome-win32-pgo\\chrome.exe", - "version_in_cs": "62.0.3194.0" + "cloud_storage_hash": "5b79a181bfbd94d8288529b0da1defa3ef097197", + "download_path": "bin\\reference_build\\chrome-win32-clang.zip", + "path_within_archive": "chrome-win32-clang\\chrome.exe", + "version_in_cs": "77.0.3822.0" } } }, @@ -30,10 +30,10 @@ "cloud_storage_bucket": "chrome-telemetry", "file_info": { "linux_x86_64": { - "cloud_storage_hash": "2592ec6f8dd56227c3c281e3cccecd6c9ba72cad", + "cloud_storage_hash": "61d68a6b00f25c964f5162f5251962468c886f3a", "download_path": "bin/reference_build/chrome-linux64.zip", "path_within_archive": "chrome-linux64/chrome", - "version_in_cs": "62.0.3192.0" + "version_in_cs": "76.0.3809.21" } } }, @@ -42,45 +42,85 @@ "cloud_storage_bucket": "chrome-telemetry", "file_info": { "android_k_armeabi-v7a": { - "cloud_storage_hash": "948c776335d3a38d6e8de0dba576e109c6b5724c", + "cloud_storage_hash": "28b913c720d56a30c092625c7862f00175a316c7", "download_path": "bin/reference_build/android_k_armeabi-v7a/ChromeStable.apk", - "version_in_cs": "63.0.3239.111" + "version_in_cs": "75.0.3770.67" }, "android_l_arm64-v8a": { - "cloud_storage_hash": "a25663aad7397002f6dfe44fb97087fdd77df119", + "cloud_storage_hash": "4b953c33c61f94c2198e8001d0d8142c6504a875", "download_path": "bin/reference_build/android_l_arm64-v8a/ChromeStable.apk", - "version_in_cs": "63.0.3239.111" + "version_in_cs": "75.0.3770.67" }, "android_l_armeabi-v7a": { - "cloud_storage_hash": "948c776335d3a38d6e8de0dba576e109c6b5724c", + "cloud_storage_hash": "28b913c720d56a30c092625c7862f00175a316c7", "download_path": "bin/reference_build/android_l_armeabi-v7a/ChromeStable.apk", - "version_in_cs": "63.0.3239.111" + "version_in_cs": "75.0.3770.67" }, + "android_n_arm64-v8a": { + "cloud_storage_hash": "84152ba8f7a25cacc79d588ed827ea75f0e4ab94", + "download_path": "bin/reference_build/android_n_arm64-v8a/Monochrome.apk", + "version_in_cs": "75.0.3770.67" + }, + "android_n_armeabi-v7a": { + "cloud_storage_hash": "656bb9e3982d0d35decd5347ced2c320a7267f33", + "download_path": "bin/reference_build/android_n_armeabi-v7a/Monochrome.apk", + "version_in_cs": "75.0.3770.67" + }, + "linux_x86_64": { + "cloud_storage_hash": "dee8469e8dcd8453efd33f3a00d7ea302a126a4b", + "download_path": "bin/reference_build/chrome-linux64.zip", + "path_within_archive": "chrome-linux64/chrome", + "version_in_cs": "75.0.3770.80" + }, + "mac_x86_64": { + "cloud_storage_hash": "16a43a1e794bb99ec1ebcd40569084985b3c6626", + "download_path": "bin/reference_builds/chrome-mac64.zip", + "path_within_archive": "chrome-mac/Google Chrome.app/Contents/MacOS/Google Chrome", + "version_in_cs": "75.0.3770.80" + }, + "win_AMD64": { + "cloud_storage_hash": "1ec52bd4164f2d93c53113a093dae9e041eb2d73", + "download_path": "bin\\reference_build\\chrome-win64-clang.zip", + "path_within_archive": "chrome-win64-clang\\chrome.exe", + "version_in_cs": "75.0.3770.80" + }, + "win_x86": { + "cloud_storage_hash": "0f9eb991ba618dc61f2063ea252f44be94c2252e", + "download_path": "bin\\reference_build\\chrome-win-clang.zip", + "path_within_archive": "chrome-win-clang\\chrome.exe", + "version_in_cs": "75.0.3770.80" + } + } + }, + "chrome_m72": { + "cloud_storage_base_folder": "binary_dependencies", + "cloud_storage_bucket": "chrome-telemetry", + "file_info": { "linux_x86_64": { - "cloud_storage_hash": "b0506e43d268eadb887ccc847695674f9d2e51a5", + "cloud_storage_hash": "537c19346b20340cc6807242e1eb6d82dfcfa2e8", "download_path": "bin/reference_build/chrome-linux64.zip", "path_within_archive": "chrome-linux64/chrome", - "version_in_cs": "63.0.3239.108" + "version_in_cs": "72.0.3626.119" }, "mac_x86_64": { - "cloud_storage_hash": "56a3de45b37b7eb563006c30a548a48928cffb39", + "cloud_storage_hash": "7f6a931f696f57561703538c6f799781d6e22e7e", "download_path": "bin/reference_builds/chrome-mac64.zip", "path_within_archive": "chrome-mac/Google Chrome.app/Contents/MacOS/Google Chrome", - "version_in_cs": "63.0.3239.108" + "version_in_cs": "72.0.3626.119" }, "win_AMD64": { - "cloud_storage_hash": "d1511334055c88fd9fa5e6e63fee666d9be8c433", - "download_path": "bin\\reference_build\\chrome-win64.zip", - "path_within_archive": "chrome-win64\\chrome.exe", - "version_in_cs": "63.0.3239.108" + "cloud_storage_hash": "563d7985c85bfe77e92b8253d0389ff8551018c7", + "download_path": "bin\\reference_build\\chrome-win64-clang.zip", + "path_within_archive": "chrome-win64-clang\\chrome.exe", + "version_in_cs": "72.0.3626.119" }, "win_x86": { - "cloud_storage_hash": "9e869b3b25ee7b682712cde6eaddc2d7fa84cc90", - "download_path": "bin\\reference_build\\chrome-win32.zip", - "path_within_archive": "chrome-win32\\chrome.exe", - "version_in_cs": "63.0.3239.108" + "cloud_storage_hash": "1802179da16e44b83bd3f0b296f9e5b0b053d59c", + "download_path": "bin\\reference_build\\chrome-win-clang.zip", + "path_within_archive": "chrome-win-clang\\chrome.exe", + "version_in_cs": "72.0.3626.119" } } } } -} +}
\ No newline at end of file diff --git a/systrace/catapult/common/py_utils/py_utils/cloud_storage.py b/systrace/catapult/common/py_utils/py_utils/cloud_storage.py index f601380..b4988c5 100644 --- a/systrace/catapult/common/py_utils/py_utils/cloud_storage.py +++ b/systrace/catapult/common/py_utils/py_utils/cloud_storage.py @@ -9,21 +9,21 @@ import contextlib import hashlib import logging import os +import re import shutil import stat import subprocess -import re import sys import tempfile import time import py_utils +from py_utils import cloud_storage_global_lock # pylint: disable=unused-import from py_utils import lock # Do a no-op import here so that cloud_storage_global_lock dep is picked up # by https://cs.chromium.org/chromium/src/build/android/test_runner.pydeps. # TODO(nedn, jbudorick): figure out a way to get rid of this ugly hack. -from py_utils import cloud_storage_global_lock # pylint: disable=unused-import logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -42,7 +42,7 @@ BUCKET_ALIASES = collections.OrderedDict(( ('output', TELEMETRY_OUTPUT), )) -BUCKET_ALIAS_NAMES = BUCKET_ALIASES.keys() +BUCKET_ALIAS_NAMES = list(BUCKET_ALIASES.keys()) _GSUTIL_PATH = os.path.join(py_utils.GetCatapultDir(), 'third_party', 'gsutil', @@ -120,6 +120,9 @@ def _EnsureExecutable(gsutil): os.chmod(gsutil, st.st_mode | stat.S_IEXEC) +def _IsRunningOnSwarming(): + return os.environ.get('SWARMING_HEADLESS') is not None + def _RunCommand(args): # On cros device, as telemetry is running as root, home will be set to /root/, # which is not writable. gsutil will attempt to create a download tracker dir @@ -132,6 +135,8 @@ def _RunCommand(args): if py_utils.IsRunningOnCrosDevice(): gsutil_env = os.environ.copy() gsutil_env['HOME'] = _CROS_GSUTIL_HOME_WAR + elif _IsRunningOnSwarming(): + gsutil_env = os.environ.copy() if os.name == 'nt': # If Windows, prepend python. Python scripts aren't directly executable. diff --git a/systrace/catapult/common/py_utils/py_utils/cloud_storage_unittest.py b/systrace/catapult/common/py_utils/py_utils/cloud_storage_unittest.py index ae2f748..7648db6 100644 --- a/systrace/catapult/common/py_utils/py_utils/cloud_storage_unittest.py +++ b/systrace/catapult/common/py_utils/py_utils/cloud_storage_unittest.py @@ -154,6 +154,19 @@ class CloudStorageFakeFsUnitTest(BaseFakeFsUnitTest): finally: cloud_storage._RunCommand = orig_run_command + @mock.patch('py_utils.cloud_storage.subprocess.Popen') + def testSwarmingUsesExistingEnv(self, mock_popen): + os.environ['SWARMING_HEADLESS'] = '1' + + mock_gsutil = mock_popen() + mock_gsutil.communicate = mock.MagicMock(return_value=('a', 'b')) + mock_gsutil.returncode = None + + cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2') + + mock_popen.assert_called_with( + mock.ANY, stderr=-1, env=os.environ, stdout=-1) + @mock.patch('py_utils.cloud_storage._FileLock') def testDisableCloudStorageIo(self, unused_lock_mock): os.environ['DISABLE_CLOUD_STORAGE_IO'] = '1' diff --git a/systrace/catapult/common/py_utils/py_utils/discover.py b/systrace/catapult/common/py_utils/py_utils/discover.py index 09d5c5e..ae8ba87 100644 --- a/systrace/catapult/common/py_utils/py_utils/discover.py +++ b/systrace/catapult/common/py_utils/py_utils/discover.py @@ -107,7 +107,7 @@ def DiscoverClasses(start_dir, # crbug.com/548652 if index_by_class_name: AssertNoKeyConflicts(classes, new_classes) - classes = dict(classes.items() + new_classes.items()) + classes = dict(list(classes.items()) + list(new_classes.items())) return classes diff --git a/systrace/catapult/common/py_utils/py_utils/discover_unittest.py b/systrace/catapult/common/py_utils/py_utils/discover_unittest.py index 137d85f..2d4fd27 100644 --- a/systrace/catapult/common/py_utils/py_utils/discover_unittest.py +++ b/systrace/catapult/common/py_utils/py_utils/discover_unittest.py @@ -1,10 +1,15 @@ # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import os import unittest from py_utils import discover +import six class DiscoverTest(unittest.TestCase): @@ -20,8 +25,8 @@ class DiscoverTest(unittest.TestCase): self._base_class, index_by_class_name=False) - actual_classes = dict((name, cls.__name__) - for name, cls in classes.iteritems()) + actual_classes = dict( + (name, cls.__name__) for name, cls in six.iteritems(classes)) expected_classes = { 'another_discover_dummyclass': 'DummyExceptionWithParameterImpl1', 'discover_dummyclass': 'DummyException', @@ -35,8 +40,8 @@ class DiscoverTest(unittest.TestCase): self._base_class, directly_constructable=True) - actual_classes = dict((name, cls.__name__) - for name, cls in classes.iteritems()) + actual_classes = dict( + (name, cls.__name__) for name, cls in six.iteritems(classes)) expected_classes = { 'dummy_exception': 'DummyException', 'dummy_exception_impl1': 'DummyExceptionImpl1', @@ -48,8 +53,8 @@ class DiscoverTest(unittest.TestCase): classes = discover.DiscoverClasses(self._start_dir, self._base_dir, self._base_class) - actual_classes = dict((name, cls.__name__) - for name, cls in classes.iteritems()) + actual_classes = dict( + (name, cls.__name__) for name, cls in six.iteritems(classes)) expected_classes = { 'dummy_exception': 'DummyException', 'dummy_exception_impl1': 'DummyExceptionImpl1', @@ -68,8 +73,8 @@ class DiscoverTest(unittest.TestCase): pattern='another*', index_by_class_name=False) - actual_classes = dict((name, cls.__name__) - for name, cls in classes.iteritems()) + actual_classes = dict( + (name, cls.__name__) for name, cls in six.iteritems(classes)) expected_classes = { 'another_discover_dummyclass': 'DummyExceptionWithParameterImpl1' } @@ -83,8 +88,8 @@ class DiscoverTest(unittest.TestCase): pattern='another*', directly_constructable=True) - actual_classes = dict((name, cls.__name__) - for name, cls in classes.iteritems()) + actual_classes = dict( + (name, cls.__name__) for name, cls in six.iteritems(classes)) expected_classes = { 'dummy_exception_impl1': 'DummyExceptionImpl1', 'dummy_exception_impl2': 'DummyExceptionImpl2', @@ -97,8 +102,8 @@ class DiscoverTest(unittest.TestCase): self._base_class, pattern='another*') - actual_classes = dict((name, cls.__name__) - for name, cls in classes.iteritems()) + actual_classes = dict( + (name, cls.__name__) for name, cls in six.iteritems(classes)) expected_classes = { 'dummy_exception_impl1': 'DummyExceptionImpl1', 'dummy_exception_impl2': 'DummyExceptionImpl2', diff --git a/systrace/catapult/common/py_utils/py_utils/exc_util.py b/systrace/catapult/common/py_utils/py_utils/exc_util.py new file mode 100644 index 0000000..538ced2 --- /dev/null +++ b/systrace/catapult/common/py_utils/py_utils/exc_util.py @@ -0,0 +1,84 @@ +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import functools +import logging +import sys + + +def BestEffort(func): + """Decorator to log and dismiss exceptions if one if already being handled. + + Note: This is largely a workaround for the lack of support of exception + chaining in Python 2.7, this decorator will no longer be needed in Python 3. + + Typical usage would be in |Close| or |Disconnect| methods, to dismiss but log + any further exceptions raised if the current execution context is already + handling an exception. For example: + + class Client(object): + def Connect(self): + # code to connect ... + + @exc_util.BestEffort + def Disconnect(self): + # code to disconnect ... + + client = Client() + try: + client.Connect() + except: + client.Disconnect() + raise + + If an exception is raised by client.Connect(), and then a second exception + is raised by client.Disconnect(), the decorator will log the second exception + and let the original one be re-raised. + + Otherwise, in Python 2.7 and without the decorator, the second exception is + the one propagated to the caller; while information about the original one, + usually more important, is completely lost. + + Note that if client.Disconnect() is called in a context where an exception + is *not* being handled, then any exceptions raised within the method will + get through and be passed on to callers for them to handle in the usual way. + + The decorator can also be used on cleanup functions meant to be called on + a finally block, however you must also include an except-raise clause to + properly signal (in Python 2.7) whether an exception is being handled; e.g.: + + @exc_util.BestEffort + def cleanup(): + # do cleanup things ... + + try: + process(thing) + except: + raise # Needed to let cleanup know if an exception is being handled. + finally: + cleanup() + + Failing to include the except-raise block has the same effect as not + including the decorator at all. Namely: exceptions during |cleanup| are + raised and swallow any prior exceptions that occurred during |process|. + """ + @functools.wraps(func) + def Wrapper(*args, **kwargs): + exc_type = sys.exc_info()[0] + if exc_type is None: + # Not currently handling an exception; let any errors raise exceptions + # as usual. + func(*args, **kwargs) + else: + # Otherwise, we are currently handling an exception, dismiss and log + # any further cascading errors. Callers are responsible to handle the + # original exception. + try: + func(*args, **kwargs) + except Exception: # pylint: disable=broad-except + logging.exception( + 'While handling a %s, the following exception was also raised:', + exc_type.__name__) + + return Wrapper diff --git a/systrace/catapult/common/py_utils/py_utils/exc_util_unittest.py b/systrace/catapult/common/py_utils/py_utils/exc_util_unittest.py new file mode 100644 index 0000000..31e3b57 --- /dev/null +++ b/systrace/catapult/common/py_utils/py_utils/exc_util_unittest.py @@ -0,0 +1,183 @@ +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import re +import sys +import unittest + +from py_utils import exc_util + + +class FakeConnectionError(Exception): + pass + + +class FakeDisconnectionError(Exception): + pass + + +class FakeProcessingError(Exception): + pass + + +class FakeCleanupError(Exception): + pass + + +class FaultyClient(object): + def __init__(self, *args): + self.failures = set(args) + self.called = set() + + def Connect(self): + self.called.add('Connect') + if FakeConnectionError in self.failures: + raise FakeConnectionError('Oops!') + + def Process(self): + self.called.add('Process') + if FakeProcessingError in self.failures: + raise FakeProcessingError('Oops!') + + @exc_util.BestEffort + def Disconnect(self): + self.called.add('Disconnect') + if FakeDisconnectionError in self.failures: + raise FakeDisconnectionError('Oops!') + + @exc_util.BestEffort + def Cleanup(self): + self.called.add('Cleanup') + if FakeCleanupError in self.failures: + raise FakeCleanupError('Oops!') + + +class ReraiseTests(unittest.TestCase): + def assertLogMatches(self, pattern): + self.assertRegexpMatches( + sys.stderr.getvalue(), pattern) # pylint: disable=no-member + + def assertLogNotMatches(self, pattern): + self.assertNotRegexpMatches( + sys.stderr.getvalue(), pattern) # pylint: disable=no-member + + def testTryRaisesExceptRaises(self): + client = FaultyClient(FakeConnectionError, FakeDisconnectionError) + + # The connection error reaches the top level, while the disconnection + # error is logged. + with self.assertRaises(FakeConnectionError): + try: + client.Connect() + except: + client.Disconnect() + raise + + self.assertLogMatches(re.compile( + r'While handling a FakeConnectionError, .* was also raised:\n' + r'Traceback \(most recent call last\):\n' + r'.*\n' + r'FakeDisconnectionError: Oops!\n', re.DOTALL)) + self.assertItemsEqual(client.called, ['Connect', 'Disconnect']) + + def testTryRaisesExceptDoesnt(self): + client = FaultyClient(FakeConnectionError) + + # The connection error reaches the top level, disconnecting did not raise + # an exception (so nothing is logged). + with self.assertRaises(FakeConnectionError): + try: + client.Connect() + except: + client.Disconnect() + raise + + self.assertLogNotMatches('FakeDisconnectionError') + self.assertItemsEqual(client.called, ['Connect', 'Disconnect']) + + def testTryPassesNoException(self): + client = FaultyClient(FakeDisconnectionError) + + # If there is no connection error, the except clause is not called (even if + # it would have raised an exception). + try: + client.Connect() + except: + client.Disconnect() + raise + + self.assertLogNotMatches('FakeConnectionError') + self.assertLogNotMatches('FakeDisconnectionError') + self.assertItemsEqual(client.called, ['Connect']) + + def testTryRaisesFinallyRaises(self): + worker = FaultyClient(FakeProcessingError, FakeCleanupError) + + # The processing error reaches the top level, the cleanup error is logged. + with self.assertRaises(FakeProcessingError): + try: + worker.Process() + except: + raise # Needed for Cleanup to know if an exception is handled. + finally: + worker.Cleanup() + + self.assertLogMatches(re.compile( + r'While handling a FakeProcessingError, .* was also raised:\n' + r'Traceback \(most recent call last\):\n' + r'.*\n' + r'FakeCleanupError: Oops!\n', re.DOTALL)) + self.assertItemsEqual(worker.called, ['Process', 'Cleanup']) + + def testTryRaisesFinallyDoesnt(self): + worker = FaultyClient(FakeProcessingError) + + # The processing error reaches the top level, the cleanup code runs fine. + with self.assertRaises(FakeProcessingError): + try: + worker.Process() + except: + raise # Needed for Cleanup to know if an exception is handled. + finally: + worker.Cleanup() + + self.assertLogNotMatches('FakeProcessingError') + self.assertLogNotMatches('FakeCleanupError') + self.assertItemsEqual(worker.called, ['Process', 'Cleanup']) + + def testTryPassesFinallyRaises(self): + worker = FaultyClient(FakeCleanupError) + + # The processing code runs fine, the cleanup code raises an exception + # which reaches the top level. + with self.assertRaises(FakeCleanupError): + try: + worker.Process() + except: + raise # Needed for Cleanup to know if an exception is handled. + finally: + worker.Cleanup() + + self.assertLogNotMatches('FakeProcessingError') + self.assertLogNotMatches('FakeCleanupError') + self.assertItemsEqual(worker.called, ['Process', 'Cleanup']) + + def testTryRaisesExceptRaisesFinallyRaises(self): + worker = FaultyClient( + FakeProcessingError, FakeDisconnectionError, FakeCleanupError) + + # Chaining try-except-finally works fine. Only the processing error reaches + # the top level; the other two are logged. + with self.assertRaises(FakeProcessingError): + try: + worker.Process() + except: + worker.Disconnect() + raise + finally: + worker.Cleanup() + + self.assertLogMatches('FakeDisconnectionError') + self.assertLogMatches('FakeCleanupError') + self.assertItemsEqual(worker.called, ['Process', 'Disconnect', 'Cleanup']) diff --git a/systrace/catapult/common/py_utils/py_utils/expectations_parser.py b/systrace/catapult/common/py_utils/py_utils/expectations_parser.py index 6fa9407..534b352 100644 --- a/systrace/catapult/common/py_utils/py_utils/expectations_parser.py +++ b/systrace/catapult/common/py_utils/py_utils/expectations_parser.py @@ -2,7 +2,11 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import re +import six class ParseError(Exception): @@ -20,9 +24,9 @@ class Expectation(object): Conditions are combined using logical and. Example: ['Mac', 'Debug'] results: List of outcomes for test. Example: ['Skip', 'Pass'] """ - assert isinstance(reason, basestring) or reason is None + assert isinstance(reason, six.string_types) or reason is None self._reason = reason - assert isinstance(test, basestring) + assert isinstance(test, six.string_types) self._test = test assert isinstance(conditions, list) self._conditions = conditions diff --git a/systrace/catapult/common/py_utils/py_utils/expectations_parser_unittest.py b/systrace/catapult/common/py_utils/py_utils/expectations_parser_unittest.py index a842c4c..523e871 100644 --- a/systrace/catapult/common/py_utils/py_utils/expectations_parser_unittest.py +++ b/systrace/catapult/common/py_utils/py_utils/expectations_parser_unittest.py @@ -3,9 +3,14 @@ # found in the LICENSE file. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import unittest from py_utils import expectations_parser +from six.moves import range # pylint: disable=redefined-builtin class TestExpectationParserTest(unittest.TestCase): diff --git a/systrace/catapult/common/py_utils/py_utils/file_util.py b/systrace/catapult/common/py_utils/py_utils/file_util.py new file mode 100644 index 0000000..b1602c9 --- /dev/null +++ b/systrace/catapult/common/py_utils/py_utils/file_util.py @@ -0,0 +1,23 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import errno +import os +import shutil + + +def CopyFileWithIntermediateDirectories(source_path, dest_path): + """Copies a file and creates intermediate directories as needed. + + Args: + source_path: Path to the source file. + dest_path: Path to the destination where the source file should be copied. + """ + assert os.path.exists(source_path) + try: + os.makedirs(os.path.dirname(dest_path)) + except OSError as e: + if e.errno != errno.EEXIST: + raise + shutil.copy(source_path, dest_path) diff --git a/systrace/catapult/common/py_utils/py_utils/file_util_unittest.py b/systrace/catapult/common/py_utils/py_utils/file_util_unittest.py new file mode 100644 index 0000000..4bb19a1 --- /dev/null +++ b/systrace/catapult/common/py_utils/py_utils/file_util_unittest.py @@ -0,0 +1,66 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import errno +import os +import shutil +import tempfile +import unittest + +from py_utils import file_util + + +class FileUtilTest(unittest.TestCase): + + def setUp(self): + self._tempdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._tempdir) + + def testCopySimple(self): + source_path = os.path.join(self._tempdir, 'source') + with open(source_path, 'w') as f: + f.write('data') + + dest_path = os.path.join(self._tempdir, 'dest') + + self.assertFalse(os.path.exists(dest_path)) + file_util.CopyFileWithIntermediateDirectories(source_path, dest_path) + self.assertTrue(os.path.exists(dest_path)) + self.assertEqual('data', open(dest_path, 'r').read()) + + def testCopyMakeDirectories(self): + source_path = os.path.join(self._tempdir, 'source') + with open(source_path, 'w') as f: + f.write('data') + + dest_path = os.path.join(self._tempdir, 'path', 'to', 'dest') + + self.assertFalse(os.path.exists(dest_path)) + file_util.CopyFileWithIntermediateDirectories(source_path, dest_path) + self.assertTrue(os.path.exists(dest_path)) + self.assertEqual('data', open(dest_path, 'r').read()) + + def testCopyOverwrites(self): + source_path = os.path.join(self._tempdir, 'source') + with open(source_path, 'w') as f: + f.write('source_data') + + dest_path = os.path.join(self._tempdir, 'dest') + with open(dest_path, 'w') as f: + f.write('existing_data') + + file_util.CopyFileWithIntermediateDirectories(source_path, dest_path) + self.assertEqual('source_data', open(dest_path, 'r').read()) + + def testRaisesError(self): + source_path = os.path.join(self._tempdir, 'source') + with open(source_path, 'w') as f: + f.write('data') + + dest_path = "" + with self.assertRaises(OSError) as cm: + file_util.CopyFileWithIntermediateDirectories(source_path, dest_path) + self.assertEqual(errno.ENOENT, cm.exception.error_code) diff --git a/systrace/catapult/common/py_utils/py_utils/lock.py b/systrace/catapult/common/py_utils/py_utils/lock.py index aa9a095..ade4d1f 100644 --- a/systrace/catapult/common/py_utils/py_utils/lock.py +++ b/systrace/catapult/common/py_utils/py_utils/lock.py @@ -14,19 +14,23 @@ class LockException(Exception): pass +# pylint: disable=import-error +# pylint: disable=wrong-import-position if os.name == 'nt': - import win32con # pylint: disable=import-error - import win32file # pylint: disable=import-error - import pywintypes # pylint: disable=import-error + import win32con + import win32file + import pywintypes LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK LOCK_SH = 0 # the default LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY _OVERLAPPED = pywintypes.OVERLAPPED() elif os.name == 'posix': - import fcntl # pylint: disable=import-error + import fcntl LOCK_EX = fcntl.LOCK_EX LOCK_SH = fcntl.LOCK_SH LOCK_NB = fcntl.LOCK_NB +# pylint: enable=import-error +# pylint: enable=wrong-import-position @contextlib.contextmanager @@ -80,7 +84,7 @@ def _LockImplWin(target_file, flags): hfile = win32file._get_osfhandle(target_file.fileno()) try: win32file.LockFileEx(hfile, flags, 0, -0x10000, _OVERLAPPED) - except pywintypes.error, exc_value: + except pywintypes.error as exc_value: if exc_value[0] == 33: raise LockException('Error trying acquiring lock of %s: %s' % (target_file.name, exc_value[2])) @@ -92,7 +96,7 @@ def _UnlockImplWin(target_file): hfile = win32file._get_osfhandle(target_file.fileno()) try: win32file.UnlockFileEx(hfile, 0, -0x10000, _OVERLAPPED) - except pywintypes.error, exc_value: + except pywintypes.error as exc_value: if exc_value[0] == 158: # error: (158, 'UnlockFileEx', 'The segment is already unlocked.') # To match the 'posix' implementation, silently ignore this error @@ -105,7 +109,7 @@ def _UnlockImplWin(target_file): def _LockImplPosix(target_file, flags): try: fcntl.flock(target_file.fileno(), flags) - except IOError, exc_value: + except IOError as exc_value: if exc_value[0] == 11 or exc_value[0] == 35: raise LockException('Error trying acquiring lock of %s: %s' % (target_file.name, exc_value[1])) diff --git a/systrace/catapult/common/py_utils/py_utils/lock_unittest.py b/systrace/catapult/common/py_utils/py_utils/lock_unittest.py index a260621..7e17e55 100644 --- a/systrace/catapult/common/py_utils/py_utils/lock_unittest.py +++ b/systrace/catapult/common/py_utils/py_utils/lock_unittest.py @@ -2,14 +2,18 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import multiprocessing import os +import tempfile import time import unittest -import tempfile - from py_utils import lock +from six.moves import range # pylint: disable=redefined-builtin def _AppendTextToFile(file_name): diff --git a/systrace/catapult/common/py_utils/py_utils/logging_util_unittest.py b/systrace/catapult/common/py_utils/py_utils/logging_util_unittest.py index a957705..eb26098 100644 --- a/systrace/catapult/common/py_utils/py_utils/logging_util_unittest.py +++ b/systrace/catapult/common/py_utils/py_utils/logging_util_unittest.py @@ -2,15 +2,19 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import logging -import StringIO import unittest +try: + from six import StringIO +except ImportError: + from io import StringIO + from py_utils import logging_util class LoggingUtilTest(unittest.TestCase): def testCapture(self): - s = StringIO.StringIO() + s = StringIO() with logging_util.CaptureLogs(s): logging.fatal('test') diff --git a/systrace/catapult/common/py_utils/py_utils/memory_debug.py b/systrace/catapult/common/py_utils/py_utils/memory_debug.py index 864725d..26f10ae 100755 --- a/systrace/catapult/common/py_utils/py_utils/memory_debug.py +++ b/systrace/catapult/common/py_utils/py_utils/memory_debug.py @@ -6,8 +6,11 @@ import heapq import logging import os -import psutil import sys +try: + import psutil +except ImportError: + psutil = None BYTE_UNITS = ['B', 'KiB', 'MiB', 'GiB'] @@ -39,6 +42,9 @@ def _LogProcessInfo(pinfo, level): def LogHostMemoryUsage(top_n=10, level=logging.INFO): + if not psutil: + logging.warning('psutil module is not found, skipping logging memory info') + return if psutil.version_info < (2, 0): logging.warning('psutil %s too old, upgrade to version 2.0 or higher' ' for memory usage information.', psutil.__version__) @@ -55,7 +61,11 @@ def LogHostMemoryUsage(top_n=10, level=logging.INFO): logging.log(level, 'Memory usage of top %i processes groups', top_n) pinfos_by_names = {} for p in psutil.process_iter(): - pinfo = _GetProcessInfo(p) + try: + pinfo = _GetProcessInfo(p) + except psutil.NoSuchProcess: + logging.exception('process %s no longer exists', p) + continue pname = pinfo['name'] if pname not in pinfos_by_names: pinfos_by_names[pname] = {'name': pname, 'total_mem_rss': 0, 'pids': []} @@ -63,7 +73,9 @@ def LogHostMemoryUsage(top_n=10, level=logging.INFO): pinfos_by_names[pname]['pids'].append(str(pinfo['pid'])) sorted_pinfo_groups = heapq.nlargest( - top_n, pinfos_by_names.values(), key=lambda item: item['total_mem_rss']) + top_n, + list(pinfos_by_names.values()), + key=lambda item: item['total_mem_rss']) for group in sorted_pinfo_groups: group['total_mem_rss_fmt'] = FormatBytes(group['total_mem_rss']) group['pids_fmt'] = ', '.join(group['pids']) diff --git a/systrace/catapult/common/py_utils/py_utils/modules_util.py b/systrace/catapult/common/py_utils/py_utils/modules_util.py new file mode 100644 index 0000000..6c1106d --- /dev/null +++ b/systrace/catapult/common/py_utils/py_utils/modules_util.py @@ -0,0 +1,35 @@ +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +from distutils import version # pylint: disable=no-name-in-module + + +def RequireVersion(module, min_version, max_version=None): + """Ensure that an imported module's version is within a required range. + + Version strings are parsed with LooseVersion, so versions like "1.8.0rc1" + (default numpy on macOS Sierra) and "2.4.13.2" (a version of OpenCV 2.x) + are allowed. + + Args: + module: An already imported python module. + min_version: The module must have this or a higher version. + max_version: Optional, the module should not have this or a higher version. + + Raises: + ImportError if the module's __version__ is not within the allowed range. + """ + module_version = version.LooseVersion(module.__version__) + min_version = version.LooseVersion(str(min_version)) + valid_version = min_version <= module_version + + if max_version is not None: + max_version = version.LooseVersion(str(max_version)) + valid_version = valid_version and (module_version < max_version) + wants_version = 'at or above %s and below %s' % (min_version, max_version) + else: + wants_version = '%s or higher' % min_version + + if not valid_version: + raise ImportError('%s has version %s, but version %s is required' % ( + module.__name__, module_version, wants_version)) diff --git a/systrace/catapult/common/py_utils/py_utils/modules_util_unittest.py b/systrace/catapult/common/py_utils/py_utils/modules_util_unittest.py new file mode 100644 index 0000000..aa05674 --- /dev/null +++ b/systrace/catapult/common/py_utils/py_utils/modules_util_unittest.py @@ -0,0 +1,41 @@ +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +import unittest + +from py_utils import modules_util + + +class FakeModule(object): + def __init__(self, name, version): + self.__name__ = name + self.__version__ = version + + +class ModulesUitlTest(unittest.TestCase): + def testRequireVersion_valid(self): + numpy = FakeModule('numpy', '2.3') + try: + modules_util.RequireVersion(numpy, '1.0') + except ImportError: + self.fail('ImportError raised unexpectedly') + + def testRequireVersion_versionTooLow(self): + numpy = FakeModule('numpy', '2.3') + with self.assertRaises(ImportError) as error: + modules_util.RequireVersion(numpy, '2.5') + self.assertEqual( + str(error.exception), + 'numpy has version 2.3, but version 2.5 or higher is required') + + def testRequireVersion_versionTooHigh(self): + numpy = FakeModule('numpy', '2.3') + with self.assertRaises(ImportError) as error: + modules_util.RequireVersion(numpy, '1.0', '2.0') + self.assertEqual( + str(error.exception), 'numpy has version 2.3, but version' + ' at or above 1.0 and below 2.0 is required') + + +if __name__ == '__main__': + unittest.main() diff --git a/systrace/catapult/common/py_utils/py_utils/refactor/__init__.py b/systrace/catapult/common/py_utils/py_utils/refactor/__init__.py index e3fbb5f..938ff68 100644 --- a/systrace/catapult/common/py_utils/py_utils/refactor/__init__.py +++ b/systrace/catapult/common/py_utils/py_utils/refactor/__init__.py @@ -12,7 +12,7 @@ import functools import multiprocessing # pylint: disable=wildcard-import -from py_utils.refactor.annotated_symbol import * +from py_utils.refactor.annotated_symbol import * # pylint: disable=redefined-builtin from py_utils.refactor.module import Module diff --git a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/__init__.py b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/__init__.py index 610bc15..1bed84b 100644 --- a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/__init__.py +++ b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/__init__.py @@ -6,7 +6,7 @@ from py_utils.refactor.annotated_symbol.class_definition import * from py_utils.refactor.annotated_symbol.function_definition import * from py_utils.refactor.annotated_symbol.import_statement import * -from py_utils.refactor.annotated_symbol.reference import * +from py_utils.refactor.annotated_symbol.reference import * # pylint: disable=redefined-builtin from py_utils.refactor import snippet @@ -55,7 +55,7 @@ def _AnnotateNode(node): if not isinstance(node, snippet.Symbol): return node - children = map(_AnnotateNode, node.children) + children = [_AnnotateNode(c) for c in node.children] for symbol_type in ANNOTATED_GROUPINGS: annotated_grouping = symbol_type.Annotate(children) diff --git a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/base_symbol.py b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/base_symbol.py index 2e28e89..5e473bc 100644 --- a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/base_symbol.py +++ b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/base_symbol.py @@ -2,7 +2,11 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function from py_utils.refactor import snippet +from six.moves import range # pylint: disable=redefined-builtin class AnnotatedSymbol(snippet.Symbol): @@ -23,7 +27,7 @@ class AnnotatedSymbol(snippet.Symbol): return super(AnnotatedSymbol, self).__setattr__(name, value) def Cut(self, child): - for i in xrange(len(self._children)): + for i in range(len(self._children)): if self._children[i] == child: self._modified = True del self._children[i] diff --git a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/class_definition.py b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/class_definition.py index 814958f..a83ac96 100644 --- a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/class_definition.py +++ b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/class_definition.py @@ -13,8 +13,6 @@ __all__ = [ class Class(base_symbol.AnnotatedSymbol): - # pylint: disable=abstract-class-not-used - @classmethod def Annotate(cls, symbol_type, children): if symbol_type != symbol.stmt: diff --git a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/function_definition.py b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/function_definition.py index 50a1672..384d3cf 100644 --- a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/function_definition.py +++ b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/function_definition.py @@ -13,8 +13,6 @@ __all__ = [ class Function(base_symbol.AnnotatedSymbol): - # pylint: disable=abstract-class-not-used - @classmethod def Annotate(cls, symbol_type, children): if symbol_type != symbol.stmt: diff --git a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/import_statement.py b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/import_statement.py index 5c38c10..6318eff 100644 --- a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/import_statement.py +++ b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/import_statement.py @@ -2,13 +2,17 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import itertools +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import keyword import symbol import token -from py_utils.refactor.annotated_symbol import base_symbol from py_utils.refactor import snippet +from py_utils.refactor.annotated_symbol import base_symbol +from six.moves import zip_longest # pylint: disable=redefined-builtin __all__ = [ @@ -39,11 +43,11 @@ class DottedName(base_symbol.AnnotatedSymbol): raise ValueError('%s is a reserved keyword.' % value_part) # If we have too many children, cut the list down to size. + # pylint: disable=attribute-defined-outside-init self._children = self._children[:len(value_parts)*2-1] # Update child nodes. - for child, value_part in itertools.izip_longest( - self._children[::2], value_parts): + for child, value_part in zip_longest(self._children[::2], value_parts): if child: # Modify existing children. This helps preserve comments and spaces. child.value = value_part @@ -82,18 +86,22 @@ class AsName(base_symbol.AnnotatedSymbol): raise ValueError('%s is a reserved keyword.' % value) if value: + # pylint: disable=access-member-before-definition if len(self.children) < 3: # If we currently have no alias, add one. + # pylint: disable=access-member-before-definition self.children.append( snippet.TokenSnippet.Create(token.NAME, 'as', (0, 1))) + # pylint: disable=access-member-before-definition self.children.append( snippet.TokenSnippet.Create(token.NAME, value, (0, 1))) else: # We already have an alias. Just update the value. + # pylint: disable=access-member-before-definition self.children[2].value = value else: # Removing the alias. Strip the "as foo". - self.children = [self.children[0]] + self.children = [self.children[0]] # pylint: disable=line-too-long, attribute-defined-outside-init class Import(base_symbol.AnnotatedSymbol): diff --git a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/reference.py b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/reference.py index 757c57f..9a273d8 100644 --- a/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/reference.py +++ b/systrace/catapult/common/py_utils/py_utils/refactor/annotated_symbol/reference.py @@ -2,12 +2,17 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import itertools +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import symbol import token -from py_utils.refactor.annotated_symbol import base_symbol from py_utils.refactor import snippet +from py_utils.refactor.annotated_symbol import base_symbol +from six.moves import range # pylint: disable=redefined-builtin +from six.moves import zip_longest # pylint: disable=redefined-builtin __all__ = [ @@ -25,7 +30,7 @@ class Reference(base_symbol.AnnotatedSymbol): if not nodes[0].children or nodes[0].children[0].type != token.NAME: return None - for i in xrange(1, len(nodes)): + for i in range(1, len(nodes)): if not nodes: break if nodes[i].type != symbol.trailer: @@ -58,11 +63,11 @@ class Reference(base_symbol.AnnotatedSymbol): value_parts = value.split('.') # If we have too many children, cut the list down to size. + # pylint: disable=attribute-defined-outside-init self._children = self._children[:len(value_parts)] # Update child nodes. - for child, value_part in itertools.izip_longest( - self._children, value_parts): + for child, value_part in zip_longest(self._children, value_parts): if child: # Modify existing children. This helps preserve comments and spaces. child.children[-1].value = value_part diff --git a/systrace/catapult/common/py_utils/py_utils/refactor/offset_token.py b/systrace/catapult/common/py_utils/py_utils/refactor/offset_token.py index 5fa953e..deca085 100644 --- a/systrace/catapult/common/py_utils/py_utils/refactor/offset_token.py +++ b/systrace/catapult/common/py_utils/py_utils/refactor/offset_token.py @@ -1,18 +1,23 @@ +# Lint as: python2, python3 # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import collections import itertools import token import tokenize +from six.moves import zip # pylint: disable=redefined-builtin def _Pairwise(iterable): """s -> (None, s0), (s0, s1), (s1, s2), (s2, s3), ...""" a, b = itertools.tee(iterable) a = itertools.chain((None,), a) - return itertools.izip(a, b) + return zip(a, b) class OffsetToken(object): diff --git a/systrace/catapult/common/py_utils/py_utils/refactor/snippet.py b/systrace/catapult/common/py_utils/py_utils/refactor/snippet.py index b98561a..7056abf 100644 --- a/systrace/catapult/common/py_utils/py_utils/refactor/snippet.py +++ b/systrace/catapult/common/py_utils/py_utils/refactor/snippet.py @@ -2,6 +2,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from __future__ import print_function + import parser import symbol import sys @@ -140,13 +142,13 @@ class TokenSnippet(Snippet): def PrintTree(self, indent=0, stream=sys.stdout): stream.write(' ' * indent) if not self.tokens: - print >> stream, self.type_name + print(self.type_name, file=stream) return - print >> stream, '%-4s' % self.type_name, repr(self.tokens[0].string) + print('%-4s' % self.type_name, repr(self.tokens[0].string), file=stream) for tok in self.tokens[1:]: stream.write(' ' * indent) - print >> stream, ' ' * max(len(self.type_name), 4), repr(tok.string) + print(' ' * max(len(self.type_name), 4), repr(tok.string), file=stream) class Symbol(Snippet): @@ -191,10 +193,10 @@ class Symbol(Snippet): # If there's only one child, collapse it onto the same line. node = self while len(node.children) == 1 and len(node.children[0].children) == 1: - print >> stream, node.type_name, + print(node.type_name, end=' ', file=stream) node = node.children[0] - print >> stream, node.type_name + print(node.type_name, file=stream) for child in node.children: child.PrintTree(indent + 2, stream) diff --git a/systrace/catapult/common/py_utils/py_utils/refactor_util/move.py b/systrace/catapult/common/py_utils/py_utils/refactor_util/move.py index d68f93b..6d0a7cb 100644 --- a/systrace/catapult/common/py_utils/py_utils/refactor_util/move.py +++ b/systrace/catapult/common/py_utils/py_utils/refactor_util/move.py @@ -2,6 +2,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from __future__ import print_function + import functools import os import sys @@ -35,7 +37,7 @@ def _Update(moves, module): if move.UpdateImportAndReferences(module, import_statement): break except NotImplementedError as e: - print >> sys.stderr, 'Error updating %s: %s' % (module.file_path, e) + print('Error updating %s: %s' % (module.file_path, e), file=sys.stderr) class _Move(object): diff --git a/systrace/catapult/common/py_utils/py_utils/retry_util.py b/systrace/catapult/common/py_utils/py_utils/retry_util.py index e5826ca..a11bd80 100644 --- a/systrace/catapult/common/py_utils/py_utils/retry_util.py +++ b/systrace/catapult/common/py_utils/py_utils/retry_util.py @@ -1,9 +1,13 @@ # Copyright 2018 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import functools import logging import time +from six.moves import range # pylint: disable=redefined-builtin def RetryOnException(exc_type, retries): @@ -42,7 +46,7 @@ def RetryOnException(exc_type, retries): def Wrapper(*args, **kwargs): wait = 1 kwargs.setdefault('retries', retries) - for _ in xrange(kwargs['retries']): + for _ in range(kwargs['retries']): try: return f(*args, **kwargs) except exc_type as exc: diff --git a/systrace/catapult/common/py_utils/py_utils/retry_util_unittest.py b/systrace/catapult/common/py_utils/py_utils/retry_util_unittest.py index 151f88e..f24577f 100644 --- a/systrace/catapult/common/py_utils/py_utils/retry_util_unittest.py +++ b/systrace/catapult/common/py_utils/py_utils/retry_util_unittest.py @@ -1,9 +1,10 @@ # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import mock import unittest +import mock + from py_utils import retry_util diff --git a/systrace/catapult/common/py_utils/py_utils/shell_util.py b/systrace/catapult/common/py_utils/py_utils/shell_util.py index 2a529c8..6af7f8e 100644 --- a/systrace/catapult/common/py_utils/py_utils/shell_util.py +++ b/systrace/catapult/common/py_utils/py_utils/shell_util.py @@ -4,6 +4,8 @@ # # Shell scripting helpers (created for Telemetry dependency roll scripts). +from __future__ import print_function + import os as _os import shutil as _shutil import subprocess as _subprocess @@ -14,12 +16,12 @@ from contextlib import contextmanager as _contextmanager def ScopedChangeDir(new_path): old_path = _os.getcwd() _os.chdir(new_path) - print '> cd', _os.getcwd() + print('> cd', _os.getcwd()) try: yield finally: _os.chdir(old_path) - print '> cd', old_path + print('> cd', old_path) @_contextmanager def ScopedTempDir(): @@ -36,5 +38,5 @@ def CallProgram(path_parts, *args, **kwargs): args = [_os.path.join(*path_parts)] + list(args) env = dict(_os.environ) env.update(kwargs) - print '>', ' '.join(args) + print('>', ' '.join(args)) _subprocess.check_call(args, env=env) diff --git a/systrace/catapult/common/py_utils/py_utils/slots_metaclass_unittest.py b/systrace/catapult/common/py_utils/py_utils/slots_metaclass_unittest.py index 79bb343..fe21b27 100644 --- a/systrace/catapult/common/py_utils/py_utils/slots_metaclass_unittest.py +++ b/systrace/catapult/common/py_utils/py_utils/slots_metaclass_unittest.py @@ -2,15 +2,21 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import unittest from py_utils import slots_metaclass +import six + class SlotsMetaclassUnittest(unittest.TestCase): def testSlotsMetaclass(self): - class NiceClass(object): - __metaclass__ = slots_metaclass.SlotsMetaclass + + class NiceClass(six.with_metaclass(slots_metaclass.SlotsMetaclass, object)): __slots__ = '_nice', def __init__(self, nice): diff --git a/systrace/catapult/common/py_utils/py_utils/tempfile_ext.py b/systrace/catapult/common/py_utils/py_utils/tempfile_ext.py index 394ad5b..ba68c52 100644 --- a/systrace/catapult/common/py_utils/py_utils/tempfile_ext.py +++ b/systrace/catapult/common/py_utils/py_utils/tempfile_ext.py @@ -2,8 +2,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - import contextlib +import os import shutil import tempfile @@ -28,3 +28,32 @@ def NamedTemporaryDirectory(suffix='', prefix='tmp', dir=None): yield d finally: shutil.rmtree(d) + + +@contextlib.contextmanager +def NamedTemporaryFile(mode='w+b', suffix='', prefix='tmp'): + """A conext manager to hold a named temporary file. + + It's similar to Python's tempfile.NamedTemporaryFile except: + - The file is _not_ deleted when you close the temporary file handle, so you + can close it and then use the name of the file to re-open it later. + - The file *is* always deleted when exiting the context managed code. + """ + with NamedTemporaryDirectory() as temp_dir: + yield tempfile.NamedTemporaryFile( + mode=mode, suffix=suffix, prefix=prefix, dir=temp_dir, delete=False) + + +@contextlib.contextmanager +def TemporaryFileName(prefix='tmp', suffix=''): + """A context manager to just get the path to a file that does not exist. + + The parent directory of the file is a newly clreated temporary directory, + and the name of the file is just `prefix + suffix`. The file istelf is not + created, you are in fact guaranteed that it does not exit. + + The entire parent directory, possibly including the named temporary file and + any sibling files, is entirely deleted when exiting the context managed code. + """ + with NamedTemporaryDirectory() as temp_dir: + yield os.path.join(temp_dir, prefix + suffix) diff --git a/systrace/catapult/common/py_utils/py_utils/tempfile_ext_unittest.py b/systrace/catapult/common/py_utils/py_utils/tempfile_ext_unittest.py index 6844623..76a0efd 100644 --- a/systrace/catapult/common/py_utils/py_utils/tempfile_ext_unittest.py +++ b/systrace/catapult/common/py_utils/py_utils/tempfile_ext_unittest.py @@ -2,14 +2,15 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import filecmp import os +import shutil from py_utils import tempfile_ext from pyfakefs import fake_filesystem_unittest class NamedTemporaryDirectoryTest(fake_filesystem_unittest.TestCase): - def setUp(self): self.setUpPyfakefs() @@ -37,3 +38,37 @@ class NamedTemporaryDirectoryTest(fake_filesystem_unittest.TestCase): self.fs.CreateDirectory(test_dir) with tempfile_ext.NamedTemporaryDirectory(dir=test_dir) as d: self.assertEquals(test_dir, os.path.dirname(d)) + + +class TemporaryFilesTest(fake_filesystem_unittest.TestCase): + def setUp(self): + self.setUpPyfakefs() + + def tearDown(self): + self.tearDownPyfakefs() + + def testNamedTemporaryFile(self): + with tempfile_ext.NamedTemporaryFile() as f: + self.assertTrue(os.path.isfile(f.name)) + f.write('<data>') + f.close() + self.assertTrue(os.path.exists(f.name)) + with open(f.name) as f2: + self.assertEqual(f2.read(), '<data>') + + self.assertFalse(os.path.exists(f.name)) + + def testTemporaryFileName(self): + with tempfile_ext.TemporaryFileName('foo') as filepath: + self.assertTrue(os.path.basename(filepath), 'foo') + self.assertFalse(os.path.exists(filepath)) + + with open(filepath, 'w') as f: + f.write('<data>') + self.assertTrue(os.path.exists(filepath)) + + shutil.copyfile(filepath, filepath + '.bak') + self.assertTrue(filecmp.cmp(filepath, filepath + '.bak')) + + self.assertFalse(os.path.exists(filepath)) + self.assertFalse(os.path.exists(os.path.dirname(filepath))) diff --git a/systrace/catapult/common/py_utils/py_utils/xvfb.py b/systrace/catapult/common/py_utils/py_utils/xvfb.py index c09f3e3..06ce7dd 100644 --- a/systrace/catapult/common/py_utils/py_utils/xvfb.py +++ b/systrace/catapult/common/py_utils/py_utils/xvfb.py @@ -10,6 +10,8 @@ import time def ShouldStartXvfb(): + # TODO(crbug.com/973847): Note that you can locally change this to return + # False to diagnose timeouts for dev server tests. return platform.system() == 'Linux' |