diff options
Diffstat (limited to 'catapult/devil/devil/utils')
| -rw-r--r-- | catapult/devil/devil/utils/cmd_helper.py | 58 | ||||
| -rwxr-xr-x | catapult/devil/devil/utils/cmd_helper_test.py | 25 | ||||
| -rw-r--r-- | catapult/devil/devil/utils/decorators.py | 17 | ||||
| -rwxr-xr-x | catapult/devil/devil/utils/decorators_test.py | 75 | ||||
| -rwxr-xr-x | catapult/devil/devil/utils/markdown_test.py | 7 | ||||
| -rw-r--r-- | catapult/devil/devil/utils/mock_calls.py | 2 | ||||
| -rw-r--r-- | catapult/devil/devil/utils/parallelizer_test.py | 33 | ||||
| -rw-r--r-- | catapult/devil/devil/utils/reraiser_thread_unittest.py | 4 | ||||
| -rw-r--r-- | catapult/devil/devil/utils/usb_hubs.py | 2 |
9 files changed, 186 insertions, 37 deletions
diff --git a/catapult/devil/devil/utils/cmd_helper.py b/catapult/devil/devil/utils/cmd_helper.py index 634c9716..3a959450 100644 --- a/catapult/devil/devil/utils/cmd_helper.py +++ b/catapult/devil/devil/utils/cmd_helper.py @@ -10,11 +10,19 @@ import pipes import select import signal import string -import StringIO import subprocess import sys import time +CATAPULT_ROOT_PATH = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..')) +SIX_PATH = os.path.join(CATAPULT_ROOT_PATH, 'third_party', 'six') + +if SIX_PATH not in sys.path: + sys.path.append(SIX_PATH) + +import six + from devil import base_error logger = logging.getLogger(__name__) @@ -23,7 +31,8 @@ _SafeShellChars = frozenset(string.ascii_letters + string.digits + '@%_-+=:,./') # Cache the string-escape codec to ensure subprocess can find it # later. Return value doesn't matter. -codecs.lookup('string-escape') +if six.PY2: + codecs.lookup('string-escape') def SingleQuote(s): @@ -102,6 +111,7 @@ def Popen(args, cwd=None, env=None): # preexec_fn isn't supported on windows. + # pylint: disable=unexpected-keyword-arg if sys.platform == 'win32': close_fds = (stdin is None and stdout is None and stderr is None) preexec_fn = None @@ -109,7 +119,8 @@ def Popen(args, close_fds = True preexec_fn = lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL) - return subprocess.Popen( + if six.PY2: + return subprocess.Popen( args=args, cwd=cwd, stdin=stdin, @@ -118,8 +129,29 @@ def Popen(args, shell=shell, close_fds=close_fds, env=env, - preexec_fn=preexec_fn) - + preexec_fn=preexec_fn + ) + else: + # opens stdout in text mode, so that caller side always get 'str', + # and there will be no type mismatch error. + # Ignore any decoding error, so that caller will not crash due to + # uncaught exception. Decoding errors are unavoidable, as we + # do not know the encoding of the output, and in some output there + # will be multiple encodings (e.g. adb logcat) + return subprocess.Popen( + args=args, + cwd=cwd, + stdin=stdin, + stdout=stdout, + stderr=stderr, + shell=shell, + close_fds=close_fds, + env=env, + preexec_fn=preexec_fn, + universal_newlines=True, + encoding='utf-8', + errors='ignore' + ) def Call(args, stdout=None, stderr=None, shell=None, cwd=None, env=None): pipe = Popen( @@ -165,7 +197,7 @@ def GetCmdOutput(args, cwd=None, shell=False, env=None): def _ValidateAndLogCommand(args, cwd, shell): - if isinstance(args, basestring): + if isinstance(args, six.string_types): if not shell: raise Exception('string args must be run with shell=True') else: @@ -283,6 +315,12 @@ class TimeoutError(base_error.BaseError): return self._output +def _read_and_decode(fd, buffer_size): + data = os.read(fd, buffer_size) + if data and six.PY3: + data = data.decode('utf-8', errors='ignore') + return data + def _IterProcessStdoutFcntl(process, iter_timeout=None, timeout=None, @@ -316,7 +354,7 @@ def _IterProcessStdoutFcntl(process, read_fds, _, _ = select.select([child_fd], [], [], iter_aware_poll_interval) if child_fd in read_fds: - data = os.read(child_fd, buffer_size) + data = _read_and_decode(child_fd, buffer_size) if not data: break yield data @@ -328,7 +366,7 @@ def _IterProcessStdoutFcntl(process, read_fds, _, _ = select.select([child_fd], [], [], iter_aware_poll_interval) if child_fd in read_fds: - data = os.read(child_fd, buffer_size) + data = _read_and_decode(child_fd, buffer_size) if data: yield data continue @@ -365,7 +403,7 @@ def _IterProcessStdoutQueue(process, # TODO(jbudorick): Pick an appropriate read size here. while True: try: - output_chunk = os.read(process.stdout.fileno(), buffer_size) + output_chunk = _read_and_decode(process.stdout.fileno(), buffer_size) except IOError: break stdout_queue.put(output_chunk, True) @@ -452,7 +490,7 @@ def GetCmdStatusAndOutputWithTimeout(args, TimeoutError on timeout. """ _ValidateAndLogCommand(args, cwd, shell) - output = StringIO.StringIO() + output = six.StringIO() process = Popen( args, cwd=cwd, diff --git a/catapult/devil/devil/utils/cmd_helper_test.py b/catapult/devil/devil/utils/cmd_helper_test.py index 57abceb4..0eeefe16 100755 --- a/catapult/devil/devil/utils/cmd_helper_test.py +++ b/catapult/devil/devil/utils/cmd_helper_test.py @@ -33,6 +33,17 @@ class CmdHelperSingleQuoteTest(unittest.TestCase): self.assertEquals(test_string, cmd_helper.GetCmdOutput(cmd, shell=True).rstrip()) +class CmdHelperGetCmdStatusAndOutputTest(unittest.TestCase): + def testGetCmdStatusAndOutput_success(self): + cmd = 'echo "Hello World"' + status, output = cmd_helper.GetCmdStatusAndOutput(cmd, shell=True) + self.assertEqual(status, 0) + self.assertEqual(output.rstrip(), "Hello World") + + def testGetCmdStatusAndOutput_unicode(self): + # pylint: disable=no-self-use + cmd = 'echo "\x80\x31Hello World\n"' + cmd_helper.GetCmdStatusAndOutput(cmd, shell=True) class CmdHelperDoubleQuoteTest(unittest.TestCase): def testDoubleQuote_basic(self): @@ -195,7 +206,7 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): # pylint: disable=protected-access _SIMPLE_OUTPUT_SEQUENCE = [ - _ProcessOutputEvent(read_contents='1\n2\n'), + _ProcessOutputEvent(read_contents=b'1\n2\n'), ] def testIterCmdOutputLines_success(self): @@ -205,6 +216,14 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): cmd_helper._IterCmdOutputLines(mock_proc, 'mock_proc'), 1): self.assertEquals(num, int(line)) + def testIterCmdOutputLines_unicode(self): + output_sequence = [ + _ProcessOutputEvent(read_contents=b'\x80\x31\nHello\n\xE2\x98\xA0') + ] + with _MockProcess(output_sequence=output_sequence) as mock_proc: + lines = list(cmd_helper._IterCmdOutputLines(mock_proc, 'mock_proc')) + self.assertEquals(lines[1], "Hello") + def testIterCmdOutputLines_exitStatusFail(self): with self.assertRaises(subprocess.CalledProcessError): with _MockProcess( @@ -238,9 +257,9 @@ class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): def testIterCmdOutputLines_delay(self): output_sequence = [ - _ProcessOutputEvent(read_contents='1\n2\n', ts=1), + _ProcessOutputEvent(read_contents=b'1\n2\n', ts=1), _ProcessOutputEvent(read_contents=None, ts=2), - _ProcessOutputEvent(read_contents='Awake', ts=10), + _ProcessOutputEvent(read_contents=b'Awake', ts=10), ] with _MockProcess(output_sequence=output_sequence) as mock_proc: for num, line in enumerate( diff --git a/catapult/devil/devil/utils/decorators.py b/catapult/devil/devil/utils/decorators.py new file mode 100644 index 00000000..5d286107 --- /dev/null +++ b/catapult/devil/devil/utils/decorators.py @@ -0,0 +1,17 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import functools + + +def Memoize(f): + """Decorator to cache return values of function.""" + memoize_dict = {} + @functools.wraps(f) + def wrapper(*args, **kwargs): + key = repr((args, kwargs)) + if key not in memoize_dict: + memoize_dict[key] = f(*args, **kwargs) + return memoize_dict[key] + return wrapper diff --git a/catapult/devil/devil/utils/decorators_test.py b/catapult/devil/devil/utils/decorators_test.py new file mode 100755 index 00000000..f81974ac --- /dev/null +++ b/catapult/devil/devil/utils/decorators_test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unit tests for decorators.py.""" + +import unittest + +from devil.utils import decorators + + +class MemoizeDecoratorTest(unittest.TestCase): + + def testFunctionExceptionNotMemoized(self): + """Tests that |Memoize| decorator does not cache exception results.""" + + class ExceptionType1(Exception): + pass + + class ExceptionType2(Exception): + pass + + @decorators.Memoize + def raiseExceptions(): + if raiseExceptions.count == 0: + raiseExceptions.count += 1 + raise ExceptionType1() + + if raiseExceptions.count == 1: + raise ExceptionType2() + raiseExceptions.count = 0 + + with self.assertRaises(ExceptionType1): + raiseExceptions() + with self.assertRaises(ExceptionType2): + raiseExceptions() + + def testFunctionResultMemoized(self): + """Tests that |Memoize| decorator caches results.""" + + @decorators.Memoize + def memoized(): + memoized.count += 1 + return memoized.count + memoized.count = 0 + + def notMemoized(): + notMemoized.count += 1 + return notMemoized.count + notMemoized.count = 0 + + self.assertEquals(memoized(), 1) + self.assertEquals(memoized(), 1) + self.assertEquals(memoized(), 1) + + self.assertEquals(notMemoized(), 1) + self.assertEquals(notMemoized(), 2) + self.assertEquals(notMemoized(), 3) + + def testFunctionMemoizedBasedOnArgs(self): + """Tests that |Memoize| caches results based on args and kwargs.""" + + @decorators.Memoize + def returnValueBasedOnArgsKwargs(a, k=0): + return a + k + + self.assertEquals(returnValueBasedOnArgsKwargs(1, 1), 2) + self.assertEquals(returnValueBasedOnArgsKwargs(1, 2), 3) + self.assertEquals(returnValueBasedOnArgsKwargs(2, 1), 3) + self.assertEquals(returnValueBasedOnArgsKwargs(3, 3), 6) + + +if __name__ == '__main__': + unittest.main() diff --git a/catapult/devil/devil/utils/markdown_test.py b/catapult/devil/devil/utils/markdown_test.py index 621d56ba..11cd46ea 100755 --- a/catapult/devil/devil/utils/markdown_test.py +++ b/catapult/devil/devil/utils/markdown_test.py @@ -103,10 +103,9 @@ class MarkdownTest(unittest.TestCase): def testLink(self): link_text = 'Devil home' link_target = ( - 'https://github.com/catapult-project/catapult/tree/master/devil') - expected = ( - '[Devil home]' - '(https://github.com/catapult-project/catapult/tree/master/devil)') + 'https://chromium.googlesource.com/catapult.git/+/HEAD/devil') + expected = ('[Devil home]' + '(https://chromium.googlesource.com/catapult.git/+/HEAD/devil)') self.assertEquals(expected, markdown.md_link(link_text, link_target)) def testLinkTextContainsBracket(self): diff --git a/catapult/devil/devil/utils/mock_calls.py b/catapult/devil/devil/utils/mock_calls.py index 2b359385..ba96658a 100644 --- a/catapult/devil/devil/utils/mock_calls.py +++ b/catapult/devil/devil/utils/mock_calls.py @@ -51,7 +51,7 @@ class TestCase(unittest.TestCase): (call.parent.name, call.parent) for call, _ in self._expected_calls) self._patched = [ test_case.patch_call(call, side_effect=do_check(call)) - for call in watched.itervalues() + for call in watched.values() ] def __enter__(self): diff --git a/catapult/devil/devil/utils/parallelizer_test.py b/catapult/devil/devil/utils/parallelizer_test.py index 7c86148c..2d8f72aa 100644 --- a/catapult/devil/devil/utils/parallelizer_test.py +++ b/catapult/devil/devil/utils/parallelizer_test.py @@ -19,6 +19,7 @@ if __name__ == '__main__': os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))) from devil.utils import parallelizer +from devil.base_error import BaseError class ParallelizerTestObject(object): @@ -58,7 +59,7 @@ class ParallelizerTestObject(object): def _write_completion_file(self): if self._completion_file_name and len(self._completion_file_name): with open(self._completion_file_name, 'w+b') as completion_file: - completion_file.write('complete') + completion_file.write(b'complete') def __getitem__(self, index): return self._thing[index] @@ -87,13 +88,13 @@ class ParallelizerTest(unittest.TestCase): self.assertEquals(expected, r) def testMutate(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] self.assertTrue(all(d.doReturnTheThing() for d in devices)) ParallelizerTestObject.parallel(devices).doSetTheThing(False).pFinish(1) self.assertTrue(not any(d.doReturnTheThing() for d in devices)) def testAllReturn(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] results = ParallelizerTestObject.parallel(devices).doReturnTheThing().pGet( 1) self.assertTrue(isinstance(results, list)) @@ -103,7 +104,7 @@ class ParallelizerTest(unittest.TestCase): def testAllRaise(self): devices = [ ParallelizerTestObject(Exception('thing %d' % i)) - for i in xrange(0, 10) + for i in range(0, 10) ] p = ParallelizerTestObject.parallel(devices).doRaiseTheThing() with self.assertRaises(Exception): @@ -117,21 +118,21 @@ class ParallelizerTest(unittest.TestCase): try: completion_files = [ tempfile.NamedTemporaryFile(delete=False) - for _ in xrange(0, parallel_device_count) + for _ in range(0, parallel_device_count) ] devices = [ ParallelizerTestObject( - i if i != exception_index else Exception(exception_msg), + i if i != exception_index else BaseError(exception_msg), completion_files[i].name) - for i in xrange(0, parallel_device_count) + for i in range(0, parallel_device_count) ] for f in completion_files: f.close() p = ParallelizerTestObject.parallel(devices) - with self.assertRaises(Exception) as e: + with self.assertRaises(BaseError) as e: p.doRaiseIfExceptionElseSleepFor(2).pGet(3) self.assertTrue(exception_msg in str(e.exception)) - for i in xrange(0, parallel_device_count): + for i in range(0, parallel_device_count): with open(completion_files[i].name) as f: if i == exception_index: self.assertEquals('', f.read()) @@ -142,7 +143,7 @@ class ParallelizerTest(unittest.TestCase): os.remove(f.name) def testReusable(self): - devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + devices = [ParallelizerTestObject(True) for _ in range(0, 10)] p = ParallelizerTestObject.parallel(devices) results = p.doReturn(True).pGet(1) self.assertTrue(all(results)) @@ -152,23 +153,23 @@ class ParallelizerTest(unittest.TestCase): results = p.doRaise(Exception('reusableTest')).pGet(1) def testContained(self): - devices = [ParallelizerTestObject(i) for i in xrange(0, 10)] + devices = [ParallelizerTestObject(i) for i in range(0, 10)] results = (ParallelizerTestObject.parallel(devices).helper. doReturnStringThing().pGet(1)) self.assertTrue(isinstance(results, list)) self.assertEquals(10, len(results)) - for i in xrange(0, 10): + for i in range(0, 10): self.assertEquals(str(i), results[i]) def testGetItem(self): - devices = [ParallelizerTestObject(range(i, i + 10)) for i in xrange(0, 10)] + devices = [ParallelizerTestObject(range(i, i + 10)) for i in range(0, 10)] results = ParallelizerTestObject.parallel(devices)[9].pGet(1) - self.assertEquals(range(9, 19), results) + self.assertEquals(list(range(9, 19)), results) class SyncParallelizerTest(unittest.TestCase): def testContextManager(self): - in_context = [False for i in xrange(10)] + in_context = [False for i in range(10)] @contextlib.contextmanager def enter_into_context(i): @@ -179,7 +180,7 @@ class SyncParallelizerTest(unittest.TestCase): in_context[i] = False parallelized_context = parallelizer.SyncParallelizer( - [enter_into_context(i) for i in xrange(10)]) + [enter_into_context(i) for i in range(10)]) with parallelized_context: self.assertTrue(all(in_context)) diff --git a/catapult/devil/devil/utils/reraiser_thread_unittest.py b/catapult/devil/devil/utils/reraiser_thread_unittest.py index eb37456c..f1fabd0f 100644 --- a/catapult/devil/devil/utils/reraiser_thread_unittest.py +++ b/catapult/devil/devil/utils/reraiser_thread_unittest.py @@ -64,7 +64,7 @@ class TestReraiserThreadGroup(unittest.TestCase): ran[i] = True group = reraiser_thread.ReraiserThreadGroup() - for i in xrange(5): + for i in range(5): group.Add(reraiser_thread.ReraiserThread(f, args=[i])) group.StartAll() group.JoinAll() @@ -76,7 +76,7 @@ class TestReraiserThreadGroup(unittest.TestCase): raise TestException group = reraiser_thread.ReraiserThreadGroup( - [reraiser_thread.ReraiserThread(f) for _ in xrange(5)]) + [reraiser_thread.ReraiserThread(f) for _ in range(5)]) group.StartAll() with self.assertRaises(TestException): group.JoinAll() diff --git a/catapult/devil/devil/utils/usb_hubs.py b/catapult/devil/devil/utils/usb_hubs.py index 313cf3f6..b83fb610 100644 --- a/catapult/devil/devil/utils/usb_hubs.py +++ b/catapult/devil/devil/utils/usb_hubs.py @@ -73,7 +73,7 @@ class HubType(object): A series of (int, USBNode) tuples giving a physical port and the Node connected to it. """ - for (virtual, physical) in mapping.iteritems(): + for (virtual, physical) in mapping.items(): if node.HasPort(virtual): if isinstance(physical, dict): for res in self._GppHelper(node.PortToDevice(virtual), physical): |