diff options
author | Chris Craik <ccraik@google.com> | 2016-03-28 13:54:49 -0700 |
---|---|---|
committer | Chris Craik <ccraik@google.com> | 2016-03-28 13:57:49 -0700 |
commit | cef7893435aa41160dd1255c43cb8498279738cc (patch) | |
tree | faf334a3dad13ef724722ff7b1d2c673257bbafd /catapult/devil | |
parent | 11c2fbcfb91714e0aac1b2e53db3a4490c89a091 (diff) | |
download | chromium-trace-cef7893435aa41160dd1255c43cb8498279738cc.tar.gz |
Update to latest catapult (e5abb7bd)android-7.1.0_r7android-7.1.0_r6android-7.1.0_r5android-7.1.0_r4android-7.1.0_r3android-7.1.0_r2android-7.1.0_r1nougat-dr1-releasenougat-dev
Change-Id: Ic610f2da8ecd564d5dd58cbc8b9a738ee74b2a06
Diffstat (limited to 'catapult/devil')
96 files changed, 16762 insertions, 0 deletions
diff --git a/catapult/devil/PRESUBMIT.py b/catapult/devil/PRESUBMIT.py new file mode 100644 index 00000000..edec3e16 --- /dev/null +++ b/catapult/devil/PRESUBMIT.py @@ -0,0 +1,85 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Presubmit script for devil. + +See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts for +details on the presubmit API built into depot_tools. +""" + + +def _RunPylint(input_api, output_api): + return input_api.RunTests(input_api.canned_checks.RunPylint( + input_api, output_api, pylintrc='pylintrc')) + + +def _RunUnitTests(input_api, output_api): + def J(*dirs): + """Returns a path relative to presubmit directory.""" + return input_api.os_path.join( + input_api.PresubmitLocalPath(), 'devil', *dirs) + + test_env = dict(input_api.environ) + test_env.update({ + 'PYTHONDONTWRITEBYTECODE': '1', + 'PYTHONPATH': ':'.join([J(), J('..')]), + }) + + return input_api.canned_checks.RunUnitTests( + input_api, + output_api, + unit_tests=[ + J('devil_env_test.py'), + J('android', 'battery_utils_test.py'), + J('android', 'device_utils_test.py'), + J('android', 'fastboot_utils_test.py'), + J('android', 'md5sum_test.py'), + J('android', 'logcat_monitor_test.py'), + J('android', 'tools', 'script_common_test.py'), + J('utils', 'cmd_helper_test.py'), + J('utils', 'timeout_retry_unittest.py'), + ], + env=test_env) + + +def _EnsureNoPylibUse(input_api, output_api): + def other_python_files(f): + this_presubmit_file = input_api.os_path.join( + input_api.PresubmitLocalPath(), 'PRESUBMIT.py') + return (f.LocalPath().endswith('.py') + and not f.AbsoluteLocalPath() == this_presubmit_file) + + changed_files = input_api.AffectedSourceFiles(other_python_files) + import_error_re = input_api.re.compile( + r'(from pylib.* import)|(import pylib)') + + errors = [] + for f in changed_files: + errors.extend( + '%s:%d' % (f.LocalPath(), line_number) + for line_number, line_text in f.ChangedContents() + if import_error_re.search(line_text)) + + if errors: + return [output_api.PresubmitError( + 'pylib modules should not be imported from devil modules.', + items=errors)] + return [] + + +def CommonChecks(input_api, output_api): + output = [] + output += _RunPylint(input_api, output_api) + output += _RunUnitTests(input_api, output_api) + output += _EnsureNoPylibUse(input_api, output_api) + return output + + +def CheckChangeOnUpload(input_api, output_api): + return CommonChecks(input_api, output_api) + + +def CheckChangeOnCommit(input_api, output_api): + return CommonChecks(input_api, output_api) + diff --git a/catapult/devil/bin/run_py_tests b/catapult/devil/bin/run_py_tests new file mode 100755 index 00000000..44ec61e8 --- /dev/null +++ b/catapult/devil/bin/run_py_tests @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import sys + +_CATAPULT_PATH = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', '..')) +_DEVIL_PATH = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..')) + +sys.path.append(_CATAPULT_PATH) +from catapult_build import run_with_typ + + +def main(): + return run_with_typ.Run(top_level_dir=_DEVIL_PATH) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/catapult/devil/devil/OWNERS b/catapult/devil/devil/OWNERS new file mode 100644 index 00000000..fd584fcd --- /dev/null +++ b/catapult/devil/devil/OWNERS @@ -0,0 +1,4 @@ +jbudorick@chromium.org +mikecase@chromium.org +perezju@chromium.org +rnephew@chromium.org diff --git a/catapult/devil/devil/README.md b/catapult/devil/devil/README.md new file mode 100644 index 00000000..b3eb5d0f --- /dev/null +++ b/catapult/devil/devil/README.md @@ -0,0 +1,17 @@ +<!-- Copyright 2015 The Chromium Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. +--> +devil +===== + +devil is a library used by the Chromium developers to interact with Android +devices. It currently supports SDK level 16 and above. + +😈 + +Contributing +============ + +Please see the [contributor's guide](https://github.com/catapult-project/catapult/blob/master/CONTRIBUTING.md). + diff --git a/catapult/devil/devil/__init__.py b/catapult/devil/devil/__init__.py new file mode 100644 index 00000000..50b23dff --- /dev/null +++ b/catapult/devil/devil/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/catapult/devil/devil/android/__init__.py b/catapult/devil/devil/android/__init__.py new file mode 100644 index 00000000..50b23dff --- /dev/null +++ b/catapult/devil/devil/android/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/catapult/devil/devil/android/apk_helper.py b/catapult/devil/devil/android/apk_helper.py new file mode 100644 index 00000000..61eeda06 --- /dev/null +++ b/catapult/devil/devil/android/apk_helper.py @@ -0,0 +1,142 @@ +# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Module containing utilities for apk packages.""" + +import re + +from devil.android.sdk import aapt + + +_MANIFEST_ATTRIBUTE_RE = re.compile( + r'\s*A: ([^\(\)= ]*)(?:\([^\(\)= ]*\))?=' + r'(?:"(.*)" \(Raw: .*\)|\(type.*?\)(.*))$') +_MANIFEST_ELEMENT_RE = re.compile(r'\s*(?:E|N): (\S*) .*$') + + +def GetPackageName(apk_path): + """Returns the package name of the apk.""" + return ApkHelper(apk_path).GetPackageName() + + +# TODO(jbudorick): Deprecate and remove this function once callers have been +# converted to ApkHelper.GetInstrumentationName +def GetInstrumentationName(apk_path): + """Returns the name of the Instrumentation in the apk.""" + return ApkHelper(apk_path).GetInstrumentationName() + + +def ToHelper(path_or_helper): + """Creates an ApkHelper unless one is already given.""" + if isinstance(path_or_helper, basestring): + return ApkHelper(path_or_helper) + return path_or_helper + + +def _ParseManifestFromApk(apk_path): + aapt_output = aapt.Dump('xmltree', apk_path, 'AndroidManifest.xml') + + parsed_manifest = {} + node_stack = [parsed_manifest] + indent = ' ' + + for line in aapt_output[1:]: + if len(line) == 0: + continue + + indent_depth = 0 + while line[(len(indent) * indent_depth):].startswith(indent): + indent_depth += 1 + + node_stack = node_stack[:indent_depth] + node = node_stack[-1] + + m = _MANIFEST_ELEMENT_RE.match(line[len(indent) * indent_depth:]) + if m: + if not m.group(1) in node: + node[m.group(1)] = {} + node_stack += [node[m.group(1)]] + continue + + m = _MANIFEST_ATTRIBUTE_RE.match(line[len(indent) * indent_depth:]) + if m: + if not m.group(1) in node: + node[m.group(1)] = [] + node[m.group(1)].append(m.group(2) or m.group(3)) + continue + + return parsed_manifest + + +class ApkHelper(object): + + def __init__(self, path): + self._apk_path = path + self._manifest = None + + @property + def path(self): + return self._apk_path + + def GetActivityName(self): + """Returns the name of the Activity in the apk.""" + manifest_info = self._GetManifest() + try: + activity = ( + manifest_info['manifest']['application']['activity'] + ['android:name'][0]) + except KeyError: + return None + if '.' not in activity: + activity = '%s.%s' % (self.GetPackageName(), activity) + elif activity.startswith('.'): + activity = '%s%s' % (self.GetPackageName(), activity) + return activity + + def GetInstrumentationName( + self, default='android.test.InstrumentationTestRunner'): + """Returns the name of the Instrumentation in the apk.""" + manifest_info = self._GetManifest() + try: + return manifest_info['manifest']['instrumentation']['android:name'][0] + except KeyError: + return default + + def GetPackageName(self): + """Returns the package name of the apk.""" + manifest_info = self._GetManifest() + try: + return manifest_info['manifest']['package'][0] + except KeyError: + raise Exception('Failed to determine package name of %s' % self._apk_path) + + def GetPermissions(self): + manifest_info = self._GetManifest() + try: + return manifest_info['manifest']['uses-permission']['android:name'] + except KeyError: + return [] + + def GetSplitName(self): + """Returns the name of the split of the apk.""" + manifest_info = self._GetManifest() + try: + return manifest_info['manifest']['split'][0] + except KeyError: + return None + + def HasIsolatedProcesses(self): + """Returns whether any services exist that use isolatedProcess=true.""" + manifest_info = self._GetManifest() + try: + services = manifest_info['manifest']['application']['service'] + return any(int(v, 0) for v in services['android:isolatedProcess']) + except KeyError: + return False + + def _GetManifest(self): + if not self._manifest: + self._manifest = _ParseManifestFromApk(self._apk_path) + return self._manifest + diff --git a/catapult/devil/devil/android/app_ui.py b/catapult/devil/devil/android/app_ui.py new file mode 100644 index 00000000..d5025f40 --- /dev/null +++ b/catapult/devil/devil/android/app_ui.py @@ -0,0 +1,213 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Provides functionality to interact with UI elements of an Android app.""" + +import re +from xml.etree import ElementTree as element_tree + +from devil.android import decorators +from devil.android import device_temp_file +from devil.utils import geometry +from devil.utils import timeout_retry + +_DEFAULT_SHORT_TIMEOUT = 10 +_DEFAULT_SHORT_RETRIES = 3 +_DEFAULT_LONG_TIMEOUT = 30 +_DEFAULT_LONG_RETRIES = 0 + +# Parse rectangle bounds given as: '[left,top][right,bottom]'. +_RE_BOUNDS = re.compile( + r'\[(?P<left>\d+),(?P<top>\d+)\]\[(?P<right>\d+),(?P<bottom>\d+)\]') + + +class _UiNode(object): + + def __init__(self, device, xml_node, package=None): + """Object to interact with a UI node from an xml snapshot. + + Note: there is usually no need to call this constructor directly. Instead, + use an AppUi object (below) to grab an xml screenshot from a device and + find nodes in it. + + Args: + device: A device_utils.DeviceUtils instance. + xml_node: An ElementTree instance of the node to interact with. + package: An optional package name for the app owning this node. + """ + self._device = device + self._xml_node = xml_node + self._package = package + + def _GetAttribute(self, key): + """Get the value of an attribute of this node.""" + return self._xml_node.attrib.get(key) + + @property + def bounds(self): + """Get a rectangle with the bounds of this UI node. + + Returns: + A geometry.Rectangle instance. + """ + d = _RE_BOUNDS.match(self._GetAttribute('bounds')).groupdict() + return geometry.Rectangle.FromDict({k: int(v) for k, v in d.iteritems()}) + + def Tap(self, point=None, dp_units=False): + """Send a tap event to the UI node. + + Args: + point: An optional geometry.Point instance indicating the location to + tap, relative to the bounds of the UI node, i.e. (0, 0) taps the + top-left corner. If ommited, the center of the node is tapped. + dp_units: If True, indicates that the coordinates of the point are given + in device-independent pixels; otherwise they are assumed to be "real" + pixels. This option has no effect when the point is ommited. + """ + if point is None: + point = self.bounds.center + else: + if dp_units: + point = (float(self._device.pixel_density) / 160) * point + point += self.bounds.top_left + + x, y = (str(int(v)) for v in point) + self._device.RunShellCommand(['input', 'tap', x, y], check_return=True) + + def __getitem__(self, key): + """Retrieve a child of this node by its index. + + Args: + key: An integer with the index of the child to retrieve. + Returns: + A UI node instance of the selected child. + Raises: + IndexError if the index is out of range. + """ + return type(self)(self._device, self._xml_node[key], package=self._package) + + def _Find(self, **kwargs): + """Find the first descendant node that matches a given criteria. + + Note: clients would usually call AppUi.GetUiNode or AppUi.WaitForUiNode + instead. + + For example: + + app = app_ui.AppUi(device, package='org.my.app') + app.GetUiNode(resource_id='some_element', text='hello') + + would retrieve the first matching node with both of the xml attributes: + + resource-id='org.my.app:id/some_element' + text='hello' + + As the example shows, if given and needed, the value of the resource_id key + is auto-completed with the package name specified in the AppUi constructor. + + Args: + Arguments are specified as key-value pairs, where keys correnspond to + attribute names in xml nodes (replacing any '-' with '_' to make them + valid identifiers). At least one argument must be supplied, and arguments + with a None value are ignored. + Returns: + A UI node instance of the first descendant node that matches ALL the + given key-value criteria; or None if no such node is found. + Raises: + TypeError if no search arguments are provided. + """ + matches_criteria = self._NodeMatcher(kwargs) + for node in self._xml_node.iter(): + if matches_criteria(node): + return type(self)(self._device, node, package=self._package) + return None + + def _NodeMatcher(self, kwargs): + # Auto-complete resource-id's using the package name if available. + resource_id = kwargs.get('resource_id') + if (resource_id is not None + and self._package is not None + and ':id/' not in resource_id): + kwargs['resource_id'] = '%s:id/%s' % (self._package, resource_id) + + criteria = [(k.replace('_', '-'), v) + for k, v in kwargs.iteritems() + if v is not None] + if not criteria: + raise TypeError('At least one search criteria should be specified') + return lambda node: all(node.get(k) == v for k, v in criteria) + + +class AppUi(object): + # timeout and retry arguments appear unused, but are handled by decorator. + # pylint: disable=unused-argument + + def __init__(self, device, package=None): + """Object to interact with the UI of an Android app. + + Args: + device: A device_utils.DeviceUtils instance. + package: An optional package name for the app. + """ + self._device = device + self._package = package + + @property + def package(self): + return self._package + + @decorators.WithTimeoutAndRetriesDefaults(_DEFAULT_SHORT_TIMEOUT, + _DEFAULT_SHORT_RETRIES) + def _GetRootUiNode(self, timeout=None, retries=None): + """Get a node pointing to the root of the UI nodes on screen. + + Note: This is currently implemented via adb calls to uiatomator and it + is *slow*, ~2 secs per call. Do not rely on low-level implementation + details that may change in the future. + + TODO(crbug.com/567217): Swap to a more efficient implementation. + + Args: + timeout: A number of seconds to wait for the uiautomator dump. + retries: Number of times to retry if the adb command fails. + Returns: + A UI node instance pointing to the root of the xml screenshot. + """ + with device_temp_file.DeviceTempFile(self._device.adb) as dtemp: + self._device.RunShellCommand(['uiautomator', 'dump', dtemp.name], + check_return=True) + xml_node = element_tree.fromstring( + self._device.ReadFile(dtemp.name, force_pull=True)) + return _UiNode(self._device, xml_node, package=self._package) + + def GetUiNode(self, **kwargs): + """Get the first node found matching a specified criteria. + + Args: + See _UiNode._Find. + Returns: + A UI node instance of the node if found, otherwise None. + """ + # pylint: disable=protected-access + return self._GetRootUiNode()._Find(**kwargs) + + @decorators.WithTimeoutAndRetriesDefaults(_DEFAULT_LONG_TIMEOUT, + _DEFAULT_LONG_RETRIES) + def WaitForUiNode(self, timeout=None, retries=None, **kwargs): + """Wait for a node matching a given criteria to appear on the screen. + + Args: + timeout: A number of seconds to wait for the matching node to appear. + retries: Number of times to retry in case of adb command errors. + For other args, to specify the search criteria, see _UiNode._Find. + Returns: + The UI node instance found. + Raises: + device_errors.CommandTimeoutError if the node is not found before the + timeout. + """ + def node_found(): + return self.GetUiNode(**kwargs) + + return timeout_retry.WaitFor(node_found) diff --git a/catapult/devil/devil/android/app_ui_test.py b/catapult/devil/devil/android/app_ui_test.py new file mode 100644 index 00000000..34729851 --- /dev/null +++ b/catapult/devil/devil/android/app_ui_test.py @@ -0,0 +1,191 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unit tests for the app_ui module.""" + +import unittest +from xml.etree import ElementTree as element_tree + +from devil import devil_env +from devil.android import app_ui +from devil.android import device_errors +from devil.utils import geometry + +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + + +MOCK_XML_LOADING = ''' +<?xml version='1.0' encoding='UTF-8' standalone='yes' ?> +<hierarchy rotation="0"> + <node bounds="[0,50][1536,178]" content-desc="Loading" + resource-id="com.example.app:id/spinner"/> +</hierarchy> +'''.strip() + + +MOCK_XML_LOADED = ''' +<?xml version='1.0' encoding='UTF-8' standalone='yes' ?> +<hierarchy rotation="0"> + <node bounds="[0,50][1536,178]" content-desc="" + resource-id="com.example.app:id/toolbar"> + <node bounds="[0,58][112,170]" content-desc="Open navigation drawer"/> + <node bounds="[121,50][1536,178]" + resource-id="com.example.app:id/actionbar_custom_view"> + <node bounds="[121,50][1424,178]" + resource-id="com.example.app:id/actionbar_title" text="Primary"/> + <node bounds="[1424,50][1536,178]" content-desc="Search" + resource-id="com.example.app:id/actionbar_search_button"/> + </node> + </node> + <node bounds="[0,178][576,1952]" resource-id="com.example.app:id/drawer"> + <node bounds="[0,178][144,1952]" + resource-id="com.example.app:id/mini_drawer"> + <node bounds="[40,254][104,318]" resource-id="com.example.app:id/avatar"/> + <node bounds="[16,354][128,466]" content-desc="Primary" + resource-id="com.example.app:id/image_view"/> + <node bounds="[16,466][128,578]" content-desc="Social" + resource-id="com.example.app:id/image_view"/> + <node bounds="[16,578][128,690]" content-desc="Promotions" + resource-id="com.example.app:id/image_view"/> + </node> + </node> +</hierarchy> +'''.strip() + + +class UiAppTest(unittest.TestCase): + + def setUp(self): + self.device = mock.Mock() + self.device.pixel_density = 320 # Each dp pixel is 2 real pixels. + self.app = app_ui.AppUi(self.device, package='com.example.app') + self._setMockXmlScreenshots([MOCK_XML_LOADED]) + + def _setMockXmlScreenshots(self, xml_docs): + """Mock self.app._GetRootUiNode to load nodes from some test xml_docs. + + Each time the method is called it will return a UI node for each string + given in |xml_docs|, or rise a time out error when the list is exhausted. + """ + # pylint: disable=protected-access + def get_mock_root_ui_node(value): + if isinstance(value, Exception): + raise value + return app_ui._UiNode( + self.device, element_tree.fromstring(value), self.app.package) + + xml_docs.append(device_errors.CommandTimeoutError('Timed out!')) + + self.app._GetRootUiNode = mock.Mock( + side_effect=(get_mock_root_ui_node(doc) for doc in xml_docs)) + + def assertNodeHasAttribs(self, node, attr): + # pylint: disable=protected-access + for key, value in attr.iteritems(): + self.assertEquals(node._GetAttribute(key), value) + + def assertTappedOnceAt(self, x, y): + self.device.RunShellCommand.assert_called_once_with( + ['input', 'tap', str(x), str(y)], check_return=True) + + def testFind_byText(self): + node = self.app.GetUiNode(text='Primary') + self.assertNodeHasAttribs(node, { + 'text': 'Primary', + 'content-desc': None, + 'resource-id': 'com.example.app:id/actionbar_title', + }) + self.assertEquals(node.bounds, geometry.Rectangle([121, 50], [1424, 178])) + + def testFind_byContentDesc(self): + node = self.app.GetUiNode(content_desc='Social') + self.assertNodeHasAttribs(node, { + 'text': None, + 'content-desc': 'Social', + 'resource-id': 'com.example.app:id/image_view', + }) + self.assertEquals(node.bounds, geometry.Rectangle([16, 466], [128, 578])) + + def testFind_byResourceId_autocompleted(self): + node = self.app.GetUiNode(resource_id='image_view') + self.assertNodeHasAttribs(node, { + 'content-desc': 'Primary', + 'resource-id': 'com.example.app:id/image_view', + }) + + def testFind_byResourceId_absolute(self): + node = self.app.GetUiNode(resource_id='com.example.app:id/image_view') + self.assertNodeHasAttribs(node, { + 'content-desc': 'Primary', + 'resource-id': 'com.example.app:id/image_view', + }) + + def testFind_byMultiple(self): + node = self.app.GetUiNode(resource_id='image_view', + content_desc='Promotions') + self.assertNodeHasAttribs(node, { + 'content-desc': 'Promotions', + 'resource-id': 'com.example.app:id/image_view', + }) + self.assertEquals(node.bounds, geometry.Rectangle([16, 578], [128, 690])) + + def testFind_notFound(self): + node = self.app.GetUiNode(resource_id='does_not_exist') + self.assertIsNone(node) + + def testFind_noArgsGiven(self): + # Same exception given by Python for a function call with not enough args. + with self.assertRaises(TypeError): + self.app.GetUiNode() + + def testGetChildren(self): + node = self.app.GetUiNode(resource_id='mini_drawer') + self.assertNodeHasAttribs( + node[0], {'resource-id': 'com.example.app:id/avatar'}) + self.assertNodeHasAttribs(node[1], {'content-desc': 'Primary'}) + self.assertNodeHasAttribs(node[2], {'content-desc': 'Social'}) + self.assertNodeHasAttribs(node[3], {'content-desc': 'Promotions'}) + with self.assertRaises(IndexError): + # pylint: disable=pointless-statement + node[4] + + def testTap_center(self): + node = self.app.GetUiNode(content_desc='Open navigation drawer') + node.Tap() + self.assertTappedOnceAt(56, 114) + + def testTap_topleft(self): + node = self.app.GetUiNode(content_desc='Open navigation drawer') + node.Tap(geometry.Point(0, 0)) + self.assertTappedOnceAt(0, 58) + + def testTap_withOffset(self): + node = self.app.GetUiNode(content_desc='Open navigation drawer') + node.Tap(geometry.Point(10, 20)) + self.assertTappedOnceAt(10, 78) + + def testTap_withOffsetInDp(self): + node = self.app.GetUiNode(content_desc='Open navigation drawer') + node.Tap(geometry.Point(10, 20), dp_units=True) + self.assertTappedOnceAt(20, 98) + + def testTap_dpUnitsIgnored(self): + node = self.app.GetUiNode(content_desc='Open navigation drawer') + node.Tap(dp_units=True) + self.assertTappedOnceAt(56, 114) # Still taps at center. + + @mock.patch('time.sleep', mock.Mock()) + def testWaitForUiNode_found(self): + self._setMockXmlScreenshots( + [MOCK_XML_LOADING, MOCK_XML_LOADING, MOCK_XML_LOADED]) + node = self.app.WaitForUiNode(resource_id='actionbar_title') + self.assertNodeHasAttribs(node, {'text': 'Primary'}) + + @mock.patch('time.sleep', mock.Mock()) + def testWaitForUiNode_notFound(self): + self._setMockXmlScreenshots( + [MOCK_XML_LOADING, MOCK_XML_LOADING, MOCK_XML_LOADING]) + with self.assertRaises(device_errors.CommandTimeoutError): + self.app.WaitForUiNode(resource_id='actionbar_title') diff --git a/catapult/devil/devil/android/battery_utils.py b/catapult/devil/devil/android/battery_utils.py new file mode 100644 index 00000000..4c8f5431 --- /dev/null +++ b/catapult/devil/devil/android/battery_utils.py @@ -0,0 +1,650 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Provides a variety of device interactions with power. +""" +# pylint: disable=unused-argument + +import collections +import contextlib +import csv +import logging + +from devil.android import decorators +from devil.android import device_errors +from devil.android import device_utils +from devil.android.sdk import version_codes +from devil.utils import timeout_retry + +_DEFAULT_TIMEOUT = 30 +_DEFAULT_RETRIES = 3 + + +_DEVICE_PROFILES = [ + { + 'name': 'Nexus 4', + 'witness_file': '/sys/module/pm8921_charger/parameters/disabled', + 'enable_command': ( + 'echo 0 > /sys/module/pm8921_charger/parameters/disabled && ' + 'dumpsys battery reset'), + 'disable_command': ( + 'echo 1 > /sys/module/pm8921_charger/parameters/disabled && ' + 'dumpsys battery set ac 0 && dumpsys battery set usb 0'), + 'charge_counter': None, + 'voltage': None, + 'current': None, + }, + { + 'name': 'Nexus 5', + # Nexus 5 + # Setting the HIZ bit of the bq24192 causes the charger to actually ignore + # energy coming from USB. Setting the power_supply offline just updates the + # Android system to reflect that. + 'witness_file': '/sys/kernel/debug/bq24192/INPUT_SRC_CONT', + 'enable_command': ( + 'echo 0x4A > /sys/kernel/debug/bq24192/INPUT_SRC_CONT && ' + 'chmod 644 /sys/class/power_supply/usb/online && ' + 'echo 1 > /sys/class/power_supply/usb/online && ' + 'dumpsys battery reset'), + 'disable_command': ( + 'echo 0xCA > /sys/kernel/debug/bq24192/INPUT_SRC_CONT && ' + 'chmod 644 /sys/class/power_supply/usb/online && ' + 'echo 0 > /sys/class/power_supply/usb/online && ' + 'dumpsys battery set ac 0 && dumpsys battery set usb 0'), + 'charge_counter': None, + 'voltage': None, + 'current': None, + }, + { + 'name': 'Nexus 6', + 'witness_file': None, + 'enable_command': ( + 'echo 1 > /sys/class/power_supply/battery/charging_enabled && ' + 'dumpsys battery reset'), + 'disable_command': ( + 'echo 0 > /sys/class/power_supply/battery/charging_enabled && ' + 'dumpsys battery set ac 0 && dumpsys battery set usb 0'), + 'charge_counter': ( + '/sys/class/power_supply/max170xx_battery/charge_counter_ext'), + 'voltage': '/sys/class/power_supply/max170xx_battery/voltage_now', + 'current': '/sys/class/power_supply/max170xx_battery/current_now', + }, + { + 'name': 'Nexus 9', + 'witness_file': None, + 'enable_command': ( + 'echo Disconnected > ' + '/sys/bus/i2c/drivers/bq2419x/0-006b/input_cable_state && ' + 'dumpsys battery reset'), + 'disable_command': ( + 'echo Connected > ' + '/sys/bus/i2c/drivers/bq2419x/0-006b/input_cable_state && ' + 'dumpsys battery set ac 0 && dumpsys battery set usb 0'), + 'charge_counter': '/sys/class/power_supply/battery/charge_counter_ext', + 'voltage': '/sys/class/power_supply/battery/voltage_now', + 'current': '/sys/class/power_supply/battery/current_now', + }, + { + 'name': 'Nexus 10', + 'witness_file': None, + 'enable_command': None, + 'disable_command': None, + 'charge_counter': None, + 'voltage': '/sys/class/power_supply/ds2784-fuelgauge/voltage_now', + 'current': '/sys/class/power_supply/ds2784-fuelgauge/current_now', + + }, +] + +# The list of useful dumpsys columns. +# Index of the column containing the format version. +_DUMP_VERSION_INDEX = 0 +# Index of the column containing the type of the row. +_ROW_TYPE_INDEX = 3 +# Index of the column containing the uid. +_PACKAGE_UID_INDEX = 4 +# Index of the column containing the application package. +_PACKAGE_NAME_INDEX = 5 +# The column containing the uid of the power data. +_PWI_UID_INDEX = 1 +# The column containing the type of consumption. Only consumption since last +# charge are of interest here. +_PWI_AGGREGATION_INDEX = 2 +_PWS_AGGREGATION_INDEX = _PWI_AGGREGATION_INDEX +# The column containing the amount of power used, in mah. +_PWI_POWER_CONSUMPTION_INDEX = 5 +_PWS_POWER_CONSUMPTION_INDEX = _PWI_POWER_CONSUMPTION_INDEX + +_MAX_CHARGE_ERROR = 20 + + +class BatteryUtils(object): + + def __init__(self, device, default_timeout=_DEFAULT_TIMEOUT, + default_retries=_DEFAULT_RETRIES): + """BatteryUtils constructor. + + Args: + device: A DeviceUtils instance. + default_timeout: An integer containing the default number of seconds to + wait for an operation to complete if no explicit value + is provided. + default_retries: An integer containing the default number or times an + operation should be retried on failure if no explicit + value is provided. + Raises: + TypeError: If it is not passed a DeviceUtils instance. + """ + if not isinstance(device, device_utils.DeviceUtils): + raise TypeError('Must be initialized with DeviceUtils object.') + self._device = device + self._cache = device.GetClientCache(self.__class__.__name__) + self._default_timeout = default_timeout + self._default_retries = default_retries + + @decorators.WithTimeoutAndRetriesFromInstance() + def SupportsFuelGauge(self, timeout=None, retries=None): + """Detect if fuel gauge chip is present. + + Args: + timeout: timeout in seconds + retries: number of retries + + Returns: + True if known fuel gauge files are present. + False otherwise. + """ + self._DiscoverDeviceProfile() + return (self._cache['profile']['enable_command'] != None + and self._cache['profile']['charge_counter'] != None) + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetFuelGaugeChargeCounter(self, timeout=None, retries=None): + """Get value of charge_counter on fuel gauge chip. + + Device must have charging disabled for this, not just battery updates + disabled. The only device that this currently works with is the nexus 5. + + Args: + timeout: timeout in seconds + retries: number of retries + + Returns: + value of charge_counter for fuel gauge chip in units of nAh. + + Raises: + device_errors.CommandFailedError: If fuel gauge chip not found. + """ + if self.SupportsFuelGauge(): + return int(self._device.ReadFile( + self._cache['profile']['charge_counter'])) + raise device_errors.CommandFailedError( + 'Unable to find fuel gauge.') + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetNetworkData(self, package, timeout=None, retries=None): + """Get network data for specific package. + + Args: + package: package name you want network data for. + timeout: timeout in seconds + retries: number of retries + + Returns: + Tuple of (sent_data, recieved_data) + None if no network data found + """ + # If device_utils clears cache, cache['uids'] doesn't exist + if 'uids' not in self._cache: + self._cache['uids'] = {} + if package not in self._cache['uids']: + self.GetPowerData() + if package not in self._cache['uids']: + logging.warning('No UID found for %s. Can\'t get network data.', + package) + return None + + network_data_path = '/proc/uid_stat/%s/' % self._cache['uids'][package] + try: + send_data = int(self._device.ReadFile(network_data_path + 'tcp_snd')) + # If ReadFile throws exception, it means no network data usage file for + # package has been recorded. Return 0 sent and 0 received. + except device_errors.AdbShellCommandFailedError: + logging.warning('No sent data found for package %s', package) + send_data = 0 + try: + recv_data = int(self._device.ReadFile(network_data_path + 'tcp_rcv')) + except device_errors.AdbShellCommandFailedError: + logging.warning('No received data found for package %s', package) + recv_data = 0 + return (send_data, recv_data) + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetPowerData(self, timeout=None, retries=None): + """Get power data for device. + + Args: + timeout: timeout in seconds + retries: number of retries + + Returns: + Dict containing system power, and a per-package power dict keyed on + package names. + { + 'system_total': 23.1, + 'per_package' : { + package_name: { + 'uid': uid, + 'data': [1,2,3] + }, + } + } + """ + if 'uids' not in self._cache: + self._cache['uids'] = {} + dumpsys_output = self._device.RunShellCommand( + ['dumpsys', 'batterystats', '-c'], + check_return=True, large_output=True) + csvreader = csv.reader(dumpsys_output) + pwi_entries = collections.defaultdict(list) + system_total = None + for entry in csvreader: + if entry[_DUMP_VERSION_INDEX] not in ['8', '9']: + # Wrong dumpsys version. + raise device_errors.DeviceVersionError( + 'Dumpsys version must be 8 or 9. %s found.' + % entry[_DUMP_VERSION_INDEX]) + if _ROW_TYPE_INDEX < len(entry) and entry[_ROW_TYPE_INDEX] == 'uid': + current_package = entry[_PACKAGE_NAME_INDEX] + if (self._cache['uids'].get(current_package) + and self._cache['uids'].get(current_package) + != entry[_PACKAGE_UID_INDEX]): + raise device_errors.CommandFailedError( + 'Package %s found multiple times with different UIDs %s and %s' + % (current_package, self._cache['uids'][current_package], + entry[_PACKAGE_UID_INDEX])) + self._cache['uids'][current_package] = entry[_PACKAGE_UID_INDEX] + elif (_PWI_POWER_CONSUMPTION_INDEX < len(entry) + and entry[_ROW_TYPE_INDEX] == 'pwi' + and entry[_PWI_AGGREGATION_INDEX] == 'l'): + pwi_entries[entry[_PWI_UID_INDEX]].append( + float(entry[_PWI_POWER_CONSUMPTION_INDEX])) + elif (_PWS_POWER_CONSUMPTION_INDEX < len(entry) + and entry[_ROW_TYPE_INDEX] == 'pws' + and entry[_PWS_AGGREGATION_INDEX] == 'l'): + # This entry should only appear once. + assert system_total is None + system_total = float(entry[_PWS_POWER_CONSUMPTION_INDEX]) + + per_package = {p: {'uid': uid, 'data': pwi_entries[uid]} + for p, uid in self._cache['uids'].iteritems()} + return {'system_total': system_total, 'per_package': per_package} + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetBatteryInfo(self, timeout=None, retries=None): + """Gets battery info for the device. + + Args: + timeout: timeout in seconds + retries: number of retries + Returns: + A dict containing various battery information as reported by dumpsys + battery. + """ + result = {} + # Skip the first line, which is just a header. + for line in self._device.RunShellCommand( + ['dumpsys', 'battery'], check_return=True)[1:]: + # If usb charging has been disabled, an extra line of header exists. + if 'UPDATES STOPPED' in line: + logging.warning('Dumpsys battery not receiving updates. ' + 'Run dumpsys battery reset if this is in error.') + elif ':' not in line: + logging.warning('Unknown line found in dumpsys battery: "%s"', line) + else: + k, v = line.split(':', 1) + result[k.strip()] = v.strip() + return result + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetCharging(self, timeout=None, retries=None): + """Gets the charging state of the device. + + Args: + timeout: timeout in seconds + retries: number of retries + Returns: + True if the device is charging, false otherwise. + """ + battery_info = self.GetBatteryInfo() + for k in ('AC powered', 'USB powered', 'Wireless powered'): + if (k in battery_info and + battery_info[k].lower() in ('true', '1', 'yes')): + return True + return False + + # TODO(rnephew): Make private when all use cases can use the context manager. + @decorators.WithTimeoutAndRetriesFromInstance() + def DisableBatteryUpdates(self, timeout=None, retries=None): + """Resets battery data and makes device appear like it is not + charging so that it will collect power data since last charge. + + Args: + timeout: timeout in seconds + retries: number of retries + + Raises: + device_errors.CommandFailedError: When resetting batterystats fails to + reset power values. + device_errors.DeviceVersionError: If device is not L or higher. + """ + def battery_updates_disabled(): + return self.GetCharging() is False + + self._ClearPowerData() + self._device.RunShellCommand(['dumpsys', 'battery', 'set', 'ac', '0'], + check_return=True) + self._device.RunShellCommand(['dumpsys', 'battery', 'set', 'usb', '0'], + check_return=True) + timeout_retry.WaitFor(battery_updates_disabled, wait_period=1) + + # TODO(rnephew): Make private when all use cases can use the context manager. + @decorators.WithTimeoutAndRetriesFromInstance() + def EnableBatteryUpdates(self, timeout=None, retries=None): + """Restarts device charging so that dumpsys no longer collects power data. + + Args: + timeout: timeout in seconds + retries: number of retries + + Raises: + device_errors.DeviceVersionError: If device is not L or higher. + """ + def battery_updates_enabled(): + return (self.GetCharging() + or not bool('UPDATES STOPPED' in self._device.RunShellCommand( + ['dumpsys', 'battery'], check_return=True))) + + self._device.RunShellCommand(['dumpsys', 'battery', 'reset'], + check_return=True) + timeout_retry.WaitFor(battery_updates_enabled, wait_period=1) + + @contextlib.contextmanager + def BatteryMeasurement(self, timeout=None, retries=None): + """Context manager that enables battery data collection. It makes + the device appear to stop charging so that dumpsys will start collecting + power data since last charge. Once the with block is exited, charging is + resumed and power data since last charge is no longer collected. + + Only for devices L and higher. + + Example usage: + with BatteryMeasurement(): + browser_actions() + get_power_data() # report usage within this block + after_measurements() # Anything that runs after power + # measurements are collected + + Args: + timeout: timeout in seconds + retries: number of retries + + Raises: + device_errors.DeviceVersionError: If device is not L or higher. + """ + if self._device.build_version_sdk < version_codes.LOLLIPOP: + raise device_errors.DeviceVersionError('Device must be L or higher.') + try: + self.DisableBatteryUpdates(timeout=timeout, retries=retries) + yield + finally: + self.EnableBatteryUpdates(timeout=timeout, retries=retries) + + def _DischargeDevice(self, percent, wait_period=120): + """Disables charging and waits for device to discharge given amount + + Args: + percent: level of charge to discharge. + + Raises: + ValueError: If percent is not between 1 and 99. + """ + battery_level = int(self.GetBatteryInfo().get('level')) + if not 0 < percent < 100: + raise ValueError('Discharge amount(%s) must be between 1 and 99' + % percent) + if battery_level is None: + logging.warning('Unable to find current battery level. Cannot discharge.') + return + # Do not discharge if it would make battery level too low. + if percent >= battery_level - 10: + logging.warning('Battery is too low or discharge amount requested is too ' + 'high. Cannot discharge phone %s percent.', percent) + return + + self._HardwareSetCharging(False) + + def device_discharged(): + self._HardwareSetCharging(True) + current_level = int(self.GetBatteryInfo().get('level')) + logging.info('current battery level: %s', current_level) + if battery_level - current_level >= percent: + return True + self._HardwareSetCharging(False) + return False + + timeout_retry.WaitFor(device_discharged, wait_period=wait_period) + + def ChargeDeviceToLevel(self, level, wait_period=60): + """Enables charging and waits for device to be charged to given level. + + Args: + level: level of charge to wait for. + wait_period: time in seconds to wait between checking. + Raises: + device_errors.DeviceChargingError: If error while charging is detected. + """ + self.SetCharging(True) + charge_status = { + 'charge_failure_count': 0, + 'last_charge_value': 0 + } + def device_charged(): + battery_level = self.GetBatteryInfo().get('level') + if battery_level is None: + logging.warning('Unable to find current battery level.') + battery_level = 100 + else: + logging.info('current battery level: %s', battery_level) + battery_level = int(battery_level) + + # Use > so that it will not reset if charge is going down. + if battery_level > charge_status['last_charge_value']: + charge_status['last_charge_value'] = battery_level + charge_status['charge_failure_count'] = 0 + else: + charge_status['charge_failure_count'] += 1 + + if (not battery_level >= level + and charge_status['charge_failure_count'] >= _MAX_CHARGE_ERROR): + raise device_errors.DeviceChargingError( + 'Device not charging properly. Current level:%s Previous level:%s' + % (battery_level, charge_status['last_charge_value'])) + return battery_level >= level + + timeout_retry.WaitFor(device_charged, wait_period=wait_period) + + def LetBatteryCoolToTemperature(self, target_temp, wait_period=180): + """Lets device sit to give battery time to cool down + Args: + temp: maximum temperature to allow in tenths of degrees c. + wait_period: time in seconds to wait between checking. + """ + def cool_device(): + temp = self.GetBatteryInfo().get('temperature') + if temp is None: + logging.warning('Unable to find current battery temperature.') + temp = 0 + else: + logging.info('Current battery temperature: %s', temp) + if int(temp) <= target_temp: + return True + else: + if self._cache['profile']['name'] == 'Nexus 5': + self._DischargeDevice(1) + return False + + self._DiscoverDeviceProfile() + self.EnableBatteryUpdates() + logging.info('Waiting for the device to cool down to %s (0.1 C)', + target_temp) + timeout_retry.WaitFor(cool_device, wait_period=wait_period) + + @decorators.WithTimeoutAndRetriesFromInstance() + def SetCharging(self, enabled, timeout=None, retries=None): + """Enables or disables charging on the device. + + Args: + enabled: A boolean indicating whether charging should be enabled or + disabled. + timeout: timeout in seconds + retries: number of retries + """ + if self.GetCharging() == enabled: + logging.warning('Device charging already in expected state: %s', enabled) + return + + self._DiscoverDeviceProfile() + if enabled: + if self._cache['profile']['enable_command']: + self._HardwareSetCharging(enabled) + else: + logging.info('Unable to enable charging via hardware. ' + 'Falling back to software enabling.') + self.EnableBatteryUpdates() + else: + if self._cache['profile']['enable_command']: + self._ClearPowerData() + self._HardwareSetCharging(enabled) + else: + logging.info('Unable to disable charging via hardware. ' + 'Falling back to software disabling.') + self.DisableBatteryUpdates() + + def _HardwareSetCharging(self, enabled, timeout=None, retries=None): + """Enables or disables charging on the device. + + Args: + enabled: A boolean indicating whether charging should be enabled or + disabled. + timeout: timeout in seconds + retries: number of retries + + Raises: + device_errors.CommandFailedError: If method of disabling charging cannot + be determined. + """ + self._DiscoverDeviceProfile() + if not self._cache['profile']['enable_command']: + raise device_errors.CommandFailedError( + 'Unable to find charging commands.') + + command = (self._cache['profile']['enable_command'] if enabled + else self._cache['profile']['disable_command']) + + def verify_charging(): + return self.GetCharging() == enabled + + self._device.RunShellCommand( + command, check_return=True, as_root=True, large_output=True) + timeout_retry.WaitFor(verify_charging, wait_period=1) + + @contextlib.contextmanager + def PowerMeasurement(self, timeout=None, retries=None): + """Context manager that enables battery power collection. + + Once the with block is exited, charging is resumed. Will attempt to disable + charging at the hardware level, and if that fails will fall back to software + disabling of battery updates. + + Only for devices L and higher. + + Example usage: + with PowerMeasurement(): + browser_actions() + get_power_data() # report usage within this block + after_measurements() # Anything that runs after power + # measurements are collected + + Args: + timeout: timeout in seconds + retries: number of retries + """ + try: + self.SetCharging(False, timeout=timeout, retries=retries) + yield + finally: + self.SetCharging(True, timeout=timeout, retries=retries) + + def _ClearPowerData(self): + """Resets battery data and makes device appear like it is not + charging so that it will collect power data since last charge. + + Returns: + True if power data cleared. + False if power data clearing is not supported (pre-L) + + Raises: + device_errors.DeviceVersionError: If power clearing is supported, + but fails. + """ + if self._device.build_version_sdk < version_codes.LOLLIPOP: + logging.warning('Dumpsys power data only available on 5.0 and above. ' + 'Cannot clear power data.') + return False + + self._device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'usb', '1'], check_return=True) + self._device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'ac', '1'], check_return=True) + self._device.RunShellCommand( + ['dumpsys', 'batterystats', '--reset'], check_return=True) + battery_data = self._device.RunShellCommand( + ['dumpsys', 'batterystats', '--charged', '-c'], + check_return=True, large_output=True) + for line in battery_data: + l = line.split(',') + if (len(l) > _PWI_POWER_CONSUMPTION_INDEX and l[_ROW_TYPE_INDEX] == 'pwi' + and l[_PWI_POWER_CONSUMPTION_INDEX] != 0): + self._device.RunShellCommand( + ['dumpsys', 'battery', 'reset'], check_return=True) + raise device_errors.CommandFailedError( + 'Non-zero pmi value found after reset.') + self._device.RunShellCommand( + ['dumpsys', 'battery', 'reset'], check_return=True) + return True + + def _DiscoverDeviceProfile(self): + """Checks and caches device information. + + Returns: + True if profile is found, false otherwise. + """ + + if 'profile' in self._cache: + return True + for profile in _DEVICE_PROFILES: + if self._device.product_model == profile['name']: + self._cache['profile'] = profile + return True + self._cache['profile'] = { + 'name': None, + 'witness_file': None, + 'enable_command': None, + 'disable_command': None, + 'charge_counter': None, + 'voltage': None, + 'current': None, + } + return False diff --git a/catapult/devil/devil/android/battery_utils_test.py b/catapult/devil/devil/android/battery_utils_test.py new file mode 100755 index 00000000..79939217 --- /dev/null +++ b/catapult/devil/devil/android/battery_utils_test.py @@ -0,0 +1,676 @@ +#!/usr/bin/env python +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Unit tests for the contents of battery_utils.py +""" + +# pylint: disable=protected-access,unused-argument + +import logging +import unittest + +from devil import devil_env +from devil.android import battery_utils +from devil.android import device_errors +from devil.android import device_utils +from devil.android import device_utils_test +from devil.utils import mock_calls + +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + +_DUMPSYS_OUTPUT = [ + '9,0,i,uid,1000,test_package1', + '9,0,i,uid,1001,test_package2', + '9,1000,l,pwi,uid,1', + '9,1001,l,pwi,uid,2', + '9,0,l,pws,1728,2000,190,207', +] + + +class BatteryUtilsTest(mock_calls.TestCase): + + _NEXUS_5 = { + 'name': 'Nexus 5', + 'witness_file': '/sys/kernel/debug/bq24192/INPUT_SRC_CONT', + 'enable_command': ( + 'echo 0x4A > /sys/kernel/debug/bq24192/INPUT_SRC_CONT && ' + 'echo 1 > /sys/class/power_supply/usb/online'), + 'disable_command': ( + 'echo 0xCA > /sys/kernel/debug/bq24192/INPUT_SRC_CONT && ' + 'chmod 644 /sys/class/power_supply/usb/online && ' + 'echo 0 > /sys/class/power_supply/usb/online'), + 'charge_counter': None, + 'voltage': None, + 'current': None, + } + + _NEXUS_6 = { + 'name': 'Nexus 6', + 'witness_file': None, + 'enable_command': None, + 'disable_command': None, + 'charge_counter': ( + '/sys/class/power_supply/max170xx_battery/charge_counter_ext'), + 'voltage': '/sys/class/power_supply/max170xx_battery/voltage_now', + 'current': '/sys/class/power_supply/max170xx_battery/current_now', + } + + _NEXUS_10 = { + 'name': 'Nexus 10', + 'witness_file': None, + 'enable_command': None, + 'disable_command': None, + 'charge_counter': ( + '/sys/class/power_supply/ds2784-fuelgauge/charge_counter_ext'), + 'voltage': '/sys/class/power_supply/ds2784-fuelgauge/voltage_now', + 'current': '/sys/class/power_supply/ds2784-fuelgauge/current_now', + } + + def ShellError(self, output=None, status=1): + def action(cmd, *args, **kwargs): + raise device_errors.AdbShellCommandFailedError( + cmd, output, status, str(self.device)) + if output is None: + output = 'Permission denied\n' + return action + + def setUp(self): + self.adb = device_utils_test._AdbWrapperMock('0123456789abcdef') + self.device = device_utils.DeviceUtils( + self.adb, default_timeout=10, default_retries=0) + self.watchMethodCalls(self.call.adb, ignore=['GetDeviceSerial']) + self.battery = battery_utils.BatteryUtils( + self.device, default_timeout=10, default_retries=0) + + +class BatteryUtilsInitTest(unittest.TestCase): + + def testInitWithDeviceUtil(self): + serial = '0fedcba987654321' + d = device_utils.DeviceUtils(serial) + b = battery_utils.BatteryUtils(d) + self.assertEqual(d, b._device) + + def testInitWithMissing_fails(self): + with self.assertRaises(TypeError): + battery_utils.BatteryUtils(None) + with self.assertRaises(TypeError): + battery_utils.BatteryUtils('') + + +class BatteryUtilsSetChargingTest(BatteryUtilsTest): + + @mock.patch('time.sleep', mock.Mock()) + def testHardwareSetCharging_enabled(self): + self.battery._cache['profile'] = self._NEXUS_5 + with self.assertCalls( + (self.call.device.RunShellCommand( + mock.ANY, check_return=True, as_root=True, large_output=True), []), + (self.call.battery.GetCharging(), False), + (self.call.battery.GetCharging(), True)): + self.battery._HardwareSetCharging(True) + + def testHardwareSetCharging_alreadyEnabled(self): + self.battery._cache['profile'] = self._NEXUS_5 + with self.assertCalls( + (self.call.device.RunShellCommand( + mock.ANY, check_return=True, as_root=True, large_output=True), []), + (self.call.battery.GetCharging(), True)): + self.battery._HardwareSetCharging(True) + + @mock.patch('time.sleep', mock.Mock()) + def testHardwareSetCharging_disabled(self): + self.battery._cache['profile'] = self._NEXUS_5 + with self.assertCalls( + (self.call.device.RunShellCommand( + mock.ANY, check_return=True, as_root=True, large_output=True), []), + (self.call.battery.GetCharging(), True), + (self.call.battery.GetCharging(), False)): + self.battery._HardwareSetCharging(False) + + +class BatteryUtilsSetBatteryMeasurementTest(BatteryUtilsTest): + + @mock.patch('time.sleep', mock.Mock()) + def testBatteryMeasurementWifi(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=22): + with self.assertCalls( + (self.call.battery._ClearPowerData(), True), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'ac', '0'], check_return=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'usb', '0'], check_return=True), + []), + (self.call.battery.GetCharging(), False), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'reset'], check_return=True), []), + (self.call.battery.GetCharging(), False), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery'], check_return=True), ['UPDATES STOPPED']), + (self.call.battery.GetCharging(), False), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery'], check_return=True), [])): + with self.battery.BatteryMeasurement(): + pass + + @mock.patch('time.sleep', mock.Mock()) + def testBatteryMeasurementUsb(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=22): + with self.assertCalls( + (self.call.battery._ClearPowerData(), True), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'ac', '0'], check_return=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'usb', '0'], check_return=True), + []), + (self.call.battery.GetCharging(), False), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'reset'], check_return=True), []), + (self.call.battery.GetCharging(), False), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery'], check_return=True), ['UPDATES STOPPED']), + (self.call.battery.GetCharging(), True)): + with self.battery.BatteryMeasurement(): + pass + + +class BatteryUtilsGetPowerData(BatteryUtilsTest): + + def testGetPowerData(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '-c'], + check_return=True, large_output=True), + _DUMPSYS_OUTPUT)): + data = self.battery.GetPowerData() + check = { + 'system_total': 2000.0, + 'per_package': { + 'test_package1': {'uid': '1000', 'data': [1.0]}, + 'test_package2': {'uid': '1001', 'data': [2.0]} + } + } + self.assertEqual(data, check) + + def testGetPowerData_packageCollisionSame(self): + self.battery._cache['uids'] = {'test_package1': '1000'} + with self.assertCall( + self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '-c'], + check_return=True, large_output=True), + _DUMPSYS_OUTPUT): + data = self.battery.GetPowerData() + check = { + 'system_total': 2000.0, + 'per_package': { + 'test_package1': {'uid': '1000', 'data': [1.0]}, + 'test_package2': {'uid': '1001', 'data': [2.0]} + } + } + self.assertEqual(data, check) + + def testGetPowerData_packageCollisionDifferent(self): + self.battery._cache['uids'] = {'test_package1': '1'} + with self.assertCall( + self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '-c'], + check_return=True, large_output=True), + _DUMPSYS_OUTPUT): + with self.assertRaises(device_errors.CommandFailedError): + self.battery.GetPowerData() + + def testGetPowerData_cacheCleared(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '-c'], + check_return=True, large_output=True), + _DUMPSYS_OUTPUT)): + self.battery._cache.clear() + data = self.battery.GetPowerData() + check = { + 'system_total': 2000.0, + 'per_package': { + 'test_package1': {'uid': '1000', 'data': [1.0]}, + 'test_package2': {'uid': '1001', 'data': [2.0]} + } + } + self.assertEqual(data, check) + + +class BatteryUtilsChargeDevice(BatteryUtilsTest): + + @mock.patch('time.sleep', mock.Mock()) + def testChargeDeviceToLevel_pass(self): + with self.assertCalls( + (self.call.battery.SetCharging(True)), + (self.call.battery.GetBatteryInfo(), {'level': '50'}), + (self.call.battery.GetBatteryInfo(), {'level': '100'})): + self.battery.ChargeDeviceToLevel(95) + + @mock.patch('time.sleep', mock.Mock()) + def testChargeDeviceToLevel_failureSame(self): + with self.assertCalls( + (self.call.battery.SetCharging(True)), + (self.call.battery.GetBatteryInfo(), {'level': '50'}), + (self.call.battery.GetBatteryInfo(), {'level': '50'}), + + (self.call.battery.GetBatteryInfo(), {'level': '50'})): + with self.assertRaises(device_errors.DeviceChargingError): + old_max = battery_utils._MAX_CHARGE_ERROR + try: + battery_utils._MAX_CHARGE_ERROR = 2 + self.battery.ChargeDeviceToLevel(95) + finally: + battery_utils._MAX_CHARGE_ERROR = old_max + + @mock.patch('time.sleep', mock.Mock()) + def testChargeDeviceToLevel_failureDischarge(self): + with self.assertCalls( + (self.call.battery.SetCharging(True)), + (self.call.battery.GetBatteryInfo(), {'level': '50'}), + (self.call.battery.GetBatteryInfo(), {'level': '49'}), + (self.call.battery.GetBatteryInfo(), {'level': '48'})): + with self.assertRaises(device_errors.DeviceChargingError): + old_max = battery_utils._MAX_CHARGE_ERROR + try: + battery_utils._MAX_CHARGE_ERROR = 2 + self.battery.ChargeDeviceToLevel(95) + finally: + battery_utils._MAX_CHARGE_ERROR = old_max + + +class BatteryUtilsDischargeDevice(BatteryUtilsTest): + + @mock.patch('time.sleep', mock.Mock()) + def testDischargeDevice_exact(self): + with self.assertCalls( + (self.call.battery.GetBatteryInfo(), {'level': '100'}), + (self.call.battery._HardwareSetCharging(False)), + (self.call.battery._HardwareSetCharging(True)), + (self.call.battery.GetBatteryInfo(), {'level': '99'})): + self.battery._DischargeDevice(1) + + @mock.patch('time.sleep', mock.Mock()) + def testDischargeDevice_over(self): + with self.assertCalls( + (self.call.battery.GetBatteryInfo(), {'level': '100'}), + (self.call.battery._HardwareSetCharging(False)), + (self.call.battery._HardwareSetCharging(True)), + (self.call.battery.GetBatteryInfo(), {'level': '50'})): + self.battery._DischargeDevice(1) + + @mock.patch('time.sleep', mock.Mock()) + def testDischargeDevice_takeslong(self): + with self.assertCalls( + (self.call.battery.GetBatteryInfo(), {'level': '100'}), + (self.call.battery._HardwareSetCharging(False)), + (self.call.battery._HardwareSetCharging(True)), + (self.call.battery.GetBatteryInfo(), {'level': '100'}), + (self.call.battery._HardwareSetCharging(False)), + (self.call.battery._HardwareSetCharging(True)), + (self.call.battery.GetBatteryInfo(), {'level': '99'}), + (self.call.battery._HardwareSetCharging(False)), + (self.call.battery._HardwareSetCharging(True)), + (self.call.battery.GetBatteryInfo(), {'level': '98'}), + (self.call.battery._HardwareSetCharging(False)), + (self.call.battery._HardwareSetCharging(True)), + (self.call.battery.GetBatteryInfo(), {'level': '97'})): + self.battery._DischargeDevice(3) + + @mock.patch('time.sleep', mock.Mock()) + def testDischargeDevice_dischargeTooClose(self): + with self.assertCalls( + (self.call.battery.GetBatteryInfo(), {'level': '100'})): + self.battery._DischargeDevice(99) + + @mock.patch('time.sleep', mock.Mock()) + def testDischargeDevice_percentageOutOfBounds(self): + with self.assertCalls( + (self.call.battery.GetBatteryInfo(), {'level': '100'})): + with self.assertRaises(ValueError): + self.battery._DischargeDevice(100) + with self.assertCalls( + (self.call.battery.GetBatteryInfo(), {'level': '100'})): + with self.assertRaises(ValueError): + self.battery._DischargeDevice(0) + + +class BatteryUtilsGetBatteryInfoTest(BatteryUtilsTest): + + def testGetBatteryInfo_normal(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'battery'], check_return=True), + [ + 'Current Battery Service state:', + ' AC powered: false', + ' USB powered: true', + ' level: 100', + ' temperature: 321', + ])): + self.assertEquals( + { + 'AC powered': 'false', + 'USB powered': 'true', + 'level': '100', + 'temperature': '321', + }, + self.battery.GetBatteryInfo()) + + def testGetBatteryInfo_nothing(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'battery'], check_return=True), [])): + self.assertEquals({}, self.battery.GetBatteryInfo()) + + +class BatteryUtilsGetChargingTest(BatteryUtilsTest): + + def testGetCharging_usb(self): + with self.assertCall( + self.call.battery.GetBatteryInfo(), {'USB powered': 'true'}): + self.assertTrue(self.battery.GetCharging()) + + def testGetCharging_usbFalse(self): + with self.assertCall( + self.call.battery.GetBatteryInfo(), {'USB powered': 'false'}): + self.assertFalse(self.battery.GetCharging()) + + def testGetCharging_ac(self): + with self.assertCall( + self.call.battery.GetBatteryInfo(), {'AC powered': 'true'}): + self.assertTrue(self.battery.GetCharging()) + + def testGetCharging_wireless(self): + with self.assertCall( + self.call.battery.GetBatteryInfo(), {'Wireless powered': 'true'}): + self.assertTrue(self.battery.GetCharging()) + + def testGetCharging_unknown(self): + with self.assertCall( + self.call.battery.GetBatteryInfo(), {'level': '42'}): + self.assertFalse(self.battery.GetCharging()) + + +class BatteryUtilsGetNetworkDataTest(BatteryUtilsTest): + + def testGetNetworkData_noDataUsage(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '-c'], + check_return=True, large_output=True), + _DUMPSYS_OUTPUT), + (self.call.device.ReadFile('/proc/uid_stat/1000/tcp_snd'), + self.ShellError()), + (self.call.device.ReadFile('/proc/uid_stat/1000/tcp_rcv'), + self.ShellError())): + self.assertEquals(self.battery.GetNetworkData('test_package1'), (0, 0)) + + def testGetNetworkData_badPackage(self): + with self.assertCall( + self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '-c'], + check_return=True, large_output=True), + _DUMPSYS_OUTPUT): + self.assertEqual(self.battery.GetNetworkData('asdf'), None) + + def testGetNetworkData_packageNotCached(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '-c'], + check_return=True, large_output=True), + _DUMPSYS_OUTPUT), + (self.call.device.ReadFile('/proc/uid_stat/1000/tcp_snd'), 1), + (self.call.device.ReadFile('/proc/uid_stat/1000/tcp_rcv'), 2)): + self.assertEqual(self.battery.GetNetworkData('test_package1'), (1, 2)) + + def testGetNetworkData_packageCached(self): + self.battery._cache['uids'] = {'test_package1': '1000'} + with self.assertCalls( + (self.call.device.ReadFile('/proc/uid_stat/1000/tcp_snd'), 1), + (self.call.device.ReadFile('/proc/uid_stat/1000/tcp_rcv'), 2)): + self.assertEqual(self.battery.GetNetworkData('test_package1'), (1, 2)) + + def testGetNetworkData_clearedCache(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '-c'], + check_return=True, large_output=True), + _DUMPSYS_OUTPUT), + (self.call.device.ReadFile('/proc/uid_stat/1000/tcp_snd'), 1), + (self.call.device.ReadFile('/proc/uid_stat/1000/tcp_rcv'), 2)): + self.battery._cache.clear() + self.assertEqual(self.battery.GetNetworkData('test_package1'), (1, 2)) + + +class BatteryUtilsLetBatteryCoolToTemperatureTest(BatteryUtilsTest): + + @mock.patch('time.sleep', mock.Mock()) + def testLetBatteryCoolToTemperature_startUnder(self): + self.battery._cache['profile'] = self._NEXUS_6 + with self.assertCalls( + (self.call.battery.EnableBatteryUpdates(), []), + (self.call.battery.GetBatteryInfo(), {'temperature': '500'})): + self.battery.LetBatteryCoolToTemperature(600) + + @mock.patch('time.sleep', mock.Mock()) + def testLetBatteryCoolToTemperature_startOver(self): + self.battery._cache['profile'] = self._NEXUS_6 + with self.assertCalls( + (self.call.battery.EnableBatteryUpdates(), []), + (self.call.battery.GetBatteryInfo(), {'temperature': '500'}), + (self.call.battery.GetBatteryInfo(), {'temperature': '400'})): + self.battery.LetBatteryCoolToTemperature(400) + + @mock.patch('time.sleep', mock.Mock()) + def testLetBatteryCoolToTemperature_nexus5Hot(self): + self.battery._cache['profile'] = self._NEXUS_5 + with self.assertCalls( + (self.call.battery.EnableBatteryUpdates(), []), + (self.call.battery.GetBatteryInfo(), {'temperature': '500'}), + (self.call.battery._DischargeDevice(1), []), + (self.call.battery.GetBatteryInfo(), {'temperature': '400'})): + self.battery.LetBatteryCoolToTemperature(400) + + @mock.patch('time.sleep', mock.Mock()) + def testLetBatteryCoolToTemperature_nexus5Cool(self): + self.battery._cache['profile'] = self._NEXUS_5 + with self.assertCalls( + (self.call.battery.EnableBatteryUpdates(), []), + (self.call.battery.GetBatteryInfo(), {'temperature': '400'})): + self.battery.LetBatteryCoolToTemperature(400) + + +class BatteryUtilsSupportsFuelGaugeTest(BatteryUtilsTest): + + def testSupportsFuelGauge_false(self): + self.battery._cache['profile'] = self._NEXUS_5 + self.assertFalse(self.battery.SupportsFuelGauge()) + + def testSupportsFuelGauge_trueMax(self): + self.battery._cache['profile'] = self._NEXUS_6 + # TODO(rnephew): Change this to assertTrue when we have support for + # disabling hardware charging on nexus 6. + self.assertFalse(self.battery.SupportsFuelGauge()) + + def testSupportsFuelGauge_trueDS(self): + self.battery._cache['profile'] = self._NEXUS_10 + # TODO(rnephew): Change this to assertTrue when we have support for + # disabling hardware charging on nexus 10. + self.assertFalse(self.battery.SupportsFuelGauge()) + + +class BatteryUtilsGetFuelGaugeChargeCounterTest(BatteryUtilsTest): + + def testGetFuelGaugeChargeCounter_noFuelGauge(self): + self.battery._cache['profile'] = self._NEXUS_5 + with self.assertRaises(device_errors.CommandFailedError): + self.battery.GetFuelGaugeChargeCounter() + + def testGetFuelGaugeChargeCounter_fuelGaugePresent(self): + self.battery._cache['profile'] = self._NEXUS_6 + with self.assertCalls( + (self.call.battery.SupportsFuelGauge(), True), + (self.call.device.ReadFile(mock.ANY), '123')): + self.assertEqual(self.battery.GetFuelGaugeChargeCounter(), 123) + + +class BatteryUtilsSetCharging(BatteryUtilsTest): + + @mock.patch('time.sleep', mock.Mock()) + def testSetCharging_softwareSetTrue(self): + self.battery._cache['profile'] = self._NEXUS_6 + with self.assertCalls( + (self.call.battery.GetCharging(), False), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'reset'], check_return=True), []), + (self.call.battery.GetCharging(), False), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery'], check_return=True), ['UPDATES STOPPED']), + (self.call.battery.GetCharging(), True)): + self.battery.SetCharging(True) + + @mock.patch('time.sleep', mock.Mock()) + def testSetCharging_softwareSetFalse(self): + self.battery._cache['profile'] = self._NEXUS_6 + with self.assertCalls( + (self.call.battery.GetCharging(), True), + (self.call.battery._ClearPowerData(), True), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'ac', '0'], check_return=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'usb', '0'], check_return=True), []), + (self.call.battery.GetCharging(), False)): + self.battery.SetCharging(False) + + @mock.patch('time.sleep', mock.Mock()) + def testSetCharging_hardwareSetTrue(self): + self.battery._cache['profile'] = self._NEXUS_5 + with self.assertCalls( + (self.call.battery.GetCharging(), False), + (self.call.battery._HardwareSetCharging(True))): + self.battery.SetCharging(True) + + @mock.patch('time.sleep', mock.Mock()) + def testSetCharging_hardwareSetFalse(self): + self.battery._cache['profile'] = self._NEXUS_5 + with self.assertCalls( + (self.call.battery.GetCharging(), True), + (self.call.battery._ClearPowerData(), True), + (self.call.battery._HardwareSetCharging(False))): + self.battery.SetCharging(False) + + def testSetCharging_expectedStateAlreadyTrue(self): + with self.assertCalls((self.call.battery.GetCharging(), True)): + self.battery.SetCharging(True) + + def testSetCharging_expectedStateAlreadyFalse(self): + with self.assertCalls((self.call.battery.GetCharging(), False)): + self.battery.SetCharging(False) + + +class BatteryUtilsPowerMeasurement(BatteryUtilsTest): + + def testPowerMeasurement_hardware(self): + self.battery._cache['profile'] = self._NEXUS_5 + with self.assertCalls( + (self.call.battery.GetCharging(), True), + (self.call.battery._ClearPowerData(), True), + (self.call.battery._HardwareSetCharging(False)), + (self.call.battery.GetCharging(), False), + (self.call.battery._HardwareSetCharging(True))): + with self.battery.PowerMeasurement(): + pass + + @mock.patch('time.sleep', mock.Mock()) + def testPowerMeasurement_software(self): + self.battery._cache['profile'] = self._NEXUS_6 + with self.assertCalls( + (self.call.battery.GetCharging(), True), + (self.call.battery._ClearPowerData(), True), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'ac', '0'], check_return=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'usb', '0'], check_return=True), []), + (self.call.battery.GetCharging(), False), + (self.call.battery.GetCharging(), False), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'reset'], check_return=True), []), + (self.call.battery.GetCharging(), False), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery'], check_return=True), ['UPDATES STOPPED']), + (self.call.battery.GetCharging(), True)): + with self.battery.PowerMeasurement(): + pass + + +class BatteryUtilsDiscoverDeviceProfile(BatteryUtilsTest): + + def testDiscoverDeviceProfile_known(self): + with self.patch_call(self.call.device.product_model, + return_value='Nexus 4'): + self.battery._DiscoverDeviceProfile() + self.assertEqual(self.battery._cache['profile']['name'], "Nexus 4") + + def testDiscoverDeviceProfile_unknown(self): + with self.patch_call(self.call.device.product_model, + return_value='Other'): + self.battery._DiscoverDeviceProfile() + self.assertEqual(self.battery._cache['profile']['name'], None) + + +class BatteryUtilsClearPowerData(BatteryUtilsTest): + + def testClearPowerData_preL(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=20): + self.assertFalse(self.battery._ClearPowerData()) + + def testClearPowerData_clearedL(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=22): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'usb', '1'], check_return=True), + []), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'ac', '1'], check_return=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '--reset'], check_return=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '--charged', '-c'], + check_return=True, large_output=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'reset'], check_return=True), [])): + self.assertTrue(self.battery._ClearPowerData()) + + def testClearPowerData_notClearedL(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=22): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'usb', '1'], check_return=True), + []), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'set', 'ac', '1'], check_return=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '--reset'], check_return=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '--charged', '-c'], + check_return=True, large_output=True), + ['9,1000,l,pwi,uid,0.0327']), + (self.call.device.RunShellCommand( + ['dumpsys', 'battery', 'reset'], check_return=True), [])): + with self.assertRaises(device_errors.CommandFailedError): + self.battery._ClearPowerData() + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.DEBUG) + unittest.main(verbosity=2) diff --git a/catapult/devil/devil/android/constants/__init__.py b/catapult/devil/devil/android/constants/__init__.py new file mode 100644 index 00000000..50b23dff --- /dev/null +++ b/catapult/devil/devil/android/constants/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/catapult/devil/devil/android/constants/chrome.py b/catapult/devil/devil/android/constants/chrome.py new file mode 100644 index 00000000..5190ff93 --- /dev/null +++ b/catapult/devil/devil/android/constants/chrome.py @@ -0,0 +1,60 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import collections + +PackageInfo = collections.namedtuple( + 'PackageInfo', + ['package', 'activity', 'cmdline_file', 'devtools_socket', 'test_package']) + +PACKAGE_INFO = { + 'chrome_document': PackageInfo( + 'com.google.android.apps.chrome.document', + 'com.google.android.apps.chrome.document.ChromeLauncherActivity', + '/data/local/chrome-command-line', + 'chrome_devtools_remote', + None), + 'chrome': PackageInfo( + 'com.google.android.apps.chrome', + 'com.google.android.apps.chrome.Main', + '/data/local/chrome-command-line', + 'chrome_devtools_remote', + 'com.google.android.apps.chrome.tests'), + 'chrome_beta': PackageInfo( + 'com.chrome.beta', + 'com.google.android.apps.chrome.Main', + '/data/local/chrome-command-line', + 'chrome_devtools_remote', + None), + 'chrome_stable': PackageInfo( + 'com.android.chrome', + 'com.google.android.apps.chrome.Main', + '/data/local/chrome-command-line', + 'chrome_devtools_remote', + None), + 'chrome_dev': PackageInfo( + 'com.chrome.dev', + 'com.google.android.apps.chrome.Main', + '/data/local/chrome-command-line', + 'chrome_devtools_remote', + None), + 'chrome_canary': PackageInfo( + 'com.chrome.canary', + 'com.google.android.apps.chrome.Main', + '/data/local/chrome-command-line', + 'chrome_devtools_remote', + None), + 'chrome_work': PackageInfo( + 'com.chrome.work', + 'com.google.android.apps.chrome.Main', + '/data/local/chrome-command-line', + 'chrome_devtools_remote', + None), + 'chromium': PackageInfo( + 'org.chromium.chrome', + 'com.google.android.apps.chrome.Main', + '/data/local/chrome-command-line', + 'chrome_devtools_remote', + 'org.chromium.chrome.tests'), +} diff --git a/catapult/devil/devil/android/constants/file_system.py b/catapult/devil/devil/android/constants/file_system.py new file mode 100644 index 00000000..bffec614 --- /dev/null +++ b/catapult/devil/devil/android/constants/file_system.py @@ -0,0 +1,5 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +TEST_EXECUTABLE_DIR = '/data/local/tmp' diff --git a/catapult/devil/devil/android/decorators.py b/catapult/devil/devil/android/decorators.py new file mode 100644 index 00000000..3844b49a --- /dev/null +++ b/catapult/devil/devil/android/decorators.py @@ -0,0 +1,176 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Function/method decorators that provide timeout and retry logic. +""" + +import functools +import itertools +import sys + +from devil.android import device_errors +from devil.utils import cmd_helper +from devil.utils import reraiser_thread +from devil.utils import timeout_retry + +DEFAULT_TIMEOUT_ATTR = '_default_timeout' +DEFAULT_RETRIES_ATTR = '_default_retries' + + +def _TimeoutRetryWrapper( + f, timeout_func, retries_func, retry_if_func=timeout_retry.AlwaysRetry, + pass_values=False): + """ Wraps a funcion with timeout and retry handling logic. + + Args: + f: The function to wrap. + timeout_func: A callable that returns the timeout value. + retries_func: A callable that returns the retries value. + pass_values: If True, passes the values returned by |timeout_func| and + |retries_func| to the wrapped function as 'timeout' and + 'retries' kwargs, respectively. + Returns: + The wrapped function. + """ + @functools.wraps(f) + def timeout_retry_wrapper(*args, **kwargs): + timeout = timeout_func(*args, **kwargs) + retries = retries_func(*args, **kwargs) + if pass_values: + kwargs['timeout'] = timeout + kwargs['retries'] = retries + + @functools.wraps(f) + def impl(): + return f(*args, **kwargs) + try: + if timeout_retry.CurrentTimeoutThreadGroup(): + # Don't wrap if there's already an outer timeout thread. + return impl() + else: + desc = '%s(%s)' % (f.__name__, ', '.join(itertools.chain( + (str(a) for a in args), + ('%s=%s' % (k, str(v)) for k, v in kwargs.iteritems())))) + return timeout_retry.Run(impl, timeout, retries, desc=desc, + retry_if_func=retry_if_func) + except reraiser_thread.TimeoutError as e: + raise device_errors.CommandTimeoutError(str(e)), None, ( + sys.exc_info()[2]) + except cmd_helper.TimeoutError as e: + raise device_errors.CommandTimeoutError(str(e)), None, ( + sys.exc_info()[2]) + return timeout_retry_wrapper + + +def WithTimeoutAndRetries(f): + """A decorator that handles timeouts and retries. + + 'timeout' and 'retries' kwargs must be passed to the function. + + Args: + f: The function to decorate. + Returns: + The decorated function. + """ + get_timeout = lambda *a, **kw: kw['timeout'] + get_retries = lambda *a, **kw: kw['retries'] + return _TimeoutRetryWrapper(f, get_timeout, get_retries) + + +def WithTimeoutAndConditionalRetries(retry_if_func): + """Returns a decorator that handles timeouts and, in some cases, retries. + + 'timeout' and 'retries' kwargs must be passed to the function. + + Args: + retry_if_func: A unary callable that takes an exception and returns + whether failures should be retried. + Returns: + The actual decorator. + """ + def decorator(f): + get_timeout = lambda *a, **kw: kw['timeout'] + get_retries = lambda *a, **kw: kw['retries'] + return _TimeoutRetryWrapper( + f, get_timeout, get_retries, retry_if_func=retry_if_func) + return decorator + + +def WithExplicitTimeoutAndRetries(timeout, retries): + """Returns a decorator that handles timeouts and retries. + + The provided |timeout| and |retries| values are always used. + + Args: + timeout: The number of seconds to wait for the decorated function to + return. Always used. + retries: The number of times the decorated function should be retried on + failure. Always used. + Returns: + The actual decorator. + """ + def decorator(f): + get_timeout = lambda *a, **kw: timeout + get_retries = lambda *a, **kw: retries + return _TimeoutRetryWrapper(f, get_timeout, get_retries) + return decorator + + +def WithTimeoutAndRetriesDefaults(default_timeout, default_retries): + """Returns a decorator that handles timeouts and retries. + + The provided |default_timeout| and |default_retries| values are used only + if timeout and retries values are not provided. + + Args: + default_timeout: The number of seconds to wait for the decorated function + to return. Only used if a 'timeout' kwarg is not passed + to the decorated function. + default_retries: The number of times the decorated function should be + retried on failure. Only used if a 'retries' kwarg is not + passed to the decorated function. + Returns: + The actual decorator. + """ + def decorator(f): + get_timeout = lambda *a, **kw: kw.get('timeout', default_timeout) + get_retries = lambda *a, **kw: kw.get('retries', default_retries) + return _TimeoutRetryWrapper(f, get_timeout, get_retries, pass_values=True) + return decorator + + +def WithTimeoutAndRetriesFromInstance( + default_timeout_name=DEFAULT_TIMEOUT_ATTR, + default_retries_name=DEFAULT_RETRIES_ATTR, + min_default_timeout=None): + """Returns a decorator that handles timeouts and retries. + + The provided |default_timeout_name| and |default_retries_name| are used to + get the default timeout value and the default retries value from the object + instance if timeout and retries values are not provided. + + Note that this should only be used to decorate methods, not functions. + + Args: + default_timeout_name: The name of the default timeout attribute of the + instance. + default_retries_name: The name of the default retries attribute of the + instance. + min_timeout: Miniumum timeout to be used when using instance timeout. + Returns: + The actual decorator. + """ + def decorator(f): + def get_timeout(inst, *_args, **kwargs): + ret = getattr(inst, default_timeout_name) + if min_default_timeout is not None: + ret = max(min_default_timeout, ret) + return kwargs.get('timeout', ret) + + def get_retries(inst, *_args, **kwargs): + return kwargs.get('retries', getattr(inst, default_retries_name)) + return _TimeoutRetryWrapper(f, get_timeout, get_retries, pass_values=True) + return decorator + diff --git a/catapult/devil/devil/android/decorators_test.py b/catapult/devil/devil/android/decorators_test.py new file mode 100644 index 00000000..f60953e1 --- /dev/null +++ b/catapult/devil/devil/android/decorators_test.py @@ -0,0 +1,332 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Unit tests for decorators.py. +""" + +# pylint: disable=W0613 + +import time +import traceback +import unittest + +from devil.android import decorators +from devil.android import device_errors +from devil.utils import reraiser_thread + +_DEFAULT_TIMEOUT = 30 +_DEFAULT_RETRIES = 3 + + +class DecoratorsTest(unittest.TestCase): + _decorated_function_called_count = 0 + + def testFunctionDecoratorDoesTimeouts(self): + """Tests that the base decorator handles the timeout logic.""" + DecoratorsTest._decorated_function_called_count = 0 + + @decorators.WithTimeoutAndRetries + def alwaysTimesOut(timeout=None, retries=None): + DecoratorsTest._decorated_function_called_count += 1 + time.sleep(100) + + start_time = time.time() + with self.assertRaises(device_errors.CommandTimeoutError): + alwaysTimesOut(timeout=1, retries=0) + elapsed_time = time.time() - start_time + self.assertTrue(elapsed_time >= 1) + self.assertEquals(1, DecoratorsTest._decorated_function_called_count) + + def testFunctionDecoratorDoesRetries(self): + """Tests that the base decorator handles the retries logic.""" + DecoratorsTest._decorated_function_called_count = 0 + + @decorators.WithTimeoutAndRetries + def alwaysRaisesCommandFailedError(timeout=None, retries=None): + DecoratorsTest._decorated_function_called_count += 1 + raise device_errors.CommandFailedError('testCommand failed') + + with self.assertRaises(device_errors.CommandFailedError): + alwaysRaisesCommandFailedError(timeout=30, retries=10) + self.assertEquals(11, DecoratorsTest._decorated_function_called_count) + + def testFunctionDecoratorRequiresParams(self): + """Tests that the base decorator requires timeout and retries params.""" + @decorators.WithTimeoutAndRetries + def requiresExplicitTimeoutAndRetries(timeout=None, retries=None): + return (timeout, retries) + + with self.assertRaises(KeyError): + requiresExplicitTimeoutAndRetries() + with self.assertRaises(KeyError): + requiresExplicitTimeoutAndRetries(timeout=10) + with self.assertRaises(KeyError): + requiresExplicitTimeoutAndRetries(retries=0) + expected_timeout = 10 + expected_retries = 1 + (actual_timeout, actual_retries) = ( + requiresExplicitTimeoutAndRetries(timeout=expected_timeout, + retries=expected_retries)) + self.assertEquals(expected_timeout, actual_timeout) + self.assertEquals(expected_retries, actual_retries) + + def testFunctionDecoratorTranslatesReraiserExceptions(self): + """Tests that the explicit decorator translates reraiser exceptions.""" + @decorators.WithTimeoutAndRetries + def alwaysRaisesProvidedException(exception, timeout=None, retries=None): + raise exception + + exception_desc = 'Reraiser thread timeout error' + with self.assertRaises(device_errors.CommandTimeoutError) as e: + alwaysRaisesProvidedException( + reraiser_thread.TimeoutError(exception_desc), + timeout=10, retries=1) + self.assertEquals(exception_desc, str(e.exception)) + + def testConditionalRetriesDecoratorRetries(self): + def do_not_retry_no_adb_error(exc): + return not isinstance(exc, device_errors.NoAdbError) + + actual_tries = [0] + + @decorators.WithTimeoutAndConditionalRetries(do_not_retry_no_adb_error) + def alwaysRaisesCommandFailedError(timeout=None, retries=None): + actual_tries[0] += 1 + raise device_errors.CommandFailedError('Command failed :(') + + with self.assertRaises(device_errors.CommandFailedError): + alwaysRaisesCommandFailedError(timeout=10, retries=10) + self.assertEquals(11, actual_tries[0]) + + def testConditionalRetriesDecoratorDoesntRetry(self): + def do_not_retry_no_adb_error(exc): + return not isinstance(exc, device_errors.NoAdbError) + + actual_tries = [0] + + @decorators.WithTimeoutAndConditionalRetries(do_not_retry_no_adb_error) + def alwaysRaisesNoAdbError(timeout=None, retries=None): + actual_tries[0] += 1 + raise device_errors.NoAdbError() + + with self.assertRaises(device_errors.NoAdbError): + alwaysRaisesNoAdbError(timeout=10, retries=10) + self.assertEquals(1, actual_tries[0]) + + def testDefaultsFunctionDecoratorDoesTimeouts(self): + """Tests that the defaults decorator handles timeout logic.""" + DecoratorsTest._decorated_function_called_count = 0 + + @decorators.WithTimeoutAndRetriesDefaults(1, 0) + def alwaysTimesOut(timeout=None, retries=None): + DecoratorsTest._decorated_function_called_count += 1 + time.sleep(100) + + start_time = time.time() + with self.assertRaises(device_errors.CommandTimeoutError): + alwaysTimesOut() + elapsed_time = time.time() - start_time + self.assertTrue(elapsed_time >= 1) + self.assertEquals(1, DecoratorsTest._decorated_function_called_count) + + DecoratorsTest._decorated_function_called_count = 0 + with self.assertRaises(device_errors.CommandTimeoutError): + alwaysTimesOut(timeout=2) + elapsed_time = time.time() - start_time + self.assertTrue(elapsed_time >= 2) + self.assertEquals(1, DecoratorsTest._decorated_function_called_count) + + def testDefaultsFunctionDecoratorDoesRetries(self): + """Tests that the defaults decorator handles retries logic.""" + DecoratorsTest._decorated_function_called_count = 0 + + @decorators.WithTimeoutAndRetriesDefaults(30, 10) + def alwaysRaisesCommandFailedError(timeout=None, retries=None): + DecoratorsTest._decorated_function_called_count += 1 + raise device_errors.CommandFailedError('testCommand failed') + + with self.assertRaises(device_errors.CommandFailedError): + alwaysRaisesCommandFailedError() + self.assertEquals(11, DecoratorsTest._decorated_function_called_count) + + DecoratorsTest._decorated_function_called_count = 0 + with self.assertRaises(device_errors.CommandFailedError): + alwaysRaisesCommandFailedError(retries=5) + self.assertEquals(6, DecoratorsTest._decorated_function_called_count) + + def testDefaultsFunctionDecoratorPassesValues(self): + """Tests that the defaults decorator passes timeout and retries kwargs.""" + @decorators.WithTimeoutAndRetriesDefaults(30, 10) + def alwaysReturnsTimeouts(timeout=None, retries=None): + return timeout + + self.assertEquals(30, alwaysReturnsTimeouts()) + self.assertEquals(120, alwaysReturnsTimeouts(timeout=120)) + + @decorators.WithTimeoutAndRetriesDefaults(30, 10) + def alwaysReturnsRetries(timeout=None, retries=None): + return retries + + self.assertEquals(10, alwaysReturnsRetries()) + self.assertEquals(1, alwaysReturnsRetries(retries=1)) + + def testDefaultsFunctionDecoratorTranslatesReraiserExceptions(self): + """Tests that the explicit decorator translates reraiser exceptions.""" + @decorators.WithTimeoutAndRetriesDefaults(30, 10) + def alwaysRaisesProvidedException(exception, timeout=None, retries=None): + raise exception + + exception_desc = 'Reraiser thread timeout error' + with self.assertRaises(device_errors.CommandTimeoutError) as e: + alwaysRaisesProvidedException( + reraiser_thread.TimeoutError(exception_desc)) + self.assertEquals(exception_desc, str(e.exception)) + + def testExplicitFunctionDecoratorDoesTimeouts(self): + """Tests that the explicit decorator handles timeout logic.""" + DecoratorsTest._decorated_function_called_count = 0 + + @decorators.WithExplicitTimeoutAndRetries(1, 0) + def alwaysTimesOut(): + DecoratorsTest._decorated_function_called_count += 1 + time.sleep(100) + + start_time = time.time() + with self.assertRaises(device_errors.CommandTimeoutError): + alwaysTimesOut() + elapsed_time = time.time() - start_time + self.assertTrue(elapsed_time >= 1) + self.assertEquals(1, DecoratorsTest._decorated_function_called_count) + + def testExplicitFunctionDecoratorDoesRetries(self): + """Tests that the explicit decorator handles retries logic.""" + DecoratorsTest._decorated_function_called_count = 0 + + @decorators.WithExplicitTimeoutAndRetries(30, 10) + def alwaysRaisesCommandFailedError(): + DecoratorsTest._decorated_function_called_count += 1 + raise device_errors.CommandFailedError('testCommand failed') + + with self.assertRaises(device_errors.CommandFailedError): + alwaysRaisesCommandFailedError() + self.assertEquals(11, DecoratorsTest._decorated_function_called_count) + + def testExplicitDecoratorTranslatesReraiserExceptions(self): + """Tests that the explicit decorator translates reraiser exceptions.""" + @decorators.WithExplicitTimeoutAndRetries(30, 10) + def alwaysRaisesProvidedException(exception): + raise exception + + exception_desc = 'Reraiser thread timeout error' + with self.assertRaises(device_errors.CommandTimeoutError) as e: + alwaysRaisesProvidedException( + reraiser_thread.TimeoutError(exception_desc)) + self.assertEquals(exception_desc, str(e.exception)) + + class _MethodDecoratorTestObject(object): + """An object suitable for testing the method decorator.""" + + def __init__(self, test_case, default_timeout=_DEFAULT_TIMEOUT, + default_retries=_DEFAULT_RETRIES): + self._test_case = test_case + self.default_timeout = default_timeout + self.default_retries = default_retries + self.function_call_counters = { + 'alwaysRaisesCommandFailedError': 0, + 'alwaysTimesOut': 0, + 'requiresExplicitTimeoutAndRetries': 0, + } + + @decorators.WithTimeoutAndRetriesFromInstance( + 'default_timeout', 'default_retries') + def alwaysTimesOut(self, timeout=None, retries=None): + self.function_call_counters['alwaysTimesOut'] += 1 + time.sleep(100) + self._test_case.assertFalse(True, msg='Failed to time out?') + + @decorators.WithTimeoutAndRetriesFromInstance( + 'default_timeout', 'default_retries') + def alwaysRaisesCommandFailedError(self, timeout=None, retries=None): + self.function_call_counters['alwaysRaisesCommandFailedError'] += 1 + raise device_errors.CommandFailedError('testCommand failed') + + # pylint: disable=no-self-use + + @decorators.WithTimeoutAndRetriesFromInstance( + 'default_timeout', 'default_retries') + def alwaysReturnsTimeout(self, timeout=None, retries=None): + return timeout + + @decorators.WithTimeoutAndRetriesFromInstance( + 'default_timeout', 'default_retries', min_default_timeout=100) + def alwaysReturnsTimeoutWithMin(self, timeout=None, retries=None): + return timeout + + @decorators.WithTimeoutAndRetriesFromInstance( + 'default_timeout', 'default_retries') + def alwaysReturnsRetries(self, timeout=None, retries=None): + return retries + + @decorators.WithTimeoutAndRetriesFromInstance( + 'default_timeout', 'default_retries') + def alwaysRaisesProvidedException(self, exception, timeout=None, + retries=None): + raise exception + + # pylint: enable=no-self-use + + def testMethodDecoratorDoesTimeout(self): + """Tests that the method decorator handles timeout logic.""" + test_obj = self._MethodDecoratorTestObject(self) + start_time = time.time() + with self.assertRaises(device_errors.CommandTimeoutError): + try: + test_obj.alwaysTimesOut(timeout=1, retries=0) + except: + traceback.print_exc() + raise + elapsed_time = time.time() - start_time + self.assertTrue(elapsed_time >= 1) + self.assertEquals(1, test_obj.function_call_counters['alwaysTimesOut']) + + def testMethodDecoratorDoesRetries(self): + """Tests that the method decorator handles retries logic.""" + test_obj = self._MethodDecoratorTestObject(self) + with self.assertRaises(device_errors.CommandFailedError): + try: + test_obj.alwaysRaisesCommandFailedError(retries=10) + except: + traceback.print_exc() + raise + self.assertEquals( + 11, test_obj.function_call_counters['alwaysRaisesCommandFailedError']) + + def testMethodDecoratorPassesValues(self): + """Tests that the method decorator passes timeout and retries kwargs.""" + test_obj = self._MethodDecoratorTestObject( + self, default_timeout=42, default_retries=31) + self.assertEquals(42, test_obj.alwaysReturnsTimeout()) + self.assertEquals(41, test_obj.alwaysReturnsTimeout(timeout=41)) + self.assertEquals(31, test_obj.alwaysReturnsRetries()) + self.assertEquals(32, test_obj.alwaysReturnsRetries(retries=32)) + + def testMethodDecoratorUsesMiniumumTimeout(self): + test_obj = self._MethodDecoratorTestObject( + self, default_timeout=42, default_retries=31) + self.assertEquals(100, test_obj.alwaysReturnsTimeoutWithMin()) + self.assertEquals(41, test_obj.alwaysReturnsTimeoutWithMin(timeout=41)) + + def testMethodDecoratorTranslatesReraiserExceptions(self): + test_obj = self._MethodDecoratorTestObject(self) + + exception_desc = 'Reraiser thread timeout error' + with self.assertRaises(device_errors.CommandTimeoutError) as e: + test_obj.alwaysRaisesProvidedException( + reraiser_thread.TimeoutError(exception_desc)) + self.assertEquals(exception_desc, str(e.exception)) + +if __name__ == '__main__': + unittest.main(verbosity=2) + diff --git a/catapult/devil/devil/android/device_blacklist.py b/catapult/devil/devil/android/device_blacklist.py new file mode 100644 index 00000000..94f9cbec --- /dev/null +++ b/catapult/devil/devil/android/device_blacklist.py @@ -0,0 +1,71 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import json +import logging +import os +import threading +import time + + +class Blacklist(object): + + def __init__(self, path): + self._blacklist_lock = threading.RLock() + self._path = path + + def Read(self): + """Reads the blacklist from the blacklist file. + + Returns: + A dict containing bad devices. + """ + with self._blacklist_lock: + if not os.path.exists(self._path): + return dict() + + with open(self._path, 'r') as f: + blacklist = json.load(f) + if not isinstance(blacklist, dict): + logging.warning('Ignoring %s: %s (a dict was expected instead)', + self._path, blacklist) + blacklist = dict() + return blacklist + + def Write(self, blacklist): + """Writes the provided blacklist to the blacklist file. + + Args: + blacklist: list of bad devices to write to the blacklist file. + """ + with self._blacklist_lock: + with open(self._path, 'w') as f: + json.dump(blacklist, f) + + def Extend(self, devices, reason='unknown'): + """Adds devices to blacklist file. + + Args: + devices: list of bad devices to be added to the blacklist file. + reason: string specifying the reason for blacklist (eg: 'unauthorized') + """ + timestamp = time.time() + event_info = { + 'timestamp': timestamp, + 'reason': reason, + } + device_dicts = {device: event_info for device in devices} + logging.info('Adding %s to blacklist %s for reason: %s', + ','.join(devices), self._path, reason) + with self._blacklist_lock: + blacklist = self.Read() + blacklist.update(device_dicts) + self.Write(blacklist) + + def Reset(self): + """Erases the blacklist file if it exists.""" + logging.info('Resetting blacklist %s', self._path) + with self._blacklist_lock: + if os.path.exists(self._path): + os.remove(self._path) diff --git a/catapult/devil/devil/android/device_errors.py b/catapult/devil/devil/android/device_errors.py new file mode 100644 index 00000000..b1b8890f --- /dev/null +++ b/catapult/devil/devil/android/device_errors.py @@ -0,0 +1,124 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Exception classes raised by AdbWrapper and DeviceUtils. +""" + +from devil import base_error +from devil.utils import cmd_helper + + +class CommandFailedError(base_error.BaseError): + """Exception for command failures.""" + + def __init__(self, message, device_serial=None): + if device_serial is not None: + message = '(device: %s) %s' % (device_serial, message) + self.device_serial = device_serial + super(CommandFailedError, self).__init__(message) + + +class _BaseCommandFailedError(CommandFailedError): + """Base Exception for adb and fastboot command failures.""" + + def __init__(self, args, output, status=None, device_serial=None, + message=None): + self.args = args + self.output = output + self.status = status + if not message: + adb_cmd = ' '.join(cmd_helper.SingleQuote(arg) for arg in self.args) + message = ['adb %s: failed ' % adb_cmd] + if status: + message.append('with exit status %s ' % self.status) + if output: + message.append('and output:\n') + message.extend('- %s\n' % line for line in output.splitlines()) + else: + message.append('and no output.') + message = ''.join(message) + super(_BaseCommandFailedError, self).__init__(message, device_serial) + + +class AdbCommandFailedError(_BaseCommandFailedError): + """Exception for adb command failures.""" + + def __init__(self, args, output, status=None, device_serial=None, + message=None): + super(AdbCommandFailedError, self).__init__( + args, output, status=status, message=message, + device_serial=device_serial) + + +class FastbootCommandFailedError(_BaseCommandFailedError): + """Exception for fastboot command failures.""" + + def __init__(self, args, output, status=None, device_serial=None, + message=None): + super(FastbootCommandFailedError, self).__init__( + args, output, status=status, message=message, + device_serial=device_serial) + + +class DeviceVersionError(CommandFailedError): + """Exception for device version failures.""" + + def __init__(self, message, device_serial=None): + super(DeviceVersionError, self).__init__(message, device_serial) + + +class AdbShellCommandFailedError(AdbCommandFailedError): + """Exception for shell command failures run via adb.""" + + def __init__(self, command, output, status, device_serial=None): + self.command = command + message = ['shell command run via adb failed on the device:\n', + ' command: %s\n' % command] + message.append(' exit status: %s\n' % status) + if output: + message.append(' output:\n') + if isinstance(output, basestring): + output_lines = output.splitlines() + else: + output_lines = output + message.extend(' - %s\n' % line for line in output_lines) + else: + message.append(" output: ''\n") + message = ''.join(message) + super(AdbShellCommandFailedError, self).__init__( + ['shell', command], output, status, device_serial, message) + + +class CommandTimeoutError(base_error.BaseError): + """Exception for command timeouts.""" + pass + + +class DeviceUnreachableError(base_error.BaseError): + """Exception for device unreachable failures.""" + pass + + +class NoDevicesError(base_error.BaseError): + """Exception for having no devices attached.""" + + def __init__(self): + super(NoDevicesError, self).__init__( + 'No devices attached.', is_infra_error=True) + + +class NoAdbError(base_error.BaseError): + """Exception for being unable to find ADB.""" + + def __init__(self, msg=None): + super(NoAdbError, self).__init__( + msg or 'Unable to find adb.', is_infra_error=True) + + +class DeviceChargingError(CommandFailedError): + """Exception for device charging errors.""" + + def __init__(self, message, device_serial=None): + super(DeviceChargingError, self).__init__(message, device_serial) diff --git a/catapult/devil/devil/android/device_list.py b/catapult/devil/devil/android/device_list.py new file mode 100644 index 00000000..0eb6acba --- /dev/null +++ b/catapult/devil/devil/android/device_list.py @@ -0,0 +1,30 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A module to keep track of devices across builds.""" + +import os + +LAST_DEVICES_FILENAME = '.last_devices' +LAST_MISSING_DEVICES_FILENAME = '.last_missing' + + +def GetPersistentDeviceList(file_name): + """Returns a list of devices. + + Args: + file_name: the file name containing a list of devices. + + Returns: List of device serial numbers that were on the bot. + """ + with open(file_name) as f: + return f.read().splitlines() + + +def WritePersistentDeviceList(file_name, device_list): + path = os.path.dirname(file_name) + if not os.path.exists(path): + os.makedirs(path) + with open(file_name, 'w') as f: + f.write('\n'.join(set(device_list))) diff --git a/catapult/devil/devil/android/device_signal.py b/catapult/devil/devil/android/device_signal.py new file mode 100644 index 00000000..2cec46d7 --- /dev/null +++ b/catapult/devil/devil/android/device_signal.py @@ -0,0 +1,41 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Defines constants for signals that should be supported on devices. + +Note: Obtained by running `kill -l` on a user device. +""" + + +SIGHUP = 1 # Hangup +SIGINT = 2 # Interrupt +SIGQUIT = 3 # Quit +SIGILL = 4 # Illegal instruction +SIGTRAP = 5 # Trap +SIGABRT = 6 # Aborted +SIGBUS = 7 # Bus error +SIGFPE = 8 # Floating point exception +SIGKILL = 9 # Killed +SIGUSR1 = 10 # User signal 1 +SIGSEGV = 11 # Segmentation fault +SIGUSR2 = 12 # User signal 2 +SIGPIPE = 13 # Broken pipe +SIGALRM = 14 # Alarm clock +SIGTERM = 15 # Terminated +SIGSTKFLT = 16 # Stack fault +SIGCHLD = 17 # Child exited +SIGCONT = 18 # Continue +SIGSTOP = 19 # Stopped (signal) +SIGTSTP = 20 # Stopped +SIGTTIN = 21 # Stopped (tty input) +SIGTTOU = 22 # Stopped (tty output) +SIGURG = 23 # Urgent I/O condition +SIGXCPU = 24 # CPU time limit exceeded +SIGXFSZ = 25 # File size limit exceeded +SIGVTALRM = 26 # Virtual timer expired +SIGPROF = 27 # Profiling timer expired +SIGWINCH = 28 # Window size changed +SIGIO = 29 # I/O possible +SIGPWR = 30 # Power failure +SIGSYS = 31 # Bad system call diff --git a/catapult/devil/devil/android/device_temp_file.py b/catapult/devil/devil/android/device_temp_file.py new file mode 100644 index 00000000..75488c50 --- /dev/null +++ b/catapult/devil/devil/android/device_temp_file.py @@ -0,0 +1,56 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A temp file that automatically gets pushed and deleted from a device.""" + +# pylint: disable=W0622 + +import posixpath +import random +import threading + +from devil.android import device_errors +from devil.utils import cmd_helper + + +class DeviceTempFile(object): + + def __init__(self, adb, suffix='', prefix='temp_file', dir='/data/local/tmp'): + """Find an unused temporary file path on the device. + + When this object is closed, the file will be deleted on the device. + + Args: + adb: An instance of AdbWrapper + suffix: The suffix of the name of the temp file. + prefix: The prefix of the name of the temp file. + dir: The directory on the device where to place the temp file. + """ + self._adb = adb + # Python's random module use 52-bit numbers according to its docs. + random_hex = hex(random.randint(0, 2 ** 52))[2:] + self.name = posixpath.join(dir, '%s-%s%s' % (prefix, random_hex, suffix)) + self.name_quoted = cmd_helper.SingleQuote(self.name) + + def close(self): + """Deletes the temporary file from the device.""" + # ignore exception if the file is already gone. + def delete_temporary_file(): + try: + self._adb.Shell('rm -f %s' % self.name_quoted, expect_status=None) + except device_errors.AdbCommandFailedError: + # file does not exist on Android version without 'rm -f' support (ICS) + pass + + # It shouldn't matter when the temp file gets deleted, so do so + # asynchronously. + threading.Thread( + target=delete_temporary_file, + name='delete_temporary_file(%s)' % self._adb.GetDeviceSerial()).start() + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() diff --git a/catapult/devil/devil/android/device_utils.py b/catapult/devil/devil/android/device_utils.py new file mode 100644 index 00000000..5cea40ba --- /dev/null +++ b/catapult/devil/devil/android/device_utils.py @@ -0,0 +1,2180 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Provides a variety of device interactions based on adb. + +Eventually, this will be based on adb_wrapper. +""" +# pylint: disable=unused-argument + +import collections +import itertools +import json +import logging +import multiprocessing +import os +import posixpath +import re +import shutil +import tempfile +import time +import zipfile + +from devil import base_error +from devil import devil_env +from devil.utils import cmd_helper +from devil.android import apk_helper +from devil.android import device_signal +from devil.android import decorators +from devil.android import device_errors +from devil.android import device_temp_file +from devil.android import install_commands +from devil.android import logcat_monitor +from devil.android import md5sum +from devil.android.sdk import adb_wrapper +from devil.android.sdk import gce_adb_wrapper +from devil.android.sdk import intent +from devil.android.sdk import keyevent +from devil.android.sdk import split_select +from devil.android.sdk import version_codes +from devil.utils import host_utils +from devil.utils import parallelizer +from devil.utils import reraiser_thread +from devil.utils import timeout_retry +from devil.utils import zip_utils + +_DEFAULT_TIMEOUT = 30 +_DEFAULT_RETRIES = 3 + +# A sentinel object for default values +# TODO(jbudorick,perezju): revisit how default values are handled by +# the timeout_retry decorators. +DEFAULT = object() + +_RESTART_ADBD_SCRIPT = """ + trap '' HUP + trap '' TERM + trap '' PIPE + function restart() { + stop adbd + start adbd + } + restart & +""" + +# Not all permissions can be set. +_PERMISSIONS_BLACKLIST = [ + 'android.permission.ACCESS_MOCK_LOCATION', + 'android.permission.ACCESS_NETWORK_STATE', + 'android.permission.ACCESS_WIFI_STATE', + 'android.permission.AUTHENTICATE_ACCOUNTS', + 'android.permission.BLUETOOTH', + 'android.permission.BLUETOOTH_ADMIN', + 'android.permission.DOWNLOAD_WITHOUT_NOTIFICATION', + 'android.permission.INTERNET', + 'android.permission.MANAGE_ACCOUNTS', + 'android.permission.MODIFY_AUDIO_SETTINGS', + 'android.permission.NFC', + 'android.permission.READ_SYNC_SETTINGS', + 'android.permission.READ_SYNC_STATS', + 'android.permission.RECEIVE_BOOT_COMPLETED', + 'android.permission.RECORD_VIDEO', + 'android.permission.RUN_INSTRUMENTATION', + 'android.permission.USE_CREDENTIALS', + 'android.permission.VIBRATE', + 'android.permission.WAKE_LOCK', + 'android.permission.WRITE_SYNC_SETTINGS', + 'com.android.browser.permission.READ_HISTORY_BOOKMARKS', + 'com.android.browser.permission.WRITE_HISTORY_BOOKMARKS', + 'com.android.launcher.permission.INSTALL_SHORTCUT', + 'com.chrome.permission.DEVICE_EXTRAS', + 'com.google.android.apps.chrome.permission.C2D_MESSAGE', + 'com.google.android.apps.chrome.permission.READ_WRITE_BOOKMARK_FOLDERS', + 'com.google.android.apps.chrome.TOS_ACKED', + 'com.google.android.c2dm.permission.RECEIVE', + 'com.google.android.providers.gsf.permission.READ_GSERVICES', + 'com.sec.enterprise.knox.MDM_CONTENT_PROVIDER', + 'org.chromium.chrome.permission.C2D_MESSAGE', + 'org.chromium.chrome.permission.READ_WRITE_BOOKMARK_FOLDERS', + 'org.chromium.chrome.TOS_ACKED', +] + +_CURRENT_FOCUS_CRASH_RE = re.compile( + r'\s*mCurrentFocus.*Application (Error|Not Responding): (\S+)}') + +_GETPROP_RE = re.compile(r'\[(.*?)\]: \[(.*?)\]') +_IPV4_ADDRESS_RE = re.compile(r'([0-9]{1,3}\.){3}[0-9]{1,3}\:[0-9]{4,5}') + + +@decorators.WithExplicitTimeoutAndRetries( + _DEFAULT_TIMEOUT, _DEFAULT_RETRIES) +def GetAVDs(): + """Returns a list of Android Virtual Devices. + + Returns: + A list containing the configured AVDs. + """ + lines = cmd_helper.GetCmdOutput([ + os.path.join(devil_env.config.LocalPath('android_sdk'), + 'tools', 'android'), + 'list', 'avd']).splitlines() + avds = [] + for line in lines: + if 'Name:' not in line: + continue + key, value = (s.strip() for s in line.split(':', 1)) + if key == 'Name': + avds.append(value) + return avds + + +@decorators.WithExplicitTimeoutAndRetries( + _DEFAULT_TIMEOUT, _DEFAULT_RETRIES) +def RestartServer(): + """Restarts the adb server. + + Raises: + CommandFailedError if we fail to kill or restart the server. + """ + def adb_killed(): + return not adb_wrapper.AdbWrapper.IsServerOnline() + + def adb_started(): + return adb_wrapper.AdbWrapper.IsServerOnline() + + adb_wrapper.AdbWrapper.KillServer() + if not timeout_retry.WaitFor(adb_killed, wait_period=1, max_tries=5): + # TODO(perezju): raise an exception after fixng http://crbug.com/442319 + logging.warning('Failed to kill adb server') + adb_wrapper.AdbWrapper.StartServer() + if not timeout_retry.WaitFor(adb_started, wait_period=1, max_tries=5): + raise device_errors.CommandFailedError('Failed to start adb server') + + +def _GetTimeStamp(): + """Return a basic ISO 8601 time stamp with the current local time.""" + return time.strftime('%Y%m%dT%H%M%S', time.localtime()) + + +def _JoinLines(lines): + # makes sure that the last line is also terminated, and is more memory + # efficient than first appending an end-line to each line and then joining + # all of them together. + return ''.join(s for line in lines for s in (line, '\n')) + + +def _IsGceInstance(serial): + return _IPV4_ADDRESS_RE.match(serial) + + +def _CreateAdbWrapper(device): + if _IsGceInstance(str(device)): + return gce_adb_wrapper.GceAdbWrapper(str(device)) + else: + if isinstance(device, adb_wrapper.AdbWrapper): + return device + else: + return adb_wrapper.AdbWrapper(device) + + +class DeviceUtils(object): + + _MAX_ADB_COMMAND_LENGTH = 512 + _MAX_ADB_OUTPUT_LENGTH = 32768 + _LAUNCHER_FOCUSED_RE = re.compile( + r'\s*mCurrentFocus.*(Launcher|launcher).*') + _VALID_SHELL_VARIABLE = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$') + + LOCAL_PROPERTIES_PATH = posixpath.join('/', 'data', 'local.prop') + + # Property in /data/local.prop that controls Java assertions. + JAVA_ASSERT_PROPERTY = 'dalvik.vm.enableassertions' + + def __init__(self, device, enable_device_files_cache=False, + default_timeout=_DEFAULT_TIMEOUT, + default_retries=_DEFAULT_RETRIES): + """DeviceUtils constructor. + + Args: + device: Either a device serial, an existing AdbWrapper instance, or an + an existing AndroidCommands instance. + enable_device_files_cache: For PushChangedFiles(), cache checksums of + pushed files rather than recomputing them on a subsequent call. + default_timeout: An integer containing the default number of seconds to + wait for an operation to complete if no explicit value is provided. + default_retries: An integer containing the default number or times an + operation should be retried on failure if no explicit value is provided. + """ + self.adb = None + if isinstance(device, basestring): + self.adb = _CreateAdbWrapper(device) + elif isinstance(device, adb_wrapper.AdbWrapper): + self.adb = device + else: + raise ValueError('Unsupported device value: %r' % device) + self._commands_installed = None + self._default_timeout = default_timeout + self._default_retries = default_retries + self._enable_device_files_cache = enable_device_files_cache + self._cache = {} + self._client_caches = {} + assert hasattr(self, decorators.DEFAULT_TIMEOUT_ATTR) + assert hasattr(self, decorators.DEFAULT_RETRIES_ATTR) + + self._ClearCache() + + def __eq__(self, other): + """Checks whether |other| refers to the same device as |self|. + + Args: + other: The object to compare to. This can be a basestring, an instance + of adb_wrapper.AdbWrapper, or an instance of DeviceUtils. + Returns: + Whether |other| refers to the same device as |self|. + """ + return self.adb.GetDeviceSerial() == str(other) + + def __lt__(self, other): + """Compares two instances of DeviceUtils. + + This merely compares their serial numbers. + + Args: + other: The instance of DeviceUtils to compare to. + Returns: + Whether |self| is less than |other|. + """ + return self.adb.GetDeviceSerial() < other.adb.GetDeviceSerial() + + def __str__(self): + """Returns the device serial.""" + return self.adb.GetDeviceSerial() + + @decorators.WithTimeoutAndRetriesFromInstance() + def IsOnline(self, timeout=None, retries=None): + """Checks whether the device is online. + + Args: + timeout: timeout in seconds + retries: number of retries + + Returns: + True if the device is online, False otherwise. + + Raises: + CommandTimeoutError on timeout. + """ + try: + return self.adb.GetState() == 'device' + except base_error.BaseError as exc: + logging.info('Failed to get state: %s', exc) + return False + + @decorators.WithTimeoutAndRetriesFromInstance() + def HasRoot(self, timeout=None, retries=None): + """Checks whether or not adbd has root privileges. + + Args: + timeout: timeout in seconds + retries: number of retries + + Returns: + True if adbd has root privileges, False otherwise. + + Raises: + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + try: + self.RunShellCommand('ls /root', check_return=True) + return True + except device_errors.AdbCommandFailedError: + return False + + def NeedsSU(self, timeout=DEFAULT, retries=DEFAULT): + """Checks whether 'su' is needed to access protected resources. + + Args: + timeout: timeout in seconds + retries: number of retries + + Returns: + True if 'su' is available on the device and is needed to to access + protected resources; False otherwise if either 'su' is not available + (e.g. because the device has a user build), or not needed (because adbd + already has root privileges). + + Raises: + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + if 'needs_su' not in self._cache: + try: + self.RunShellCommand( + '%s && ! ls /root' % self._Su('ls /root'), check_return=True, + timeout=self._default_timeout if timeout is DEFAULT else timeout, + retries=self._default_retries if retries is DEFAULT else retries) + self._cache['needs_su'] = True + except device_errors.AdbCommandFailedError: + self._cache['needs_su'] = False + return self._cache['needs_su'] + + def _Su(self, command): + if self.build_version_sdk >= version_codes.MARSHMALLOW: + return 'su 0 %s' % command + return 'su -c %s' % command + + @decorators.WithTimeoutAndRetriesFromInstance() + def EnableRoot(self, timeout=None, retries=None): + """Restarts adbd with root privileges. + + Args: + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandFailedError if root could not be enabled. + CommandTimeoutError on timeout. + """ + if self.IsUserBuild(): + raise device_errors.CommandFailedError( + 'Cannot enable root in user builds.', str(self)) + if 'needs_su' in self._cache: + del self._cache['needs_su'] + self.adb.Root() + self.WaitUntilFullyBooted() + + @decorators.WithTimeoutAndRetriesFromInstance() + def IsUserBuild(self, timeout=None, retries=None): + """Checks whether or not the device is running a user build. + + Args: + timeout: timeout in seconds + retries: number of retries + + Returns: + True if the device is running a user build, False otherwise (i.e. if + it's running a userdebug build). + + Raises: + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + return self.build_type == 'user' + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetExternalStoragePath(self, timeout=None, retries=None): + """Get the device's path to its SD card. + + Args: + timeout: timeout in seconds + retries: number of retries + + Returns: + The device's path to its SD card. + + Raises: + CommandFailedError if the external storage path could not be determined. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + if 'external_storage' in self._cache: + return self._cache['external_storage'] + + value = self.RunShellCommand('echo $EXTERNAL_STORAGE', + single_line=True, + check_return=True) + if not value: + raise device_errors.CommandFailedError('$EXTERNAL_STORAGE is not set', + str(self)) + self._cache['external_storage'] = value + return value + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetApplicationPaths(self, package, timeout=None, retries=None): + """Get the paths of the installed apks on the device for the given package. + + Args: + package: Name of the package. + + Returns: + List of paths to the apks on the device for the given package. + """ + return self._GetApplicationPathsInternal(package) + + def _GetApplicationPathsInternal(self, package, skip_cache=False): + cached_result = self._cache['package_apk_paths'].get(package) + if cached_result is not None and not skip_cache: + if package in self._cache['package_apk_paths_to_verify']: + self._cache['package_apk_paths_to_verify'].remove(package) + # Don't verify an app that is not thought to be installed. We are + # concerned only with apps we think are installed having been + # uninstalled manually. + if cached_result and not self.PathExists(cached_result): + cached_result = None + self._cache['package_apk_checksums'].pop(package, 0) + if cached_result is not None: + return list(cached_result) + # 'pm path' is liable to incorrectly exit with a nonzero number starting + # in Lollipop. + # TODO(jbudorick): Check if this is fixed as new Android versions are + # released to put an upper bound on this. + should_check_return = (self.build_version_sdk < version_codes.LOLLIPOP) + output = self.RunShellCommand( + ['pm', 'path', package], check_return=should_check_return) + apks = [] + for line in output: + if not line.startswith('package:'): + raise device_errors.CommandFailedError( + 'pm path returned: %r' % '\n'.join(output), str(self)) + apks.append(line[len('package:'):]) + self._cache['package_apk_paths'][package] = list(apks) + return apks + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetApplicationVersion(self, package, timeout=None, retries=None): + """Get the version name of a package installed on the device. + + Args: + package: Name of the package. + + Returns: + A string with the version name or None if the package is not found + on the device. + """ + output = self.RunShellCommand( + ['dumpsys', 'package', package], check_return=True) + if not output: + return None + for line in output: + line = line.strip() + if line.startswith('versionName='): + return line[len('versionName='):] + raise device_errors.CommandFailedError( + 'Version name for %s not found on dumpsys output' % package, str(self)) + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetApplicationDataDirectory(self, package, timeout=None, retries=None): + """Get the data directory on the device for the given package. + + Args: + package: Name of the package. + + Returns: + The package's data directory, or None if the package doesn't exist on the + device. + """ + try: + output = self._RunPipedShellCommand( + 'pm dump %s | grep dataDir=' % cmd_helper.SingleQuote(package)) + for line in output: + _, _, dataDir = line.partition('dataDir=') + if dataDir: + return dataDir + except device_errors.CommandFailedError: + logging.exception('Could not find data directory for %s', package) + return None + + @decorators.WithTimeoutAndRetriesFromInstance() + def WaitUntilFullyBooted(self, wifi=False, timeout=None, retries=None): + """Wait for the device to fully boot. + + This means waiting for the device to boot, the package manager to be + available, and the SD card to be ready. It can optionally mean waiting + for wifi to come up, too. + + Args: + wifi: A boolean indicating if we should wait for wifi to come up or not. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandFailedError on failure. + CommandTimeoutError if one of the component waits times out. + DeviceUnreachableError if the device becomes unresponsive. + """ + def sd_card_ready(): + try: + self.RunShellCommand(['test', '-d', self.GetExternalStoragePath()], + check_return=True) + return True + except device_errors.AdbCommandFailedError: + return False + + def pm_ready(): + try: + return self._GetApplicationPathsInternal('android', skip_cache=True) + except device_errors.CommandFailedError: + return False + + def boot_completed(): + return self.GetProp('sys.boot_completed', cache=False) == '1' + + def wifi_enabled(): + return 'Wi-Fi is enabled' in self.RunShellCommand(['dumpsys', 'wifi'], + check_return=False) + + self.adb.WaitForDevice() + timeout_retry.WaitFor(sd_card_ready) + timeout_retry.WaitFor(pm_ready) + timeout_retry.WaitFor(boot_completed) + if wifi: + timeout_retry.WaitFor(wifi_enabled) + + REBOOT_DEFAULT_TIMEOUT = 10 * _DEFAULT_TIMEOUT + + @decorators.WithTimeoutAndRetriesFromInstance( + min_default_timeout=REBOOT_DEFAULT_TIMEOUT) + def Reboot(self, block=True, wifi=False, timeout=None, retries=None): + """Reboot the device. + + Args: + block: A boolean indicating if we should wait for the reboot to complete. + wifi: A boolean indicating if we should wait for wifi to be enabled after + the reboot. The option has no effect unless |block| is also True. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + def device_offline(): + return not self.IsOnline() + + self.adb.Reboot() + self._ClearCache() + timeout_retry.WaitFor(device_offline, wait_period=1) + if block: + self.WaitUntilFullyBooted(wifi=wifi) + + INSTALL_DEFAULT_TIMEOUT = 4 * _DEFAULT_TIMEOUT + + @decorators.WithTimeoutAndRetriesFromInstance( + min_default_timeout=INSTALL_DEFAULT_TIMEOUT) + def Install(self, apk, allow_downgrade=False, reinstall=False, + permissions=None, timeout=None, retries=None): + """Install an APK. + + Noop if an identical APK is already installed. + + Args: + apk: An ApkHelper instance or string containing the path to the APK. + allow_downgrade: A boolean indicating if we should allow downgrades. + reinstall: A boolean indicating if we should keep any existing app data. + permissions: Set of permissions to set. If not set, finds permissions with + apk helper. To set no permissions, pass []. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandFailedError if the installation fails. + CommandTimeoutError if the installation times out. + DeviceUnreachableError on missing device. + """ + self._InstallInternal(apk, None, allow_downgrade=allow_downgrade, + reinstall=reinstall, permissions=permissions) + + @decorators.WithTimeoutAndRetriesFromInstance( + min_default_timeout=INSTALL_DEFAULT_TIMEOUT) + def InstallSplitApk(self, base_apk, split_apks, allow_downgrade=False, + reinstall=False, allow_cached_props=False, + permissions=None, timeout=None, retries=None): + """Install a split APK. + + Noop if all of the APK splits are already installed. + + Args: + base_apk: An ApkHelper instance or string containing the path to the base + APK. + split_apks: A list of strings of paths of all of the APK splits. + allow_downgrade: A boolean indicating if we should allow downgrades. + reinstall: A boolean indicating if we should keep any existing app data. + allow_cached_props: Whether to use cached values for device properties. + permissions: Set of permissions to set. If not set, finds permissions with + apk helper. To set no permissions, pass []. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandFailedError if the installation fails. + CommandTimeoutError if the installation times out. + DeviceUnreachableError on missing device. + DeviceVersionError if device SDK is less than Android L. + """ + self._InstallInternal(base_apk, split_apks, reinstall=reinstall, + allow_cached_props=allow_cached_props, + permissions=permissions, + allow_downgrade=allow_downgrade) + + def _InstallInternal(self, base_apk, split_apks, allow_downgrade=False, + reinstall=False, allow_cached_props=False, + permissions=None): + if split_apks: + self._CheckSdkLevel(version_codes.LOLLIPOP) + + base_apk = apk_helper.ToHelper(base_apk) + + all_apks = [base_apk.path] + if split_apks: + all_apks += split_select.SelectSplits( + self, base_apk.path, split_apks, allow_cached_props=allow_cached_props) + if len(all_apks) == 1: + logging.warning('split-select did not select any from %s', split_apks) + + package_name = base_apk.GetPackageName() + device_apk_paths = self._GetApplicationPathsInternal(package_name) + + apks_to_install = None + host_checksums = None + if not device_apk_paths: + apks_to_install = all_apks + elif len(device_apk_paths) > 1 and not split_apks: + logging.warning( + 'Installing non-split APK when split APK was previously installed') + apks_to_install = all_apks + elif len(device_apk_paths) == 1 and split_apks: + logging.warning( + 'Installing split APK when non-split APK was previously installed') + apks_to_install = all_apks + else: + try: + apks_to_install, host_checksums = ( + self._ComputeStaleApks(package_name, all_apks)) + except EnvironmentError as e: + logging.warning('Error calculating md5: %s', e) + apks_to_install, host_checksums = all_apks, None + if apks_to_install and not reinstall: + self.Uninstall(package_name) + apks_to_install = all_apks + + if apks_to_install: + # Assume that we won't know the resulting device state. + self._cache['package_apk_paths'].pop(package_name, 0) + self._cache['package_apk_checksums'].pop(package_name, 0) + if split_apks: + partial = package_name if len(apks_to_install) < len(all_apks) else None + self.adb.InstallMultiple( + apks_to_install, partial=partial, reinstall=reinstall, + allow_downgrade=allow_downgrade) + else: + self.adb.Install( + base_apk.path, reinstall=reinstall, allow_downgrade=allow_downgrade) + if (permissions is None + and self.build_version_sdk >= version_codes.MARSHMALLOW): + permissions = base_apk.GetPermissions() + self.GrantPermissions(package_name, permissions) + # Upon success, we know the device checksums, but not their paths. + if host_checksums is not None: + self._cache['package_apk_checksums'][package_name] = host_checksums + else: + # Running adb install terminates running instances of the app, so to be + # consistent, we explicitly terminate it when skipping the install. + self.ForceStop(package_name) + + @decorators.WithTimeoutAndRetriesFromInstance() + def Uninstall(self, package_name, keep_data=False, timeout=None, + retries=None): + """Remove the app |package_name| from the device. + + This is a no-op if the app is not already installed. + + Args: + package_name: The package to uninstall. + keep_data: (optional) Whether to keep the data and cache directories. + timeout: Timeout in seconds. + retries: Number of retries. + + Raises: + CommandFailedError if the uninstallation fails. + CommandTimeoutError if the uninstallation times out. + DeviceUnreachableError on missing device. + """ + installed = self._GetApplicationPathsInternal(package_name) + if not installed: + return + try: + self.adb.Uninstall(package_name, keep_data) + self._cache['package_apk_paths'][package_name] = [] + self._cache['package_apk_checksums'][package_name] = set() + except: + # Clear cache since we can't be sure of the state. + self._cache['package_apk_paths'].pop(package_name, 0) + self._cache['package_apk_checksums'].pop(package_name, 0) + raise + + def _CheckSdkLevel(self, required_sdk_level): + """Raises an exception if the device does not have the required SDK level. + """ + if self.build_version_sdk < required_sdk_level: + raise device_errors.DeviceVersionError( + ('Requires SDK level %s, device is SDK level %s' % + (required_sdk_level, self.build_version_sdk)), + device_serial=self.adb.GetDeviceSerial()) + + @decorators.WithTimeoutAndRetriesFromInstance() + def RunShellCommand(self, cmd, check_return=False, cwd=None, env=None, + as_root=False, single_line=False, large_output=False, + timeout=None, retries=None): + """Run an ADB shell command. + + The command to run |cmd| should be a sequence of program arguments or else + a single string. + + When |cmd| is a sequence, it is assumed to contain the name of the command + to run followed by its arguments. In this case, arguments are passed to the + command exactly as given, without any further processing by the shell. This + allows to easily pass arguments containing spaces or special characters + without having to worry about getting quoting right. Whenever possible, it + is recomended to pass |cmd| as a sequence. + + When |cmd| is given as a string, it will be interpreted and run by the + shell on the device. + + This behaviour is consistent with that of command runners in cmd_helper as + well as Python's own subprocess.Popen. + + TODO(perezju) Change the default of |check_return| to True when callers + have switched to the new behaviour. + + Args: + cmd: A string with the full command to run on the device, or a sequence + containing the command and its arguments. + check_return: A boolean indicating whether or not the return code should + be checked. + cwd: The device directory in which the command should be run. + env: The environment variables with which the command should be run. + as_root: A boolean indicating whether the shell command should be run + with root privileges. + single_line: A boolean indicating if only a single line of output is + expected. + large_output: Uses a work-around for large shell command output. Without + this large output will be truncated. + timeout: timeout in seconds + retries: number of retries + + Returns: + If single_line is False, the output of the command as a list of lines, + otherwise, a string with the unique line of output emmited by the command + (with the optional newline at the end stripped). + + Raises: + AdbCommandFailedError if check_return is True and the exit code of + the command run on the device is non-zero. + CommandFailedError if single_line is True but the output contains two or + more lines. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + def env_quote(key, value): + if not DeviceUtils._VALID_SHELL_VARIABLE.match(key): + raise KeyError('Invalid shell variable name %r' % key) + # using double quotes here to allow interpolation of shell variables + return '%s=%s' % (key, cmd_helper.DoubleQuote(value)) + + def run(cmd): + return self.adb.Shell(cmd) + + def handle_check_return(cmd): + try: + return run(cmd) + except device_errors.AdbCommandFailedError as exc: + if check_return: + raise + else: + return exc.output + + def handle_large_command(cmd): + if len(cmd) < self._MAX_ADB_COMMAND_LENGTH: + return handle_check_return(cmd) + else: + with device_temp_file.DeviceTempFile(self.adb, suffix='.sh') as script: + self._WriteFileWithPush(script.name, cmd) + logging.info('Large shell command will be run from file: %s ...', + cmd[:self._MAX_ADB_COMMAND_LENGTH]) + return handle_check_return('sh %s' % script.name_quoted) + + def handle_large_output(cmd, large_output_mode): + if large_output_mode: + with device_temp_file.DeviceTempFile(self.adb) as large_output_file: + cmd = '( %s )>%s' % (cmd, large_output_file.name) + logging.debug('Large output mode enabled. Will write output to ' + 'device and read results from file.') + handle_large_command(cmd) + return self.ReadFile(large_output_file.name, force_pull=True) + else: + try: + return handle_large_command(cmd) + except device_errors.AdbCommandFailedError as exc: + if exc.status is None: + logging.exception('No output found for %s', cmd) + logging.warning('Attempting to run in large_output mode.') + logging.warning('Use RunShellCommand(..., large_output=True) for ' + 'shell commands that expect a lot of output.') + return handle_large_output(cmd, True) + else: + raise + + if not isinstance(cmd, basestring): + cmd = ' '.join(cmd_helper.SingleQuote(s) for s in cmd) + if env: + env = ' '.join(env_quote(k, v) for k, v in env.iteritems()) + cmd = '%s %s' % (env, cmd) + if cwd: + cmd = 'cd %s && %s' % (cmd_helper.SingleQuote(cwd), cmd) + if as_root and self.NeedsSU(): + # "su -c sh -c" allows using shell features in |cmd| + cmd = self._Su('sh -c %s' % cmd_helper.SingleQuote(cmd)) + + output = handle_large_output(cmd, large_output).splitlines() + + if single_line: + if not output: + return '' + elif len(output) == 1: + return output[0] + else: + msg = 'one line of output was expected, but got: %s' + raise device_errors.CommandFailedError(msg % output, str(self)) + else: + return output + + def _RunPipedShellCommand(self, script, **kwargs): + PIPESTATUS_LEADER = 'PIPESTATUS: ' + + script += '; echo "%s${PIPESTATUS[@]}"' % PIPESTATUS_LEADER + kwargs['check_return'] = True + output = self.RunShellCommand(script, **kwargs) + pipestatus_line = output[-1] + + if not pipestatus_line.startswith(PIPESTATUS_LEADER): + logging.error('Pipe exit statuses of shell script missing.') + raise device_errors.AdbShellCommandFailedError( + script, output, status=None, + device_serial=self.adb.GetDeviceSerial()) + + output = output[:-1] + statuses = [ + int(s) for s in pipestatus_line[len(PIPESTATUS_LEADER):].split()] + if any(statuses): + raise device_errors.AdbShellCommandFailedError( + script, output, status=statuses, + device_serial=self.adb.GetDeviceSerial()) + return output + + @decorators.WithTimeoutAndRetriesFromInstance() + def KillAll(self, process_name, exact=False, signum=device_signal.SIGKILL, + as_root=False, blocking=False, quiet=False, + timeout=None, retries=None): + """Kill all processes with the given name on the device. + + Args: + process_name: A string containing the name of the process to kill. + exact: A boolean indicating whether to kill all processes matching + the string |process_name| exactly, or all of those which contain + |process_name| as a substring. Defaults to False. + signum: An integer containing the signal number to send to kill. Defaults + to SIGKILL (9). + as_root: A boolean indicating whether the kill should be executed with + root privileges. + blocking: A boolean indicating whether we should wait until all processes + with the given |process_name| are dead. + quiet: A boolean indicating whether to ignore the fact that no processes + to kill were found. + timeout: timeout in seconds + retries: number of retries + + Returns: + The number of processes attempted to kill. + + Raises: + CommandFailedError if no process was killed and |quiet| is False. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + procs_pids = self.GetPids(process_name) + if exact: + procs_pids = {process_name: procs_pids.get(process_name, [])} + pids = set(itertools.chain(*procs_pids.values())) + if not pids: + if quiet: + return 0 + else: + raise device_errors.CommandFailedError( + 'No process "%s"' % process_name, str(self)) + + logging.info( + 'KillAll(%r, ...) attempting to kill the following:', process_name) + for name, ids in procs_pids.iteritems(): + for i in ids: + logging.info(' %05s %s', str(i), name) + + cmd = ['kill', '-%d' % signum] + sorted(pids) + self.RunShellCommand(cmd, as_root=as_root, check_return=True) + + def all_pids_killed(): + procs_pids_remain = self.GetPids(process_name) + return not pids.intersection(itertools.chain(*procs_pids_remain.values())) + + if blocking: + timeout_retry.WaitFor(all_pids_killed, wait_period=0.1) + + return len(pids) + + @decorators.WithTimeoutAndRetriesFromInstance() + def StartActivity(self, intent_obj, blocking=False, trace_file_name=None, + force_stop=False, timeout=None, retries=None): + """Start package's activity on the device. + + Args: + intent_obj: An Intent object to send. + blocking: A boolean indicating whether we should wait for the activity to + finish launching. + trace_file_name: If present, a string that both indicates that we want to + profile the activity and contains the path to which the + trace should be saved. + force_stop: A boolean indicating whether we should stop the activity + before starting it. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandFailedError if the activity could not be started. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + cmd = ['am', 'start'] + if blocking: + cmd.append('-W') + if trace_file_name: + cmd.extend(['--start-profiler', trace_file_name]) + if force_stop: + cmd.append('-S') + cmd.extend(intent_obj.am_args) + for line in self.RunShellCommand(cmd, check_return=True): + if line.startswith('Error:'): + raise device_errors.CommandFailedError(line, str(self)) + + @decorators.WithTimeoutAndRetriesFromInstance() + def StartInstrumentation(self, component, finish=True, raw=False, + extras=None, timeout=None, retries=None): + if extras is None: + extras = {} + + cmd = ['am', 'instrument'] + if finish: + cmd.append('-w') + if raw: + cmd.append('-r') + for k, v in extras.iteritems(): + cmd.extend(['-e', str(k), str(v)]) + cmd.append(component) + + # Store the package name in a shell variable to help the command stay under + # the _MAX_ADB_COMMAND_LENGTH limit. + package = component.split('/')[0] + shell_snippet = 'p=%s;%s' % (package, + cmd_helper.ShrinkToSnippet(cmd, 'p', package)) + return self.RunShellCommand(shell_snippet, check_return=True, + large_output=True) + + @decorators.WithTimeoutAndRetriesFromInstance() + def BroadcastIntent(self, intent_obj, timeout=None, retries=None): + """Send a broadcast intent. + + Args: + intent: An Intent to broadcast. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + cmd = ['am', 'broadcast'] + intent_obj.am_args + self.RunShellCommand(cmd, check_return=True) + + @decorators.WithTimeoutAndRetriesFromInstance() + def GoHome(self, timeout=None, retries=None): + """Return to the home screen and obtain launcher focus. + + This command launches the home screen and attempts to obtain + launcher focus until the timeout is reached. + + Args: + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + def is_launcher_focused(): + output = self.RunShellCommand(['dumpsys', 'window', 'windows'], + check_return=True, large_output=True) + return any(self._LAUNCHER_FOCUSED_RE.match(l) for l in output) + + def dismiss_popups(): + # There is a dialog present; attempt to get rid of it. + # Not all dialogs can be dismissed with back. + self.SendKeyEvent(keyevent.KEYCODE_ENTER) + self.SendKeyEvent(keyevent.KEYCODE_BACK) + return is_launcher_focused() + + # If Home is already focused, return early to avoid unnecessary work. + if is_launcher_focused(): + return + + self.StartActivity( + intent.Intent(action='android.intent.action.MAIN', + category='android.intent.category.HOME'), + blocking=True) + + if not is_launcher_focused(): + timeout_retry.WaitFor(dismiss_popups, wait_period=1) + + @decorators.WithTimeoutAndRetriesFromInstance() + def ForceStop(self, package, timeout=None, retries=None): + """Close the application. + + Args: + package: A string containing the name of the package to stop. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + cmd = 'p=%s;if [[ "$(ps)" = *$p* ]]; then am force-stop $p; fi' + self.RunShellCommand(cmd % package, check_return=True) + + @decorators.WithTimeoutAndRetriesFromInstance() + def ClearApplicationState( + self, package, permissions=None, timeout=None, retries=None): + """Clear all state for the given package. + + Args: + package: A string containing the name of the package to stop. + permissions: List of permissions to set after clearing data. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + # Check that the package exists before clearing it for android builds below + # JB MR2. Necessary because calling pm clear on a package that doesn't exist + # may never return. + if ((self.build_version_sdk >= version_codes.JELLY_BEAN_MR2) + or self._GetApplicationPathsInternal(package)): + self.RunShellCommand(['pm', 'clear', package], check_return=True) + self.GrantPermissions(package, permissions) + + @decorators.WithTimeoutAndRetriesFromInstance() + def SendKeyEvent(self, keycode, timeout=None, retries=None): + """Sends a keycode to the device. + + See the devil.android.sdk.keyevent module for suitable keycode values. + + Args: + keycode: A integer keycode to send to the device. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + self.RunShellCommand(['input', 'keyevent', format(keycode, 'd')], + check_return=True) + + PUSH_CHANGED_FILES_DEFAULT_TIMEOUT = 10 * _DEFAULT_TIMEOUT + + @decorators.WithTimeoutAndRetriesFromInstance( + min_default_timeout=PUSH_CHANGED_FILES_DEFAULT_TIMEOUT) + def PushChangedFiles(self, host_device_tuples, timeout=None, + retries=None, delete_device_stale=False): + """Push files to the device, skipping files that don't need updating. + + When a directory is pushed, it is traversed recursively on the host and + all files in it are pushed to the device as needed. + Additionally, if delete_device_stale option is True, + files that exist on the device but don't exist on the host are deleted. + + Args: + host_device_tuples: A list of (host_path, device_path) tuples, where + |host_path| is an absolute path of a file or directory on the host + that should be minimially pushed to the device, and |device_path| is + an absolute path of the destination on the device. + timeout: timeout in seconds + retries: number of retries + delete_device_stale: option to delete stale files on device + + Raises: + CommandFailedError on failure. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + + all_changed_files = [] + all_stale_files = [] + missing_dirs = [] + cache_commit_funcs = [] + for h, d in host_device_tuples: + assert os.path.isabs(h) and posixpath.isabs(d) + changed_files, up_to_date_files, stale_files, cache_commit_func = ( + self._GetChangedAndStaleFiles(h, d, delete_device_stale)) + all_changed_files += changed_files + all_stale_files += stale_files + cache_commit_funcs.append(cache_commit_func) + if (os.path.isdir(h) and changed_files and not up_to_date_files + and not stale_files): + missing_dirs.append(d) + + if delete_device_stale and all_stale_files: + self.RunShellCommand(['rm', '-f'] + all_stale_files, + check_return=True) + + if all_changed_files: + if missing_dirs: + self.RunShellCommand(['mkdir', '-p'] + missing_dirs, check_return=True) + self._PushFilesImpl(host_device_tuples, all_changed_files) + for func in cache_commit_funcs: + func() + + def _GetChangedAndStaleFiles(self, host_path, device_path, track_stale=False): + """Get files to push and delete + + Args: + host_path: an absolute path of a file or directory on the host + device_path: an absolute path of a file or directory on the device + track_stale: whether to bother looking for stale files (slower) + + Returns: + a three-element tuple + 1st element: a list of (host_files_path, device_files_path) tuples to push + 2nd element: a list of host_files_path that are up-to-date + 3rd element: a list of stale files under device_path, or [] when + track_stale == False + """ + try: + # Length calculations below assume no trailing /. + host_path = host_path.rstrip('/') + device_path = device_path.rstrip('/') + + specific_device_paths = [device_path] + ignore_other_files = not track_stale and os.path.isdir(host_path) + if ignore_other_files: + specific_device_paths = [] + for root, _, filenames in os.walk(host_path): + relative_dir = root[len(host_path) + 1:] + specific_device_paths.extend( + posixpath.join(device_path, relative_dir, f) for f in filenames) + + def calculate_host_checksums(): + return md5sum.CalculateHostMd5Sums([host_path]) + + def calculate_device_checksums(): + if self._enable_device_files_cache: + cache_entry = self._cache['device_path_checksums'].get(device_path) + if cache_entry and cache_entry[0] == ignore_other_files: + return dict(cache_entry[1]) + + sums = md5sum.CalculateDeviceMd5Sums(specific_device_paths, self) + + cache_entry = [ignore_other_files, sums] + self._cache['device_path_checksums'][device_path] = cache_entry + return dict(sums) + + host_checksums, device_checksums = reraiser_thread.RunAsync(( + calculate_host_checksums, + calculate_device_checksums)) + except EnvironmentError as e: + logging.warning('Error calculating md5: %s', e) + return ([(host_path, device_path)], [], [], lambda: 0) + + to_push = [] + up_to_date = [] + to_delete = [] + if os.path.isfile(host_path): + host_checksum = host_checksums.get(host_path) + device_checksum = device_checksums.get(device_path) + if host_checksum == device_checksum: + up_to_date.append(host_path) + else: + to_push.append((host_path, device_path)) + else: + for host_abs_path, host_checksum in host_checksums.iteritems(): + device_abs_path = posixpath.join( + device_path, os.path.relpath(host_abs_path, host_path)) + device_checksum = device_checksums.pop(device_abs_path, None) + if device_checksum == host_checksum: + up_to_date.append(host_abs_path) + else: + to_push.append((host_abs_path, device_abs_path)) + to_delete = device_checksums.keys() + + def cache_commit_func(): + new_sums = {posixpath.join(device_path, path[len(host_path) + 1:]): val + for path, val in host_checksums.iteritems()} + cache_entry = [ignore_other_files, new_sums] + self._cache['device_path_checksums'][device_path] = cache_entry + + return (to_push, up_to_date, to_delete, cache_commit_func) + + def _ComputeDeviceChecksumsForApks(self, package_name): + ret = self._cache['package_apk_checksums'].get(package_name) + if ret is None: + device_paths = self._GetApplicationPathsInternal(package_name) + file_to_checksums = md5sum.CalculateDeviceMd5Sums(device_paths, self) + ret = set(file_to_checksums.values()) + self._cache['package_apk_checksums'][package_name] = ret + return ret + + def _ComputeStaleApks(self, package_name, host_apk_paths): + def calculate_host_checksums(): + return md5sum.CalculateHostMd5Sums(host_apk_paths) + + def calculate_device_checksums(): + return self._ComputeDeviceChecksumsForApks(package_name) + + host_checksums, device_checksums = reraiser_thread.RunAsync(( + calculate_host_checksums, calculate_device_checksums)) + stale_apks = [k for (k, v) in host_checksums.iteritems() + if v not in device_checksums] + return stale_apks, set(host_checksums.values()) + + def _PushFilesImpl(self, host_device_tuples, files): + if not files: + return + + size = sum(host_utils.GetRecursiveDiskUsage(h) for h, _ in files) + file_count = len(files) + dir_size = sum(host_utils.GetRecursiveDiskUsage(h) + for h, _ in host_device_tuples) + dir_file_count = 0 + for h, _ in host_device_tuples: + if os.path.isdir(h): + dir_file_count += sum(len(f) for _r, _d, f in os.walk(h)) + else: + dir_file_count += 1 + + push_duration = self._ApproximateDuration( + file_count, file_count, size, False) + dir_push_duration = self._ApproximateDuration( + len(host_device_tuples), dir_file_count, dir_size, False) + zip_duration = self._ApproximateDuration(1, 1, size, True) + + if dir_push_duration < push_duration and dir_push_duration < zip_duration: + self._PushChangedFilesIndividually(host_device_tuples) + elif push_duration < zip_duration: + self._PushChangedFilesIndividually(files) + elif self._commands_installed is False: + # Already tried and failed to install unzip command. + self._PushChangedFilesIndividually(files) + elif not self._PushChangedFilesZipped( + files, [d for _, d in host_device_tuples]): + self._PushChangedFilesIndividually(files) + + def _MaybeInstallCommands(self): + if self._commands_installed is None: + try: + if not install_commands.Installed(self): + install_commands.InstallCommands(self) + self._commands_installed = True + except device_errors.CommandFailedError as e: + logging.warning('unzip not available: %s', str(e)) + self._commands_installed = False + return self._commands_installed + + @staticmethod + def _ApproximateDuration(adb_calls, file_count, byte_count, is_zipping): + # We approximate the time to push a set of files to a device as: + # t = c1 * a + c2 * f + c3 + b / c4 + b / (c5 * c6), where + # t: total time (sec) + # c1: adb call time delay (sec) + # a: number of times adb is called (unitless) + # c2: push time delay (sec) + # f: number of files pushed via adb (unitless) + # c3: zip time delay (sec) + # c4: zip rate (bytes/sec) + # b: total number of bytes (bytes) + # c5: transfer rate (bytes/sec) + # c6: compression ratio (unitless) + + # All of these are approximations. + ADB_CALL_PENALTY = 0.1 # seconds + ADB_PUSH_PENALTY = 0.01 # seconds + ZIP_PENALTY = 2.0 # seconds + ZIP_RATE = 10000000.0 # bytes / second + TRANSFER_RATE = 2000000.0 # bytes / second + COMPRESSION_RATIO = 2.0 # unitless + + adb_call_time = ADB_CALL_PENALTY * adb_calls + adb_push_setup_time = ADB_PUSH_PENALTY * file_count + if is_zipping: + zip_time = ZIP_PENALTY + byte_count / ZIP_RATE + transfer_time = byte_count / (TRANSFER_RATE * COMPRESSION_RATIO) + else: + zip_time = 0 + transfer_time = byte_count / TRANSFER_RATE + return adb_call_time + adb_push_setup_time + zip_time + transfer_time + + def _PushChangedFilesIndividually(self, files): + for h, d in files: + self.adb.Push(h, d) + + def _PushChangedFilesZipped(self, files, dirs): + with tempfile.NamedTemporaryFile(suffix='.zip') as zip_file: + zip_proc = multiprocessing.Process( + target=DeviceUtils._CreateDeviceZip, + args=(zip_file.name, files)) + zip_proc.start() + try: + # While it's zipping, ensure the unzip command exists on the device. + if not self._MaybeInstallCommands(): + zip_proc.terminate() + return False + + # Warm up NeedsSU cache while we're still zipping. + self.NeedsSU() + with device_temp_file.DeviceTempFile( + self.adb, suffix='.zip') as device_temp: + zip_proc.join() + self.adb.Push(zip_file.name, device_temp.name) + quoted_dirs = ' '.join(cmd_helper.SingleQuote(d) for d in dirs) + self.RunShellCommand( + 'unzip %s&&chmod -R 777 %s' % (device_temp.name, quoted_dirs), + as_root=True, + env={'PATH': '%s:$PATH' % install_commands.BIN_DIR}, + check_return=True) + finally: + if zip_proc.is_alive(): + zip_proc.terminate() + return True + + @staticmethod + def _CreateDeviceZip(zip_path, host_device_tuples): + with zipfile.ZipFile(zip_path, 'w') as zip_file: + for host_path, device_path in host_device_tuples: + zip_utils.WriteToZipFile(zip_file, host_path, device_path) + + # TODO(nednguyen): remove this and migrate the callsite to PathExists(). + @decorators.WithTimeoutAndRetriesFromInstance() + def FileExists(self, device_path, timeout=None, retries=None): + """Checks whether the given file exists on the device. + + Arguments are the same as PathExists. + """ + return self.PathExists(device_path, timeout=timeout, retries=retries) + + @decorators.WithTimeoutAndRetriesFromInstance() + def PathExists(self, device_paths, as_root=False, timeout=None, retries=None): + """Checks whether the given path(s) exists on the device. + + Args: + device_path: A string containing the absolute path to the file on the + device, or an iterable of paths to check. + as_root: Whether root permissions should be use to check for the existence + of the given path(s). + timeout: timeout in seconds + retries: number of retries + + Returns: + True if the all given paths exist on the device, False otherwise. + + Raises: + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + paths = device_paths + if isinstance(paths, basestring): + paths = (paths,) + condition = ' -a '.join('-e %s' % cmd_helper.SingleQuote(p) for p in paths) + cmd = 'test %s' % condition + try: + self.RunShellCommand(cmd, as_root=as_root, check_return=True, + timeout=timeout, retries=retries) + return True + except device_errors.CommandFailedError: + return False + + @decorators.WithTimeoutAndRetriesFromInstance() + def PullFile(self, device_path, host_path, timeout=None, retries=None): + """Pull a file from the device. + + Args: + device_path: A string containing the absolute path of the file to pull + from the device. + host_path: A string containing the absolute path of the destination on + the host. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandFailedError on failure. + CommandTimeoutError on timeout. + """ + # Create the base dir if it doesn't exist already + dirname = os.path.dirname(host_path) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname) + self.adb.Pull(device_path, host_path) + + def _ReadFileWithPull(self, device_path): + try: + d = tempfile.mkdtemp() + host_temp_path = os.path.join(d, 'tmp_ReadFileWithPull') + self.adb.Pull(device_path, host_temp_path) + with open(host_temp_path, 'r') as host_temp: + return host_temp.read() + finally: + if os.path.exists(d): + shutil.rmtree(d) + + _LS_RE = re.compile( + r'(?P<perms>\S+) (?:(?P<inodes>\d+) +)?(?P<owner>\S+) +(?P<group>\S+) +' + r'(?:(?P<size>\d+) +)?(?P<date>\S+) +(?P<time>\S+) +(?P<name>.+)$') + + @decorators.WithTimeoutAndRetriesFromInstance() + def ReadFile(self, device_path, as_root=False, force_pull=False, + timeout=None, retries=None): + """Reads the contents of a file from the device. + + Args: + device_path: A string containing the absolute path of the file to read + from the device. + as_root: A boolean indicating whether the read should be executed with + root privileges. + force_pull: A boolean indicating whether to force the operation to be + performed by pulling a file from the device. The default is, when the + contents are short, to retrieve the contents using cat instead. + timeout: timeout in seconds + retries: number of retries + + Returns: + The contents of |device_path| as a string. Contents are intepreted using + universal newlines, so the caller will see them encoded as '\n'. Also, + all lines will be terminated. + + Raises: + AdbCommandFailedError if the file can't be read. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + def get_size(path): + # TODO(jbudorick): Implement a generic version of Stat() that handles + # as_root=True, then switch this implementation to use that. + ls_out = self.RunShellCommand(['ls', '-l', device_path], as_root=as_root, + check_return=True) + file_name = posixpath.basename(device_path) + for line in ls_out: + m = self._LS_RE.match(line) + if m and file_name == posixpath.basename(m.group('name')): + return int(m.group('size')) + logging.warning('Could not determine size of %s.', device_path) + return None + + if (not force_pull + and 0 < get_size(device_path) <= self._MAX_ADB_OUTPUT_LENGTH): + return _JoinLines(self.RunShellCommand( + ['cat', device_path], as_root=as_root, check_return=True)) + elif as_root and self.NeedsSU(): + with device_temp_file.DeviceTempFile(self.adb) as device_temp: + cmd = 'SRC=%s DEST=%s;cp "$SRC" "$DEST" && chmod 666 "$DEST"' % ( + cmd_helper.SingleQuote(device_path), + cmd_helper.SingleQuote(device_temp.name)) + self.RunShellCommand(cmd, as_root=True, check_return=True) + return self._ReadFileWithPull(device_temp.name) + else: + return self._ReadFileWithPull(device_path) + + def _WriteFileWithPush(self, device_path, contents): + with tempfile.NamedTemporaryFile() as host_temp: + host_temp.write(contents) + host_temp.flush() + self.adb.Push(host_temp.name, device_path) + + @decorators.WithTimeoutAndRetriesFromInstance() + def WriteFile(self, device_path, contents, as_root=False, force_push=False, + timeout=None, retries=None): + """Writes |contents| to a file on the device. + + Args: + device_path: A string containing the absolute path to the file to write + on the device. + contents: A string containing the data to write to the device. + as_root: A boolean indicating whether the write should be executed with + root privileges (if available). + force_push: A boolean indicating whether to force the operation to be + performed by pushing a file to the device. The default is, when the + contents are short, to pass the contents using a shell script instead. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandFailedError if the file could not be written on the device. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + if not force_push and len(contents) < self._MAX_ADB_COMMAND_LENGTH: + # If the contents are small, for efficieny we write the contents with + # a shell command rather than pushing a file. + cmd = 'echo -n %s > %s' % (cmd_helper.SingleQuote(contents), + cmd_helper.SingleQuote(device_path)) + self.RunShellCommand(cmd, as_root=as_root, check_return=True) + elif as_root and self.NeedsSU(): + # Adb does not allow to "push with su", so we first push to a temp file + # on a safe location, and then copy it to the desired location with su. + with device_temp_file.DeviceTempFile(self.adb) as device_temp: + self._WriteFileWithPush(device_temp.name, contents) + # Here we need 'cp' rather than 'mv' because the temp and + # destination files might be on different file systems (e.g. + # on internal storage and an external sd card). + self.RunShellCommand(['cp', device_temp.name, device_path], + as_root=True, check_return=True) + else: + # If root is not needed, we can push directly to the desired location. + self._WriteFileWithPush(device_path, contents) + + @decorators.WithTimeoutAndRetriesFromInstance() + def Ls(self, device_path, timeout=None, retries=None): + """Lists the contents of a directory on the device. + + Args: + device_path: A string containing the path of the directory on the device + to list. + timeout: timeout in seconds + retries: number of retries + + Returns: + A list of pairs (filename, stat) for each file found in the directory, + where the stat object has the properties: st_mode, st_size, and st_time. + + Raises: + AdbCommandFailedError if |device_path| does not specify a valid and + accessible directory in the device. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + return self.adb.Ls(device_path) + + @decorators.WithTimeoutAndRetriesFromInstance() + def Stat(self, device_path, timeout=None, retries=None): + """Get the stat attributes of a file or directory on the device. + + Args: + device_path: A string containing the path of from which to get attributes + on the device. + timeout: timeout in seconds + retries: number of retries + + Returns: + A stat object with the properties: st_mode, st_size, and st_time + + Raises: + CommandFailedError if device_path cannot be found on the device. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + dirname, target = device_path.rsplit('/', 1) + for filename, stat in self.adb.Ls(dirname): + if filename == target: + return stat + raise device_errors.CommandFailedError( + 'Cannot find file or directory: %r' % device_path, str(self)) + + @decorators.WithTimeoutAndRetriesFromInstance() + def SetJavaAsserts(self, enabled, timeout=None, retries=None): + """Enables or disables Java asserts. + + Args: + enabled: A boolean indicating whether Java asserts should be enabled + or disabled. + timeout: timeout in seconds + retries: number of retries + + Returns: + True if the device-side property changed and a restart is required as a + result, False otherwise. + + Raises: + CommandTimeoutError on timeout. + """ + def find_property(lines, property_name): + for index, line in enumerate(lines): + if line.strip() == '': + continue + key_value = tuple(s.strip() for s in line.split('=', 1)) + if len(key_value) != 2: + continue + key, value = key_value + if key == property_name: + return index, value + return None, '' + + new_value = 'all' if enabled else '' + + # First ensure the desired property is persisted. + try: + properties = self.ReadFile(self.LOCAL_PROPERTIES_PATH).splitlines() + except device_errors.CommandFailedError: + properties = [] + index, value = find_property(properties, self.JAVA_ASSERT_PROPERTY) + if new_value != value: + if new_value: + new_line = '%s=%s' % (self.JAVA_ASSERT_PROPERTY, new_value) + if index is None: + properties.append(new_line) + else: + properties[index] = new_line + else: + assert index is not None # since new_value == '' and new_value != value + properties.pop(index) + self.WriteFile(self.LOCAL_PROPERTIES_PATH, _JoinLines(properties)) + + # Next, check the current runtime value is what we need, and + # if not, set it and report that a reboot is required. + value = self.GetProp(self.JAVA_ASSERT_PROPERTY) + if new_value != value: + self.SetProp(self.JAVA_ASSERT_PROPERTY, new_value) + return True + else: + return False + + def GetLanguage(self, cache=False): + """Returns the language setting on the device. + Args: + cache: Whether to use cached properties when available. + """ + return self.GetProp('persist.sys.language', cache=cache) + + def GetCountry(self, cache=False): + """Returns the country setting on the device. + + Args: + cache: Whether to use cached properties when available. + """ + return self.GetProp('persist.sys.country', cache=cache) + + @property + def screen_density(self): + """Returns the screen density of the device.""" + DPI_TO_DENSITY = { + 120: 'ldpi', + 160: 'mdpi', + 240: 'hdpi', + 320: 'xhdpi', + 480: 'xxhdpi', + 640: 'xxxhdpi', + } + return DPI_TO_DENSITY.get(self.pixel_density, 'tvdpi') + + @property + def pixel_density(self): + return int(self.GetProp('ro.sf.lcd_density', cache=True)) + + @property + def build_description(self): + """Returns the build description of the system. + + For example: + nakasi-user 4.4.4 KTU84P 1227136 release-keys + """ + return self.GetProp('ro.build.description', cache=True) + + @property + def build_fingerprint(self): + """Returns the build fingerprint of the system. + + For example: + google/nakasi/grouper:4.4.4/KTU84P/1227136:user/release-keys + """ + return self.GetProp('ro.build.fingerprint', cache=True) + + @property + def build_id(self): + """Returns the build ID of the system (e.g. 'KTU84P').""" + return self.GetProp('ro.build.id', cache=True) + + @property + def build_product(self): + """Returns the build product of the system (e.g. 'grouper').""" + return self.GetProp('ro.build.product', cache=True) + + @property + def build_type(self): + """Returns the build type of the system (e.g. 'user').""" + return self.GetProp('ro.build.type', cache=True) + + @property + def build_version_sdk(self): + """Returns the build version sdk of the system as a number (e.g. 19). + + For version code numbers see: + http://developer.android.com/reference/android/os/Build.VERSION_CODES.html + + For named constants see devil.android.sdk.version_codes + + Raises: + CommandFailedError if the build version sdk is not a number. + """ + value = self.GetProp('ro.build.version.sdk', cache=True) + try: + return int(value) + except ValueError: + raise device_errors.CommandFailedError( + 'Invalid build version sdk: %r' % value) + + @property + def product_cpu_abi(self): + """Returns the product cpu abi of the device (e.g. 'armeabi-v7a').""" + return self.GetProp('ro.product.cpu.abi', cache=True) + + @property + def product_model(self): + """Returns the name of the product model (e.g. 'Nexus 7').""" + return self.GetProp('ro.product.model', cache=True) + + @property + def product_name(self): + """Returns the product name of the device (e.g. 'nakasi').""" + return self.GetProp('ro.product.name', cache=True) + + @property + def product_board(self): + """Returns the product board name of the device (e.g. 'shamu').""" + return self.GetProp('ro.product.board', cache=True) + + def GetProp(self, property_name, cache=False, timeout=DEFAULT, + retries=DEFAULT): + """Gets a property from the device. + + Args: + property_name: A string containing the name of the property to get from + the device. + cache: Whether to use cached properties when available. + timeout: timeout in seconds + retries: number of retries + + Returns: + The value of the device's |property_name| property. + + Raises: + CommandTimeoutError on timeout. + """ + assert isinstance(property_name, basestring), ( + "property_name is not a string: %r" % property_name) + + prop_cache = self._cache['getprop'] + if cache: + if property_name not in prop_cache: + # It takes ~120ms to query a single property, and ~130ms to query all + # properties. So, when caching we always query all properties. + output = self.RunShellCommand( + ['getprop'], check_return=True, large_output=True, + timeout=self._default_timeout if timeout is DEFAULT else timeout, + retries=self._default_retries if retries is DEFAULT else retries) + prop_cache.clear() + for key, value in _GETPROP_RE.findall(''.join(output)): + prop_cache[key] = value + if property_name not in prop_cache: + prop_cache[property_name] = '' + else: + # timeout and retries are handled down at run shell, because we don't + # want to apply them in the other branch when reading from the cache + value = self.RunShellCommand( + ['getprop', property_name], single_line=True, check_return=True, + timeout=self._default_timeout if timeout is DEFAULT else timeout, + retries=self._default_retries if retries is DEFAULT else retries) + prop_cache[property_name] = value + return prop_cache[property_name] + + @decorators.WithTimeoutAndRetriesFromInstance() + def SetProp(self, property_name, value, check=False, timeout=None, + retries=None): + """Sets a property on the device. + + Args: + property_name: A string containing the name of the property to set on + the device. + value: A string containing the value to set to the property on the + device. + check: A boolean indicating whether to check that the property was + successfully set on the device. + timeout: timeout in seconds + retries: number of retries + + Raises: + CommandFailedError if check is true and the property was not correctly + set on the device (e.g. because it is not rooted). + CommandTimeoutError on timeout. + """ + assert isinstance(property_name, basestring), ( + "property_name is not a string: %r" % property_name) + assert isinstance(value, basestring), "value is not a string: %r" % value + + self.RunShellCommand(['setprop', property_name, value], check_return=True) + prop_cache = self._cache['getprop'] + if property_name in prop_cache: + del prop_cache[property_name] + # TODO(perezju) remove the option and make the check mandatory, but using a + # single shell script to both set- and getprop. + if check and value != self.GetProp(property_name, cache=False): + raise device_errors.CommandFailedError( + 'Unable to set property %r on the device to %r' + % (property_name, value), str(self)) + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetABI(self, timeout=None, retries=None): + """Gets the device main ABI. + + Args: + timeout: timeout in seconds + retries: number of retries + + Returns: + The device's main ABI name. + + Raises: + CommandTimeoutError on timeout. + """ + return self.GetProp('ro.product.cpu.abi', cache=True) + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetPids(self, process_name, timeout=None, retries=None): + """Returns the PIDs of processes with the given name. + + Note that the |process_name| is often the package name. + + Args: + process_name: A string containing the process name to get the PIDs for. + timeout: timeout in seconds + retries: number of retries + + Returns: + A dict mapping process name to a list of PIDs for each process that + contained the provided |process_name|. + + Raises: + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + procs_pids = collections.defaultdict(list) + try: + ps_output = self._RunPipedShellCommand( + 'ps | grep -F %s' % cmd_helper.SingleQuote(process_name)) + except device_errors.AdbShellCommandFailedError as e: + if e.status and isinstance(e.status, list) and not e.status[0]: + # If ps succeeded but grep failed, there were no processes with the + # given name. + return procs_pids + else: + raise + + for line in ps_output: + try: + ps_data = line.split() + if process_name in ps_data[-1]: + pid, process = ps_data[1], ps_data[-1] + procs_pids[process].append(pid) + except IndexError: + pass + return procs_pids + + @decorators.WithTimeoutAndRetriesFromInstance() + def TakeScreenshot(self, host_path=None, timeout=None, retries=None): + """Takes a screenshot of the device. + + Args: + host_path: A string containing the path on the host to save the + screenshot to. If None, a file name in the current + directory will be generated. + timeout: timeout in seconds + retries: number of retries + + Returns: + The name of the file on the host to which the screenshot was saved. + + Raises: + CommandFailedError on failure. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + if not host_path: + host_path = os.path.abspath('screenshot-%s-%s.png' % ( + self.adb.GetDeviceSerial(), _GetTimeStamp())) + with device_temp_file.DeviceTempFile(self.adb, suffix='.png') as device_tmp: + self.RunShellCommand(['/system/bin/screencap', '-p', device_tmp.name], + check_return=True) + self.PullFile(device_tmp.name, host_path) + return host_path + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetMemoryUsageForPid(self, pid, timeout=None, retries=None): + """Gets the memory usage for the given PID. + + Args: + pid: PID of the process. + timeout: timeout in seconds + retries: number of retries + + Returns: + A dict containing memory usage statistics for the PID. May include: + Size, Rss, Pss, Shared_Clean, Shared_Dirty, Private_Clean, + Private_Dirty, VmHWM + + Raises: + CommandTimeoutError on timeout. + """ + result = collections.defaultdict(int) + + try: + result.update(self._GetMemoryUsageForPidFromSmaps(pid)) + except device_errors.CommandFailedError: + logging.exception('Error getting memory usage from smaps') + + try: + result.update(self._GetMemoryUsageForPidFromStatus(pid)) + except device_errors.CommandFailedError: + logging.exception('Error getting memory usage from status') + + return result + + @decorators.WithTimeoutAndRetriesFromInstance() + def DismissCrashDialogIfNeeded(self, timeout=None, retries=None): + """Dismiss the error/ANR dialog if present. + + Returns: Name of the crashed package if a dialog is focused, + None otherwise. + """ + def _FindFocusedWindow(): + match = None + # TODO(jbudorick): Try to grep the output on the device instead of using + # large_output if/when DeviceUtils exposes a public interface for piped + # shell command handling. + for line in self.RunShellCommand(['dumpsys', 'window', 'windows'], + check_return=True, large_output=True): + match = re.match(_CURRENT_FOCUS_CRASH_RE, line) + if match: + break + return match + + match = _FindFocusedWindow() + if not match: + return None + package = match.group(2) + logging.warning('Trying to dismiss %s dialog for %s', *match.groups()) + self.SendKeyEvent(keyevent.KEYCODE_DPAD_RIGHT) + self.SendKeyEvent(keyevent.KEYCODE_DPAD_RIGHT) + self.SendKeyEvent(keyevent.KEYCODE_ENTER) + match = _FindFocusedWindow() + if match: + logging.error('Still showing a %s dialog for %s', *match.groups()) + return package + + def _GetMemoryUsageForPidFromSmaps(self, pid): + SMAPS_COLUMNS = ( + 'Size', 'Rss', 'Pss', 'Shared_Clean', 'Shared_Dirty', 'Private_Clean', + 'Private_Dirty') + + showmap_out = self._RunPipedShellCommand( + 'showmap %d | grep TOTAL' % int(pid), as_root=True) + + split_totals = showmap_out[-1].split() + if (not split_totals + or len(split_totals) != 9 + or split_totals[-1] != 'TOTAL'): + raise device_errors.CommandFailedError( + 'Invalid output from showmap: %s' % '\n'.join(showmap_out)) + + return dict(itertools.izip(SMAPS_COLUMNS, (int(n) for n in split_totals))) + + def _GetMemoryUsageForPidFromStatus(self, pid): + for line in self.ReadFile( + '/proc/%s/status' % str(pid), as_root=True).splitlines(): + if line.startswith('VmHWM:'): + return {'VmHWM': int(line.split()[1])} + raise device_errors.CommandFailedError( + 'Could not find memory peak value for pid %s', str(pid)) + + def GetLogcatMonitor(self, *args, **kwargs): + """Returns a new LogcatMonitor associated with this device. + + Parameters passed to this function are passed directly to + |logcat_monitor.LogcatMonitor| and are documented there. + """ + return logcat_monitor.LogcatMonitor(self.adb, *args, **kwargs) + + def GetClientCache(self, client_name): + """Returns client cache.""" + if client_name not in self._client_caches: + self._client_caches[client_name] = {} + return self._client_caches[client_name] + + def _ClearCache(self): + """Clears all caches.""" + for client in self._client_caches: + self._client_caches[client].clear() + self._cache = { + # Map of packageId -> list of on-device .apk paths + 'package_apk_paths': {}, + # Set of packageId that were loaded from LoadCacheData and not yet + # verified. + 'package_apk_paths_to_verify': set(), + # Map of packageId -> set of on-device .apk checksums + 'package_apk_checksums': {}, + # Map of property_name -> value + 'getprop': {}, + # Map of device_path -> [ignore_other_files, map of path->checksum] + 'device_path_checksums': {}, + } + + def LoadCacheData(self, data): + """Initializes the cache from data created using DumpCacheData.""" + obj = json.loads(data) + self._cache['package_apk_paths'] = obj.get('package_apk_paths', {}) + # When using a cache across script invokations, verify that apps have + # not been uninstalled. + self._cache['package_apk_paths_to_verify'] = set( + self._cache['package_apk_paths'].iterkeys()) + + package_apk_checksums = obj.get('package_apk_checksums', {}) + for k, v in package_apk_checksums.iteritems(): + package_apk_checksums[k] = set(v) + self._cache['package_apk_checksums'] = package_apk_checksums + device_path_checksums = obj.get('device_path_checksums', {}) + self._cache['device_path_checksums'] = device_path_checksums + + def DumpCacheData(self): + """Dumps the current cache state to a string.""" + obj = {} + obj['package_apk_paths'] = self._cache['package_apk_paths'] + obj['package_apk_checksums'] = self._cache['package_apk_checksums'] + # JSON can't handle sets. + for k, v in obj['package_apk_checksums'].iteritems(): + obj['package_apk_checksums'][k] = list(v) + obj['device_path_checksums'] = self._cache['device_path_checksums'] + return json.dumps(obj, separators=(',', ':')) + + @classmethod + def parallel(cls, devices, async=False): + """Creates a Parallelizer to operate over the provided list of devices. + + Args: + devices: A list of either DeviceUtils instances or objects from + from which DeviceUtils instances can be constructed. If None, + all attached devices will be used. + async: If true, returns a Parallelizer that runs operations + asynchronously. + + Returns: + A Parallelizer operating over |devices|. + + Raises: + device_errors.NoDevicesError: If no devices are passed. + """ + if not devices: + raise device_errors.NoDevicesError() + + devices = [d if isinstance(d, cls) else cls(d) for d in devices] + if async: + return parallelizer.Parallelizer(devices) + else: + return parallelizer.SyncParallelizer(devices) + + @classmethod + def HealthyDevices(cls, blacklist=None, **kwargs): + blacklisted_devices = blacklist.Read() if blacklist else [] + + def blacklisted(adb): + if adb.GetDeviceSerial() in blacklisted_devices: + logging.warning('Device %s is blacklisted.', adb.GetDeviceSerial()) + return True + return False + + devices = [] + for adb in adb_wrapper.AdbWrapper.Devices(): + if not blacklisted(adb): + devices.append(cls(_CreateAdbWrapper(adb), **kwargs)) + return devices + + @decorators.WithTimeoutAndRetriesFromInstance() + def RestartAdbd(self, timeout=None, retries=None): + logging.info('Restarting adbd on device.') + with device_temp_file.DeviceTempFile(self.adb, suffix='.sh') as script: + self.WriteFile(script.name, _RESTART_ADBD_SCRIPT) + self.RunShellCommand(['source', script.name], as_root=True) + self.adb.WaitForDevice() + + @decorators.WithTimeoutAndRetriesFromInstance() + def GrantPermissions(self, package, permissions, timeout=None, retries=None): + # Permissions only need to be set on M and above because of the changes to + # the permission model. + if not permissions or self.build_version_sdk < version_codes.MARSHMALLOW: + return + logging.info('Setting permissions for %s.', package) + permissions = [p for p in permissions if p not in _PERMISSIONS_BLACKLIST] + if ('android.permission.WRITE_EXTERNAL_STORAGE' in permissions + and 'android.permission.READ_EXTERNAL_STORAGE' not in permissions): + permissions.append('android.permission.READ_EXTERNAL_STORAGE') + cmd = '&&'.join('pm grant %s %s' % (package, p) for p in permissions) + if cmd: + output = self.RunShellCommand(cmd, check_return=True) + if output: + logging.warning('Possible problem when granting permissions. Blacklist ' + 'may need to be updated.') + for line in output: + logging.warning(' %s', line) + + @decorators.WithTimeoutAndRetriesFromInstance() + def IsScreenOn(self, timeout=None, retries=None): + """Determines if screen is on. + + Dumpsys input_method exposes screen on/off state. Below is an explination of + the states. + + Pre-L: + On: mScreenOn=true + Off: mScreenOn=false + L+: + On: mInteractive=true + Off: mInteractive=false + + Returns: + True if screen is on, false if it is off. + + Raises: + device_errors.CommandFailedError: If screen state cannot be found. + """ + if self.build_version_sdk < version_codes.LOLLIPOP: + input_check = 'mScreenOn' + check_value = 'mScreenOn=true' + else: + input_check = 'mInteractive' + check_value = 'mInteractive=true' + dumpsys_out = self._RunPipedShellCommand( + 'dumpsys input_method | grep %s' % input_check) + if not dumpsys_out: + raise device_errors.CommandFailedError( + 'Unable to detect screen state', str(self)) + return check_value in dumpsys_out[0] + + @decorators.WithTimeoutAndRetriesFromInstance() + def SetScreen(self, on, timeout=None, retries=None): + """Turns screen on and off. + + Args: + on: bool to decide state to switch to. True = on False = off. + """ + def screen_test(): + return self.IsScreenOn() == on + + if screen_test(): + logging.info('Screen already in expected state.') + return + self.RunShellCommand('input keyevent 26') + timeout_retry.WaitFor(screen_test, wait_period=1) diff --git a/catapult/devil/devil/android/device_utils_devicetest.py b/catapult/devil/devil/android/device_utils_devicetest.py new file mode 100755 index 00000000..9a503738 --- /dev/null +++ b/catapult/devil/devil/android/device_utils_devicetest.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Unit tests for the contents of device_utils.py (mostly DeviceUtils). +The test will invoke real devices +""" + +import os +import tempfile +import unittest + +from devil.android import device_utils +from devil.android.sdk import adb_wrapper +from devil.utils import cmd_helper + +_OLD_CONTENTS = "foo" +_NEW_CONTENTS = "bar" +_DEVICE_DIR = "/data/local/tmp/device_utils_test" +_SUB_DIR = "sub" +_SUB_DIR1 = "sub1" +_SUB_DIR2 = "sub2" + + +class DeviceUtilsPushDeleteFilesTest(unittest.TestCase): + + def setUp(self): + devices = adb_wrapper.AdbWrapper.Devices() + assert devices, 'A device must be attached' + self.adb = devices[0] + self.adb.WaitForDevice() + self.device = device_utils.DeviceUtils( + self.adb, default_timeout=10, default_retries=0) + + @staticmethod + def _MakeTempFile(contents): + """Make a temporary file with the given contents. + + Args: + contents: string to write to the temporary file. + + Returns: + the tuple contains the absolute path to the file and the file name + """ + fi, path = tempfile.mkstemp(text=True) + with os.fdopen(fi, 'w') as f: + f.write(contents) + file_name = os.path.basename(path) + return (path, file_name) + + @staticmethod + def _MakeTempFileGivenDir(directory, contents): + """Make a temporary file under the given directory + with the given contents + + Args: + directory: the temp directory to create the file + contents: string to write to the temp file + + Returns: + the list contains the absolute path to the file and the file name + """ + fi, path = tempfile.mkstemp(dir=directory, text=True) + with os.fdopen(fi, 'w') as f: + f.write(contents) + file_name = os.path.basename(path) + return (path, file_name) + + @staticmethod + def _ChangeTempFile(path, contents): + with os.open(path, 'w') as f: + f.write(contents) + + @staticmethod + def _DeleteTempFile(path): + os.remove(path) + + def testPushChangedFiles_noFileChange(self): + (host_file_path, file_name) = self._MakeTempFile(_OLD_CONTENTS) + device_file_path = "%s/%s" % (_DEVICE_DIR, file_name) + self.adb.Push(host_file_path, device_file_path) + self.device.PushChangedFiles([(host_file_path, device_file_path)]) + result = self.device.RunShellCommand(['cat', device_file_path], + single_line=True) + self.assertEqual(_OLD_CONTENTS, result) + + cmd_helper.RunCmd(['rm', host_file_path]) + self.device.RunShellCommand(['rm', '-rf', _DEVICE_DIR]) + + def testPushChangedFiles_singleFileChange(self): + (host_file_path, file_name) = self._MakeTempFile(_OLD_CONTENTS) + device_file_path = "%s/%s" % (_DEVICE_DIR, file_name) + self.adb.Push(host_file_path, device_file_path) + + with open(host_file_path, 'w') as f: + f.write(_NEW_CONTENTS) + self.device.PushChangedFiles([(host_file_path, device_file_path)]) + result = self.device.RunShellCommand(['cat', device_file_path], + single_line=True) + self.assertEqual(_NEW_CONTENTS, result) + + cmd_helper.RunCmd(['rm', host_file_path]) + self.device.RunShellCommand(['rm', '-rf', _DEVICE_DIR]) + + def testDeleteFiles(self): + host_tmp_dir = tempfile.mkdtemp() + (host_file_path, file_name) = self._MakeTempFileGivenDir( + host_tmp_dir, _OLD_CONTENTS) + + device_file_path = "%s/%s" % (_DEVICE_DIR, file_name) + self.adb.Push(host_file_path, device_file_path) + + cmd_helper.RunCmd(['rm', host_file_path]) + self.device.PushChangedFiles([(host_tmp_dir, _DEVICE_DIR)], + delete_device_stale=True) + result = self.device.RunShellCommand(['ls', _DEVICE_DIR], single_line=True) + self.assertEqual('', result) + + cmd_helper.RunCmd(['rm', '-rf', host_tmp_dir]) + self.device.RunShellCommand(['rm', '-rf', _DEVICE_DIR]) + + def testPushAndDeleteFiles_noSubDir(self): + host_tmp_dir = tempfile.mkdtemp() + (host_file_path1, file_name1) = self._MakeTempFileGivenDir( + host_tmp_dir, _OLD_CONTENTS) + (host_file_path2, file_name2) = self._MakeTempFileGivenDir( + host_tmp_dir, _OLD_CONTENTS) + + device_file_path1 = "%s/%s" % (_DEVICE_DIR, file_name1) + device_file_path2 = "%s/%s" % (_DEVICE_DIR, file_name2) + self.adb.Push(host_file_path1, device_file_path1) + self.adb.Push(host_file_path2, device_file_path2) + + with open(host_file_path1, 'w') as f: + f.write(_NEW_CONTENTS) + cmd_helper.RunCmd(['rm', host_file_path2]) + + self.device.PushChangedFiles([(host_tmp_dir, _DEVICE_DIR)], + delete_device_stale=True) + result = self.device.RunShellCommand(['cat', device_file_path1], + single_line=True) + self.assertEqual(_NEW_CONTENTS, result) + result = self.device.RunShellCommand(['ls', _DEVICE_DIR], single_line=True) + self.assertEqual(file_name1, result) + + self.device.RunShellCommand(['rm', '-rf', _DEVICE_DIR]) + cmd_helper.RunCmd(['rm', '-rf', host_tmp_dir]) + + def testPushAndDeleteFiles_SubDir(self): + host_tmp_dir = tempfile.mkdtemp() + host_sub_dir1 = "%s/%s" % (host_tmp_dir, _SUB_DIR1) + host_sub_dir2 = "%s/%s/%s" % (host_tmp_dir, _SUB_DIR, _SUB_DIR2) + cmd_helper.RunCmd(['mkdir', '-p', host_sub_dir1]) + cmd_helper.RunCmd(['mkdir', '-p', host_sub_dir2]) + + (host_file_path1, file_name1) = self._MakeTempFileGivenDir( + host_tmp_dir, _OLD_CONTENTS) + (host_file_path2, file_name2) = self._MakeTempFileGivenDir( + host_tmp_dir, _OLD_CONTENTS) + (host_file_path3, file_name3) = self._MakeTempFileGivenDir( + host_sub_dir1, _OLD_CONTENTS) + (host_file_path4, file_name4) = self._MakeTempFileGivenDir( + host_sub_dir2, _OLD_CONTENTS) + + device_file_path1 = "%s/%s" % (_DEVICE_DIR, file_name1) + device_file_path2 = "%s/%s" % (_DEVICE_DIR, file_name2) + device_file_path3 = "%s/%s/%s" % (_DEVICE_DIR, _SUB_DIR1, file_name3) + device_file_path4 = "%s/%s/%s/%s" % (_DEVICE_DIR, _SUB_DIR, + _SUB_DIR2, file_name4) + + self.adb.Push(host_file_path1, device_file_path1) + self.adb.Push(host_file_path2, device_file_path2) + self.adb.Push(host_file_path3, device_file_path3) + self.adb.Push(host_file_path4, device_file_path4) + + with open(host_file_path1, 'w') as f: + f.write(_NEW_CONTENTS) + cmd_helper.RunCmd(['rm', host_file_path2]) + cmd_helper.RunCmd(['rm', host_file_path4]) + + self.device.PushChangedFiles([(host_tmp_dir, _DEVICE_DIR)], + delete_device_stale=True) + result = self.device.RunShellCommand(['cat', device_file_path1], + single_line=True) + self.assertEqual(_NEW_CONTENTS, result) + + result = self.device.RunShellCommand(['ls', _DEVICE_DIR]) + self.assertIn(file_name1, result) + self.assertIn(_SUB_DIR1, result) + self.assertIn(_SUB_DIR, result) + self.assertEqual(3, len(result)) + + result = self.device.RunShellCommand(['cat', device_file_path3], + single_line=True) + self.assertEqual(_OLD_CONTENTS, result) + + result = self.device.RunShellCommand(["ls", "%s/%s/%s" + % (_DEVICE_DIR, _SUB_DIR, _SUB_DIR2)], + single_line=True) + self.assertEqual('', result) + + self.device.RunShellCommand(['rm', '-rf', _DEVICE_DIR]) + cmd_helper.RunCmd(['rm', '-rf', host_tmp_dir]) + + def testRestartAdbd(self): + old_adbd_pid = self.device.RunShellCommand( + ['ps', '|', 'grep', 'adbd'])[1].split()[1] + self.device.RestartAdbd() + new_adbd_pid = self.device.RunShellCommand( + ['ps', '|', 'grep', 'adbd'])[1].split()[1] + self.assertNotEqual(old_adbd_pid, new_adbd_pid) + + +if __name__ == '__main__': + unittest.main() diff --git a/catapult/devil/devil/android/device_utils_test.py b/catapult/devil/devil/android/device_utils_test.py new file mode 100755 index 00000000..38849eca --- /dev/null +++ b/catapult/devil/devil/android/device_utils_test.py @@ -0,0 +1,2312 @@ +#!/usr/bin/env python +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Unit tests for the contents of device_utils.py (mostly DeviceUtils). +""" + +# pylint: disable=protected-access +# pylint: disable=unused-argument + +import logging +import unittest + +from devil import devil_env +from devil.android import device_errors +from devil.android import device_signal +from devil.android import device_utils +from devil.android.sdk import adb_wrapper +from devil.android.sdk import intent +from devil.android.sdk import version_codes +from devil.utils import cmd_helper +from devil.utils import mock_calls + +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + + +class _MockApkHelper(object): + + def __init__(self, path, package_name, perms=None): + self.path = path + self.package_name = package_name + self.perms = perms + + def GetPackageName(self): + return self.package_name + + def GetPermissions(self): + return self.perms + + +class DeviceUtilsInitTest(unittest.TestCase): + + def testInitWithStr(self): + serial_as_str = str('0123456789abcdef') + d = device_utils.DeviceUtils('0123456789abcdef') + self.assertEqual(serial_as_str, d.adb.GetDeviceSerial()) + + def testInitWithUnicode(self): + serial_as_unicode = unicode('fedcba9876543210') + d = device_utils.DeviceUtils(serial_as_unicode) + self.assertEqual(serial_as_unicode, d.adb.GetDeviceSerial()) + + def testInitWithAdbWrapper(self): + serial = '123456789abcdef0' + a = adb_wrapper.AdbWrapper(serial) + d = device_utils.DeviceUtils(a) + self.assertEqual(serial, d.adb.GetDeviceSerial()) + + def testInitWithMissing_fails(self): + with self.assertRaises(ValueError): + device_utils.DeviceUtils(None) + with self.assertRaises(ValueError): + device_utils.DeviceUtils('') + + +class DeviceUtilsGetAVDsTest(mock_calls.TestCase): + + def testGetAVDs(self): + mocked_attrs = { + 'android_sdk': '/my/sdk/path' + } + with mock.patch('devil.devil_env._Environment.LocalPath', + mock.Mock(side_effect=lambda a: mocked_attrs[a])): + with self.assertCall( + mock.call.devil.utils.cmd_helper.GetCmdOutput( + [mock.ANY, 'list', 'avd']), + 'Available Android Virtual Devices:\n' + ' Name: my_android5.0\n' + ' Path: /some/path/to/.android/avd/my_android5.0.avd\n' + ' Target: Android 5.0 (API level 21)\n' + ' Tag/ABI: default/x86\n' + ' Skin: WVGA800\n'): + self.assertEquals(['my_android5.0'], device_utils.GetAVDs()) + + +class DeviceUtilsRestartServerTest(mock_calls.TestCase): + + @mock.patch('time.sleep', mock.Mock()) + def testRestartServer_succeeds(self): + with self.assertCalls( + mock.call.devil.android.sdk.adb_wrapper.AdbWrapper.KillServer(), + (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutput( + ['pgrep', 'adb']), + (1, '')), + mock.call.devil.android.sdk.adb_wrapper.AdbWrapper.StartServer(), + (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutput( + ['pgrep', 'adb']), + (1, '')), + (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutput( + ['pgrep', 'adb']), + (0, '123\n'))): + device_utils.RestartServer() + + +class MockTempFile(object): + + def __init__(self, name='/tmp/some/file'): + self.file = mock.MagicMock(spec=file) + self.file.name = name + self.file.name_quoted = cmd_helper.SingleQuote(name) + + def __enter__(self): + return self.file + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + @property + def name(self): + return self.file.name + + +class _PatchedFunction(object): + + def __init__(self, patched=None, mocked=None): + self.patched = patched + self.mocked = mocked + + +def _AdbWrapperMock(test_serial, is_ready=True): + adb = mock.Mock(spec=adb_wrapper.AdbWrapper) + adb.__str__ = mock.Mock(return_value=test_serial) + adb.GetDeviceSerial.return_value = test_serial + adb.is_ready = is_ready + return adb + + +class DeviceUtilsTest(mock_calls.TestCase): + + def setUp(self): + self.adb = _AdbWrapperMock('0123456789abcdef') + self.device = device_utils.DeviceUtils( + self.adb, default_timeout=10, default_retries=0) + self.watchMethodCalls(self.call.adb, ignore=['GetDeviceSerial']) + + def AdbCommandError(self, args=None, output=None, status=None, msg=None): + if args is None: + args = ['[unspecified]'] + return mock.Mock(side_effect=device_errors.AdbCommandFailedError( + args, output, status, msg, str(self.device))) + + def CommandError(self, msg=None): + if msg is None: + msg = 'Command failed' + return mock.Mock(side_effect=device_errors.CommandFailedError( + msg, str(self.device))) + + def ShellError(self, output=None, status=1): + def action(cmd, *args, **kwargs): + raise device_errors.AdbShellCommandFailedError( + cmd, output, status, str(self.device)) + if output is None: + output = 'Permission denied\n' + return action + + def TimeoutError(self, msg=None): + if msg is None: + msg = 'Operation timed out' + return mock.Mock(side_effect=device_errors.CommandTimeoutError( + msg, str(self.device))) + + +class DeviceUtilsEqTest(DeviceUtilsTest): + + def testEq_equal_deviceUtils(self): + other = device_utils.DeviceUtils(_AdbWrapperMock('0123456789abcdef')) + self.assertTrue(self.device == other) + self.assertTrue(other == self.device) + + def testEq_equal_adbWrapper(self): + other = adb_wrapper.AdbWrapper('0123456789abcdef') + self.assertTrue(self.device == other) + self.assertTrue(other == self.device) + + def testEq_equal_string(self): + other = '0123456789abcdef' + self.assertTrue(self.device == other) + self.assertTrue(other == self.device) + + def testEq_devicesNotEqual(self): + other = device_utils.DeviceUtils(_AdbWrapperMock('0123456789abcdee')) + self.assertFalse(self.device == other) + self.assertFalse(other == self.device) + + def testEq_identity(self): + self.assertTrue(self.device == self.device) + + def testEq_serialInList(self): + devices = [self.device] + self.assertTrue('0123456789abcdef' in devices) + + +class DeviceUtilsLtTest(DeviceUtilsTest): + + def testLt_lessThan(self): + other = device_utils.DeviceUtils(_AdbWrapperMock('ffffffffffffffff')) + self.assertTrue(self.device < other) + self.assertTrue(other > self.device) + + def testLt_greaterThan_lhs(self): + other = device_utils.DeviceUtils(_AdbWrapperMock('0000000000000000')) + self.assertFalse(self.device < other) + self.assertFalse(other > self.device) + + def testLt_equal(self): + other = device_utils.DeviceUtils(_AdbWrapperMock('0123456789abcdef')) + self.assertFalse(self.device < other) + self.assertFalse(other > self.device) + + def testLt_sorted(self): + devices = [ + device_utils.DeviceUtils(_AdbWrapperMock('ffffffffffffffff')), + device_utils.DeviceUtils(_AdbWrapperMock('0000000000000000')), + ] + sorted_devices = sorted(devices) + self.assertEquals('0000000000000000', + sorted_devices[0].adb.GetDeviceSerial()) + self.assertEquals('ffffffffffffffff', + sorted_devices[1].adb.GetDeviceSerial()) + + +class DeviceUtilsStrTest(DeviceUtilsTest): + + def testStr_returnsSerial(self): + with self.assertCalls( + (self.call.adb.GetDeviceSerial(), '0123456789abcdef')): + self.assertEqual('0123456789abcdef', str(self.device)) + + +class DeviceUtilsIsOnlineTest(DeviceUtilsTest): + + def testIsOnline_true(self): + with self.assertCall(self.call.adb.GetState(), 'device'): + self.assertTrue(self.device.IsOnline()) + + def testIsOnline_false(self): + with self.assertCall(self.call.adb.GetState(), 'offline'): + self.assertFalse(self.device.IsOnline()) + + def testIsOnline_error(self): + with self.assertCall(self.call.adb.GetState(), self.CommandError()): + self.assertFalse(self.device.IsOnline()) + + +class DeviceUtilsHasRootTest(DeviceUtilsTest): + + def testHasRoot_true(self): + with self.assertCall(self.call.adb.Shell('ls /root'), 'foo\n'): + self.assertTrue(self.device.HasRoot()) + + def testHasRoot_false(self): + with self.assertCall(self.call.adb.Shell('ls /root'), self.ShellError()): + self.assertFalse(self.device.HasRoot()) + + +class DeviceUtilsEnableRootTest(DeviceUtilsTest): + + def testEnableRoot_succeeds(self): + with self.assertCalls( + (self.call.device.IsUserBuild(), False), + self.call.adb.Root(), + self.call.device.WaitUntilFullyBooted()): + self.device.EnableRoot() + + def testEnableRoot_userBuild(self): + with self.assertCalls( + (self.call.device.IsUserBuild(), True)): + with self.assertRaises(device_errors.CommandFailedError): + self.device.EnableRoot() + + def testEnableRoot_rootFails(self): + with self.assertCalls( + (self.call.device.IsUserBuild(), False), + (self.call.adb.Root(), self.CommandError())): + with self.assertRaises(device_errors.CommandFailedError): + self.device.EnableRoot() + + +class DeviceUtilsIsUserBuildTest(DeviceUtilsTest): + + def testIsUserBuild_yes(self): + with self.assertCall( + self.call.device.GetProp('ro.build.type', cache=True), 'user'): + self.assertTrue(self.device.IsUserBuild()) + + def testIsUserBuild_no(self): + with self.assertCall( + self.call.device.GetProp('ro.build.type', cache=True), 'userdebug'): + self.assertFalse(self.device.IsUserBuild()) + + +class DeviceUtilsGetExternalStoragePathTest(DeviceUtilsTest): + + def testGetExternalStoragePath_succeeds(self): + with self.assertCall( + self.call.adb.Shell('echo $EXTERNAL_STORAGE'), '/fake/storage/path\n'): + self.assertEquals('/fake/storage/path', + self.device.GetExternalStoragePath()) + + def testGetExternalStoragePath_fails(self): + with self.assertCall(self.call.adb.Shell('echo $EXTERNAL_STORAGE'), '\n'): + with self.assertRaises(device_errors.CommandFailedError): + self.device.GetExternalStoragePath() + + +class DeviceUtilsGetApplicationPathsInternalTest(DeviceUtilsTest): + + def testGetApplicationPathsInternal_exists(self): + with self.assertCalls( + (self.call.device.GetProp('ro.build.version.sdk', cache=True), '19'), + (self.call.device.RunShellCommand( + ['pm', 'path', 'android'], check_return=True), + ['package:/path/to/android.apk'])): + self.assertEquals(['/path/to/android.apk'], + self.device._GetApplicationPathsInternal('android')) + + def testGetApplicationPathsInternal_notExists(self): + with self.assertCalls( + (self.call.device.GetProp('ro.build.version.sdk', cache=True), '19'), + (self.call.device.RunShellCommand( + ['pm', 'path', 'not.installed.app'], check_return=True), + '')): + self.assertEquals([], + self.device._GetApplicationPathsInternal('not.installed.app')) + + def testGetApplicationPathsInternal_fails(self): + with self.assertCalls( + (self.call.device.GetProp('ro.build.version.sdk', cache=True), '19'), + (self.call.device.RunShellCommand( + ['pm', 'path', 'android'], check_return=True), + self.CommandError('ERROR. Is package manager running?\n'))): + with self.assertRaises(device_errors.CommandFailedError): + self.device._GetApplicationPathsInternal('android') + + +class DeviceUtils_GetApplicationVersionTest(DeviceUtilsTest): + + def test_GetApplicationVersion_exists(self): + with self.assertCalls( + (self.call.adb.Shell('dumpsys package com.android.chrome'), + 'Packages:\n' + ' Package [com.android.chrome] (3901ecfb):\n' + ' userId=1234 gids=[123, 456, 789]\n' + ' pkg=Package{1fecf634 com.android.chrome}\n' + ' versionName=45.0.1234.7\n')): + self.assertEquals('45.0.1234.7', + self.device.GetApplicationVersion('com.android.chrome')) + + def test_GetApplicationVersion_notExists(self): + with self.assertCalls( + (self.call.adb.Shell('dumpsys package com.android.chrome'), '')): + self.assertEquals(None, + self.device.GetApplicationVersion('com.android.chrome')) + + def test_GetApplicationVersion_fails(self): + with self.assertCalls( + (self.call.adb.Shell('dumpsys package com.android.chrome'), + 'Packages:\n' + ' Package [com.android.chrome] (3901ecfb):\n' + ' userId=1234 gids=[123, 456, 789]\n' + ' pkg=Package{1fecf634 com.android.chrome}\n')): + with self.assertRaises(device_errors.CommandFailedError): + self.device.GetApplicationVersion('com.android.chrome') + + +class DeviceUtilsGetApplicationDataDirectoryTest(DeviceUtilsTest): + + def testGetApplicationDataDirectory_exists(self): + with self.assertCall( + self.call.device._RunPipedShellCommand( + 'pm dump foo.bar.baz | grep dataDir='), + ['dataDir=/data/data/foo.bar.baz']): + self.assertEquals( + '/data/data/foo.bar.baz', + self.device.GetApplicationDataDirectory('foo.bar.baz')) + + def testGetApplicationDataDirectory_notExists(self): + with self.assertCall( + self.call.device._RunPipedShellCommand( + 'pm dump foo.bar.baz | grep dataDir='), + self.ShellError()): + self.assertIsNone(self.device.GetApplicationDataDirectory('foo.bar.baz')) + + +@mock.patch('time.sleep', mock.Mock()) +class DeviceUtilsWaitUntilFullyBootedTest(DeviceUtilsTest): + + def testWaitUntilFullyBooted_succeedsNoWifi(self): + with self.assertCalls( + self.call.adb.WaitForDevice(), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), '/fake/storage/path'), + (self.call.adb.Shell('test -d /fake/storage/path'), ''), + # pm_ready + (self.call.device._GetApplicationPathsInternal('android', + skip_cache=True), + ['package:/some/fake/path']), + # boot_completed + (self.call.device.GetProp('sys.boot_completed', cache=False), '1')): + self.device.WaitUntilFullyBooted(wifi=False) + + def testWaitUntilFullyBooted_succeedsWithWifi(self): + with self.assertCalls( + self.call.adb.WaitForDevice(), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), '/fake/storage/path'), + (self.call.adb.Shell('test -d /fake/storage/path'), ''), + # pm_ready + (self.call.device._GetApplicationPathsInternal('android', + skip_cache=True), + ['package:/some/fake/path']), + # boot_completed + (self.call.device.GetProp('sys.boot_completed', cache=False), '1'), + # wifi_enabled + (self.call.adb.Shell('dumpsys wifi'), + 'stuff\nWi-Fi is enabled\nmore stuff\n')): + self.device.WaitUntilFullyBooted(wifi=True) + + def testWaitUntilFullyBooted_deviceNotInitiallyAvailable(self): + with self.assertCalls( + self.call.adb.WaitForDevice(), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), self.AdbCommandError()), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), self.AdbCommandError()), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), self.AdbCommandError()), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), self.AdbCommandError()), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), '/fake/storage/path'), + (self.call.adb.Shell('test -d /fake/storage/path'), ''), + # pm_ready + (self.call.device._GetApplicationPathsInternal('android', + skip_cache=True), + ['package:/some/fake/path']), + # boot_completed + (self.call.device.GetProp('sys.boot_completed', cache=False), '1')): + self.device.WaitUntilFullyBooted(wifi=False) + + def testWaitUntilFullyBooted_sdCardReadyFails_noPath(self): + with self.assertCalls( + self.call.adb.WaitForDevice(), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), self.CommandError())): + with self.assertRaises(device_errors.CommandFailedError): + self.device.WaitUntilFullyBooted(wifi=False) + + def testWaitUntilFullyBooted_sdCardReadyFails_notExists(self): + with self.assertCalls( + self.call.adb.WaitForDevice(), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), '/fake/storage/path'), + (self.call.adb.Shell('test -d /fake/storage/path'), self.ShellError()), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), '/fake/storage/path'), + (self.call.adb.Shell('test -d /fake/storage/path'), self.ShellError()), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), '/fake/storage/path'), + (self.call.adb.Shell('test -d /fake/storage/path'), + self.TimeoutError())): + with self.assertRaises(device_errors.CommandTimeoutError): + self.device.WaitUntilFullyBooted(wifi=False) + + def testWaitUntilFullyBooted_devicePmFails(self): + with self.assertCalls( + self.call.adb.WaitForDevice(), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), '/fake/storage/path'), + (self.call.adb.Shell('test -d /fake/storage/path'), ''), + # pm_ready + (self.call.device._GetApplicationPathsInternal('android', + skip_cache=True), + self.CommandError()), + # pm_ready + (self.call.device._GetApplicationPathsInternal('android', + skip_cache=True), + self.CommandError()), + # pm_ready + (self.call.device._GetApplicationPathsInternal('android', + skip_cache=True), + self.TimeoutError())): + with self.assertRaises(device_errors.CommandTimeoutError): + self.device.WaitUntilFullyBooted(wifi=False) + + def testWaitUntilFullyBooted_bootFails(self): + with self.assertCalls( + self.call.adb.WaitForDevice(), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), '/fake/storage/path'), + (self.call.adb.Shell('test -d /fake/storage/path'), ''), + # pm_ready + (self.call.device._GetApplicationPathsInternal('android', + skip_cache=True), + ['package:/some/fake/path']), + # boot_completed + (self.call.device.GetProp('sys.boot_completed', cache=False), '0'), + # boot_completed + (self.call.device.GetProp('sys.boot_completed', cache=False), '0'), + # boot_completed + (self.call.device.GetProp('sys.boot_completed', cache=False), + self.TimeoutError())): + with self.assertRaises(device_errors.CommandTimeoutError): + self.device.WaitUntilFullyBooted(wifi=False) + + def testWaitUntilFullyBooted_wifiFails(self): + with self.assertCalls( + self.call.adb.WaitForDevice(), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), '/fake/storage/path'), + (self.call.adb.Shell('test -d /fake/storage/path'), ''), + # pm_ready + (self.call.device._GetApplicationPathsInternal('android', + skip_cache=True), + ['package:/some/fake/path']), + # boot_completed + (self.call.device.GetProp('sys.boot_completed', cache=False), '1'), + # wifi_enabled + (self.call.adb.Shell('dumpsys wifi'), 'stuff\nmore stuff\n'), + # wifi_enabled + (self.call.adb.Shell('dumpsys wifi'), 'stuff\nmore stuff\n'), + # wifi_enabled + (self.call.adb.Shell('dumpsys wifi'), self.TimeoutError())): + with self.assertRaises(device_errors.CommandTimeoutError): + self.device.WaitUntilFullyBooted(wifi=True) + + +@mock.patch('time.sleep', mock.Mock()) +class DeviceUtilsRebootTest(DeviceUtilsTest): + + def testReboot_nonBlocking(self): + with self.assertCalls( + self.call.adb.Reboot(), + (self.call.device.IsOnline(), True), + (self.call.device.IsOnline(), False)): + self.device.Reboot(block=False) + + def testReboot_blocking(self): + with self.assertCalls( + self.call.adb.Reboot(), + (self.call.device.IsOnline(), True), + (self.call.device.IsOnline(), False), + self.call.device.WaitUntilFullyBooted(wifi=False)): + self.device.Reboot(block=True) + + def testReboot_blockUntilWifi(self): + with self.assertCalls( + self.call.adb.Reboot(), + (self.call.device.IsOnline(), True), + (self.call.device.IsOnline(), False), + self.call.device.WaitUntilFullyBooted(wifi=True)): + self.device.Reboot(block=True, wifi=True) + + +class DeviceUtilsInstallTest(DeviceUtilsTest): + + mock_apk = _MockApkHelper('/fake/test/app.apk', 'test.package', ['p1']) + + def testInstall_noPriorInstall(self): + with self.patch_call(self.call.device.build_version_sdk, return_value=23): + with self.assertCalls( + (self.call.device._GetApplicationPathsInternal('test.package'), []), + self.call.adb.Install('/fake/test/app.apk', reinstall=False, + allow_downgrade=False), + (self.call.device.GrantPermissions('test.package', ['p1']), [])): + self.device.Install(DeviceUtilsInstallTest.mock_apk, retries=0) + + def testInstall_permissionsPreM(self): + with self.patch_call(self.call.device.build_version_sdk, return_value=20): + with self.assertCalls( + (self.call.device._GetApplicationPathsInternal('test.package'), []), + (self.call.adb.Install('/fake/test/app.apk', reinstall=False, + allow_downgrade=False))): + self.device.Install(DeviceUtilsInstallTest.mock_apk, retries=0) + + def testInstall_findPermissions(self): + with self.patch_call(self.call.device.build_version_sdk, return_value=23): + with self.assertCalls( + (self.call.device._GetApplicationPathsInternal('test.package'), []), + (self.call.adb.Install('/fake/test/app.apk', reinstall=False, + allow_downgrade=False)), + (self.call.device.GrantPermissions('test.package', ['p1']), [])): + self.device.Install(DeviceUtilsInstallTest.mock_apk, retries=0) + + def testInstall_passPermissions(self): + with self.assertCalls( + (self.call.device._GetApplicationPathsInternal('test.package'), []), + (self.call.adb.Install('/fake/test/app.apk', reinstall=False, + allow_downgrade=False)), + (self.call.device.GrantPermissions('test.package', ['p1', 'p2']), [])): + self.device.Install(DeviceUtilsInstallTest.mock_apk, retries=0, + permissions=['p1', 'p2']) + + def testInstall_differentPriorInstall(self): + with self.assertCalls( + (self.call.device._GetApplicationPathsInternal('test.package'), + ['/fake/data/app/test.package.apk']), + (self.call.device._ComputeStaleApks('test.package', + ['/fake/test/app.apk']), + (['/fake/test/app.apk'], None)), + self.call.device.Uninstall('test.package'), + self.call.adb.Install('/fake/test/app.apk', reinstall=False, + allow_downgrade=False)): + self.device.Install(DeviceUtilsInstallTest.mock_apk, retries=0, + permissions=[]) + + def testInstall_differentPriorInstall_reinstall(self): + with self.assertCalls( + (self.call.device._GetApplicationPathsInternal('test.package'), + ['/fake/data/app/test.package.apk']), + (self.call.device._ComputeStaleApks('test.package', + ['/fake/test/app.apk']), + (['/fake/test/app.apk'], None)), + self.call.adb.Install('/fake/test/app.apk', reinstall=True, + allow_downgrade=False)): + self.device.Install(DeviceUtilsInstallTest.mock_apk, + reinstall=True, retries=0, permissions=[]) + + def testInstall_identicalPriorInstall_reinstall(self): + with self.assertCalls( + (self.call.device._GetApplicationPathsInternal('test.package'), + ['/fake/data/app/test.package.apk']), + (self.call.device._ComputeStaleApks('test.package', + ['/fake/test/app.apk']), + ([], None)), + (self.call.device.ForceStop('test.package'))): + self.device.Install(DeviceUtilsInstallTest.mock_apk, + reinstall=True, retries=0, permissions=[]) + + def testInstall_fails(self): + with self.assertCalls( + (self.call.device._GetApplicationPathsInternal('test.package'), []), + (self.call.adb.Install('/fake/test/app.apk', reinstall=False, + allow_downgrade=False), + self.CommandError('Failure\r\n'))): + with self.assertRaises(device_errors.CommandFailedError): + self.device.Install(DeviceUtilsInstallTest.mock_apk, retries=0) + + def testInstall_downgrade(self): + with self.assertCalls( + (self.call.device._GetApplicationPathsInternal('test.package'), + ['/fake/data/app/test.package.apk']), + (self.call.device._ComputeStaleApks('test.package', + ['/fake/test/app.apk']), + (['/fake/test/app.apk'], None)), + self.call.adb.Install('/fake/test/app.apk', reinstall=True, + allow_downgrade=True)): + self.device.Install(DeviceUtilsInstallTest.mock_apk, + reinstall=True, retries=0, permissions=[], allow_downgrade=True) + + +class DeviceUtilsInstallSplitApkTest(DeviceUtilsTest): + + mock_apk = _MockApkHelper('base.apk', 'test.package', ['p1']) + + def testInstallSplitApk_noPriorInstall(self): + with self.assertCalls( + (self.call.device._CheckSdkLevel(21)), + (mock.call.devil.android.sdk.split_select.SelectSplits( + self.device, 'base.apk', + ['split1.apk', 'split2.apk', 'split3.apk'], + allow_cached_props=False), + ['split2.apk']), + (self.call.device._GetApplicationPathsInternal('test.package'), []), + (self.call.adb.InstallMultiple( + ['base.apk', 'split2.apk'], partial=None, reinstall=False, + allow_downgrade=False))): + self.device.InstallSplitApk(DeviceUtilsInstallSplitApkTest.mock_apk, + ['split1.apk', 'split2.apk', 'split3.apk'], permissions=[], retries=0) + + def testInstallSplitApk_partialInstall(self): + with self.assertCalls( + (self.call.device._CheckSdkLevel(21)), + (mock.call.devil.android.sdk.split_select.SelectSplits( + self.device, 'base.apk', + ['split1.apk', 'split2.apk', 'split3.apk'], + allow_cached_props=False), + ['split2.apk']), + (self.call.device._GetApplicationPathsInternal('test.package'), + ['base-on-device.apk', 'split2-on-device.apk']), + (self.call.device._ComputeStaleApks('test.package', + ['base.apk', 'split2.apk']), + (['split2.apk'], None)), + (self.call.adb.InstallMultiple( + ['split2.apk'], partial='test.package', reinstall=True, + allow_downgrade=False))): + self.device.InstallSplitApk(DeviceUtilsInstallSplitApkTest.mock_apk, + ['split1.apk', 'split2.apk', 'split3.apk'], + reinstall=True, permissions=[], retries=0) + + def testInstallSplitApk_downgrade(self): + with self.assertCalls( + (self.call.device._CheckSdkLevel(21)), + (mock.call.devil.android.sdk.split_select.SelectSplits( + self.device, 'base.apk', + ['split1.apk', 'split2.apk', 'split3.apk'], + allow_cached_props=False), + ['split2.apk']), + (self.call.device._GetApplicationPathsInternal('test.package'), + ['base-on-device.apk', 'split2-on-device.apk']), + (self.call.device._ComputeStaleApks('test.package', + ['base.apk', 'split2.apk']), + (['split2.apk'], None)), + (self.call.adb.InstallMultiple( + ['split2.apk'], partial='test.package', reinstall=True, + allow_downgrade=True))): + self.device.InstallSplitApk(DeviceUtilsInstallSplitApkTest.mock_apk, + ['split1.apk', 'split2.apk', 'split3.apk'], + reinstall=True, permissions=[], retries=0, + allow_downgrade=True) + + +class DeviceUtilsUninstallTest(DeviceUtilsTest): + + def testUninstall_callsThrough(self): + with self.assertCalls( + (self.call.device._GetApplicationPathsInternal('test.package'), + ['/path.apk']), + self.call.adb.Uninstall('test.package', True)): + self.device.Uninstall('test.package', True) + + def testUninstall_noop(self): + with self.assertCalls( + (self.call.device._GetApplicationPathsInternal('test.package'), [])): + self.device.Uninstall('test.package', True) + + +class DeviceUtilsSuTest(DeviceUtilsTest): + + def testSu_preM(self): + with self.patch_call( + self.call.device.build_version_sdk, + return_value=version_codes.LOLLIPOP_MR1): + self.assertEquals('su -c foo', self.device._Su('foo')) + + def testSu_mAndAbove(self): + with self.patch_call( + self.call.device.build_version_sdk, + return_value=version_codes.MARSHMALLOW): + self.assertEquals('su 0 foo', self.device._Su('foo')) + + +class DeviceUtilsRunShellCommandTest(DeviceUtilsTest): + + def setUp(self): + super(DeviceUtilsRunShellCommandTest, self).setUp() + self.device.NeedsSU = mock.Mock(return_value=False) + + def testRunShellCommand_commandAsList(self): + with self.assertCall(self.call.adb.Shell('pm list packages'), ''): + self.device.RunShellCommand(['pm', 'list', 'packages']) + + def testRunShellCommand_commandAsListQuoted(self): + with self.assertCall(self.call.adb.Shell("echo 'hello world' '$10'"), ''): + self.device.RunShellCommand(['echo', 'hello world', '$10']) + + def testRunShellCommand_commandAsString(self): + with self.assertCall(self.call.adb.Shell('echo "$VAR"'), ''): + self.device.RunShellCommand('echo "$VAR"') + + def testNewRunShellImpl_withEnv(self): + with self.assertCall( + self.call.adb.Shell('VAR=some_string echo "$VAR"'), ''): + self.device.RunShellCommand('echo "$VAR"', env={'VAR': 'some_string'}) + + def testNewRunShellImpl_withEnvQuoted(self): + with self.assertCall( + self.call.adb.Shell('PATH="$PATH:/other/path" run_this'), ''): + self.device.RunShellCommand('run_this', env={'PATH': '$PATH:/other/path'}) + + def testNewRunShellImpl_withEnv_failure(self): + with self.assertRaises(KeyError): + self.device.RunShellCommand('some_cmd', env={'INVALID NAME': 'value'}) + + def testNewRunShellImpl_withCwd(self): + with self.assertCall(self.call.adb.Shell('cd /some/test/path && ls'), ''): + self.device.RunShellCommand('ls', cwd='/some/test/path') + + def testNewRunShellImpl_withCwdQuoted(self): + with self.assertCall( + self.call.adb.Shell("cd '/some test/path with/spaces' && ls"), ''): + self.device.RunShellCommand('ls', cwd='/some test/path with/spaces') + + def testRunShellCommand_withHugeCmd(self): + payload = 'hi! ' * 1024 + expected_cmd = "echo '%s'" % payload + with self.assertCalls( + (mock.call.devil.android.device_temp_file.DeviceTempFile( + self.adb, suffix='.sh'), MockTempFile('/sdcard/temp-123.sh')), + self.call.device._WriteFileWithPush('/sdcard/temp-123.sh', expected_cmd), + (self.call.adb.Shell('sh /sdcard/temp-123.sh'), payload + '\n')): + self.assertEquals([payload], + self.device.RunShellCommand(['echo', payload])) + + def testRunShellCommand_withHugeCmdAndSU(self): + payload = 'hi! ' * 1024 + expected_cmd_without_su = """sh -c 'echo '"'"'%s'"'"''""" % payload + expected_cmd = 'su -c %s' % expected_cmd_without_su + with self.assertCalls( + (self.call.device.NeedsSU(), True), + (self.call.device._Su(expected_cmd_without_su), expected_cmd), + (mock.call.devil.android.device_temp_file.DeviceTempFile( + self.adb, suffix='.sh'), MockTempFile('/sdcard/temp-123.sh')), + self.call.device._WriteFileWithPush('/sdcard/temp-123.sh', expected_cmd), + (self.call.adb.Shell('sh /sdcard/temp-123.sh'), payload + '\n')): + self.assertEquals( + [payload], + self.device.RunShellCommand(['echo', payload], as_root=True)) + + def testRunShellCommand_withSu(self): + expected_cmd_without_su = "sh -c 'setprop service.adb.root 0'" + expected_cmd = 'su -c %s' % expected_cmd_without_su + with self.assertCalls( + (self.call.device.NeedsSU(), True), + (self.call.device._Su(expected_cmd_without_su), expected_cmd), + (self.call.adb.Shell(expected_cmd), '')): + self.device.RunShellCommand('setprop service.adb.root 0', as_root=True) + + def testRunShellCommand_manyLines(self): + cmd = 'ls /some/path' + with self.assertCall(self.call.adb.Shell(cmd), 'file1\nfile2\nfile3\n'): + self.assertEquals(['file1', 'file2', 'file3'], + self.device.RunShellCommand(cmd)) + + def testRunShellCommand_singleLine_success(self): + cmd = 'echo $VALUE' + with self.assertCall(self.call.adb.Shell(cmd), 'some value\n'): + self.assertEquals('some value', + self.device.RunShellCommand(cmd, single_line=True)) + + def testRunShellCommand_singleLine_successEmptyLine(self): + cmd = 'echo $VALUE' + with self.assertCall(self.call.adb.Shell(cmd), '\n'): + self.assertEquals('', + self.device.RunShellCommand(cmd, single_line=True)) + + def testRunShellCommand_singleLine_successWithoutEndLine(self): + cmd = 'echo -n $VALUE' + with self.assertCall(self.call.adb.Shell(cmd), 'some value'): + self.assertEquals('some value', + self.device.RunShellCommand(cmd, single_line=True)) + + def testRunShellCommand_singleLine_successNoOutput(self): + cmd = 'echo -n $VALUE' + with self.assertCall(self.call.adb.Shell(cmd), ''): + self.assertEquals('', + self.device.RunShellCommand(cmd, single_line=True)) + + def testRunShellCommand_singleLine_failTooManyLines(self): + cmd = 'echo $VALUE' + with self.assertCall(self.call.adb.Shell(cmd), + 'some value\nanother value\n'): + with self.assertRaises(device_errors.CommandFailedError): + self.device.RunShellCommand(cmd, single_line=True) + + def testRunShellCommand_checkReturn_success(self): + cmd = 'echo $ANDROID_DATA' + output = '/data\n' + with self.assertCall(self.call.adb.Shell(cmd), output): + self.assertEquals([output.rstrip()], + self.device.RunShellCommand(cmd, check_return=True)) + + def testRunShellCommand_checkReturn_failure(self): + cmd = 'ls /root' + output = 'opendir failed, Permission denied\n' + with self.assertCall(self.call.adb.Shell(cmd), self.ShellError(output)): + with self.assertRaises(device_errors.AdbCommandFailedError): + self.device.RunShellCommand(cmd, check_return=True) + + def testRunShellCommand_checkReturn_disabled(self): + cmd = 'ls /root' + output = 'opendir failed, Permission denied\n' + with self.assertCall(self.call.adb.Shell(cmd), self.ShellError(output)): + self.assertEquals([output.rstrip()], + self.device.RunShellCommand(cmd, check_return=False)) + + def testRunShellCommand_largeOutput_enabled(self): + cmd = 'echo $VALUE' + temp_file = MockTempFile('/sdcard/temp-123') + cmd_redirect = '( %s )>%s' % (cmd, temp_file.name) + with self.assertCalls( + (mock.call.devil.android.device_temp_file.DeviceTempFile(self.adb), + temp_file), + (self.call.adb.Shell(cmd_redirect)), + (self.call.device.ReadFile(temp_file.name, force_pull=True), + 'something')): + self.assertEquals( + ['something'], + self.device.RunShellCommand( + cmd, large_output=True, check_return=True)) + + def testRunShellCommand_largeOutput_disabledNoTrigger(self): + cmd = 'something' + with self.assertCall(self.call.adb.Shell(cmd), self.ShellError('')): + with self.assertRaises(device_errors.AdbCommandFailedError): + self.device.RunShellCommand(cmd, check_return=True) + + def testRunShellCommand_largeOutput_disabledTrigger(self): + cmd = 'echo $VALUE' + temp_file = MockTempFile('/sdcard/temp-123') + cmd_redirect = '( %s )>%s' % (cmd, temp_file.name) + with self.assertCalls( + (self.call.adb.Shell(cmd), self.ShellError('', None)), + (mock.call.devil.android.device_temp_file.DeviceTempFile(self.adb), + temp_file), + (self.call.adb.Shell(cmd_redirect)), + (self.call.device.ReadFile(mock.ANY, force_pull=True), + 'something')): + self.assertEquals(['something'], + self.device.RunShellCommand(cmd, check_return=True)) + + +class DeviceUtilsRunPipedShellCommandTest(DeviceUtilsTest): + + def testRunPipedShellCommand_success(self): + with self.assertCall( + self.call.device.RunShellCommand( + 'ps | grep foo; echo "PIPESTATUS: ${PIPESTATUS[@]}"', + check_return=True), + ['This line contains foo', 'PIPESTATUS: 0 0']): + self.assertEquals(['This line contains foo'], + self.device._RunPipedShellCommand('ps | grep foo')) + + def testRunPipedShellCommand_firstCommandFails(self): + with self.assertCall( + self.call.device.RunShellCommand( + 'ps | grep foo; echo "PIPESTATUS: ${PIPESTATUS[@]}"', + check_return=True), + ['PIPESTATUS: 1 0']): + with self.assertRaises(device_errors.AdbShellCommandFailedError) as ec: + self.device._RunPipedShellCommand('ps | grep foo') + self.assertEquals([1, 0], ec.exception.status) + + def testRunPipedShellCommand_secondCommandFails(self): + with self.assertCall( + self.call.device.RunShellCommand( + 'ps | grep foo; echo "PIPESTATUS: ${PIPESTATUS[@]}"', + check_return=True), + ['PIPESTATUS: 0 1']): + with self.assertRaises(device_errors.AdbShellCommandFailedError) as ec: + self.device._RunPipedShellCommand('ps | grep foo') + self.assertEquals([0, 1], ec.exception.status) + + def testRunPipedShellCommand_outputCutOff(self): + with self.assertCall( + self.call.device.RunShellCommand( + 'ps | grep foo; echo "PIPESTATUS: ${PIPESTATUS[@]}"', + check_return=True), + ['foo.bar'] * 256 + ['foo.ba']): + with self.assertRaises(device_errors.AdbShellCommandFailedError) as ec: + self.device._RunPipedShellCommand('ps | grep foo') + self.assertIs(None, ec.exception.status) + + +@mock.patch('time.sleep', mock.Mock()) +class DeviceUtilsKillAllTest(DeviceUtilsTest): + + def testKillAll_noMatchingProcessesFailure(self): + with self.assertCall(self.call.device.GetPids('test_process'), {}): + with self.assertRaises(device_errors.CommandFailedError): + self.device.KillAll('test_process') + + def testKillAll_noMatchingProcessesQuiet(self): + with self.assertCall(self.call.device.GetPids('test_process'), {}): + self.assertEqual(0, self.device.KillAll('test_process', quiet=True)) + + def testKillAll_nonblocking(self): + with self.assertCalls( + (self.call.device.GetPids('some.process'), + {'some.process': ['1234'], 'some.processing.thing': ['5678']}), + (self.call.adb.Shell('kill -9 1234 5678'), '')): + self.assertEquals( + 2, self.device.KillAll('some.process', blocking=False)) + + def testKillAll_blocking(self): + with self.assertCalls( + (self.call.device.GetPids('some.process'), + {'some.process': ['1234'], 'some.processing.thing': ['5678']}), + (self.call.adb.Shell('kill -9 1234 5678'), ''), + (self.call.device.GetPids('some.process'), + {'some.processing.thing': ['5678']}), + (self.call.device.GetPids('some.process'), + {'some.process': ['1111']})): # Other instance with different pid. + self.assertEquals( + 2, self.device.KillAll('some.process', blocking=True)) + + def testKillAll_exactNonblocking(self): + with self.assertCalls( + (self.call.device.GetPids('some.process'), + {'some.process': ['1234'], 'some.processing.thing': ['5678']}), + (self.call.adb.Shell('kill -9 1234'), '')): + self.assertEquals( + 1, self.device.KillAll('some.process', exact=True, blocking=False)) + + def testKillAll_exactBlocking(self): + with self.assertCalls( + (self.call.device.GetPids('some.process'), + {'some.process': ['1234'], 'some.processing.thing': ['5678']}), + (self.call.adb.Shell('kill -9 1234'), ''), + (self.call.device.GetPids('some.process'), + {'some.process': ['1234'], 'some.processing.thing': ['5678']}), + (self.call.device.GetPids('some.process'), + {'some.processing.thing': ['5678']})): + self.assertEquals( + 1, self.device.KillAll('some.process', exact=True, blocking=True)) + + def testKillAll_root(self): + with self.assertCalls( + (self.call.device.GetPids('some.process'), {'some.process': ['1234']}), + (self.call.device.NeedsSU(), True), + (self.call.device._Su("sh -c 'kill -9 1234'"), + "su -c sh -c 'kill -9 1234'"), + (self.call.adb.Shell("su -c sh -c 'kill -9 1234'"), '')): + self.assertEquals( + 1, self.device.KillAll('some.process', as_root=True)) + + def testKillAll_sigterm(self): + with self.assertCalls( + (self.call.device.GetPids('some.process'), + {'some.process': ['1234']}), + (self.call.adb.Shell('kill -15 1234'), '')): + self.assertEquals( + 1, self.device.KillAll('some.process', signum=device_signal.SIGTERM)) + + def testKillAll_multipleInstances(self): + with self.assertCalls( + (self.call.device.GetPids('some.process'), + {'some.process': ['1234', '4567']}), + (self.call.adb.Shell('kill -15 1234 4567'), '')): + self.assertEquals( + 2, self.device.KillAll('some.process', signum=device_signal.SIGTERM)) + + +class DeviceUtilsStartActivityTest(DeviceUtilsTest): + + def testStartActivity_actionOnly(self): + test_intent = intent.Intent(action='android.intent.action.VIEW') + with self.assertCall( + self.call.adb.Shell('am start ' + '-a android.intent.action.VIEW'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent) + + def testStartActivity_success(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main') + with self.assertCall( + self.call.adb.Shell('am start ' + '-a android.intent.action.VIEW ' + '-n test.package/.Main'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent) + + def testStartActivity_failure(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main') + with self.assertCall( + self.call.adb.Shell('am start ' + '-a android.intent.action.VIEW ' + '-n test.package/.Main'), + 'Error: Failed to start test activity'): + with self.assertRaises(device_errors.CommandFailedError): + self.device.StartActivity(test_intent) + + def testStartActivity_blocking(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main') + with self.assertCall( + self.call.adb.Shell('am start ' + '-W ' + '-a android.intent.action.VIEW ' + '-n test.package/.Main'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent, blocking=True) + + def testStartActivity_withCategory(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main', + category='android.intent.category.HOME') + with self.assertCall( + self.call.adb.Shell('am start ' + '-a android.intent.action.VIEW ' + '-c android.intent.category.HOME ' + '-n test.package/.Main'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent) + + def testStartActivity_withMultipleCategories(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main', + category=['android.intent.category.HOME', + 'android.intent.category.BROWSABLE']) + with self.assertCall( + self.call.adb.Shell('am start ' + '-a android.intent.action.VIEW ' + '-c android.intent.category.HOME ' + '-c android.intent.category.BROWSABLE ' + '-n test.package/.Main'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent) + + def testStartActivity_withData(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main', + data='http://www.google.com/') + with self.assertCall( + self.call.adb.Shell('am start ' + '-a android.intent.action.VIEW ' + '-d http://www.google.com/ ' + '-n test.package/.Main'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent) + + def testStartActivity_withStringExtra(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main', + extras={'foo': 'test'}) + with self.assertCall( + self.call.adb.Shell('am start ' + '-a android.intent.action.VIEW ' + '-n test.package/.Main ' + '--es foo test'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent) + + def testStartActivity_withBoolExtra(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main', + extras={'foo': True}) + with self.assertCall( + self.call.adb.Shell('am start ' + '-a android.intent.action.VIEW ' + '-n test.package/.Main ' + '--ez foo True'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent) + + def testStartActivity_withIntExtra(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main', + extras={'foo': 123}) + with self.assertCall( + self.call.adb.Shell('am start ' + '-a android.intent.action.VIEW ' + '-n test.package/.Main ' + '--ei foo 123'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent) + + def testStartActivity_withTraceFile(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main') + with self.assertCall( + self.call.adb.Shell('am start ' + '--start-profiler test_trace_file.out ' + '-a android.intent.action.VIEW ' + '-n test.package/.Main'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent, + trace_file_name='test_trace_file.out') + + def testStartActivity_withForceStop(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main') + with self.assertCall( + self.call.adb.Shell('am start ' + '-S ' + '-a android.intent.action.VIEW ' + '-n test.package/.Main'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent, force_stop=True) + + def testStartActivity_withFlags(self): + test_intent = intent.Intent(action='android.intent.action.VIEW', + package='test.package', + activity='.Main', + flags='0x10000000') + with self.assertCall( + self.call.adb.Shell('am start ' + '-a android.intent.action.VIEW ' + '-n test.package/.Main ' + '-f 0x10000000'), + 'Starting: Intent { act=android.intent.action.VIEW }'): + self.device.StartActivity(test_intent) + + +class DeviceUtilsStartInstrumentationTest(DeviceUtilsTest): + + def testStartInstrumentation_nothing(self): + with self.assertCalls( + self.call.device.RunShellCommand( + 'p=test.package;am instrument "$p"/.TestInstrumentation', + check_return=True, large_output=True)): + self.device.StartInstrumentation( + 'test.package/.TestInstrumentation', + finish=False, raw=False, extras=None) + + def testStartInstrumentation_finish(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + 'p=test.package;am instrument -w "$p"/.TestInstrumentation', + check_return=True, large_output=True), + ['OK (1 test)'])): + output = self.device.StartInstrumentation( + 'test.package/.TestInstrumentation', + finish=True, raw=False, extras=None) + self.assertEquals(['OK (1 test)'], output) + + def testStartInstrumentation_raw(self): + with self.assertCalls( + self.call.device.RunShellCommand( + 'p=test.package;am instrument -r "$p"/.TestInstrumentation', + check_return=True, large_output=True)): + self.device.StartInstrumentation( + 'test.package/.TestInstrumentation', + finish=False, raw=True, extras=None) + + def testStartInstrumentation_extras(self): + with self.assertCalls( + self.call.device.RunShellCommand( + 'p=test.package;am instrument -e "$p".foo Foo -e bar \'Val \'"$p" ' + '"$p"/.TestInstrumentation', + check_return=True, large_output=True)): + self.device.StartInstrumentation( + 'test.package/.TestInstrumentation', + finish=False, raw=False, extras={'test.package.foo': 'Foo', + 'bar': 'Val test.package'}) + + +class DeviceUtilsBroadcastIntentTest(DeviceUtilsTest): + + def testBroadcastIntent_noExtras(self): + test_intent = intent.Intent(action='test.package.with.an.INTENT') + with self.assertCall( + self.call.adb.Shell('am broadcast -a test.package.with.an.INTENT'), + 'Broadcasting: Intent { act=test.package.with.an.INTENT } '): + self.device.BroadcastIntent(test_intent) + + def testBroadcastIntent_withExtra(self): + test_intent = intent.Intent(action='test.package.with.an.INTENT', + extras={'foo': 'bar value'}) + with self.assertCall( + self.call.adb.Shell( + "am broadcast -a test.package.with.an.INTENT --es foo 'bar value'"), + 'Broadcasting: Intent { act=test.package.with.an.INTENT } '): + self.device.BroadcastIntent(test_intent) + + def testBroadcastIntent_withExtra_noValue(self): + test_intent = intent.Intent(action='test.package.with.an.INTENT', + extras={'foo': None}) + with self.assertCall( + self.call.adb.Shell( + 'am broadcast -a test.package.with.an.INTENT --esn foo'), + 'Broadcasting: Intent { act=test.package.with.an.INTENT } '): + self.device.BroadcastIntent(test_intent) + + +class DeviceUtilsGoHomeTest(DeviceUtilsTest): + + def testGoHome_popupsExist(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), []), + (self.call.device.RunShellCommand( + ['am', 'start', '-W', '-a', 'android.intent.action.MAIN', + '-c', 'android.intent.category.HOME'], check_return=True), + 'Starting: Intent { act=android.intent.action.MAIN }\r\n'''), + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), []), + (self.call.device.RunShellCommand( + ['input', 'keyevent', '66'], check_return=True)), + (self.call.device.RunShellCommand( + ['input', 'keyevent', '4'], check_return=True)), + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), + ['mCurrentFocus Launcher'])): + self.device.GoHome() + + def testGoHome_willRetry(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), []), + (self.call.device.RunShellCommand( + ['am', 'start', '-W', '-a', 'android.intent.action.MAIN', + '-c', 'android.intent.category.HOME'], check_return=True), + 'Starting: Intent { act=android.intent.action.MAIN }\r\n'''), + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), []), + (self.call.device.RunShellCommand( + ['input', 'keyevent', '66'], check_return=True,)), + (self.call.device.RunShellCommand( + ['input', 'keyevent', '4'], check_return=True)), + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), []), + (self.call.device.RunShellCommand( + ['input', 'keyevent', '66'], check_return=True)), + (self.call.device.RunShellCommand( + ['input', 'keyevent', '4'], check_return=True)), + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), + self.TimeoutError())): + with self.assertRaises(device_errors.CommandTimeoutError): + self.device.GoHome() + + def testGoHome_alreadyFocused(self): + with self.assertCall( + self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), + ['mCurrentFocus Launcher']): + self.device.GoHome() + + def testGoHome_alreadyFocusedAlternateCase(self): + with self.assertCall( + self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), + [' mCurrentFocus .launcher/.']): + self.device.GoHome() + + def testGoHome_obtainsFocusAfterGoingHome(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), []), + (self.call.device.RunShellCommand( + ['am', 'start', '-W', '-a', 'android.intent.action.MAIN', + '-c', 'android.intent.category.HOME'], check_return=True), + 'Starting: Intent { act=android.intent.action.MAIN }\r\n'''), + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), + ['mCurrentFocus Launcher'])): + self.device.GoHome() + + +class DeviceUtilsForceStopTest(DeviceUtilsTest): + + def testForceStop(self): + with self.assertCall( + self.call.adb.Shell('p=test.package;if [[ "$(ps)" = *$p* ]]; then ' + 'am force-stop $p; fi'), + ''): + self.device.ForceStop('test.package') + + +class DeviceUtilsClearApplicationStateTest(DeviceUtilsTest): + + def testClearApplicationState_setPermissions(self): + with self.assertCalls( + (self.call.device.GetProp('ro.build.version.sdk', cache=True), '17'), + (self.call.device._GetApplicationPathsInternal('this.package.exists'), + ['/data/app/this.package.exists.apk']), + (self.call.device.RunShellCommand( + ['pm', 'clear', 'this.package.exists'], + check_return=True), + ['Success']), + (self.call.device.GrantPermissions( + 'this.package.exists', ['p1']), [])): + self.device.ClearApplicationState( + 'this.package.exists', permissions=['p1']) + + def testClearApplicationState_packageDoesntExist(self): + with self.assertCalls( + (self.call.device.GetProp('ro.build.version.sdk', cache=True), '11'), + (self.call.device._GetApplicationPathsInternal('does.not.exist'), + [])): + self.device.ClearApplicationState('does.not.exist') + + def testClearApplicationState_packageDoesntExistOnAndroidJBMR2OrAbove(self): + with self.assertCalls( + (self.call.device.GetProp('ro.build.version.sdk', cache=True), '18'), + (self.call.device.RunShellCommand( + ['pm', 'clear', 'this.package.does.not.exist'], + check_return=True), + ['Failed'])): + self.device.ClearApplicationState('this.package.does.not.exist') + + def testClearApplicationState_packageExists(self): + with self.assertCalls( + (self.call.device.GetProp('ro.build.version.sdk', cache=True), '17'), + (self.call.device._GetApplicationPathsInternal('this.package.exists'), + ['/data/app/this.package.exists.apk']), + (self.call.device.RunShellCommand( + ['pm', 'clear', 'this.package.exists'], + check_return=True), + ['Success'])): + self.device.ClearApplicationState('this.package.exists') + + def testClearApplicationState_packageExistsOnAndroidJBMR2OrAbove(self): + with self.assertCalls( + (self.call.device.GetProp('ro.build.version.sdk', cache=True), '18'), + (self.call.device.RunShellCommand( + ['pm', 'clear', 'this.package.exists'], + check_return=True), + ['Success'])): + self.device.ClearApplicationState('this.package.exists') + + +class DeviceUtilsSendKeyEventTest(DeviceUtilsTest): + + def testSendKeyEvent(self): + with self.assertCall(self.call.adb.Shell('input keyevent 66'), ''): + self.device.SendKeyEvent(66) + + +class DeviceUtilsPushChangedFilesIndividuallyTest(DeviceUtilsTest): + + def testPushChangedFilesIndividually_empty(self): + test_files = [] + with self.assertCalls(): + self.device._PushChangedFilesIndividually(test_files) + + def testPushChangedFilesIndividually_single(self): + test_files = [('/test/host/path', '/test/device/path')] + with self.assertCalls(self.call.adb.Push(*test_files[0])): + self.device._PushChangedFilesIndividually(test_files) + + def testPushChangedFilesIndividually_multiple(self): + test_files = [ + ('/test/host/path/file1', '/test/device/path/file1'), + ('/test/host/path/file2', '/test/device/path/file2')] + with self.assertCalls( + self.call.adb.Push(*test_files[0]), + self.call.adb.Push(*test_files[1])): + self.device._PushChangedFilesIndividually(test_files) + + +class DeviceUtilsPushChangedFilesZippedTest(DeviceUtilsTest): + + def testPushChangedFilesZipped_noUnzipCommand(self): + test_files = [('/test/host/path/file1', '/test/device/path/file1')] + mock_zip_temp = mock.mock_open() + mock_zip_temp.return_value.name = '/test/temp/file/tmp.zip' + with self.assertCalls( + (mock.call.tempfile.NamedTemporaryFile(suffix='.zip'), mock_zip_temp), + (mock.call.multiprocessing.Process( + target=device_utils.DeviceUtils._CreateDeviceZip, + args=('/test/temp/file/tmp.zip', test_files)), mock.Mock()), + (self.call.device._MaybeInstallCommands(), False)): + self.assertFalse(self.device._PushChangedFilesZipped(test_files, + ['/test/dir'])) + + def _testPushChangedFilesZipped_spec(self, test_files): + mock_zip_temp = mock.mock_open() + mock_zip_temp.return_value.name = '/test/temp/file/tmp.zip' + with self.assertCalls( + (mock.call.tempfile.NamedTemporaryFile(suffix='.zip'), mock_zip_temp), + (mock.call.multiprocessing.Process( + target=device_utils.DeviceUtils._CreateDeviceZip, + args=('/test/temp/file/tmp.zip', test_files)), mock.Mock()), + (self.call.device._MaybeInstallCommands(), True), + (self.call.device.NeedsSU(), True), + (mock.call.devil.android.device_temp_file.DeviceTempFile(self.adb, + suffix='.zip'), + MockTempFile('/test/sdcard/foo123.zip')), + self.call.adb.Push( + '/test/temp/file/tmp.zip', '/test/sdcard/foo123.zip'), + self.call.device.RunShellCommand( + 'unzip /test/sdcard/foo123.zip&&chmod -R 777 /test/dir', + as_root=True, + env={'PATH': '/data/local/tmp/bin:$PATH'}, + check_return=True)): + self.assertTrue(self.device._PushChangedFilesZipped(test_files, + ['/test/dir'])) + + def testPushChangedFilesZipped_single(self): + self._testPushChangedFilesZipped_spec( + [('/test/host/path/file1', '/test/device/path/file1')]) + + def testPushChangedFilesZipped_multiple(self): + self._testPushChangedFilesZipped_spec( + [('/test/host/path/file1', '/test/device/path/file1'), + ('/test/host/path/file2', '/test/device/path/file2')]) + + +class DeviceUtilsPathExistsTest(DeviceUtilsTest): + + def testPathExists_pathExists(self): + with self.assertCall( + self.call.device.RunShellCommand( + "test -e '/path/file exists'", + as_root=False, check_return=True, timeout=10, retries=0), + []): + self.assertTrue(self.device.PathExists('/path/file exists')) + + def testPathExists_multiplePathExists(self): + with self.assertCall( + self.call.device.RunShellCommand( + "test -e '/path 1' -a -e /path2", + as_root=False, check_return=True, timeout=10, retries=0), + []): + self.assertTrue(self.device.PathExists(('/path 1', '/path2'))) + + def testPathExists_pathDoesntExist(self): + with self.assertCall( + self.call.device.RunShellCommand( + "test -e /path/file.not.exists", + as_root=False, check_return=True, timeout=10, retries=0), + self.ShellError()): + self.assertFalse(self.device.PathExists('/path/file.not.exists')) + + def testPathExists_asRoot(self): + with self.assertCall( + self.call.device.RunShellCommand( + "test -e /root/path/exists", + as_root=True, check_return=True, timeout=10, retries=0), + self.ShellError()): + self.assertFalse( + self.device.PathExists('/root/path/exists', as_root=True)) + + def testFileExists_pathDoesntExist(self): + with self.assertCall( + self.call.device.RunShellCommand( + "test -e /path/file.not.exists", + as_root=False, check_return=True, timeout=10, retries=0), + self.ShellError()): + self.assertFalse(self.device.FileExists('/path/file.not.exists')) + + +class DeviceUtilsPullFileTest(DeviceUtilsTest): + + def testPullFile_existsOnDevice(self): + with mock.patch('os.path.exists', return_value=True): + with self.assertCall( + self.call.adb.Pull('/data/app/test.file.exists', + '/test/file/host/path')): + self.device.PullFile('/data/app/test.file.exists', + '/test/file/host/path') + + def testPullFile_doesntExistOnDevice(self): + with mock.patch('os.path.exists', return_value=True): + with self.assertCall( + self.call.adb.Pull('/data/app/test.file.does.not.exist', + '/test/file/host/path'), + self.CommandError('remote object does not exist')): + with self.assertRaises(device_errors.CommandFailedError): + self.device.PullFile('/data/app/test.file.does.not.exist', + '/test/file/host/path') + + +class DeviceUtilsReadFileTest(DeviceUtilsTest): + + def testReadFileWithPull_success(self): + tmp_host_dir = '/tmp/dir/on.host/' + tmp_host = MockTempFile('/tmp/dir/on.host/tmp_ReadFileWithPull') + tmp_host.file.read.return_value = 'some interesting contents' + with self.assertCalls( + (mock.call.tempfile.mkdtemp(), tmp_host_dir), + (self.call.adb.Pull('/path/to/device/file', mock.ANY)), + (mock.call.__builtin__.open(mock.ANY, 'r'), tmp_host), + (mock.call.os.path.exists(tmp_host_dir), True), + (mock.call.shutil.rmtree(tmp_host_dir), None)): + self.assertEquals('some interesting contents', + self.device._ReadFileWithPull('/path/to/device/file')) + tmp_host.file.read.assert_called_once_with() + + def testReadFileWithPull_rejected(self): + tmp_host_dir = '/tmp/dir/on.host/' + with self.assertCalls( + (mock.call.tempfile.mkdtemp(), tmp_host_dir), + (self.call.adb.Pull('/path/to/device/file', mock.ANY), + self.CommandError()), + (mock.call.os.path.exists(tmp_host_dir), True), + (mock.call.shutil.rmtree(tmp_host_dir), None)): + with self.assertRaises(device_errors.CommandFailedError): + self.device._ReadFileWithPull('/path/to/device/file') + + def testReadFile_exists(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['ls', '-l', '/read/this/test/file'], + as_root=False, check_return=True), + ['-rw-rw---- root foo 256 1970-01-01 00:00 file']), + (self.call.device.RunShellCommand( + ['cat', '/read/this/test/file'], + as_root=False, check_return=True), + ['this is a test file'])): + self.assertEqual('this is a test file\n', + self.device.ReadFile('/read/this/test/file')) + + def testReadFile_exists2(self): + # Same as testReadFile_exists, but uses Android N ls output. + with self.assertCalls( + (self.call.device.RunShellCommand( + ['ls', '-l', '/read/this/test/file'], + as_root=False, check_return=True), + ['-rw-rw-rw- 1 root root 256 2016-03-15 03:27 /read/this/test/file']), + (self.call.device.RunShellCommand( + ['cat', '/read/this/test/file'], + as_root=False, check_return=True), + ['this is a test file'])): + self.assertEqual('this is a test file\n', + self.device.ReadFile('/read/this/test/file')) + + def testReadFile_doesNotExist(self): + with self.assertCall( + self.call.device.RunShellCommand( + ['ls', '-l', '/this/file/does.not.exist'], + as_root=False, check_return=True), + self.CommandError('File does not exist')): + with self.assertRaises(device_errors.CommandFailedError): + self.device.ReadFile('/this/file/does.not.exist') + + def testReadFile_zeroSize(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['ls', '-l', '/this/file/has/zero/size'], + as_root=False, check_return=True), + ['-r--r--r-- root foo 0 1970-01-01 00:00 zero_size_file']), + (self.call.device._ReadFileWithPull('/this/file/has/zero/size'), + 'but it has contents\n')): + self.assertEqual('but it has contents\n', + self.device.ReadFile('/this/file/has/zero/size')) + + def testReadFile_withSU(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['ls', '-l', '/this/file/can.be.read.with.su'], + as_root=True, check_return=True), + ['-rw------- root root 256 1970-01-01 00:00 can.be.read.with.su']), + (self.call.device.RunShellCommand( + ['cat', '/this/file/can.be.read.with.su'], + as_root=True, check_return=True), + ['this is a test file', 'read with su'])): + self.assertEqual( + 'this is a test file\nread with su\n', + self.device.ReadFile('/this/file/can.be.read.with.su', + as_root=True)) + + def testReadFile_withPull(self): + contents = 'a' * 123456 + with self.assertCalls( + (self.call.device.RunShellCommand( + ['ls', '-l', '/read/this/big/test/file'], + as_root=False, check_return=True), + ['-rw-rw---- root foo 123456 1970-01-01 00:00 file']), + (self.call.device._ReadFileWithPull('/read/this/big/test/file'), + contents)): + self.assertEqual( + contents, self.device.ReadFile('/read/this/big/test/file')) + + def testReadFile_withPullAndSU(self): + contents = 'b' * 123456 + with self.assertCalls( + (self.call.device.RunShellCommand( + ['ls', '-l', '/this/big/file/can.be.read.with.su'], + as_root=True, check_return=True), + ['-rw------- root root 123456 1970-01-01 00:00 can.be.read.with.su']), + (self.call.device.NeedsSU(), True), + (mock.call.devil.android.device_temp_file.DeviceTempFile(self.adb), + MockTempFile('/sdcard/tmp/on.device')), + self.call.device.RunShellCommand( + 'SRC=/this/big/file/can.be.read.with.su DEST=/sdcard/tmp/on.device;' + 'cp "$SRC" "$DEST" && chmod 666 "$DEST"', + as_root=True, check_return=True), + (self.call.device._ReadFileWithPull('/sdcard/tmp/on.device'), + contents)): + self.assertEqual( + contents, + self.device.ReadFile('/this/big/file/can.be.read.with.su', + as_root=True)) + + def testReadFile_forcePull(self): + contents = 'a' * 123456 + with self.assertCall( + self.call.device._ReadFileWithPull('/read/this/big/test/file'), + contents): + self.assertEqual( + contents, + self.device.ReadFile('/read/this/big/test/file', force_pull=True)) + + +class DeviceUtilsWriteFileTest(DeviceUtilsTest): + + def testWriteFileWithPush_success(self): + tmp_host = MockTempFile('/tmp/file/on.host') + contents = 'some interesting contents' + with self.assertCalls( + (mock.call.tempfile.NamedTemporaryFile(), tmp_host), + self.call.adb.Push('/tmp/file/on.host', '/path/to/device/file')): + self.device._WriteFileWithPush('/path/to/device/file', contents) + tmp_host.file.write.assert_called_once_with(contents) + + def testWriteFileWithPush_rejected(self): + tmp_host = MockTempFile('/tmp/file/on.host') + contents = 'some interesting contents' + with self.assertCalls( + (mock.call.tempfile.NamedTemporaryFile(), tmp_host), + (self.call.adb.Push('/tmp/file/on.host', '/path/to/device/file'), + self.CommandError())): + with self.assertRaises(device_errors.CommandFailedError): + self.device._WriteFileWithPush('/path/to/device/file', contents) + + def testWriteFile_withPush(self): + contents = 'some large contents ' * 26 # 20 * 26 = 520 chars + with self.assertCalls( + self.call.device._WriteFileWithPush('/path/to/device/file', contents)): + self.device.WriteFile('/path/to/device/file', contents) + + def testWriteFile_withPushForced(self): + contents = 'tiny contents' + with self.assertCalls( + self.call.device._WriteFileWithPush('/path/to/device/file', contents)): + self.device.WriteFile('/path/to/device/file', contents, force_push=True) + + def testWriteFile_withPushAndSU(self): + contents = 'some large contents ' * 26 # 20 * 26 = 520 chars + with self.assertCalls( + (self.call.device.NeedsSU(), True), + (mock.call.devil.android.device_temp_file.DeviceTempFile(self.adb), + MockTempFile('/sdcard/tmp/on.device')), + self.call.device._WriteFileWithPush('/sdcard/tmp/on.device', contents), + self.call.device.RunShellCommand( + ['cp', '/sdcard/tmp/on.device', '/path/to/device/file'], + as_root=True, check_return=True)): + self.device.WriteFile('/path/to/device/file', contents, as_root=True) + + def testWriteFile_withEcho(self): + with self.assertCall(self.call.adb.Shell( + "echo -n the.contents > /test/file/to.write"), ''): + self.device.WriteFile('/test/file/to.write', 'the.contents') + + def testWriteFile_withEchoAndQuotes(self): + with self.assertCall(self.call.adb.Shell( + "echo -n 'the contents' > '/test/file/to write'"), ''): + self.device.WriteFile('/test/file/to write', 'the contents') + + def testWriteFile_withEchoAndSU(self): + expected_cmd_without_su = "sh -c 'echo -n contents > /test/file'" + expected_cmd = 'su -c %s' % expected_cmd_without_su + with self.assertCalls( + (self.call.device.NeedsSU(), True), + (self.call.device._Su(expected_cmd_without_su), expected_cmd), + (self.call.adb.Shell(expected_cmd), + '')): + self.device.WriteFile('/test/file', 'contents', as_root=True) + + +class DeviceUtilsLsTest(DeviceUtilsTest): + + def testLs_directory(self): + result = [('.', adb_wrapper.DeviceStat(16889, 4096, 1417436123)), + ('..', adb_wrapper.DeviceStat(16873, 4096, 12382237)), + ('testfile.txt', adb_wrapper.DeviceStat(33206, 3, 1417436122))] + with self.assertCalls( + (self.call.adb.Ls('/data/local/tmp'), result)): + self.assertEquals(result, + self.device.Ls('/data/local/tmp')) + + def testLs_nothing(self): + with self.assertCalls( + (self.call.adb.Ls('/data/local/tmp/testfile.txt'), [])): + self.assertEquals([], + self.device.Ls('/data/local/tmp/testfile.txt')) + + +class DeviceUtilsStatTest(DeviceUtilsTest): + + def testStat_file(self): + result = [('.', adb_wrapper.DeviceStat(16889, 4096, 1417436123)), + ('..', adb_wrapper.DeviceStat(16873, 4096, 12382237)), + ('testfile.txt', adb_wrapper.DeviceStat(33206, 3, 1417436122))] + with self.assertCalls( + (self.call.adb.Ls('/data/local/tmp'), result)): + self.assertEquals(adb_wrapper.DeviceStat(33206, 3, 1417436122), + self.device.Stat('/data/local/tmp/testfile.txt')) + + def testStat_directory(self): + result = [('.', adb_wrapper.DeviceStat(16873, 4096, 12382237)), + ('..', adb_wrapper.DeviceStat(16873, 4096, 12382237)), + ('tmp', adb_wrapper.DeviceStat(16889, 4096, 1417436123))] + with self.assertCalls( + (self.call.adb.Ls('/data/local'), result)): + self.assertEquals(adb_wrapper.DeviceStat(16889, 4096, 1417436123), + self.device.Stat('/data/local/tmp')) + + def testStat_doesNotExist(self): + result = [('.', adb_wrapper.DeviceStat(16889, 4096, 1417436123)), + ('..', adb_wrapper.DeviceStat(16873, 4096, 12382237)), + ('testfile.txt', adb_wrapper.DeviceStat(33206, 3, 1417436122))] + with self.assertCalls( + (self.call.adb.Ls('/data/local/tmp'), result)): + with self.assertRaises(device_errors.CommandFailedError): + self.device.Stat('/data/local/tmp/does.not.exist.txt') + + +class DeviceUtilsSetJavaAssertsTest(DeviceUtilsTest): + + def testSetJavaAsserts_enable(self): + with self.assertCalls( + (self.call.device.ReadFile(self.device.LOCAL_PROPERTIES_PATH), + 'some.example.prop=with an example value\n' + 'some.other.prop=value_ok\n'), + self.call.device.WriteFile( + self.device.LOCAL_PROPERTIES_PATH, + 'some.example.prop=with an example value\n' + 'some.other.prop=value_ok\n' + 'dalvik.vm.enableassertions=all\n'), + (self.call.device.GetProp('dalvik.vm.enableassertions'), ''), + self.call.device.SetProp('dalvik.vm.enableassertions', 'all')): + self.assertTrue(self.device.SetJavaAsserts(True)) + + def testSetJavaAsserts_disable(self): + with self.assertCalls( + (self.call.device.ReadFile(self.device.LOCAL_PROPERTIES_PATH), + 'some.example.prop=with an example value\n' + 'dalvik.vm.enableassertions=all\n' + 'some.other.prop=value_ok\n'), + self.call.device.WriteFile( + self.device.LOCAL_PROPERTIES_PATH, + 'some.example.prop=with an example value\n' + 'some.other.prop=value_ok\n'), + (self.call.device.GetProp('dalvik.vm.enableassertions'), 'all'), + self.call.device.SetProp('dalvik.vm.enableassertions', '')): + self.assertTrue(self.device.SetJavaAsserts(False)) + + def testSetJavaAsserts_alreadyEnabled(self): + with self.assertCalls( + (self.call.device.ReadFile(self.device.LOCAL_PROPERTIES_PATH), + 'some.example.prop=with an example value\n' + 'dalvik.vm.enableassertions=all\n' + 'some.other.prop=value_ok\n'), + (self.call.device.GetProp('dalvik.vm.enableassertions'), 'all')): + self.assertFalse(self.device.SetJavaAsserts(True)) + + def testSetJavaAsserts_malformedLocalProp(self): + with self.assertCalls( + (self.call.device.ReadFile(self.device.LOCAL_PROPERTIES_PATH), + 'some.example.prop=with an example value\n' + 'malformed_property\n' + 'dalvik.vm.enableassertions=all\n' + 'some.other.prop=value_ok\n'), + (self.call.device.GetProp('dalvik.vm.enableassertions'), 'all')): + self.assertFalse(self.device.SetJavaAsserts(True)) + + +class DeviceUtilsGetPropTest(DeviceUtilsTest): + + def testGetProp_exists(self): + with self.assertCall( + self.call.device.RunShellCommand( + ['getprop', 'test.property'], check_return=True, single_line=True, + timeout=self.device._default_timeout, + retries=self.device._default_retries), + 'property_value'): + self.assertEqual('property_value', + self.device.GetProp('test.property')) + + def testGetProp_doesNotExist(self): + with self.assertCall( + self.call.device.RunShellCommand( + ['getprop', 'property.does.not.exist'], + check_return=True, single_line=True, + timeout=self.device._default_timeout, + retries=self.device._default_retries), + ''): + self.assertEqual('', self.device.GetProp('property.does.not.exist')) + + def testGetProp_cachedRoProp(self): + with self.assertCall( + self.call.device.RunShellCommand( + ['getprop'], check_return=True, large_output=True, + timeout=self.device._default_timeout, + retries=self.device._default_retries), + ['[ro.build.type]: [userdebug]']): + self.assertEqual('userdebug', + self.device.GetProp('ro.build.type', cache=True)) + self.assertEqual('userdebug', + self.device.GetProp('ro.build.type', cache=True)) + + def testGetProp_retryAndCache(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['getprop'], check_return=True, large_output=True, + timeout=self.device._default_timeout, + retries=3), + ['[ro.build.type]: [userdebug]'])): + self.assertEqual('userdebug', + self.device.GetProp('ro.build.type', + cache=True, retries=3)) + self.assertEqual('userdebug', + self.device.GetProp('ro.build.type', + cache=True, retries=3)) + + +class DeviceUtilsSetPropTest(DeviceUtilsTest): + + def testSetProp(self): + with self.assertCall( + self.call.device.RunShellCommand( + ['setprop', 'test.property', 'test value'], check_return=True)): + self.device.SetProp('test.property', 'test value') + + def testSetProp_check_succeeds(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['setprop', 'test.property', 'new_value'], check_return=True)), + (self.call.device.GetProp('test.property', cache=False), 'new_value')): + self.device.SetProp('test.property', 'new_value', check=True) + + def testSetProp_check_fails(self): + with self.assertCalls( + (self.call.device.RunShellCommand( + ['setprop', 'test.property', 'new_value'], check_return=True)), + (self.call.device.GetProp('test.property', cache=False), 'old_value')): + with self.assertRaises(device_errors.CommandFailedError): + self.device.SetProp('test.property', 'new_value', check=True) + + +class DeviceUtilsGetPidsTest(DeviceUtilsTest): + + def testGetPids_noMatches(self): + with self.assertCall( + self.call.device._RunPipedShellCommand('ps | grep -F does.not.match'), + []): + self.assertEqual({}, self.device.GetPids('does.not.match')) + + def testGetPids_oneMatch(self): + with self.assertCall( + self.call.device._RunPipedShellCommand('ps | grep -F one.match'), + ['user 1001 100 1024 1024 ffffffff 00000000 one.match']): + self.assertEqual( + {'one.match': ['1001']}, + self.device.GetPids('one.match')) + + def testGetPids_multipleMatches(self): + with self.assertCall( + self.call.device._RunPipedShellCommand('ps | grep -F match'), + ['user 1001 100 1024 1024 ffffffff 00000000 one.match', + 'user 1002 100 1024 1024 ffffffff 00000000 two.match', + 'user 1003 100 1024 1024 ffffffff 00000000 three.match']): + self.assertEqual( + {'one.match': ['1001'], + 'two.match': ['1002'], + 'three.match': ['1003']}, + self.device.GetPids('match')) + + def testGetPids_exactMatch(self): + with self.assertCall( + self.call.device._RunPipedShellCommand('ps | grep -F exact.match'), + ['user 1000 100 1024 1024 ffffffff 00000000 not.exact.match', + 'user 1234 100 1024 1024 ffffffff 00000000 exact.match']): + self.assertEqual( + {'not.exact.match': ['1000'], 'exact.match': ['1234']}, + self.device.GetPids('exact.match')) + + def testGetPids_quotable(self): + with self.assertCall( + self.call.device._RunPipedShellCommand("ps | grep -F 'my$process'"), + ['user 1234 100 1024 1024 ffffffff 00000000 my$process']): + self.assertEqual( + {'my$process': ['1234']}, self.device.GetPids('my$process')) + + def testGetPids_multipleInstances(self): + with self.assertCall( + self.call.device._RunPipedShellCommand('ps | grep -F foo'), + ['user 1000 100 1024 1024 ffffffff 00000000 foo', + 'user 1234 100 1024 1024 ffffffff 00000000 foo']): + self.assertEqual( + {'foo': ['1000', '1234']}, + self.device.GetPids('foo')) + + +class DeviceUtilsTakeScreenshotTest(DeviceUtilsTest): + + def testTakeScreenshot_fileNameProvided(self): + with self.assertCalls( + (mock.call.devil.android.device_temp_file.DeviceTempFile( + self.adb, suffix='.png'), + MockTempFile('/tmp/path/temp-123.png')), + (self.call.adb.Shell('/system/bin/screencap -p /tmp/path/temp-123.png'), + ''), + self.call.device.PullFile('/tmp/path/temp-123.png', + '/test/host/screenshot.png')): + self.device.TakeScreenshot('/test/host/screenshot.png') + + +class DeviceUtilsGetMemoryUsageForPidTest(DeviceUtilsTest): + + def setUp(self): + super(DeviceUtilsGetMemoryUsageForPidTest, self).setUp() + + def testGetMemoryUsageForPid_validPid(self): + with self.assertCalls( + (self.call.device._RunPipedShellCommand( + 'showmap 1234 | grep TOTAL', as_root=True), + ['100 101 102 103 104 105 106 107 TOTAL']), + (self.call.device.ReadFile('/proc/1234/status', as_root=True), + 'VmHWM: 1024 kB\n')): + self.assertEqual( + { + 'Size': 100, + 'Rss': 101, + 'Pss': 102, + 'Shared_Clean': 103, + 'Shared_Dirty': 104, + 'Private_Clean': 105, + 'Private_Dirty': 106, + 'VmHWM': 1024 + }, + self.device.GetMemoryUsageForPid(1234)) + + def testGetMemoryUsageForPid_noSmaps(self): + with self.assertCalls( + (self.call.device._RunPipedShellCommand( + 'showmap 4321 | grep TOTAL', as_root=True), + ['cannot open /proc/4321/smaps: No such file or directory']), + (self.call.device.ReadFile('/proc/4321/status', as_root=True), + 'VmHWM: 1024 kb\n')): + self.assertEquals({'VmHWM': 1024}, self.device.GetMemoryUsageForPid(4321)) + + def testGetMemoryUsageForPid_noStatus(self): + with self.assertCalls( + (self.call.device._RunPipedShellCommand( + 'showmap 4321 | grep TOTAL', as_root=True), + ['100 101 102 103 104 105 106 107 TOTAL']), + (self.call.device.ReadFile('/proc/4321/status', as_root=True), + self.CommandError())): + self.assertEquals( + { + 'Size': 100, + 'Rss': 101, + 'Pss': 102, + 'Shared_Clean': 103, + 'Shared_Dirty': 104, + 'Private_Clean': 105, + 'Private_Dirty': 106, + }, + self.device.GetMemoryUsageForPid(4321)) + + +class DeviceUtilsDismissCrashDialogIfNeededTest(DeviceUtilsTest): + + def testDismissCrashDialogIfNeeded_crashedPageckageNotFound(self): + sample_dumpsys_output = ''' +WINDOW MANAGER WINDOWS (dumpsys window windows) + Window #11 Window{f8b647a u0 SearchPanel}: + mDisplayId=0 mSession=Session{8 94:122} mClient=android.os.BinderProxy@1ba5 + mOwnerUid=100 mShowToOwnerOnly=false package=com.android.systemui appop=NONE + mAttrs=WM.LayoutParams{(0,0)(fillxfill) gr=#53 sim=#31 ty=2024 fl=100 + Requested w=1080 h=1920 mLayoutSeq=426 + mBaseLayer=211000 mSubLayer=0 mAnimLayer=211000+0=211000 mLastLayer=211000 +''' + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), sample_dumpsys_output.split('\n'))): + package_name = self.device.DismissCrashDialogIfNeeded() + self.assertIsNone(package_name) + + def testDismissCrashDialogIfNeeded_crashedPageckageFound(self): + sample_dumpsys_output = ''' +WINDOW MANAGER WINDOWS (dumpsys window windows) + Window #11 Window{f8b647a u0 SearchPanel}: + mDisplayId=0 mSession=Session{8 94:122} mClient=android.os.BinderProxy@1ba5 + mOwnerUid=102 mShowToOwnerOnly=false package=com.android.systemui appop=NONE + mAttrs=WM.LayoutParams{(0,0)(fillxfill) gr=#53 sim=#31 ty=2024 fl=100 + Requested w=1080 h=1920 mLayoutSeq=426 + mBaseLayer=211000 mSubLayer=0 mAnimLayer=211000+0=211000 mLastLayer=211000 + mHasPermanentDpad=false + mCurrentFocus=Window{3a27740f u0 Application Error: com.android.chrome} + mFocusedApp=AppWindowToken{470af6f token=Token{272ec24e ActivityRecord{t894}}} +''' + with self.assertCalls( + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), sample_dumpsys_output.split('\n')), + (self.call.device.RunShellCommand( + ['input', 'keyevent', '22'], check_return=True)), + (self.call.device.RunShellCommand( + ['input', 'keyevent', '22'], check_return=True)), + (self.call.device.RunShellCommand( + ['input', 'keyevent', '66'], check_return=True)), + (self.call.device.RunShellCommand( + ['dumpsys', 'window', 'windows'], check_return=True, + large_output=True), [])): + package_name = self.device.DismissCrashDialogIfNeeded() + self.assertEqual(package_name, 'com.android.chrome') + + +class DeviceUtilsClientCache(DeviceUtilsTest): + + def testClientCache_twoCaches(self): + self.device._cache['test'] = 0 + client_cache_one = self.device.GetClientCache('ClientOne') + client_cache_one['test'] = 1 + client_cache_two = self.device.GetClientCache('ClientTwo') + client_cache_two['test'] = 2 + self.assertEqual(self.device._cache['test'], 0) + self.assertEqual(client_cache_one, {'test': 1}) + self.assertEqual(client_cache_two, {'test': 2}) + self.device._ClearCache() + self.assertTrue('test' not in self.device._cache) + self.assertEqual(client_cache_one, {}) + self.assertEqual(client_cache_two, {}) + + def testClientCache_multipleInstances(self): + client_cache_one = self.device.GetClientCache('ClientOne') + client_cache_one['test'] = 1 + client_cache_two = self.device.GetClientCache('ClientOne') + self.assertEqual(client_cache_one, {'test': 1}) + self.assertEqual(client_cache_two, {'test': 1}) + self.device._ClearCache() + self.assertEqual(client_cache_one, {}) + self.assertEqual(client_cache_two, {}) + + +class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): + + def testHealthyDevices_emptyBlacklist(self): + test_serials = ['0123456789abcdef', 'fedcba9876543210'] + with self.assertCalls( + (mock.call.devil.android.sdk.adb_wrapper.AdbWrapper.Devices(), + [_AdbWrapperMock(s) for s in test_serials])): + blacklist = mock.NonCallableMock(**{'Read.return_value': []}) + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) + for serial, device in zip(test_serials, devices): + self.assertTrue(isinstance(device, device_utils.DeviceUtils)) + self.assertEquals(serial, device.adb.GetDeviceSerial()) + + def testHealthyDevices_blacklist(self): + test_serials = ['0123456789abcdef', 'fedcba9876543210'] + with self.assertCalls( + (mock.call.devil.android.sdk.adb_wrapper.AdbWrapper.Devices(), + [_AdbWrapperMock(s) for s in test_serials])): + blacklist = mock.NonCallableMock( + **{'Read.return_value': ['fedcba9876543210']}) + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) + self.assertEquals(1, len(devices)) + self.assertTrue(isinstance(devices[0], device_utils.DeviceUtils)) + self.assertEquals('0123456789abcdef', devices[0].adb.GetDeviceSerial()) + + +class DeviceUtilsRestartAdbdTest(DeviceUtilsTest): + + def testAdbdRestart(self): + mock_temp_file = '/sdcard/temp-123.sh' + with self.assertCalls( + (mock.call.devil.android.device_temp_file.DeviceTempFile( + self.adb, suffix='.sh'), MockTempFile(mock_temp_file)), + self.call.device.WriteFile(mock.ANY, mock.ANY), + (self.call.device.RunShellCommand( + ['source', mock_temp_file], as_root=True)), + self.call.adb.WaitForDevice()): + self.device.RestartAdbd() + + +class DeviceUtilsGrantPermissionsTest(DeviceUtilsTest): + + def testGrantPermissions_none(self): + self.device.GrantPermissions('package', []) + + def testGrantPermissions_underM(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.LOLLIPOP): + self.device.GrantPermissions('package', ['p1']) + + def testGrantPermissions_one(self): + permissions_cmd = 'pm grant package p1' + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.MARSHMALLOW): + with self.assertCalls( + (self.call.device.RunShellCommand( + permissions_cmd, check_return=True), [])): + self.device.GrantPermissions('package', ['p1']) + + def testGrantPermissions_multiple(self): + permissions_cmd = 'pm grant package p1&&pm grant package p2' + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.MARSHMALLOW): + with self.assertCalls( + (self.call.device.RunShellCommand( + permissions_cmd, check_return=True), [])): + self.device.GrantPermissions('package', ['p1', 'p2']) + + def testGrantPermissions_WriteExtrnalStorage(self): + permissions_cmd = ( + 'pm grant package android.permission.WRITE_EXTERNAL_STORAGE&&' + 'pm grant package android.permission.READ_EXTERNAL_STORAGE') + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.MARSHMALLOW): + with self.assertCalls( + (self.call.device.RunShellCommand( + permissions_cmd, check_return=True), [])): + self.device.GrantPermissions( + 'package', ['android.permission.WRITE_EXTERNAL_STORAGE']) + + def testGrantPermissions_BlackList(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.MARSHMALLOW): + self.device.GrantPermissions( + 'package', ['android.permission.ACCESS_MOCK_LOCATION']) + + +class DeviecUtilsIsScreenOn(DeviceUtilsTest): + + _L_SCREEN_ON = ['test=test mInteractive=true'] + _K_SCREEN_ON = ['test=test mScreenOn=true'] + _L_SCREEN_OFF = ['mInteractive=false'] + _K_SCREEN_OFF = ['mScreenOn=false'] + + def testIsScreenOn_onPreL(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.KITKAT): + with self.assertCalls( + (self.call.device._RunPipedShellCommand( + 'dumpsys input_method | grep mScreenOn'), self._K_SCREEN_ON)): + self.assertTrue(self.device.IsScreenOn()) + + def testIsScreenOn_onL(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.LOLLIPOP): + with self.assertCalls( + (self.call.device._RunPipedShellCommand( + 'dumpsys input_method | grep mInteractive'), self._L_SCREEN_ON)): + self.assertTrue(self.device.IsScreenOn()) + + def testIsScreenOn_offPreL(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.KITKAT): + with self.assertCalls( + (self.call.device._RunPipedShellCommand( + 'dumpsys input_method | grep mScreenOn'), self._K_SCREEN_OFF)): + self.assertFalse(self.device.IsScreenOn()) + + def testIsScreenOn_offL(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.LOLLIPOP): + with self.assertCalls( + (self.call.device._RunPipedShellCommand( + 'dumpsys input_method | grep mInteractive'), self._L_SCREEN_OFF)): + self.assertFalse(self.device.IsScreenOn()) + + def testIsScreenOn_noOutput(self): + with self.patch_call(self.call.device.build_version_sdk, + return_value=version_codes.LOLLIPOP): + with self.assertCalls( + (self.call.device._RunPipedShellCommand( + 'dumpsys input_method | grep mInteractive'), [])): + with self.assertRaises(device_errors.CommandFailedError): + self.device.IsScreenOn() + + +class DeviecUtilsSetScreen(DeviceUtilsTest): + + @mock.patch('time.sleep', mock.Mock()) + def testSetScren_alreadySet(self): + with self.assertCalls( + (self.call.device.IsScreenOn(), False)): + self.device.SetScreen(False) + + @mock.patch('time.sleep', mock.Mock()) + def testSetScreen_on(self): + with self.assertCalls( + (self.call.device.IsScreenOn(), False), + (self.call.device.RunShellCommand('input keyevent 26'), []), + (self.call.device.IsScreenOn(), True)): + self.device.SetScreen(True) + + @mock.patch('time.sleep', mock.Mock()) + def testSetScreen_off(self): + with self.assertCalls( + (self.call.device.IsScreenOn(), True), + (self.call.device.RunShellCommand('input keyevent 26'), []), + (self.call.device.IsScreenOn(), False)): + self.device.SetScreen(False) + + @mock.patch('time.sleep', mock.Mock()) + def testSetScreen_slow(self): + with self.assertCalls( + (self.call.device.IsScreenOn(), True), + (self.call.device.RunShellCommand('input keyevent 26'), []), + (self.call.device.IsScreenOn(), True), + (self.call.device.IsScreenOn(), True), + (self.call.device.IsScreenOn(), False)): + self.device.SetScreen(False) + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.DEBUG) + unittest.main(verbosity=2) diff --git a/catapult/devil/devil/android/fastboot_utils.py b/catapult/devil/devil/android/fastboot_utils.py new file mode 100644 index 00000000..f1287d1a --- /dev/null +++ b/catapult/devil/devil/android/fastboot_utils.py @@ -0,0 +1,246 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Provides a variety of device interactions based on fastboot.""" +# pylint: disable=unused-argument + +import contextlib +import fnmatch +import logging +import os +import re + +from devil.android import decorators +from devil.android import device_errors +from devil.android.sdk import fastboot +from devil.utils import timeout_retry + +_DEFAULT_TIMEOUT = 30 +_DEFAULT_RETRIES = 3 +_FASTBOOT_REBOOT_TIMEOUT = 10 * _DEFAULT_TIMEOUT +ALL_PARTITIONS = [ + 'bootloader', + 'radio', + 'boot', + 'recovery', + 'system', + 'userdata', + 'cache', +] + + +class FastbootUtils(object): + + _FASTBOOT_WAIT_TIME = 1 + _RESTART_WHEN_FLASHING = ['bootloader', 'radio'] + _BOARD_VERIFICATION_FILE = 'android-info.txt' + _FLASH_IMAGE_FILES = { + 'bootloader': 'bootloader*.img', + 'radio': 'radio*.img', + 'boot': 'boot.img', + 'recovery': 'recovery.img', + 'system': 'system.img', + 'userdata': 'userdata.img', + 'cache': 'cache.img', + } + + def __init__(self, device, fastbooter=None, default_timeout=_DEFAULT_TIMEOUT, + default_retries=_DEFAULT_RETRIES): + """FastbootUtils constructor. + + Example Usage to flash a device: + fastboot = fastboot_utils.FastbootUtils(device) + fastboot.FlashDevice('/path/to/build/directory') + + Args: + device: A DeviceUtils instance. + fastbooter: Optional fastboot object. If none is passed, one will + be created. + default_timeout: An integer containing the default number of seconds to + wait for an operation to complete if no explicit value is provided. + default_retries: An integer containing the default number or times an + operation should be retried on failure if no explicit value is provided. + """ + self._device = device + self._board = device.product_board + self._serial = str(device) + self._default_timeout = default_timeout + self._default_retries = default_retries + if fastbooter: + self.fastboot = fastbooter + else: + self.fastboot = fastboot.Fastboot(self._serial) + + @decorators.WithTimeoutAndRetriesFromInstance() + def WaitForFastbootMode(self, timeout=None, retries=None): + """Wait for device to boot into fastboot mode. + + This waits for the device serial to show up in fastboot devices output. + """ + def fastboot_mode(): + return self._serial in self.fastboot.Devices() + + timeout_retry.WaitFor(fastboot_mode, wait_period=self._FASTBOOT_WAIT_TIME) + + @decorators.WithTimeoutAndRetriesFromInstance( + min_default_timeout=_FASTBOOT_REBOOT_TIMEOUT) + def EnableFastbootMode(self, timeout=None, retries=None): + """Reboots phone into fastboot mode. + + Roots phone if needed, then reboots phone into fastboot mode and waits. + """ + self._device.EnableRoot() + self._device.adb.Reboot(to_bootloader=True) + self.WaitForFastbootMode() + + @decorators.WithTimeoutAndRetriesFromInstance( + min_default_timeout=_FASTBOOT_REBOOT_TIMEOUT) + def Reboot(self, bootloader=False, timeout=None, retries=None): + """Reboots out of fastboot mode. + + It reboots the phone either back into fastboot, or to a regular boot. It + then blocks until the device is ready. + + Args: + bootloader: If set to True, reboots back into bootloader. + """ + if bootloader: + self.fastboot.RebootBootloader() + self.WaitForFastbootMode() + else: + self.fastboot.Reboot() + self._device.WaitUntilFullyBooted(timeout=_FASTBOOT_REBOOT_TIMEOUT) + + def _VerifyBoard(self, directory): + """Validate as best as possible that the android build matches the device. + + Goes through build files and checks if the board name is mentioned in the + |self._BOARD_VERIFICATION_FILE| or in the build archive. + + Args: + directory: directory where build files are located. + """ + files = os.listdir(directory) + board_regex = re.compile(r'require board=(\w+)') + if self._BOARD_VERIFICATION_FILE in files: + with open(os.path.join(directory, self._BOARD_VERIFICATION_FILE)) as f: + for line in f: + m = board_regex.match(line) + if m: + board_name = m.group(1) + if board_name == self._board: + return True + elif board_name: + return False + else: + logging.warning('No board type found in %s.', + self._BOARD_VERIFICATION_FILE) + else: + logging.warning('%s not found. Unable to use it to verify device.', + self._BOARD_VERIFICATION_FILE) + + zip_regex = re.compile(r'.*%s.*\.zip' % re.escape(self._board)) + for f in files: + if zip_regex.match(f): + return True + + return False + + def _FindAndVerifyPartitionsAndImages(self, partitions, directory): + """Validate partitions and images. + + Validate all partition names and partition directories. Cannot stop mid + flash so its important to validate everything first. + + Args: + Partitions: partitions to be tested. + directory: directory containing the images. + + Returns: + Dictionary with exact partition, image name mapping. + """ + files = os.listdir(directory) + + def find_file(pattern): + for filename in files: + if fnmatch.fnmatch(filename, pattern): + return os.path.join(directory, filename) + raise device_errors.FastbootCommandFailedError( + 'Failed to flash device. Counld not find image for %s.', pattern) + + return {name: find_file(self._FLASH_IMAGE_FILES[name]) + for name in partitions} + + def _FlashPartitions(self, partitions, directory, wipe=False, force=False): + """Flashes all given partiitons with all given images. + + Args: + partitions: List of partitions to flash. + directory: Directory where all partitions can be found. + wipe: If set to true, will automatically detect if cache and userdata + partitions are sent, and if so ignore them. + force: boolean to decide to ignore board name safety checks. + + Raises: + device_errors.CommandFailedError(): If image cannot be found or if bad + partition name is give. + """ + if not self._VerifyBoard(directory): + if force: + logging.warning('Could not verify build is meant to be installed on ' + 'the current device type, but force flag is set. ' + 'Flashing device. Possibly dangerous operation.') + else: + raise device_errors.CommandFailedError( + 'Could not verify build is meant to be installed on the current ' + 'device type. Run again with force=True to force flashing with an ' + 'unverified board.') + + flash_image_files = self._FindAndVerifyPartitionsAndImages(partitions, + directory) + for partition in partitions: + if partition in ['cache', 'userdata'] and not wipe: + logging.info( + 'Not flashing in wipe mode. Skipping partition %s.', partition) + else: + logging.info( + 'Flashing %s with %s', partition, flash_image_files[partition]) + self.fastboot.Flash(partition, flash_image_files[partition]) + if partition in self._RESTART_WHEN_FLASHING: + self.Reboot(bootloader=True) + + @contextlib.contextmanager + def FastbootMode(self, timeout=None, retries=None): + """Context manager that enables fastboot mode, and reboots after. + + Example usage: + with FastbootMode(): + Flash Device + # Anything that runs after flashing. + """ + self.EnableFastbootMode() + self.fastboot.SetOemOffModeCharge(False) + try: + yield self + finally: + self.fastboot.SetOemOffModeCharge(True) + self.Reboot() + + def FlashDevice(self, directory, partitions=None, wipe=False): + """Flash device with build in |directory|. + + Directory must contain bootloader, radio, boot, recovery, system, userdata, + and cache .img files from an android build. This is a dangerous operation so + use with care. + + Args: + fastboot: A FastbootUtils instance. + directory: Directory with build files. + wipe: Wipes cache and userdata if set to true. + partitions: List of partitions to flash. Defaults to all. + """ + if partitions is None: + partitions = ALL_PARTITIONS + with self.FastbootMode(): + self._FlashPartitions(partitions, directory, wipe=wipe) diff --git a/catapult/devil/devil/android/fastboot_utils_test.py b/catapult/devil/devil/android/fastboot_utils_test.py new file mode 100755 index 00000000..8e6fc88b --- /dev/null +++ b/catapult/devil/devil/android/fastboot_utils_test.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Unit tests for the contents of fastboot_utils.py +""" + +# pylint: disable=protected-access,unused-argument + +import io +import logging +import unittest + +from devil import devil_env +from devil.android import device_errors +from devil.android import device_utils +from devil.android import fastboot_utils +from devil.android.sdk import fastboot +from devil.utils import mock_calls + +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + +_BOARD = 'board_type' +_SERIAL = '0123456789abcdef' +_PARTITIONS = ['cache', 'userdata', 'system', 'bootloader', 'radio'] +_IMAGES = { + 'cache': 'cache.img', + 'userdata': 'userdata.img', + 'system': 'system.img', + 'bootloader': 'bootloader.img', + 'radio': 'radio.img', +} +_VALID_FILES = [_BOARD + '.zip', 'android-info.txt'] +_INVALID_FILES = ['test.zip', 'android-info.txt'] + + +class MockFile(object): + + def __init__(self, name='/tmp/some/file'): + self.file = mock.MagicMock(spec=file) + self.file.name = name + + def __enter__(self): + return self.file + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + @property + def name(self): + return self.file.name + + +def _FastbootWrapperMock(test_serial): + fastbooter = mock.Mock(spec=fastboot.Fastboot) + fastbooter.__str__ = mock.Mock(return_value=test_serial) + fastbooter.Devices.return_value = [test_serial] + return fastbooter + + +def _DeviceUtilsMock(test_serial): + device = mock.Mock(spec=device_utils.DeviceUtils) + device.__str__ = mock.Mock(return_value=test_serial) + device.product_board = mock.Mock(return_value=_BOARD) + device.adb = mock.Mock() + return device + + +class FastbootUtilsTest(mock_calls.TestCase): + + def setUp(self): + self.device_utils_mock = _DeviceUtilsMock(_SERIAL) + self.fastboot_wrapper = _FastbootWrapperMock(_SERIAL) + self.fastboot = fastboot_utils.FastbootUtils( + self.device_utils_mock, fastbooter=self.fastboot_wrapper, + default_timeout=2, default_retries=0) + self.fastboot._board = _BOARD + + +class FastbootUtilsInitTest(FastbootUtilsTest): + + def testInitWithDeviceUtil(self): + f = fastboot_utils.FastbootUtils(self.device_utils_mock) + self.assertEqual(str(self.device_utils_mock), str(f._device)) + + def testInitWithMissing_fails(self): + with self.assertRaises(AttributeError): + fastboot_utils.FastbootUtils(None) + with self.assertRaises(AttributeError): + fastboot_utils.FastbootUtils('') + + +class FastbootUtilsWaitForFastbootMode(FastbootUtilsTest): + + # If this test fails by timing out after 1 second. + @mock.patch('time.sleep', mock.Mock()) + def testWaitForFastbootMode(self): + self.fastboot.WaitForFastbootMode() + + +class FastbootUtilsEnableFastbootMode(FastbootUtilsTest): + + def testEnableFastbootMode(self): + with self.assertCalls( + self.call.fastboot._device.EnableRoot(), + self.call.fastboot._device.adb.Reboot(to_bootloader=True), + self.call.fastboot.WaitForFastbootMode()): + self.fastboot.EnableFastbootMode() + + +class FastbootUtilsReboot(FastbootUtilsTest): + + def testReboot_bootloader(self): + with self.assertCalls( + self.call.fastboot.fastboot.RebootBootloader(), + self.call.fastboot.WaitForFastbootMode()): + self.fastboot.Reboot(bootloader=True) + + def testReboot_normal(self): + with self.assertCalls( + self.call.fastboot.fastboot.Reboot(), + self.call.fastboot._device.WaitUntilFullyBooted(timeout=mock.ANY)): + self.fastboot.Reboot() + + +class FastbootUtilsFlashPartitions(FastbootUtilsTest): + + def testFlashPartitions_wipe(self): + with self.assertCalls( + (self.call.fastboot._VerifyBoard('test'), True), + (self.call.fastboot._FindAndVerifyPartitionsAndImages( + _PARTITIONS, 'test'), _IMAGES), + (self.call.fastboot.fastboot.Flash('cache', 'cache.img')), + (self.call.fastboot.fastboot.Flash('userdata', 'userdata.img')), + (self.call.fastboot.fastboot.Flash('system', 'system.img')), + (self.call.fastboot.fastboot.Flash('bootloader', 'bootloader.img')), + (self.call.fastboot.Reboot(bootloader=True)), + (self.call.fastboot.fastboot.Flash('radio', 'radio.img')), + (self.call.fastboot.Reboot(bootloader=True))): + self.fastboot._FlashPartitions(_PARTITIONS, 'test', wipe=True) + + def testFlashPartitions_noWipe(self): + with self.assertCalls( + (self.call.fastboot._VerifyBoard('test'), True), + (self.call.fastboot._FindAndVerifyPartitionsAndImages( + _PARTITIONS, 'test'), _IMAGES), + (self.call.fastboot.fastboot.Flash('system', 'system.img')), + (self.call.fastboot.fastboot.Flash('bootloader', 'bootloader.img')), + (self.call.fastboot.Reboot(bootloader=True)), + (self.call.fastboot.fastboot.Flash('radio', 'radio.img')), + (self.call.fastboot.Reboot(bootloader=True))): + self.fastboot._FlashPartitions(_PARTITIONS, 'test') + + +class FastbootUtilsFastbootMode(FastbootUtilsTest): + + def testFastbootMode_good(self): + with self.assertCalls( + self.call.fastboot.EnableFastbootMode(), + self.call.fastboot.fastboot.SetOemOffModeCharge(False), + self.call.fastboot.fastboot.SetOemOffModeCharge(True), + self.call.fastboot.Reboot()): + with self.fastboot.FastbootMode() as fbm: + self.assertEqual(self.fastboot, fbm) + + def testFastbootMode_exception(self): + with self.assertCalls( + self.call.fastboot.EnableFastbootMode(), + self.call.fastboot.fastboot.SetOemOffModeCharge(False), + self.call.fastboot.fastboot.SetOemOffModeCharge(True), + self.call.fastboot.Reboot()): + with self.assertRaises(NotImplementedError): + with self.fastboot.FastbootMode() as fbm: + self.assertEqual(self.fastboot, fbm) + raise NotImplementedError + + def testFastbootMode_exceptionInEnableFastboot(self): + self.fastboot.EnableFastbootMode = mock.Mock() + self.fastboot.EnableFastbootMode.side_effect = NotImplementedError + with self.assertRaises(NotImplementedError): + with self.fastboot.FastbootMode(): + pass + + +class FastbootUtilsVerifyBoard(FastbootUtilsTest): + + def testVerifyBoard_bothValid(self): + mock_file = io.StringIO(u'require board=%s\n' % _BOARD) + with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch('os.listdir', return_value=_VALID_FILES): + self.assertTrue(self.fastboot._VerifyBoard('test')) + + def testVerifyBoard_BothNotValid(self): + mock_file = io.StringIO(u'abc') + with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch('os.listdir', return_value=_INVALID_FILES): + self.assertFalse(self.assertFalse(self.fastboot._VerifyBoard('test'))) + + def testVerifyBoard_FileNotFoundZipValid(self): + with mock.patch('os.listdir', return_value=[_BOARD + '.zip']): + self.assertTrue(self.fastboot._VerifyBoard('test')) + + def testVerifyBoard_ZipNotFoundFileValid(self): + mock_file = io.StringIO(u'require board=%s\n' % _BOARD) + with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch('os.listdir', return_value=['android-info.txt']): + self.assertTrue(self.fastboot._VerifyBoard('test')) + + def testVerifyBoard_zipNotValidFileIs(self): + mock_file = io.StringIO(u'require board=%s\n' % _BOARD) + with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch('os.listdir', return_value=_INVALID_FILES): + self.assertTrue(self.fastboot._VerifyBoard('test')) + + def testVerifyBoard_fileNotValidZipIs(self): + mock_file = io.StringIO(u'require board=WrongBoard') + with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch('os.listdir', return_value=_VALID_FILES): + self.assertFalse(self.fastboot._VerifyBoard('test')) + + def testVerifyBoard_noBoardInFileValidZip(self): + mock_file = io.StringIO(u'Regex wont match') + with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch('os.listdir', return_value=_VALID_FILES): + self.assertTrue(self.fastboot._VerifyBoard('test')) + + def testVerifyBoard_noBoardInFileInvalidZip(self): + mock_file = io.StringIO(u'Regex wont match') + with mock.patch('__builtin__.open', return_value=mock_file, create=True): + with mock.patch('os.listdir', return_value=_INVALID_FILES): + self.assertFalse(self.fastboot._VerifyBoard('test')) + + +class FastbootUtilsFindAndVerifyPartitionsAndImages(FastbootUtilsTest): + + def testFindAndVerifyPartitionsAndImages_valid(self): + PARTITIONS = [ + 'bootloader', 'radio', 'boot', 'recovery', 'system', 'userdata', 'cache' + ] + files = [ + 'bootloader-test-.img', + 'radio123.img', + 'boot.img', + 'recovery.img', + 'system.img', + 'userdata.img', + 'cache.img' + ] + return_check = { + 'bootloader': 'test/bootloader-test-.img', + 'radio': 'test/radio123.img', + 'boot': 'test/boot.img', + 'recovery': 'test/recovery.img', + 'system': 'test/system.img', + 'userdata': 'test/userdata.img', + 'cache': 'test/cache.img', + } + + with mock.patch('os.listdir', return_value=files): + return_value = self.fastboot._FindAndVerifyPartitionsAndImages( + PARTITIONS, 'test') + self.assertDictEqual(return_value, return_check) + + def testFindAndVerifyPartitionsAndImages_badPartition(self): + with mock.patch('os.listdir', return_value=['test']): + with self.assertRaises(KeyError): + self.fastboot._FindAndVerifyPartitionsAndImages(['test'], 'test') + + def testFindAndVerifyPartitionsAndImages_noFile(self): + with mock.patch('os.listdir', return_value=['test']): + with self.assertRaises(device_errors.FastbootCommandFailedError): + self.fastboot._FindAndVerifyPartitionsAndImages(['cache'], 'test') + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.DEBUG) + unittest.main(verbosity=2) diff --git a/catapult/devil/devil/android/flag_changer.py b/catapult/devil/devil/android/flag_changer.py new file mode 100644 index 00000000..4267f117 --- /dev/null +++ b/catapult/devil/devil/android/flag_changer.py @@ -0,0 +1,182 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import logging + +from devil.android import device_errors + + +class FlagChanger(object): + """Changes the flags Chrome runs with. + + Flags can be temporarily set for a particular set of unit tests. These + tests should call Restore() to revert the flags to their original state + once the tests have completed. + """ + + def __init__(self, device, cmdline_file): + """Initializes the FlagChanger and records the original arguments. + + Args: + device: A DeviceUtils instance. + cmdline_file: Path to the command line file on the device. + """ + self._device = device + + # Unrooted devices have limited access to the file system. + # Place files in /data/local/tmp/ rather than /data/local/ + if not device.HasRoot() and not '/data/local/tmp/' in cmdline_file: + self._cmdline_file = cmdline_file.replace('/data/local/', + '/data/local/tmp/') + else: + self._cmdline_file = cmdline_file + + stored_flags = '' + if self._device.PathExists(self._cmdline_file): + try: + stored_flags = self._device.ReadFile(self._cmdline_file).strip() + except device_errors.CommandFailedError: + pass + # Store the flags as a set to facilitate adding and removing flags. + self._state_stack = [set(self._TokenizeFlags(stored_flags))] + + def ReplaceFlags(self, flags): + """Replaces the flags in the command line with the ones provided. + Saves the current flags state on the stack, so a call to Restore will + change the state back to the one preceeding the call to ReplaceFlags. + + Args: + flags: A sequence of command line flags to set, eg. ['--single-process']. + Note: this should include flags only, not the name of a command + to run (ie. there is no need to start the sequence with 'chrome'). + """ + new_flags = set(flags) + self._state_stack.append(new_flags) + self._UpdateCommandLineFile() + + def AddFlags(self, flags): + """Appends flags to the command line if they aren't already there. + Saves the current flags state on the stack, so a call to Restore will + change the state back to the one preceeding the call to AddFlags. + + Args: + flags: A sequence of flags to add on, eg. ['--single-process']. + """ + self.PushFlags(add=flags) + + def RemoveFlags(self, flags): + """Removes flags from the command line, if they exist. + Saves the current flags state on the stack, so a call to Restore will + change the state back to the one preceeding the call to RemoveFlags. + + Note that calling RemoveFlags after AddFlags will result in having + two nested states. + + Args: + flags: A sequence of flags to remove, eg. ['--single-process']. Note + that we expect a complete match when removing flags; if you want + to remove a switch with a value, you must use the exact string + used to add it in the first place. + """ + self.PushFlags(remove=flags) + + def PushFlags(self, add=None, remove=None): + """Appends and removes flags to/from the command line if they aren't already + there. Saves the current flags state on the stack, so a call to Restore + will change the state back to the one preceeding the call to PushFlags. + + Args: + add: A list of flags to add on, eg. ['--single-process']. + remove: A list of flags to remove, eg. ['--single-process']. Note that we + expect a complete match when removing flags; if you want to remove + a switch with a value, you must use the exact string used to add + it in the first place. + """ + new_flags = self._state_stack[-1].copy() + if add: + new_flags.update(add) + if remove: + new_flags.difference_update(remove) + self.ReplaceFlags(new_flags) + + def Restore(self): + """Restores the flags to their state prior to the last AddFlags or + RemoveFlags call. + """ + # The initial state must always remain on the stack. + assert len(self._state_stack) > 1, ( + "Mismatch between calls to Add/RemoveFlags and Restore") + self._state_stack.pop() + self._UpdateCommandLineFile() + + def _UpdateCommandLineFile(self): + """Writes out the command line to the file, or removes it if empty.""" + current_flags = list(self._state_stack[-1]) + logging.info('Current flags: %s', current_flags) + # Root is not required to write to /data/local/tmp/. + use_root = '/data/local/tmp/' not in self._cmdline_file + if current_flags: + # The first command line argument doesn't matter as we are not actually + # launching the chrome executable using this command line. + cmd_line = ' '.join(['_'] + current_flags) + self._device.WriteFile( + self._cmdline_file, cmd_line, as_root=use_root) + file_contents = self._device.ReadFile( + self._cmdline_file, as_root=use_root).rstrip() + assert file_contents == cmd_line, ( + 'Failed to set the command line file at %s' % self._cmdline_file) + else: + self._device.RunShellCommand('rm ' + self._cmdline_file, + as_root=use_root) + assert not self._device.FileExists(self._cmdline_file), ( + 'Failed to remove the command line file at %s' % self._cmdline_file) + + @staticmethod + def _TokenizeFlags(line): + """Changes the string containing the command line into a list of flags. + + Follows similar logic to CommandLine.java::tokenizeQuotedArguments: + * Flags are split using whitespace, unless the whitespace is within a + pair of quotation marks. + * Unlike the Java version, we keep the quotation marks around switch + values since we need them to re-create the file when new flags are + appended. + + Args: + line: A string containing the entire command line. The first token is + assumed to be the program name. + """ + if not line: + return [] + + tokenized_flags = [] + current_flag = "" + within_quotations = False + + # Move through the string character by character and build up each flag + # along the way. + for c in line.strip(): + if c is '"': + if len(current_flag) > 0 and current_flag[-1] == '\\': + # Last char was a backslash; pop it, and treat this " as a literal. + current_flag = current_flag[0:-1] + '"' + else: + within_quotations = not within_quotations + current_flag += c + elif not within_quotations and (c is ' ' or c is '\t'): + if current_flag is not "": + tokenized_flags.append(current_flag) + current_flag = "" + else: + current_flag += c + + # Tack on the last flag. + if not current_flag: + if within_quotations: + logging.warn('Unterminated quoted argument: ' + line) + else: + tokenized_flags.append(current_flag) + + # Return everything but the program name. + return tokenized_flags[1:] diff --git a/catapult/devil/devil/android/forwarder.py b/catapult/devil/devil/android/forwarder.py new file mode 100644 index 00000000..21f52236 --- /dev/null +++ b/catapult/devil/devil/android/forwarder.py @@ -0,0 +1,344 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# pylint: disable=W0212 + +import fcntl +import logging +import os +import psutil + +from devil import base_error +from devil import devil_env +from devil.android.constants import file_system +from devil.android.valgrind_tools import base_tool +from devil.utils import cmd_helper + + +def _GetProcessStartTime(pid): + return psutil.Process(pid).create_time + + +class _FileLock(object): + """With statement-aware implementation of a file lock. + + File locks are needed for cross-process synchronization when the + multiprocessing Python module is used. + """ + + def __init__(self, path): + self._fd = -1 + self._path = path + + def __enter__(self): + self._fd = os.open(self._path, os.O_RDONLY | os.O_CREAT) + if self._fd < 0: + raise Exception('Could not open file %s for reading' % self._path) + fcntl.flock(self._fd, fcntl.LOCK_EX) + + def __exit__(self, _exception_type, _exception_value, traceback): + fcntl.flock(self._fd, fcntl.LOCK_UN) + os.close(self._fd) + + +class HostForwarderError(base_error.BaseError): + """Exception for failures involving host_forwarder.""" + + def __init__(self, message): + super(HostForwarderError, self).__init__(message) + + +class Forwarder(object): + """Thread-safe class to manage port forwards from the device to the host.""" + + _DEVICE_FORWARDER_FOLDER = (file_system.TEST_EXECUTABLE_DIR + + '/forwarder/') + _DEVICE_FORWARDER_PATH = (file_system.TEST_EXECUTABLE_DIR + + '/forwarder/device_forwarder') + _LOCK_PATH = '/tmp/chrome.forwarder.lock' + # Defined in host_forwarder_main.cc + _HOST_FORWARDER_LOG = '/tmp/host_forwarder_log' + + _instance = None + + @staticmethod + def Map(port_pairs, device, tool=None): + """Runs the forwarder. + + Args: + port_pairs: A list of tuples (device_port, host_port) to forward. Note + that you can specify 0 as a device_port, in which case a + port will by dynamically assigned on the device. You can + get the number of the assigned port using the + DevicePortForHostPort method. + device: A DeviceUtils instance. + tool: Tool class to use to get wrapper, if necessary, for executing the + forwarder (see valgrind_tools.py). + + Raises: + Exception on failure to forward the port. + """ + if not tool: + tool = base_tool.BaseTool() + with _FileLock(Forwarder._LOCK_PATH): + instance = Forwarder._GetInstanceLocked(tool) + instance._InitDeviceLocked(device, tool) + + device_serial = str(device) + redirection_commands = [ + ['--adb=' + devil_env.config.FetchPath('adb'), + '--serial-id=' + device_serial, + '--map', str(device_port), str(host_port)] + for device_port, host_port in port_pairs] + logging.info('Forwarding using commands: %s', redirection_commands) + + for redirection_command in redirection_commands: + try: + (exit_code, output) = cmd_helper.GetCmdStatusAndOutput( + [instance._host_forwarder_path] + redirection_command) + except OSError as e: + if e.errno == 2: + raise HostForwarderError( + 'Unable to start host forwarder. ' + 'Make sure you have built host_forwarder.') + else: raise + if exit_code != 0: + Forwarder._KillDeviceLocked(device, tool) + # Log alive forwarders + ps_out = device.RunShellCommand(['ps']) + logging.info('Currently running device_forwarders:') + for line in ps_out: + if 'device_forwarder' in line: + logging.info(' %s', line) + raise HostForwarderError( + '%s exited with %d:\n%s' % (instance._host_forwarder_path, + exit_code, '\n'.join(output))) + tokens = output.split(':') + if len(tokens) != 2: + raise HostForwarderError( + 'Unexpected host forwarder output "%s", ' + 'expected "device_port:host_port"' % output) + device_port = int(tokens[0]) + host_port = int(tokens[1]) + serial_with_port = (device_serial, device_port) + instance._device_to_host_port_map[serial_with_port] = host_port + instance._host_to_device_port_map[host_port] = serial_with_port + logging.info('Forwarding device port: %d to host port: %d.', + device_port, host_port) + + @staticmethod + def UnmapDevicePort(device_port, device): + """Unmaps a previously forwarded device port. + + Args: + device: A DeviceUtils instance. + device_port: A previously forwarded port (through Map()). + """ + with _FileLock(Forwarder._LOCK_PATH): + Forwarder._UnmapDevicePortLocked(device_port, device) + + @staticmethod + def UnmapAllDevicePorts(device): + """Unmaps all the previously forwarded ports for the provided device. + + Args: + device: A DeviceUtils instance. + port_pairs: A list of tuples (device_port, host_port) to unmap. + """ + with _FileLock(Forwarder._LOCK_PATH): + if not Forwarder._instance: + return + adb_serial = str(device) + if adb_serial not in Forwarder._instance._initialized_devices: + return + port_map = Forwarder._GetInstanceLocked( + None)._device_to_host_port_map + for (device_serial, device_port) in port_map.keys(): + if adb_serial == device_serial: + Forwarder._UnmapDevicePortLocked(device_port, device) + # There are no more ports mapped, kill the device_forwarder. + tool = base_tool.BaseTool() + Forwarder._KillDeviceLocked(device, tool) + + @staticmethod + def DevicePortForHostPort(host_port): + """Returns the device port that corresponds to a given host port.""" + with _FileLock(Forwarder._LOCK_PATH): + _, device_port = Forwarder._GetInstanceLocked( + None)._host_to_device_port_map.get(host_port) + return device_port + + @staticmethod + def RemoveHostLog(): + if os.path.exists(Forwarder._HOST_FORWARDER_LOG): + os.unlink(Forwarder._HOST_FORWARDER_LOG) + + @staticmethod + def GetHostLog(): + if not os.path.exists(Forwarder._HOST_FORWARDER_LOG): + return '' + with file(Forwarder._HOST_FORWARDER_LOG, 'r') as f: + return f.read() + + @staticmethod + def _GetInstanceLocked(tool): + """Returns the singleton instance. + + Note that the global lock must be acquired before calling this method. + + Args: + tool: Tool class to use to get wrapper, if necessary, for executing the + forwarder (see valgrind_tools.py). + """ + if not Forwarder._instance: + Forwarder._instance = Forwarder(tool) + return Forwarder._instance + + def __init__(self, tool): + """Constructs a new instance of Forwarder. + + Note that Forwarder is a singleton therefore this constructor should be + called only once. + + Args: + tool: Tool class to use to get wrapper, if necessary, for executing the + forwarder (see valgrind_tools.py). + """ + assert not Forwarder._instance + self._tool = tool + self._initialized_devices = set() + self._device_to_host_port_map = dict() + self._host_to_device_port_map = dict() + self._host_forwarder_path = devil_env.config.FetchPath('forwarder_host') + assert os.path.exists(self._host_forwarder_path), 'Please build forwarder2' + self._InitHostLocked() + + @staticmethod + def _UnmapDevicePortLocked(device_port, device): + """Internal method used by UnmapDevicePort(). + + Note that the global lock must be acquired before calling this method. + """ + instance = Forwarder._GetInstanceLocked(None) + serial = str(device) + serial_with_port = (serial, device_port) + if not serial_with_port in instance._device_to_host_port_map: + logging.error('Trying to unmap non-forwarded port %d', device_port) + return + redirection_command = ['--adb=' + devil_env.config.FetchPath('adb'), + '--serial-id=' + serial, + '--unmap', str(device_port)] + logging.info('Undo forwarding using command: %s', redirection_command) + (exit_code, output) = cmd_helper.GetCmdStatusAndOutput( + [instance._host_forwarder_path] + redirection_command) + if exit_code != 0: + logging.error( + '%s exited with %d:\n%s', + instance._host_forwarder_path, exit_code, '\n'.join(output)) + host_port = instance._device_to_host_port_map[serial_with_port] + del instance._device_to_host_port_map[serial_with_port] + del instance._host_to_device_port_map[host_port] + + @staticmethod + def _GetPidForLock(): + """Returns the PID used for host_forwarder initialization. + + The PID of the "sharder" is used to handle multiprocessing. The "sharder" + is the initial process that forks that is the parent process. + """ + return os.getpgrp() + + def _InitHostLocked(self): + """Initializes the host forwarder daemon. + + Note that the global lock must be acquired before calling this method. This + method kills any existing host_forwarder process that could be stale. + """ + # See if the host_forwarder daemon was already initialized by a concurrent + # process or thread (in case multi-process sharding is not used). + pid_for_lock = Forwarder._GetPidForLock() + fd = os.open(Forwarder._LOCK_PATH, os.O_RDWR | os.O_CREAT) + with os.fdopen(fd, 'r+') as pid_file: + pid_with_start_time = pid_file.readline() + if pid_with_start_time: + (pid, process_start_time) = pid_with_start_time.split(':') + if pid == str(pid_for_lock): + if process_start_time == str(_GetProcessStartTime(pid_for_lock)): + return + self._KillHostLocked() + pid_file.seek(0) + pid_file.write( + '%s:%s' % (pid_for_lock, str(_GetProcessStartTime(pid_for_lock)))) + pid_file.truncate() + + def _InitDeviceLocked(self, device, tool): + """Initializes the device_forwarder daemon for a specific device (once). + + Note that the global lock must be acquired before calling this method. This + method kills any existing device_forwarder daemon on the device that could + be stale, pushes the latest version of the daemon (to the device) and starts + it. + + Args: + device: A DeviceUtils instance. + tool: Tool class to use to get wrapper, if necessary, for executing the + forwarder (see valgrind_tools.py). + """ + device_serial = str(device) + if device_serial in self._initialized_devices: + return + Forwarder._KillDeviceLocked(device, tool) + forwarder_device_path_on_host = devil_env.config.FetchPath( + 'forwarder_device', device=device) + forwarder_device_path_on_device = ( + Forwarder._DEVICE_FORWARDER_FOLDER + if os.path.isdir(forwarder_device_path_on_host) + else Forwarder._DEVICE_FORWARDER_PATH) + device.PushChangedFiles([( + forwarder_device_path_on_host, + forwarder_device_path_on_device)]) + + cmd = '%s %s' % (tool.GetUtilWrapper(), Forwarder._DEVICE_FORWARDER_PATH) + device.RunShellCommand( + cmd, env={'LD_LIBRARY_PATH': Forwarder._DEVICE_FORWARDER_FOLDER}, + check_return=True) + self._initialized_devices.add(device_serial) + + def _KillHostLocked(self): + """Kills the forwarder process running on the host. + + Note that the global lock must be acquired before calling this method. + """ + logging.info('Killing host_forwarder.') + (exit_code, output) = cmd_helper.GetCmdStatusAndOutput( + [self._host_forwarder_path, '--kill-server']) + if exit_code != 0: + (exit_code, output) = cmd_helper.GetCmdStatusAndOutput( + ['pkill', '-9', 'host_forwarder']) + if exit_code != 0: + raise HostForwarderError( + '%s exited with %d:\n%s' % (self._host_forwarder_path, exit_code, + '\n'.join(output))) + + @staticmethod + def _KillDeviceLocked(device, tool): + """Kills the forwarder process running on the device. + + Note that the global lock must be acquired before calling this method. + + Args: + device: Instance of DeviceUtils for talking to the device. + tool: Wrapper tool (e.g. valgrind) that can be used to execute the device + forwarder (see valgrind_tools.py). + """ + logging.info('Killing device_forwarder.') + Forwarder._instance._initialized_devices.discard(str(device)) + if not device.FileExists(Forwarder._DEVICE_FORWARDER_PATH): + return + + cmd = '%s %s --kill-server' % (tool.GetUtilWrapper(), + Forwarder._DEVICE_FORWARDER_PATH) + device.RunShellCommand( + cmd, env={'LD_LIBRARY_PATH': Forwarder._DEVICE_FORWARDER_FOLDER}, + check_return=True) diff --git a/catapult/devil/devil/android/install_commands.py b/catapult/devil/devil/android/install_commands.py new file mode 100644 index 00000000..5a06bf3f --- /dev/null +++ b/catapult/devil/devil/android/install_commands.py @@ -0,0 +1,57 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import posixpath + +from devil import devil_env +from devil.android import device_errors +from devil.android.constants import file_system + +BIN_DIR = '%s/bin' % file_system.TEST_EXECUTABLE_DIR +_FRAMEWORK_DIR = '%s/framework' % file_system.TEST_EXECUTABLE_DIR + +_COMMANDS = { + 'unzip': 'org.chromium.android.commands.unzip.Unzip', +} + +_SHELL_COMMAND_FORMAT = ( +"""#!/system/bin/sh +base=%s +export CLASSPATH=$base/framework/chromium_commands.jar +exec app_process $base/bin %s $@ +""") + + +def Installed(device): + paths = [posixpath.join(BIN_DIR, c) for c in _COMMANDS] + paths.append(posixpath.join(_FRAMEWORK_DIR, 'chromium_commands.jar')) + return device.PathExists(paths) + + +def InstallCommands(device): + if device.IsUserBuild(): + raise device_errors.CommandFailedError( + 'chromium_commands currently requires a userdebug build.', + device_serial=device.adb.GetDeviceSerial()) + + chromium_commands_jar_path = devil_env.config.FetchPath('chromium_commands') + if not os.path.exists(chromium_commands_jar_path): + raise device_errors.CommandFailedError( + '%s not found. Please build chromium_commands.' + % chromium_commands_jar_path) + + device.RunShellCommand(['mkdir', BIN_DIR, _FRAMEWORK_DIR]) + for command, main_class in _COMMANDS.iteritems(): + shell_command = _SHELL_COMMAND_FORMAT % ( + file_system.TEST_EXECUTABLE_DIR, main_class) + shell_file = '%s/%s' % (BIN_DIR, command) + device.WriteFile(shell_file, shell_command) + device.RunShellCommand( + ['chmod', '755', shell_file], check_return=True) + + device.adb.Push( + chromium_commands_jar_path, + '%s/chromium_commands.jar' % _FRAMEWORK_DIR) + diff --git a/catapult/devil/devil/android/logcat_monitor.py b/catapult/devil/devil/android/logcat_monitor.py new file mode 100644 index 00000000..9ec94125 --- /dev/null +++ b/catapult/devil/devil/android/logcat_monitor.py @@ -0,0 +1,242 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# pylint: disable=unused-argument + +import errno +import logging +import os +import re +import shutil +import tempfile +import threading +import time + +from devil.android import decorators +from devil.android import device_errors +from devil.android.sdk import adb_wrapper +from devil.utils import reraiser_thread + + +class LogcatMonitor(object): + + _RECORD_THREAD_JOIN_WAIT = 2.0 + _WAIT_TIME = 0.2 + _THREADTIME_RE_FORMAT = ( + r'(?P<date>\S*) +(?P<time>\S*) +(?P<proc_id>%s) +(?P<thread_id>%s) +' + r'(?P<log_level>%s) +(?P<component>%s) *: +(?P<message>%s)$') + + def __init__(self, adb, clear=True, filter_specs=None, output_file=None): + """Create a LogcatMonitor instance. + + Args: + adb: An instance of adb_wrapper.AdbWrapper. + clear: If True, clear the logcat when monitoring starts. + filter_specs: An optional list of '<tag>[:priority]' strings. + output_file: File path to save recorded logcat. + """ + if isinstance(adb, adb_wrapper.AdbWrapper): + self._adb = adb + else: + raise ValueError('Unsupported type passed for argument "device"') + self._clear = clear + self._filter_specs = filter_specs + self._output_file = output_file + self._record_file = None + self._record_file_lock = threading.Lock() + self._record_thread = None + self._stop_recording_event = threading.Event() + + @property + def output_file(self): + return self._output_file + + @decorators.WithTimeoutAndRetriesDefaults(10, 0) + def WaitFor(self, success_regex, failure_regex=None, timeout=None, + retries=None): + """Wait for a matching logcat line or until a timeout occurs. + + This will attempt to match lines in the logcat against both |success_regex| + and |failure_regex| (if provided). Note that this calls re.search on each + logcat line, not re.match, so the provided regular expressions don't have + to match an entire line. + + Args: + success_regex: The regular expression to search for. + failure_regex: An optional regular expression that, if hit, causes this + to stop looking for a match. Can be None. + timeout: timeout in seconds + retries: number of retries + + Returns: + A match object if |success_regex| matches a part of a logcat line, or + None if |failure_regex| matches a part of a logcat line. + Raises: + CommandFailedError on logcat failure (NOT on a |failure_regex| match). + CommandTimeoutError if no logcat line matching either |success_regex| or + |failure_regex| is found in |timeout| seconds. + DeviceUnreachableError if the device becomes unreachable. + LogcatMonitorCommandError when calling |WaitFor| while not recording + logcat. + """ + if self._record_thread is None: + raise LogcatMonitorCommandError( + 'Must be recording logcat when calling |WaitFor|', + device_serial=str(self._adb)) + if isinstance(success_regex, basestring): + success_regex = re.compile(success_regex) + if isinstance(failure_regex, basestring): + failure_regex = re.compile(failure_regex) + + logging.debug('Waiting %d seconds for "%s"', timeout, success_regex.pattern) + + # NOTE This will continue looping until: + # - success_regex matches a line, in which case the match object is + # returned. + # - failure_regex matches a line, in which case None is returned + # - the timeout is hit, in which case a CommandTimeoutError is raised. + with open(self._record_file.name, 'r') as f: + while True: + line = f.readline() + if line: + m = success_regex.search(line) + if m: + return m + if failure_regex and failure_regex.search(line): + return None + else: + time.sleep(self._WAIT_TIME) + + def FindAll(self, message_regex, proc_id=None, thread_id=None, log_level=None, + component=None): + """Finds all lines in the logcat that match the provided constraints. + + Args: + message_regex: The regular expression that the <message> section must + match. + proc_id: The process ID to match. If None, matches any process ID. + thread_id: The thread ID to match. If None, matches any thread ID. + log_level: The log level to match. If None, matches any log level. + component: The component to match. If None, matches any component. + + Raises: + LogcatMonitorCommandError when calling |FindAll| before recording logcat. + + Yields: + A match object for each matching line in the logcat. The match object + will always contain, in addition to groups defined in |message_regex|, + the following named groups: 'date', 'time', 'proc_id', 'thread_id', + 'log_level', 'component', and 'message'. + """ + if self._record_file is None: + raise LogcatMonitorCommandError( + 'Must have recorded or be recording a logcat to call |FindAll|', + device_serial=str(self._adb)) + if proc_id is None: + proc_id = r'\d+' + if thread_id is None: + thread_id = r'\d+' + if log_level is None: + log_level = r'[VDIWEF]' + if component is None: + component = r'[^\s:]+' + # pylint: disable=protected-access + threadtime_re = re.compile( + type(self)._THREADTIME_RE_FORMAT % ( + proc_id, thread_id, log_level, component, message_regex)) + + with open(self._record_file.name, 'r') as f: + for line in f: + m = re.match(threadtime_re, line) + if m: + yield m + + def _StartRecording(self): + """Starts recording logcat to file. + + Function spawns a thread that records logcat to file and will not die + until |StopRecording| is called. + """ + def record_to_file(): + # Write the log with line buffering so the consumer sees each individual + # line. + for data in self._adb.Logcat(filter_specs=self._filter_specs, + logcat_format='threadtime'): + with self._record_file_lock: + if self._stop_recording_event.isSet(): + return + if self._record_file and not self._record_file.closed: + self._record_file.write(data + '\n') + + self._stop_recording_event.clear() + if not self._record_thread: + self._record_thread = reraiser_thread.ReraiserThread(record_to_file) + self._record_thread.start() + + def _StopRecording(self): + """Finish recording logcat.""" + if self._record_thread: + self._stop_recording_event.set() + self._record_thread.join(timeout=self._RECORD_THREAD_JOIN_WAIT) + self._record_thread.ReraiseIfException() + self._record_thread = None + + def Start(self): + """Starts the logcat monitor. + + Clears the logcat if |clear| was set in |__init__|. + """ + if self._clear: + self._adb.Logcat(clear=True) + if not self._record_file: + self._record_file = tempfile.NamedTemporaryFile(mode='a', bufsize=1) + self._StartRecording() + + def Stop(self): + """Stops the logcat monitor. + + Stops recording the logcat. Copies currently recorded logcat to + |self._output_file|. + """ + self._StopRecording() + with self._record_file_lock: + if self._record_file and self._output_file: + try: + os.makedirs(os.path.dirname(self._output_file)) + except OSError as e: + if e.errno != errno.EEXIST: + raise + shutil.copy(self._record_file.name, self._output_file) + + def Close(self): + """Closes logcat recording file. + + Should be called when finished using the logcat monitor. + """ + with self._record_file_lock: + if self._record_file: + self._record_file.close() + self._record_file = None + + def __enter__(self): + """Starts the logcat monitor.""" + self.Start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Stops the logcat monitor.""" + self.Stop() + + def __del__(self): + """Closes logcat recording file in case |Close| was never called.""" + with self._record_file_lock: + if self._record_file: + logging.warning( + 'Need to call |Close| on the logcat monitor when done!') + self._record_file.close() + + +class LogcatMonitorCommandError(device_errors.CommandFailedError): + """Exception for errors with logcat monitor commands.""" + pass diff --git a/catapult/devil/devil/android/logcat_monitor_test.py b/catapult/devil/devil/android/logcat_monitor_test.py new file mode 100755 index 00000000..8fb4d74b --- /dev/null +++ b/catapult/devil/devil/android/logcat_monitor_test.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# pylint: disable=protected-access + +import itertools +import threading +import unittest + +from devil import devil_env +from devil.android import logcat_monitor +from devil.android.sdk import adb_wrapper + +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + + +def _CreateTestLog(raw_logcat=None): + test_adb = adb_wrapper.AdbWrapper('0123456789abcdef') + test_adb.Logcat = mock.Mock(return_value=(l for l in raw_logcat)) + test_log = logcat_monitor.LogcatMonitor(test_adb, clear=False) + return test_log + + +class LogcatMonitorTest(unittest.TestCase): + + _TEST_THREADTIME_LOGCAT_DATA = [ + '01-01 01:02:03.456 7890 0987 V LogcatMonitorTest: ' + 'verbose logcat monitor test message 1', + '01-01 01:02:03.457 8901 1098 D LogcatMonitorTest: ' + 'debug logcat monitor test message 2', + '01-01 01:02:03.458 9012 2109 I LogcatMonitorTest: ' + 'info logcat monitor test message 3', + '01-01 01:02:03.459 0123 3210 W LogcatMonitorTest: ' + 'warning logcat monitor test message 4', + '01-01 01:02:03.460 1234 4321 E LogcatMonitorTest: ' + 'error logcat monitor test message 5', + '01-01 01:02:03.461 2345 5432 F LogcatMonitorTest: ' + 'fatal logcat monitor test message 6', + '01-01 01:02:03.462 3456 6543 D LogcatMonitorTest: ' + 'last line' + ] + + def assertIterEqual(self, expected_iter, actual_iter): + for expected, actual in itertools.izip_longest(expected_iter, actual_iter): + self.assertIsNotNone( + expected, + msg='actual has unexpected elements starting with %s' % str(actual)) + self.assertIsNotNone( + actual, + msg='actual is missing elements starting with %s' % str(expected)) + self.assertEqual(actual.group('proc_id'), expected[0]) + self.assertEqual(actual.group('thread_id'), expected[1]) + self.assertEqual(actual.group('log_level'), expected[2]) + self.assertEqual(actual.group('component'), expected[3]) + self.assertEqual(actual.group('message'), expected[4]) + + with self.assertRaises(StopIteration): + next(actual_iter) + with self.assertRaises(StopIteration): + next(expected_iter) + + @mock.patch('time.sleep', mock.Mock()) + def testWaitFor_success(self): + test_log = _CreateTestLog( + raw_logcat=type(self)._TEST_THREADTIME_LOGCAT_DATA) + test_log.Start() + actual_match = test_log.WaitFor(r'.*(fatal|error) logcat monitor.*', None) + self.assertTrue(actual_match) + self.assertEqual( + '01-01 01:02:03.460 1234 4321 E LogcatMonitorTest: ' + 'error logcat monitor test message 5', + actual_match.group(0)) + self.assertEqual('error', actual_match.group(1)) + test_log.Stop() + test_log.Close() + + @mock.patch('time.sleep', mock.Mock()) + def testWaitFor_failure(self): + test_log = _CreateTestLog( + raw_logcat=type(self)._TEST_THREADTIME_LOGCAT_DATA) + test_log.Start() + actual_match = test_log.WaitFor( + r'.*My Success Regex.*', r'.*(fatal|error) logcat monitor.*') + self.assertIsNone(actual_match) + test_log.Stop() + test_log.Close() + + @mock.patch('time.sleep', mock.Mock()) + def testWaitFor_buffering(self): + # Simulate an adb log stream which does not complete until the test tells it + # to. This checks that the log matcher can receive individual lines from the + # log reader thread even if adb is not producing enough output to fill an + # entire file io buffer. + finished_lock = threading.Lock() + finished_lock.acquire() + + def LogGenerator(): + for line in type(self)._TEST_THREADTIME_LOGCAT_DATA: + yield line + finished_lock.acquire() + + test_adb = adb_wrapper.AdbWrapper('0123456789abcdef') + test_adb.Logcat = mock.Mock(return_value=LogGenerator()) + test_log = logcat_monitor.LogcatMonitor(test_adb, clear=False) + test_log.Start() + + actual_match = test_log.WaitFor(r'.*last line.*', None) + finished_lock.release() + self.assertTrue(actual_match) + test_log.Stop() + test_log.Close() + + @mock.patch('time.sleep', mock.Mock()) + def testFindAll_defaults(self): + test_log = _CreateTestLog( + raw_logcat=type(self)._TEST_THREADTIME_LOGCAT_DATA) + test_log.Start() + test_log.WaitFor(r'.*last line.*', None) + test_log.Stop() + expected_results = [ + ('7890', '0987', 'V', 'LogcatMonitorTest', + 'verbose logcat monitor test message 1'), + ('8901', '1098', 'D', 'LogcatMonitorTest', + 'debug logcat monitor test message 2'), + ('9012', '2109', 'I', 'LogcatMonitorTest', + 'info logcat monitor test message 3'), + ('0123', '3210', 'W', 'LogcatMonitorTest', + 'warning logcat monitor test message 4'), + ('1234', '4321', 'E', 'LogcatMonitorTest', + 'error logcat monitor test message 5'), + ('2345', '5432', 'F', 'LogcatMonitorTest', + 'fatal logcat monitor test message 6')] + actual_results = test_log.FindAll(r'\S* logcat monitor test message \d') + self.assertIterEqual(iter(expected_results), actual_results) + test_log.Close() + + @mock.patch('time.sleep', mock.Mock()) + def testFindAll_defaults_miss(self): + test_log = _CreateTestLog( + raw_logcat=type(self)._TEST_THREADTIME_LOGCAT_DATA) + test_log.Start() + test_log.WaitFor(r'.*last line.*', None) + test_log.Stop() + expected_results = [] + actual_results = test_log.FindAll(r'\S* nothing should match this \d') + self.assertIterEqual(iter(expected_results), actual_results) + test_log.Close() + + @mock.patch('time.sleep', mock.Mock()) + def testFindAll_filterProcId(self): + test_log = _CreateTestLog( + raw_logcat=type(self)._TEST_THREADTIME_LOGCAT_DATA) + test_log.Start() + test_log.WaitFor(r'.*last line.*', None) + test_log.Stop() + actual_results = test_log.FindAll( + r'\S* logcat monitor test message \d', proc_id=1234) + expected_results = [ + ('1234', '4321', 'E', 'LogcatMonitorTest', + 'error logcat monitor test message 5')] + self.assertIterEqual(iter(expected_results), actual_results) + test_log.Close() + + @mock.patch('time.sleep', mock.Mock()) + def testFindAll_filterThreadId(self): + test_log = _CreateTestLog( + raw_logcat=type(self)._TEST_THREADTIME_LOGCAT_DATA) + test_log.Start() + test_log.WaitFor(r'.*last line.*', None) + test_log.Stop() + actual_results = test_log.FindAll( + r'\S* logcat monitor test message \d', thread_id=2109) + expected_results = [ + ('9012', '2109', 'I', 'LogcatMonitorTest', + 'info logcat monitor test message 3')] + self.assertIterEqual(iter(expected_results), actual_results) + test_log.Close() + + @mock.patch('time.sleep', mock.Mock()) + def testFindAll_filterLogLevel(self): + test_log = _CreateTestLog( + raw_logcat=type(self)._TEST_THREADTIME_LOGCAT_DATA) + test_log.Start() + test_log.WaitFor(r'.*last line.*', None) + test_log.Stop() + actual_results = test_log.FindAll( + r'\S* logcat monitor test message \d', log_level=r'[DW]') + expected_results = [ + ('8901', '1098', 'D', 'LogcatMonitorTest', + 'debug logcat monitor test message 2'), + ('0123', '3210', 'W', 'LogcatMonitorTest', + 'warning logcat monitor test message 4') + ] + self.assertIterEqual(iter(expected_results), actual_results) + test_log.Close() + + @mock.patch('time.sleep', mock.Mock()) + def testFindAll_filterComponent(self): + test_log = _CreateTestLog( + raw_logcat=type(self)._TEST_THREADTIME_LOGCAT_DATA) + test_log.Start() + test_log.WaitFor(r'.*last line.*', None) + test_log.Stop() + actual_results = test_log.FindAll(r'.*', component='LogcatMonitorTest') + expected_results = [ + ('7890', '0987', 'V', 'LogcatMonitorTest', + 'verbose logcat monitor test message 1'), + ('8901', '1098', 'D', 'LogcatMonitorTest', + 'debug logcat monitor test message 2'), + ('9012', '2109', 'I', 'LogcatMonitorTest', + 'info logcat monitor test message 3'), + ('0123', '3210', 'W', 'LogcatMonitorTest', + 'warning logcat monitor test message 4'), + ('1234', '4321', 'E', 'LogcatMonitorTest', + 'error logcat monitor test message 5'), + ('2345', '5432', 'F', 'LogcatMonitorTest', + 'fatal logcat monitor test message 6'), + ('3456', '6543', 'D', 'LogcatMonitorTest', + 'last line') + ] + self.assertIterEqual(iter(expected_results), actual_results) + test_log.Close() + + +if __name__ == '__main__': + unittest.main(verbosity=2) + diff --git a/catapult/devil/devil/android/md5sum.py b/catapult/devil/devil/android/md5sum.py new file mode 100644 index 00000000..52706461 --- /dev/null +++ b/catapult/devil/devil/android/md5sum.py @@ -0,0 +1,120 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import posixpath +import re + +from devil import devil_env +from devil.android import device_errors +from devil.utils import cmd_helper + +MD5SUM_DEVICE_LIB_PATH = '/data/local/tmp/md5sum' +MD5SUM_DEVICE_BIN_PATH = MD5SUM_DEVICE_LIB_PATH + '/md5sum_bin' + +_STARTS_WITH_CHECKSUM_RE = re.compile(r'^\s*[0-9a-fA-F]{32}\s+') + + +def CalculateHostMd5Sums(paths): + """Calculates the MD5 sum value for all items in |paths|. + + Directories are traversed recursively and the MD5 sum of each file found is + reported in the result. + + Args: + paths: A list of host paths to md5sum. + Returns: + A dict mapping file paths to their respective md5sum checksums. + """ + if isinstance(paths, basestring): + paths = [paths] + + md5sum_bin_host_path = devil_env.config.FetchPath('md5sum_host') + if not os.path.exists(md5sum_bin_host_path): + raise IOError('File not built: %s' % md5sum_bin_host_path) + out = cmd_helper.GetCmdOutput( + [md5sum_bin_host_path] + [os.path.realpath(p) for p in paths]) + + return _ParseMd5SumOutput(out.splitlines()) + + +def CalculateDeviceMd5Sums(paths, device): + """Calculates the MD5 sum value for all items in |paths|. + + Directories are traversed recursively and the MD5 sum of each file found is + reported in the result. + + Args: + paths: A list of device paths to md5sum. + Returns: + A dict mapping file paths to their respective md5sum checksums. + """ + if not paths: + return {} + + if isinstance(paths, basestring): + paths = [paths] + # Allow generators + paths = list(paths) + + md5sum_dist_path = devil_env.config.FetchPath('md5sum_device', device=device) + + if os.path.isdir(md5sum_dist_path): + md5sum_dist_bin_path = os.path.join(md5sum_dist_path, 'md5sum_bin') + else: + md5sum_dist_bin_path = md5sum_dist_path + + if not os.path.exists(md5sum_dist_path): + raise IOError('File not built: %s' % md5sum_dist_path) + md5sum_file_size = os.path.getsize(md5sum_dist_bin_path) + + # For better performance, make the script as small as possible to try and + # avoid needing to write to an intermediary file (which RunShellCommand will + # do if necessary). + md5sum_script = 'a=%s;' % MD5SUM_DEVICE_BIN_PATH + # Check if the binary is missing or has changed (using its file size as an + # indicator), and trigger a (re-)push via the exit code. + md5sum_script += '! [[ $(ls -l $a) = *%d* ]]&&exit 2;' % md5sum_file_size + # Make sure it can find libbase.so + md5sum_script += 'export LD_LIBRARY_PATH=%s;' % MD5SUM_DEVICE_LIB_PATH + if len(paths) > 1: + prefix = posixpath.commonprefix(paths) + if len(prefix) > 4: + md5sum_script += 'p="%s";' % prefix + paths = ['$p"%s"' % p[len(prefix):] for p in paths] + + md5sum_script += ';'.join('$a %s' % p for p in paths) + # Don't fail the script if the last md5sum fails (due to file not found) + # Note: ":" is equivalent to "true". + md5sum_script += ';:' + try: + out = device.RunShellCommand(md5sum_script, check_return=True) + except device_errors.AdbShellCommandFailedError as e: + # Push the binary only if it is found to not exist + # (faster than checking up-front). + if e.status == 2: + # If files were previously pushed as root (adbd running as root), trying + # to re-push as non-root causes the push command to report success, but + # actually fail. So, wipe the directory first. + device.RunShellCommand(['rm', '-rf', MD5SUM_DEVICE_LIB_PATH], + as_root=True, check_return=True) + if os.path.isdir(md5sum_dist_path): + device.adb.Push(md5sum_dist_path, MD5SUM_DEVICE_LIB_PATH) + else: + mkdir_cmd = 'a=%s;[[ -e $a ]] || mkdir $a' % MD5SUM_DEVICE_LIB_PATH + device.RunShellCommand(mkdir_cmd, check_return=True) + device.adb.Push(md5sum_dist_bin_path, MD5SUM_DEVICE_BIN_PATH) + + out = device.RunShellCommand(md5sum_script, check_return=True) + else: + raise + + return _ParseMd5SumOutput(out) + + +def _ParseMd5SumOutput(out): + hash_and_path = (l.split(None, 1) for l in out + if l and _STARTS_WITH_CHECKSUM_RE.match(l)) + return dict((p, h) for h, p in hash_and_path) + diff --git a/catapult/devil/devil/android/md5sum_test.py b/catapult/devil/devil/android/md5sum_test.py new file mode 100755 index 00000000..c9b49545 --- /dev/null +++ b/catapult/devil/devil/android/md5sum_test.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import unittest + +from devil import devil_env +from devil.android import device_errors +from devil.android import md5sum + +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + +TEST_OUT_DIR = os.path.join('test', 'out', 'directory') +HOST_MD5_EXECUTABLE = os.path.join(TEST_OUT_DIR, 'md5sum_bin_host') +MD5_DIST = os.path.join(TEST_OUT_DIR, 'md5sum_dist') + + +class Md5SumTest(unittest.TestCase): + + def setUp(self): + mocked_attrs = { + 'md5sum_host': HOST_MD5_EXECUTABLE, + 'md5sum_device': MD5_DIST, + } + self._patchers = [ + mock.patch('devil.devil_env._Environment.FetchPath', + mock.Mock(side_effect=lambda a, device=None: mocked_attrs[a])), + mock.patch('os.path.exists', + new=mock.Mock(return_value=True)), + ] + for p in self._patchers: + p.start() + + def tearDown(self): + for p in self._patchers: + p.stop() + + def testCalculateHostMd5Sums_singlePath(self): + test_path = '/test/host/file.dat' + mock_get_cmd_output = mock.Mock( + return_value='0123456789abcdeffedcba9876543210 /test/host/file.dat') + with mock.patch('devil.utils.cmd_helper.GetCmdOutput', + new=mock_get_cmd_output): + out = md5sum.CalculateHostMd5Sums(test_path) + self.assertEquals(1, len(out)) + self.assertTrue('/test/host/file.dat' in out) + self.assertEquals('0123456789abcdeffedcba9876543210', + out['/test/host/file.dat']) + mock_get_cmd_output.assert_called_once_with( + [HOST_MD5_EXECUTABLE, '/test/host/file.dat']) + + def testCalculateHostMd5Sums_list(self): + test_paths = ['/test/host/file0.dat', '/test/host/file1.dat'] + mock_get_cmd_output = mock.Mock( + return_value='0123456789abcdeffedcba9876543210 /test/host/file0.dat\n' + '123456789abcdef00fedcba987654321 /test/host/file1.dat\n') + with mock.patch('devil.utils.cmd_helper.GetCmdOutput', + new=mock_get_cmd_output): + out = md5sum.CalculateHostMd5Sums(test_paths) + self.assertEquals(2, len(out)) + self.assertTrue('/test/host/file0.dat' in out) + self.assertEquals('0123456789abcdeffedcba9876543210', + out['/test/host/file0.dat']) + self.assertTrue('/test/host/file1.dat' in out) + self.assertEquals('123456789abcdef00fedcba987654321', + out['/test/host/file1.dat']) + mock_get_cmd_output.assert_called_once_with( + [HOST_MD5_EXECUTABLE, '/test/host/file0.dat', + '/test/host/file1.dat']) + + def testCalculateHostMd5Sums_generator(self): + test_paths = ('/test/host/' + p for p in ['file0.dat', 'file1.dat']) + mock_get_cmd_output = mock.Mock( + return_value='0123456789abcdeffedcba9876543210 /test/host/file0.dat\n' + '123456789abcdef00fedcba987654321 /test/host/file1.dat\n') + with mock.patch('devil.utils.cmd_helper.GetCmdOutput', + new=mock_get_cmd_output): + out = md5sum.CalculateHostMd5Sums(test_paths) + self.assertEquals(2, len(out)) + self.assertTrue('/test/host/file0.dat' in out) + self.assertEquals('0123456789abcdeffedcba9876543210', + out['/test/host/file0.dat']) + self.assertTrue('/test/host/file1.dat' in out) + self.assertEquals('123456789abcdef00fedcba987654321', + out['/test/host/file1.dat']) + mock_get_cmd_output.assert_called_once_with( + [HOST_MD5_EXECUTABLE, '/test/host/file0.dat', '/test/host/file1.dat']) + + def testCalculateDeviceMd5Sums_noPaths(self): + device = mock.NonCallableMock() + device.RunShellCommand = mock.Mock(side_effect=Exception()) + + out = md5sum.CalculateDeviceMd5Sums([], device) + self.assertEquals(0, len(out)) + + def testCalculateDeviceMd5Sums_singlePath(self): + test_path = '/storage/emulated/legacy/test/file.dat' + + device = mock.NonCallableMock() + device_md5sum_output = [ + '0123456789abcdeffedcba9876543210 ' + '/storage/emulated/legacy/test/file.dat', + ] + device.RunShellCommand = mock.Mock(return_value=device_md5sum_output) + + with mock.patch('os.path.getsize', return_value=1337): + out = md5sum.CalculateDeviceMd5Sums(test_path, device) + self.assertEquals(1, len(out)) + self.assertTrue('/storage/emulated/legacy/test/file.dat' in out) + self.assertEquals('0123456789abcdeffedcba9876543210', + out['/storage/emulated/legacy/test/file.dat']) + self.assertEquals(1, len(device.RunShellCommand.call_args_list)) + + def testCalculateDeviceMd5Sums_list(self): + test_path = ['/storage/emulated/legacy/test/file0.dat', + '/storage/emulated/legacy/test/file1.dat'] + device = mock.NonCallableMock() + device_md5sum_output = [ + '0123456789abcdeffedcba9876543210 ' + '/storage/emulated/legacy/test/file0.dat', + '123456789abcdef00fedcba987654321 ' + '/storage/emulated/legacy/test/file1.dat', + ] + device.RunShellCommand = mock.Mock(return_value=device_md5sum_output) + + with mock.patch('os.path.getsize', return_value=1337): + out = md5sum.CalculateDeviceMd5Sums(test_path, device) + self.assertEquals(2, len(out)) + self.assertTrue('/storage/emulated/legacy/test/file0.dat' in out) + self.assertEquals('0123456789abcdeffedcba9876543210', + out['/storage/emulated/legacy/test/file0.dat']) + self.assertTrue('/storage/emulated/legacy/test/file1.dat' in out) + self.assertEquals('123456789abcdef00fedcba987654321', + out['/storage/emulated/legacy/test/file1.dat']) + self.assertEquals(1, len(device.RunShellCommand.call_args_list)) + + def testCalculateDeviceMd5Sums_generator(self): + test_path = ('/storage/emulated/legacy/test/file%d.dat' % n + for n in xrange(0, 2)) + + device = mock.NonCallableMock() + device_md5sum_output = [ + '0123456789abcdeffedcba9876543210 ' + '/storage/emulated/legacy/test/file0.dat', + '123456789abcdef00fedcba987654321 ' + '/storage/emulated/legacy/test/file1.dat', + ] + device.RunShellCommand = mock.Mock(return_value=device_md5sum_output) + + with mock.patch('os.path.getsize', return_value=1337): + out = md5sum.CalculateDeviceMd5Sums(test_path, device) + self.assertEquals(2, len(out)) + self.assertTrue('/storage/emulated/legacy/test/file0.dat' in out) + self.assertEquals('0123456789abcdeffedcba9876543210', + out['/storage/emulated/legacy/test/file0.dat']) + self.assertTrue('/storage/emulated/legacy/test/file1.dat' in out) + self.assertEquals('123456789abcdef00fedcba987654321', + out['/storage/emulated/legacy/test/file1.dat']) + self.assertEquals(1, len(device.RunShellCommand.call_args_list)) + + def testCalculateDeviceMd5Sums_singlePath_linkerWarning(self): + # See crbug/479966 + test_path = '/storage/emulated/legacy/test/file.dat' + + device = mock.NonCallableMock() + device_md5sum_output = [ + 'WARNING: linker: /data/local/tmp/md5sum/md5sum_bin: ' + 'unused DT entry: type 0x1d arg 0x15db', + 'THIS_IS_NOT_A_VALID_CHECKSUM_ZZZ some random text', + '0123456789abcdeffedcba9876543210 ' + '/storage/emulated/legacy/test/file.dat', + ] + device.RunShellCommand = mock.Mock(return_value=device_md5sum_output) + + with mock.patch('os.path.getsize', return_value=1337): + out = md5sum.CalculateDeviceMd5Sums(test_path, device) + self.assertEquals(1, len(out)) + self.assertTrue('/storage/emulated/legacy/test/file.dat' in out) + self.assertEquals('0123456789abcdeffedcba9876543210', + out['/storage/emulated/legacy/test/file.dat']) + self.assertEquals(1, len(device.RunShellCommand.call_args_list)) + + def testCalculateDeviceMd5Sums_list_fileMissing(self): + test_path = ['/storage/emulated/legacy/test/file0.dat', + '/storage/emulated/legacy/test/file1.dat'] + device = mock.NonCallableMock() + device_md5sum_output = [ + '0123456789abcdeffedcba9876543210 ' + '/storage/emulated/legacy/test/file0.dat', + '[0819/203513:ERROR:md5sum.cc(25)] Could not open file asdf', + '', + ] + device.RunShellCommand = mock.Mock(return_value=device_md5sum_output) + + with mock.patch('os.path.getsize', return_value=1337): + out = md5sum.CalculateDeviceMd5Sums(test_path, device) + self.assertEquals(1, len(out)) + self.assertTrue('/storage/emulated/legacy/test/file0.dat' in out) + self.assertEquals('0123456789abcdeffedcba9876543210', + out['/storage/emulated/legacy/test/file0.dat']) + self.assertEquals(1, len(device.RunShellCommand.call_args_list)) + + def testCalculateDeviceMd5Sums_requiresBinary(self): + test_path = '/storage/emulated/legacy/test/file.dat' + + device = mock.NonCallableMock() + device.adb = mock.NonCallableMock() + device.adb.Push = mock.Mock() + device_md5sum_output = [ + 'WARNING: linker: /data/local/tmp/md5sum/md5sum_bin: ' + 'unused DT entry: type 0x1d arg 0x15db', + 'THIS_IS_NOT_A_VALID_CHECKSUM_ZZZ some random text', + '0123456789abcdeffedcba9876543210 ' + '/storage/emulated/legacy/test/file.dat', + ] + error = device_errors.AdbShellCommandFailedError('cmd', 'out', 2) + device.RunShellCommand = mock.Mock( + side_effect=(error, '', device_md5sum_output)) + + with mock.patch('os.path.isdir', return_value=True), ( + mock.patch('os.path.getsize', return_value=1337)): + out = md5sum.CalculateDeviceMd5Sums(test_path, device) + self.assertEquals(1, len(out)) + self.assertTrue('/storage/emulated/legacy/test/file.dat' in out) + self.assertEquals('0123456789abcdeffedcba9876543210', + out['/storage/emulated/legacy/test/file.dat']) + self.assertEquals(3, len(device.RunShellCommand.call_args_list)) + device.adb.Push.assert_called_once_with( + 'test/out/directory/md5sum_dist', '/data/local/tmp/md5sum') + + +if __name__ == '__main__': + unittest.main(verbosity=2) + diff --git a/catapult/devil/devil/android/perf/__init__.py b/catapult/devil/devil/android/perf/__init__.py new file mode 100644 index 00000000..50b23dff --- /dev/null +++ b/catapult/devil/devil/android/perf/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/catapult/devil/devil/android/perf/cache_control.py b/catapult/devil/devil/android/perf/cache_control.py new file mode 100644 index 00000000..7bd0a4e7 --- /dev/null +++ b/catapult/devil/devil/android/perf/cache_control.py @@ -0,0 +1,16 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +class CacheControl(object): + _DROP_CACHES = '/proc/sys/vm/drop_caches' + + def __init__(self, device): + self._device = device + + def DropRamCaches(self): + """Drops the filesystem ram caches for performance testing.""" + self._device.RunShellCommand('sync', as_root=True) + self._device.WriteFile(CacheControl._DROP_CACHES, '3', as_root=True) + diff --git a/catapult/devil/devil/android/perf/perf_control.py b/catapult/devil/devil/android/perf/perf_control.py new file mode 100644 index 00000000..af1d52c3 --- /dev/null +++ b/catapult/devil/devil/android/perf/perf_control.py @@ -0,0 +1,156 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import atexit +import logging + +from devil.android import device_errors + + +class PerfControl(object): + """Provides methods for setting the performance mode of a device.""" + _CPU_PATH = '/sys/devices/system/cpu' + _KERNEL_MAX = '/sys/devices/system/cpu/kernel_max' + + def __init__(self, device): + self._device = device + # this will raise an AdbCommandFailedError if no CPU files are found + self._cpu_files = self._device.RunShellCommand( + 'ls -d cpu[0-9]*', cwd=self._CPU_PATH, check_return=True, as_root=True) + assert self._cpu_files, 'Failed to detect CPUs.' + self._cpu_file_list = ' '.join(self._cpu_files) + logging.info('CPUs found: %s', self._cpu_file_list) + self._have_mpdecision = self._device.FileExists('/system/bin/mpdecision') + + def SetHighPerfMode(self): + """Sets the highest stable performance mode for the device.""" + try: + self._device.EnableRoot() + except device_errors.CommandFailedError: + message = 'Need root for performance mode. Results may be NOISY!!' + logging.warning(message) + # Add an additional warning at exit, such that it's clear that any results + # may be different/noisy (due to the lack of intended performance mode). + atexit.register(logging.warning, message) + return + + product_model = self._device.product_model + # TODO(epenner): Enable on all devices (http://crbug.com/383566) + if 'Nexus 4' == product_model: + self._ForceAllCpusOnline(True) + if not self._AllCpusAreOnline(): + logging.warning('Failed to force CPUs online. Results may be NOISY!') + self._SetScalingGovernorInternal('performance') + elif 'Nexus 5' == product_model: + self._ForceAllCpusOnline(True) + if not self._AllCpusAreOnline(): + logging.warning('Failed to force CPUs online. Results may be NOISY!') + self._SetScalingGovernorInternal('performance') + self._SetScalingMaxFreq(1190400) + self._SetMaxGpuClock(200000000) + else: + self._SetScalingGovernorInternal('performance') + + def SetPerfProfilingMode(self): + """Enables all cores for reliable perf profiling.""" + self._ForceAllCpusOnline(True) + self._SetScalingGovernorInternal('performance') + if not self._AllCpusAreOnline(): + if not self._device.HasRoot(): + raise RuntimeError('Need root to force CPUs online.') + raise RuntimeError('Failed to force CPUs online.') + + def SetDefaultPerfMode(self): + """Sets the performance mode for the device to its default mode.""" + if not self._device.HasRoot(): + return + product_model = self._device.product_model + if 'Nexus 5' == product_model: + if self._AllCpusAreOnline(): + self._SetScalingMaxFreq(2265600) + self._SetMaxGpuClock(450000000) + + governor_mode = { + 'GT-I9300': 'pegasusq', + 'Galaxy Nexus': 'interactive', + 'Nexus 4': 'ondemand', + 'Nexus 5': 'ondemand', + 'Nexus 7': 'interactive', + 'Nexus 10': 'interactive' + }.get(product_model, 'ondemand') + self._SetScalingGovernorInternal(governor_mode) + self._ForceAllCpusOnline(False) + + def GetCpuInfo(self): + online = (output.rstrip() == '1' and status == 0 + for (_, output, status) in self._ForEachCpu('cat "$CPU/online"')) + governor = (output.rstrip() if status == 0 else None + for (_, output, status) + in self._ForEachCpu('cat "$CPU/cpufreq/scaling_governor"')) + return zip(self._cpu_files, online, governor) + + def _ForEachCpu(self, cmd): + script = '; '.join([ + 'for CPU in %s' % self._cpu_file_list, + 'do %s' % cmd, + 'echo -n "%~%$?%~%"', + 'done' + ]) + output = self._device.RunShellCommand( + script, cwd=self._CPU_PATH, check_return=True, as_root=True) + output = '\n'.join(output).split('%~%') + return zip(self._cpu_files, output[0::2], (int(c) for c in output[1::2])) + + def _WriteEachCpuFile(self, path, value): + results = self._ForEachCpu( + 'test -e "$CPU/{path}" && echo {value} > "$CPU/{path}"'.format( + path=path, value=value)) + cpus = ' '.join(cpu for (cpu, _, status) in results if status == 0) + if cpus: + logging.info('Successfully set %s to %r on: %s', path, value, cpus) + else: + logging.warning('Failed to set %s to %r on any cpus', path, value) + + def _SetScalingGovernorInternal(self, value): + self._WriteEachCpuFile('cpufreq/scaling_governor', value) + + def _SetScalingMaxFreq(self, value): + self._WriteEachCpuFile('cpufreq/scaling_max_freq', '%d' % value) + + def _SetMaxGpuClock(self, value): + self._device.WriteFile('/sys/class/kgsl/kgsl-3d0/max_gpuclk', + str(value), + as_root=True) + + def _AllCpusAreOnline(self): + results = self._ForEachCpu('cat "$CPU/online"') + # TODO(epenner): Investigate why file may be missing + # (http://crbug.com/397118) + return all(output.rstrip() == '1' and status == 0 + for (cpu, output, status) in results + if cpu != 'cpu0') + + def _ForceAllCpusOnline(self, force_online): + """Enable all CPUs on a device. + + Some vendors (or only Qualcomm?) hot-plug their CPUs, which can add noise + to measurements: + - In perf, samples are only taken for the CPUs that are online when the + measurement is started. + - The scaling governor can't be set for an offline CPU and frequency scaling + on newly enabled CPUs adds noise to both perf and tracing measurements. + + It appears Qualcomm is the only vendor that hot-plugs CPUs, and on Qualcomm + this is done by "mpdecision". + + """ + if self._have_mpdecision: + script = 'stop mpdecision' if force_online else 'start mpdecision' + self._device.RunShellCommand(script, check_return=True, as_root=True) + + if not self._have_mpdecision and not self._AllCpusAreOnline(): + logging.warning('Unexpected cpu hot plugging detected.') + + if force_online: + self._ForEachCpu('echo 1 > "$CPU/online"') diff --git a/catapult/devil/devil/android/perf/perf_control_devicetest.py b/catapult/devil/devil/android/perf/perf_control_devicetest.py new file mode 100644 index 00000000..71bf3fbc --- /dev/null +++ b/catapult/devil/devil/android/perf/perf_control_devicetest.py @@ -0,0 +1,39 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# pylint: disable=W0212 + +import os +import sys +import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) + +from devil.android import device_utils +from devil.android.perf import perf_control + + +class TestPerfControl(unittest.TestCase): + + def setUp(self): + if not os.getenv('BUILDTYPE'): + os.environ['BUILDTYPE'] = 'Debug' + + devices = device_utils.DeviceUtils.HealthyDevices(blacklist=None) + self.assertGreater(len(devices), 0, 'No device attached!') + self._device = devices[0] + + def testHighPerfMode(self): + perf = perf_control.PerfControl(self._device) + try: + perf.SetPerfProfilingMode() + cpu_info = perf.GetCpuInfo() + self.assertEquals(len(perf._cpu_files), len(cpu_info)) + for _, online, governor in cpu_info: + self.assertTrue(online) + self.assertEquals('performance', governor) + finally: + perf.SetDefaultPerfMode() + +if __name__ == '__main__': + unittest.main() diff --git a/catapult/devil/devil/android/perf/surface_stats_collector.py b/catapult/devil/devil/android/perf/surface_stats_collector.py new file mode 100644 index 00000000..49372ad2 --- /dev/null +++ b/catapult/devil/devil/android/perf/surface_stats_collector.py @@ -0,0 +1,183 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import Queue +import threading + + +# Log marker containing SurfaceTexture timestamps. +_SURFACE_TEXTURE_TIMESTAMPS_MESSAGE = 'SurfaceTexture update timestamps' +_SURFACE_TEXTURE_TIMESTAMP_RE = r'\d+' + + +class SurfaceStatsCollector(object): + """Collects surface stats for a SurfaceView from the output of SurfaceFlinger. + + Args: + device: A DeviceUtils instance. + """ + + def __init__(self, device): + self._device = device + self._collector_thread = None + self._surface_before = None + self._get_data_event = None + self._data_queue = None + self._stop_event = None + self._warn_about_empty_data = True + + def DisableWarningAboutEmptyData(self): + self._warn_about_empty_data = False + + def Start(self): + assert not self._collector_thread + + if self._ClearSurfaceFlingerLatencyData(): + self._get_data_event = threading.Event() + self._stop_event = threading.Event() + self._data_queue = Queue.Queue() + self._collector_thread = threading.Thread(target=self._CollectorThread) + self._collector_thread.start() + else: + raise Exception('SurfaceFlinger not supported on this device.') + + def Stop(self): + assert self._collector_thread + (refresh_period, timestamps) = self._GetDataFromThread() + if self._collector_thread: + self._stop_event.set() + self._collector_thread.join() + self._collector_thread = None + return (refresh_period, timestamps) + + def _CollectorThread(self): + last_timestamp = 0 + timestamps = [] + retries = 0 + + while not self._stop_event.is_set(): + self._get_data_event.wait(1) + try: + refresh_period, new_timestamps = self._GetSurfaceFlingerFrameData() + if refresh_period is None or timestamps is None: + retries += 1 + if retries < 3: + continue + if last_timestamp: + # Some data has already been collected, but either the app + # was closed or there's no new data. Signal the main thread and + # wait. + self._data_queue.put((None, None)) + self._stop_event.wait() + break + raise Exception('Unable to get surface flinger latency data') + + timestamps += [timestamp for timestamp in new_timestamps + if timestamp > last_timestamp] + if len(timestamps): + last_timestamp = timestamps[-1] + + if self._get_data_event.is_set(): + self._get_data_event.clear() + self._data_queue.put((refresh_period, timestamps)) + timestamps = [] + except Exception as e: + # On any error, before aborting, put the exception into _data_queue to + # prevent the main thread from waiting at _data_queue.get() infinitely. + self._data_queue.put(e) + raise + + def _GetDataFromThread(self): + self._get_data_event.set() + ret = self._data_queue.get() + if isinstance(ret, Exception): + raise ret + return ret + + def _ClearSurfaceFlingerLatencyData(self): + """Clears the SurfaceFlinger latency data. + + Returns: + True if SurfaceFlinger latency is supported by the device, otherwise + False. + """ + # The command returns nothing if it is supported, otherwise returns many + # lines of result just like 'dumpsys SurfaceFlinger'. + results = self._device.RunShellCommand( + 'dumpsys SurfaceFlinger --latency-clear SurfaceView') + return not len(results) + + def GetSurfaceFlingerPid(self): + results = self._device.RunShellCommand('ps | grep surfaceflinger') + if not results: + raise Exception('Unable to get surface flinger process id') + pid = results[0].split()[1] + return pid + + def _GetSurfaceFlingerFrameData(self): + """Returns collected SurfaceFlinger frame timing data. + + Returns: + A tuple containing: + - The display's nominal refresh period in milliseconds. + - A list of timestamps signifying frame presentation times in + milliseconds. + The return value may be (None, None) if there was no data collected (for + example, if the app was closed before the collector thread has finished). + """ + # adb shell dumpsys SurfaceFlinger --latency <window name> + # prints some information about the last 128 frames displayed in + # that window. + # The data returned looks like this: + # 16954612 + # 7657467895508 7657482691352 7657493499756 + # 7657484466553 7657499645964 7657511077881 + # 7657500793457 7657516600576 7657527404785 + # (...) + # + # The first line is the refresh period (here 16.95 ms), it is followed + # by 128 lines w/ 3 timestamps in nanosecond each: + # A) when the app started to draw + # B) the vsync immediately preceding SF submitting the frame to the h/w + # C) timestamp immediately after SF submitted that frame to the h/w + # + # The difference between the 1st and 3rd timestamp is the frame-latency. + # An interesting data is when the frame latency crosses a refresh period + # boundary, this can be calculated this way: + # + # ceil((C - A) / refresh-period) + # + # (each time the number above changes, we have a "jank"). + # If this happens a lot during an animation, the animation appears + # janky, even if it runs at 60 fps in average. + # + # We use the special "SurfaceView" window name because the statistics for + # the activity's main window are not updated when the main web content is + # composited into a SurfaceView. + results = self._device.RunShellCommand( + 'dumpsys SurfaceFlinger --latency SurfaceView') + if not len(results): + return (None, None) + + timestamps = [] + nanoseconds_per_millisecond = 1e6 + refresh_period = long(results[0]) / nanoseconds_per_millisecond + + # If a fence associated with a frame is still pending when we query the + # latency data, SurfaceFlinger gives the frame a timestamp of INT64_MAX. + # Since we only care about completed frames, we will ignore any timestamps + # with this value. + pending_fence_timestamp = (1 << 63) - 1 + + for line in results[1:]: + fields = line.split() + if len(fields) != 3: + continue + timestamp = long(fields[1]) + if timestamp == pending_fence_timestamp: + continue + timestamp /= nanoseconds_per_millisecond + timestamps.append(timestamp) + + return (refresh_period, timestamps) diff --git a/catapult/devil/devil/android/perf/thermal_throttle.py b/catapult/devil/devil/android/perf/thermal_throttle.py new file mode 100644 index 00000000..9aad4bb3 --- /dev/null +++ b/catapult/devil/devil/android/perf/thermal_throttle.py @@ -0,0 +1,132 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import logging + + +class OmapThrottlingDetector(object): + """Class to detect and track thermal throttling on an OMAP 4.""" + OMAP_TEMP_FILE = ('/sys/devices/platform/omap/omap_temp_sensor.0/' + 'temperature') + + @staticmethod + def IsSupported(device): + return device.FileExists(OmapThrottlingDetector.OMAP_TEMP_FILE) + + def __init__(self, device): + self._device = device + + @staticmethod + def BecameThrottled(log_line): + return 'omap_thermal_throttle' in log_line + + @staticmethod + def BecameUnthrottled(log_line): + return 'omap_thermal_unthrottle' in log_line + + @staticmethod + def GetThrottlingTemperature(log_line): + if 'throttle_delayed_work_fn' in log_line: + return float([s for s in log_line.split() if s.isdigit()][0]) / 1000.0 + + def GetCurrentTemperature(self): + tempdata = self._device.ReadFile(OmapThrottlingDetector.OMAP_TEMP_FILE) + return float(tempdata) / 1000.0 + + +class ExynosThrottlingDetector(object): + """Class to detect and track thermal throttling on an Exynos 5.""" + @staticmethod + def IsSupported(device): + return device.FileExists('/sys/bus/exynos5-core') + + def __init__(self, device): + pass + + @staticmethod + def BecameThrottled(log_line): + return 'exynos_tmu: Throttling interrupt' in log_line + + @staticmethod + def BecameUnthrottled(log_line): + return 'exynos_thermal_unthrottle: not throttling' in log_line + + @staticmethod + def GetThrottlingTemperature(_log_line): + return None + + @staticmethod + def GetCurrentTemperature(): + return None + + +class ThermalThrottle(object): + """Class to detect and track thermal throttling. + + Usage: + Wait for IsThrottled() to be False before running test + After running test call HasBeenThrottled() to find out if the + test run was affected by thermal throttling. + """ + + def __init__(self, device): + self._device = device + self._throttled = False + self._detector = None + if OmapThrottlingDetector.IsSupported(device): + self._detector = OmapThrottlingDetector(device) + elif ExynosThrottlingDetector.IsSupported(device): + self._detector = ExynosThrottlingDetector(device) + + def HasBeenThrottled(self): + """True if there has been any throttling since the last call to + HasBeenThrottled or IsThrottled. + """ + return self._ReadLog() + + def IsThrottled(self): + """True if currently throttled.""" + self._ReadLog() + return self._throttled + + def _ReadLog(self): + if not self._detector: + return False + has_been_throttled = False + serial_number = str(self._device) + log = self._device.RunShellCommand('dmesg -c') + degree_symbol = unichr(0x00B0) + for line in log: + if self._detector.BecameThrottled(line): + if not self._throttled: + logging.warning('>>> Device %s thermally throttled', serial_number) + self._throttled = True + has_been_throttled = True + elif self._detector.BecameUnthrottled(line): + if self._throttled: + logging.warning('>>> Device %s thermally unthrottled', serial_number) + self._throttled = False + has_been_throttled = True + temperature = self._detector.GetThrottlingTemperature(line) + if temperature is not None: + logging.info(u'Device %s thermally throttled at %3.1f%sC', + serial_number, temperature, degree_symbol) + + if logging.getLogger().isEnabledFor(logging.DEBUG): + # Print current temperature of CPU SoC. + temperature = self._detector.GetCurrentTemperature() + if temperature is not None: + logging.debug(u'Current SoC temperature of %s = %3.1f%sC', + serial_number, temperature, degree_symbol) + + # Print temperature of battery, to give a system temperature + dumpsys_log = self._device.RunShellCommand('dumpsys battery') + for line in dumpsys_log: + if 'temperature' in line: + btemp = float([s for s in line.split() if s.isdigit()][0]) / 10.0 + logging.debug(u'Current battery temperature of %s = %3.1f%sC', + serial_number, btemp, degree_symbol) + + return has_been_throttled + diff --git a/catapult/devil/devil/android/ports.py b/catapult/devil/devil/android/ports.py new file mode 100644 index 00000000..4783082c --- /dev/null +++ b/catapult/devil/devil/android/ports.py @@ -0,0 +1,178 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Functions that deal with local and device ports.""" + +import contextlib +import fcntl +import httplib +import logging +import os +import socket +import traceback + +# The net test server is started from port 10201. +_TEST_SERVER_PORT_FIRST = 10201 +_TEST_SERVER_PORT_LAST = 30000 +# A file to record next valid port of test server. +_TEST_SERVER_PORT_FILE = '/tmp/test_server_port' +_TEST_SERVER_PORT_LOCKFILE = '/tmp/test_server_port.lock' + + +# The following two methods are used to allocate the port source for various +# types of test servers. Because some net-related tests can be run on shards at +# same time, it's important to have a mechanism to allocate the port +# process-safe. In here, we implement the safe port allocation by leveraging +# flock. +def ResetTestServerPortAllocation(): + """Resets the port allocation to start from TEST_SERVER_PORT_FIRST. + + Returns: + Returns True if reset successes. Otherwise returns False. + """ + try: + with open(_TEST_SERVER_PORT_FILE, 'w') as fp: + fp.write('%d' % _TEST_SERVER_PORT_FIRST) + if os.path.exists(_TEST_SERVER_PORT_LOCKFILE): + os.unlink(_TEST_SERVER_PORT_LOCKFILE) + return True + except Exception: # pylint: disable=broad-except + logging.exception('Error while resetting port allocation') + return False + + +def AllocateTestServerPort(): + """Allocates a port incrementally. + + Returns: + Returns a valid port which should be in between TEST_SERVER_PORT_FIRST and + TEST_SERVER_PORT_LAST. Returning 0 means no more valid port can be used. + """ + port = 0 + ports_tried = [] + try: + fp_lock = open(_TEST_SERVER_PORT_LOCKFILE, 'w') + fcntl.flock(fp_lock, fcntl.LOCK_EX) + # Get current valid port and calculate next valid port. + if not os.path.exists(_TEST_SERVER_PORT_FILE): + ResetTestServerPortAllocation() + with open(_TEST_SERVER_PORT_FILE, 'r+') as fp: + port = int(fp.read()) + ports_tried.append(port) + while not IsHostPortAvailable(port): + port += 1 + ports_tried.append(port) + if (port > _TEST_SERVER_PORT_LAST or + port < _TEST_SERVER_PORT_FIRST): + port = 0 + else: + fp.seek(0, os.SEEK_SET) + fp.write('%d' % (port + 1)) + except Exception: # pylint: disable=broad-except + logging.exception('ERror while allocating port') + finally: + if fp_lock: + fcntl.flock(fp_lock, fcntl.LOCK_UN) + fp_lock.close() + if port: + logging.info('Allocate port %d for test server.', port) + else: + logging.error('Could not allocate port for test server. ' + 'List of ports tried: %s', str(ports_tried)) + return port + + +def IsHostPortAvailable(host_port): + """Checks whether the specified host port is available. + + Args: + host_port: Port on host to check. + + Returns: + True if the port on host is available, otherwise returns False. + """ + s = socket.socket() + try: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(('', host_port)) + s.close() + return True + except socket.error: + return False + + +def IsDevicePortUsed(device, device_port, state=''): + """Checks whether the specified device port is used or not. + + Args: + device: A DeviceUtils instance. + device_port: Port on device we want to check. + state: String of the specified state. Default is empty string, which + means any state. + + Returns: + True if the port on device is already used, otherwise returns False. + """ + base_urls = ('127.0.0.1:%d' % device_port, 'localhost:%d' % device_port) + netstat_results = device.RunShellCommand( + ['netstat', '-a'], check_return=True, large_output=True) + for single_connect in netstat_results: + # Column 3 is the local address which we want to check with. + connect_results = single_connect.split() + if connect_results[0] != 'tcp': + continue + if len(connect_results) < 6: + raise Exception('Unexpected format while parsing netstat line: ' + + single_connect) + is_state_match = connect_results[5] == state if state else True + if connect_results[3] in base_urls and is_state_match: + return True + return False + + +def IsHttpServerConnectable(host, port, tries=3, command='GET', path='/', + expected_read='', timeout=2): + """Checks whether the specified http server is ready to serve request or not. + + Args: + host: Host name of the HTTP server. + port: Port number of the HTTP server. + tries: How many times we want to test the connection. The default value is + 3. + command: The http command we use to connect to HTTP server. The default + command is 'GET'. + path: The path we use when connecting to HTTP server. The default path is + '/'. + expected_read: The content we expect to read from the response. The default + value is ''. + timeout: Timeout (in seconds) for each http connection. The default is 2s. + + Returns: + Tuple of (connect status, client error). connect status is a boolean value + to indicate whether the server is connectable. client_error is the error + message the server returns when connect status is false. + """ + assert tries >= 1 + for i in xrange(0, tries): + client_error = None + try: + with contextlib.closing(httplib.HTTPConnection( + host, port, timeout=timeout)) as http: + # Output some debug information when we have tried more than 2 times. + http.set_debuglevel(i >= 2) + http.request(command, path) + r = http.getresponse() + content = r.read() + if r.status == 200 and r.reason == 'OK' and content == expected_read: + return (True, '') + client_error = ('Bad response: %s %s version %s\n ' % + (r.status, r.reason, r.version) + + '\n '.join([': '.join(h) for h in r.getheaders()])) + except (httplib.HTTPException, socket.error) as e: + # Probably too quick connecting: try again. + exception_error_msgs = traceback.format_exception_only(type(e), e) + if exception_error_msgs: + client_error = ''.join(exception_error_msgs) + # Only returns last client_error. + return (False, client_error or 'Timeout') diff --git a/catapult/devil/devil/android/sdk/__init__.py b/catapult/devil/devil/android/sdk/__init__.py new file mode 100644 index 00000000..f95d3b27 --- /dev/null +++ b/catapult/devil/devil/android/sdk/__init__.py @@ -0,0 +1,6 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# This package is intended for modules that are very tightly coupled to +# tools or APIs from the Android SDK. diff --git a/catapult/devil/devil/android/sdk/aapt.py b/catapult/devil/devil/android/sdk/aapt.py new file mode 100644 index 00000000..7ae3a938 --- /dev/null +++ b/catapult/devil/devil/android/sdk/aapt.py @@ -0,0 +1,43 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This module wraps the Android Asset Packaging Tool.""" + +from devil.android.sdk import build_tools +from devil.utils import cmd_helper +from devil.utils import lazy + + +_aapt_path = lazy.WeakConstant(lambda: build_tools.GetPath('aapt')) + + +def _RunAaptCmd(args): + """Runs an aapt command. + + Args: + args: A list of arguments for aapt. + + Returns: + The output of the command. + """ + cmd = [_aapt_path.read()] + args + status, output = cmd_helper.GetCmdStatusAndOutput(cmd) + if status != 0: + raise Exception('Failed running aapt command: "%s" with output "%s".' % + (' '.join(cmd), output)) + return output + + +def Dump(what, apk, assets=None): + """Returns the output of the aapt dump command. + + Args: + what: What you want to dump. + apk: Path to apk you want to dump information for. + assets: List of assets in apk you want to dump information for. + """ + assets = assets or [] + if isinstance(assets, basestring): + assets = [assets] + return _RunAaptCmd(['dump', what, apk] + assets).splitlines() diff --git a/catapult/devil/devil/android/sdk/adb_compatibility_devicetest.py b/catapult/devil/devil/android/sdk/adb_compatibility_devicetest.py new file mode 100755 index 00000000..de08e21a --- /dev/null +++ b/catapult/devil/devil/android/sdk/adb_compatibility_devicetest.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import signal +import sys +import unittest + +from devil import devil_env +from devil.android.sdk import adb_wrapper +from devil.utils import cmd_helper +from devil.utils import timeout_retry + +_PYMOCK_PATH = os.path.abspath(os.path.join( + os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, + 'third_party', 'mock')) +with devil_env.SysPath(_PYMOCK_PATH): + import mock # pylint: disable=import-error + + +_ADB_PATH = os.environ.get('ADB_PATH', 'adb') + + +def _hostAdbPids(): + ps_status, ps_output = cmd_helper.GetCmdStatusAndOutput( + ['pgrep', '-l', 'adb']) + if ps_status != 0: + return [] + + pids_and_names = (line.split() for line in ps_output.splitlines()) + return [int(pid) for pid, name in pids_and_names + if name == 'adb'] + + +@mock.patch('devil.android.sdk.adb_wrapper.AdbWrapper.GetAdbPath', + return_value=_ADB_PATH) +class AdbCompatibilityTest(unittest.TestCase): + + def testStartServer(self, *_args): + # Manually kill off any instances of adb. + adb_pids = _hostAdbPids() + for p in adb_pids: + os.kill(p, signal.SIGKILL) + + self.assertIsNotNone( + timeout_retry.WaitFor( + lambda: not _hostAdbPids(), wait_period=0.1, max_tries=10)) + + # start the adb server + start_server_status, _ = cmd_helper.GetCmdStatusAndOutput( + [_ADB_PATH, 'start-server']) + + # verify that the server is now online + self.assertEquals(0, start_server_status) + self.assertIsNotNone( + timeout_retry.WaitFor( + lambda: bool(_hostAdbPids()), wait_period=0.1, max_tries=10)) + + def testKillServer(self, *_args): + adb_pids = _hostAdbPids() + if not adb_pids: + adb_wrapper.AdbWrapper.StartServer() + + adb_pids = _hostAdbPids() + self.assertEqual(1, len(adb_pids)) + + kill_server_status, _ = cmd_helper.GetCmdStatusAndOutput( + [_ADB_PATH, 'kill-server']) + self.assertEqual(0, kill_server_status) + + adb_pids = _hostAdbPids() + self.assertEqual(0, len(adb_pids)) + + # TODO(jbudorick): Implement tests for the following: + # taskset -c + # devices [-l] + # push + # pull + # shell + # ls + # logcat [-c] [-d] [-v] [-b] + # forward [--remove] [--list] + # jdwp + # install [-l] [-r] [-s] [-d] + # install-multiple [-l] [-r] [-s] [-d] [-p] + # uninstall [-k] + # backup -f [-apk] [-shared] [-nosystem] [-all] + # restore + # wait-for-device + # get-state (BROKEN IN THE M SDK) + # get-devpath + # remount + # reboot + # reboot-bootloader + # root + # emu + + @classmethod + def tearDownClass(cls): + version_status, version_output = cmd_helper.GetCmdStatusAndOutput( + [_ADB_PATH, 'version']) + if version_status != 0: + version = ['(unable to determine version)'] + else: + version = version_output.splitlines() + + print + print 'tested %s' % _ADB_PATH + for l in version: + print ' %s' % l + + +if __name__ == '__main__': + sys.exit(unittest.main()) diff --git a/catapult/devil/devil/android/sdk/adb_wrapper.py b/catapult/devil/devil/android/sdk/adb_wrapper.py new file mode 100644 index 00000000..a65ab7cb --- /dev/null +++ b/catapult/devil/devil/android/sdk/adb_wrapper.py @@ -0,0 +1,704 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This module wraps Android's adb tool. + +This is a thin wrapper around the adb interface. Any additional complexity +should be delegated to a higher level (ex. DeviceUtils). +""" + +import collections +import errno +import logging +import os +import re + +from devil import devil_env +from devil.android import decorators +from devil.android import device_errors +from devil.utils import cmd_helper +from devil.utils import lazy +from devil.utils import timeout_retry + +with devil_env.SysPath(devil_env.DEPENDENCY_MANAGER_PATH): + import dependency_manager # pylint: disable=import-error + + +_DEFAULT_TIMEOUT = 30 +_DEFAULT_RETRIES = 2 + +_EMULATOR_RE = re.compile(r'^emulator-[0-9]+$') + +_READY_STATE = 'device' + + +def VerifyLocalFileExists(path): + """Verifies a local file exists. + + Args: + path: Path to the local file. + + Raises: + IOError: If the file doesn't exist. + """ + if not os.path.exists(path): + raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), path) + + +def _FindAdb(): + try: + return devil_env.config.LocalPath('adb') + except dependency_manager.NoPathFoundError: + pass + + try: + return os.path.join(devil_env.config.LocalPath('android_sdk'), + 'platform-tools', 'adb') + except dependency_manager.NoPathFoundError: + pass + + try: + return devil_env.config.FetchPath('adb') + except dependency_manager.NoPathFoundError: + raise device_errors.NoAdbError() + + +def _ShouldRetryAdbCmd(exc): + return not isinstance(exc, device_errors.NoAdbError) + + +DeviceStat = collections.namedtuple('DeviceStat', + ['st_mode', 'st_size', 'st_time']) + + +class AdbWrapper(object): + """A wrapper around a local Android Debug Bridge executable.""" + + _adb_path = lazy.WeakConstant(_FindAdb) + + def __init__(self, device_serial): + """Initializes the AdbWrapper. + + Args: + device_serial: The device serial number as a string. + """ + if not device_serial: + raise ValueError('A device serial must be specified') + self._device_serial = str(device_serial) + + @classmethod + def GetAdbPath(cls): + return cls._adb_path.read() + + @classmethod + def _BuildAdbCmd(cls, args, device_serial, cpu_affinity=None): + if cpu_affinity is not None: + cmd = ['taskset', '-c', str(cpu_affinity)] + else: + cmd = [] + cmd.append(cls.GetAdbPath()) + if device_serial is not None: + cmd.extend(['-s', device_serial]) + cmd.extend(args) + return cmd + + # pylint: disable=unused-argument + @classmethod + @decorators.WithTimeoutAndConditionalRetries(_ShouldRetryAdbCmd) + def _RunAdbCmd(cls, args, timeout=None, retries=None, device_serial=None, + check_error=True, cpu_affinity=None): + # pylint: disable=no-member + try: + status, output = cmd_helper.GetCmdStatusAndOutputWithTimeout( + cls._BuildAdbCmd(args, device_serial, cpu_affinity=cpu_affinity), + timeout_retry.CurrentTimeoutThreadGroup().GetRemainingTime()) + except OSError as e: + if e.errno in (errno.ENOENT, errno.ENOEXEC): + raise device_errors.NoAdbError(msg=str(e)) + else: + raise + + if status != 0: + raise device_errors.AdbCommandFailedError( + args, output, status, device_serial) + # This catches some errors, including when the device drops offline; + # unfortunately adb is very inconsistent with error reporting so many + # command failures present differently. + if check_error and output.startswith('error:'): + raise device_errors.AdbCommandFailedError(args, output) + return output + # pylint: enable=unused-argument + + def _RunDeviceAdbCmd(self, args, timeout, retries, check_error=True): + """Runs an adb command on the device associated with this object. + + Args: + args: A list of arguments to adb. + timeout: Timeout in seconds. + retries: Number of retries. + check_error: Check that the command doesn't return an error message. This + does NOT check the exit status of shell commands. + + Returns: + The output of the command. + """ + return self._RunAdbCmd(args, timeout=timeout, retries=retries, + device_serial=self._device_serial, + check_error=check_error) + + def _IterRunDeviceAdbCmd(self, args, timeout): + """Runs an adb command and returns an iterator over its output lines. + + Args: + args: A list of arguments to adb. + timeout: Timeout in seconds. + + Yields: + The output of the command line by line. + """ + return cmd_helper.IterCmdOutputLines( + self._BuildAdbCmd(args, self._device_serial), timeout=timeout) + + def __eq__(self, other): + """Consider instances equal if they refer to the same device. + + Args: + other: The instance to compare equality with. + + Returns: + True if the instances are considered equal, false otherwise. + """ + return self._device_serial == str(other) + + def __str__(self): + """The string representation of an instance. + + Returns: + The device serial number as a string. + """ + return self._device_serial + + def __repr__(self): + return '%s(\'%s\')' % (self.__class__.__name__, self) + + # pylint: disable=unused-argument + @classmethod + def IsServerOnline(cls): + status, output = cmd_helper.GetCmdStatusAndOutput(['pgrep', 'adb']) + output = [int(x) for x in output.split()] + logging.info('PIDs for adb found: %r', output) + return status == 0 + # pylint: enable=unused-argument + + @classmethod + def KillServer(cls, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + cls._RunAdbCmd(['kill-server'], timeout=timeout, retries=retries) + + @classmethod + def StartServer(cls, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + # CPU affinity is used to reduce adb instability http://crbug.com/268450 + cls._RunAdbCmd(['start-server'], timeout=timeout, retries=retries, + cpu_affinity=0) + + @classmethod + def GetDevices(cls, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + """DEPRECATED. Refer to Devices(...) below.""" + # TODO(jbudorick): Remove this function once no more clients are using it. + return cls.Devices(timeout=timeout, retries=retries) + + @classmethod + def Devices(cls, desired_state=_READY_STATE, long_list=False, + timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + """Get the list of active attached devices. + + Args: + desired_state: If not None, limit the devices returned to only those + in the given state. + long_list: Whether to use the long listing format. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + + Yields: + AdbWrapper instances. + """ + lines = cls._RawDevices(long_list=long_list, timeout=timeout, + retries=retries) + if long_list: + return [ + [AdbWrapper(line[0])] + line[1:] + for line in lines + if (len(line) >= 2 and (not desired_state or line[1] == desired_state)) + ] + else: + return [ + AdbWrapper(line[0]) + for line in lines + if (len(line) == 2 and (not desired_state or line[1] == desired_state)) + ] + + @classmethod + def _RawDevices(cls, long_list=False, timeout=_DEFAULT_TIMEOUT, + retries=_DEFAULT_RETRIES): + cmd = ['devices'] + if long_list: + cmd.append('-l') + output = cls._RunAdbCmd(cmd, timeout=timeout, retries=retries) + return [line.split() for line in output.splitlines()[1:]] + + def GetDeviceSerial(self): + """Gets the device serial number associated with this object. + + Returns: + Device serial number as a string. + """ + return self._device_serial + + def Push(self, local, remote, timeout=60 * 5, retries=_DEFAULT_RETRIES): + """Pushes a file from the host to the device. + + Args: + local: Path on the host filesystem. + remote: Path on the device filesystem. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + VerifyLocalFileExists(local) + self._RunDeviceAdbCmd(['push', local, remote], timeout, retries) + + def Pull(self, remote, local, timeout=60 * 5, retries=_DEFAULT_RETRIES): + """Pulls a file from the device to the host. + + Args: + remote: Path on the device filesystem. + local: Path on the host filesystem. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + cmd = ['pull', remote, local] + self._RunDeviceAdbCmd(cmd, timeout, retries) + try: + VerifyLocalFileExists(local) + except IOError: + raise device_errors.AdbCommandFailedError( + cmd, 'File not found on host: %s' % local, device_serial=str(self)) + + def Shell(self, command, expect_status=0, timeout=_DEFAULT_TIMEOUT, + retries=_DEFAULT_RETRIES): + """Runs a shell command on the device. + + Args: + command: A string with the shell command to run. + expect_status: (optional) Check that the command's exit status matches + this value. Default is 0. If set to None the test is skipped. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + + Returns: + The output of the shell command as a string. + + Raises: + device_errors.AdbCommandFailedError: If the exit status doesn't match + |expect_status|. + """ + if expect_status is None: + args = ['shell', command] + else: + args = ['shell', '( %s );echo %%$?' % command.rstrip()] + output = self._RunDeviceAdbCmd(args, timeout, retries, check_error=False) + if expect_status is not None: + output_end = output.rfind('%') + if output_end < 0: + # causes the status string to become empty and raise a ValueError + output_end = len(output) + + try: + status = int(output[output_end + 1:]) + except ValueError: + logging.warning('exit status of shell command %r missing.', command) + raise device_errors.AdbShellCommandFailedError( + command, output, status=None, device_serial=self._device_serial) + output = output[:output_end] + if status != expect_status: + raise device_errors.AdbShellCommandFailedError( + command, output, status=status, device_serial=self._device_serial) + return output + + def IterShell(self, command, timeout): + """Runs a shell command and returns an iterator over its output lines. + + Args: + command: A string with the shell command to run. + timeout: Timeout in seconds. + + Yields: + The output of the command line by line. + """ + args = ['shell', command] + return cmd_helper.IterCmdOutputLines( + self._BuildAdbCmd(args, self._device_serial), timeout=timeout) + + def Ls(self, path, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + """List the contents of a directory on the device. + + Args: + path: Path on the device filesystem. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + + Returns: + A list of pairs (filename, stat) for each file found in the directory, + where the stat object has the properties: st_mode, st_size, and st_time. + + Raises: + AdbCommandFailedError if |path| does not specify a valid and accessible + directory in the device, or the output of "adb ls" command is less + than four columns + """ + def ParseLine(line, cmd): + cols = line.split(None, 3) + if len(cols) < 4: + raise device_errors.AdbCommandFailedError( + cmd, line, "the output should be 4 columns, but is only %d columns" + % len(cols), device_serial=self._device_serial) + filename = cols.pop() + stat = DeviceStat(*[int(num, base=16) for num in cols]) + return (filename, stat) + + cmd = ['ls', path] + lines = self._RunDeviceAdbCmd( + cmd, timeout=timeout, retries=retries).splitlines() + if lines: + return [ParseLine(line, cmd) for line in lines] + else: + raise device_errors.AdbCommandFailedError( + cmd, 'path does not specify an accessible directory in the device', + device_serial=self._device_serial) + + def Logcat(self, clear=False, dump=False, filter_specs=None, + logcat_format=None, ring_buffer=None, timeout=None, + retries=_DEFAULT_RETRIES): + """Get an iterable over the logcat output. + + Args: + clear: If true, clear the logcat. + dump: If true, dump the current logcat contents. + filter_specs: If set, a list of specs to filter the logcat. + logcat_format: If set, the format in which the logcat should be output. + Options include "brief", "process", "tag", "thread", "raw", "time", + "threadtime", and "long" + ring_buffer: If set, a list of alternate ring buffers to request. + Options include "main", "system", "radio", "events", "crash" or "all". + The default is equivalent to ["main", "system", "crash"]. + timeout: (optional) If set, timeout per try in seconds. If clear or dump + is set, defaults to _DEFAULT_TIMEOUT. + retries: (optional) If clear or dump is set, the number of retries to + attempt. Otherwise, does nothing. + + Yields: + logcat output line by line. + """ + cmd = ['logcat'] + use_iter = True + if clear: + cmd.append('-c') + use_iter = False + if dump: + cmd.append('-d') + use_iter = False + if logcat_format: + cmd.extend(['-v', logcat_format]) + if ring_buffer: + for buffer_name in ring_buffer: + cmd.extend(['-b', buffer_name]) + if filter_specs: + cmd.extend(filter_specs) + + if use_iter: + return self._IterRunDeviceAdbCmd(cmd, timeout) + else: + timeout = timeout if timeout is not None else _DEFAULT_TIMEOUT + return self._RunDeviceAdbCmd(cmd, timeout, retries).splitlines() + + def Forward(self, local, remote, timeout=_DEFAULT_TIMEOUT, + retries=_DEFAULT_RETRIES): + """Forward socket connections from the local socket to the remote socket. + + Sockets are specified by one of: + tcp:<port> + localabstract:<unix domain socket name> + localreserved:<unix domain socket name> + localfilesystem:<unix domain socket name> + dev:<character device name> + jdwp:<process pid> (remote only) + + Args: + local: The host socket. + remote: The device socket. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + self._RunDeviceAdbCmd(['forward', str(local), str(remote)], timeout, + retries) + + def ForwardRemove(self, local, timeout=_DEFAULT_TIMEOUT, + retries=_DEFAULT_RETRIES): + """Remove a forward socket connection. + + Args: + local: The host socket. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + self._RunDeviceAdbCmd(['forward', '--remove', str(local)], timeout, + retries) + + def ForwardList(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + """List all currently forwarded socket connections. + + Args: + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + return self._RunDeviceAdbCmd(['forward', '--list'], timeout, retries) + + def JDWP(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + """List of PIDs of processes hosting a JDWP transport. + + Args: + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + + Returns: + A list of PIDs as strings. + """ + return [a.strip() for a in + self._RunDeviceAdbCmd(['jdwp'], timeout, retries).split('\n')] + + def Install(self, apk_path, forward_lock=False, allow_downgrade=False, + reinstall=False, sd_card=False, timeout=60 * 2, + retries=_DEFAULT_RETRIES): + """Install an apk on the device. + + Args: + apk_path: Host path to the APK file. + forward_lock: (optional) If set forward-locks the app. + allow_downgrade: (optional) If set, allows for downgrades. + reinstall: (optional) If set reinstalls the app, keeping its data. + sd_card: (optional) If set installs on the SD card. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + VerifyLocalFileExists(apk_path) + cmd = ['install'] + if forward_lock: + cmd.append('-l') + if reinstall: + cmd.append('-r') + if sd_card: + cmd.append('-s') + if allow_downgrade: + cmd.append('-d') + cmd.append(apk_path) + output = self._RunDeviceAdbCmd(cmd, timeout, retries) + if 'Success' not in output: + raise device_errors.AdbCommandFailedError( + cmd, output, device_serial=self._device_serial) + + def InstallMultiple(self, apk_paths, forward_lock=False, reinstall=False, + sd_card=False, allow_downgrade=False, partial=False, + timeout=60 * 2, retries=_DEFAULT_RETRIES): + """Install an apk with splits on the device. + + Args: + apk_paths: Host path to the APK file. + forward_lock: (optional) If set forward-locks the app. + reinstall: (optional) If set reinstalls the app, keeping its data. + sd_card: (optional) If set installs on the SD card. + allow_downgrade: (optional) Allow versionCode downgrade. + partial: (optional) Package ID if apk_paths doesn't include all .apks. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + for path in apk_paths: + VerifyLocalFileExists(path) + cmd = ['install-multiple'] + if forward_lock: + cmd.append('-l') + if reinstall: + cmd.append('-r') + if sd_card: + cmd.append('-s') + if allow_downgrade: + cmd.append('-d') + if partial: + cmd.extend(('-p', partial)) + cmd.extend(apk_paths) + output = self._RunDeviceAdbCmd(cmd, timeout, retries) + if 'Success' not in output: + raise device_errors.AdbCommandFailedError( + cmd, output, device_serial=self._device_serial) + + def Uninstall(self, package, keep_data=False, timeout=_DEFAULT_TIMEOUT, + retries=_DEFAULT_RETRIES): + """Remove the app |package| from the device. + + Args: + package: The package to uninstall. + keep_data: (optional) If set keep the data and cache directories. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + cmd = ['uninstall'] + if keep_data: + cmd.append('-k') + cmd.append(package) + output = self._RunDeviceAdbCmd(cmd, timeout, retries) + if 'Failure' in output: + raise device_errors.AdbCommandFailedError( + cmd, output, device_serial=self._device_serial) + + def Backup(self, path, packages=None, apk=False, shared=False, + nosystem=True, include_all=False, timeout=_DEFAULT_TIMEOUT, + retries=_DEFAULT_RETRIES): + """Write an archive of the device's data to |path|. + + Args: + path: Local path to store the backup file. + packages: List of to packages to be backed up. + apk: (optional) If set include the .apk files in the archive. + shared: (optional) If set buckup the device's SD card. + nosystem: (optional) If set exclude system applications. + include_all: (optional) If set back up all installed applications and + |packages| is optional. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + cmd = ['backup', '-f', path] + if apk: + cmd.append('-apk') + if shared: + cmd.append('-shared') + if nosystem: + cmd.append('-nosystem') + if include_all: + cmd.append('-all') + if packages: + cmd.extend(packages) + assert bool(packages) ^ bool(include_all), ( + 'Provide \'packages\' or set \'include_all\' but not both.') + ret = self._RunDeviceAdbCmd(cmd, timeout, retries) + VerifyLocalFileExists(path) + return ret + + def Restore(self, path, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + """Restore device contents from the backup archive. + + Args: + path: Host path to the backup archive. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + VerifyLocalFileExists(path) + self._RunDeviceAdbCmd(['restore'] + [path], timeout, retries) + + def WaitForDevice(self, timeout=60 * 5, retries=_DEFAULT_RETRIES): + """Block until the device is online. + + Args: + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + self._RunDeviceAdbCmd(['wait-for-device'], timeout, retries) + + def GetState(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + """Get device state. + + Args: + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + + Returns: + One of 'offline', 'bootloader', or 'device'. + """ + # TODO(jbudorick): Revert to using get-state once it doesn't cause a + # a protocol fault. + # return self._RunDeviceAdbCmd(['get-state'], timeout, retries).strip() + + lines = self._RawDevices(timeout=timeout, retries=retries) + for line in lines: + if len(line) >= 2 and line[0] == self._device_serial: + return line[1] + return 'offline' + + def GetDevPath(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + """Gets the device path. + + Args: + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + + Returns: + The device path (e.g. usb:3-4) + """ + return self._RunDeviceAdbCmd(['get-devpath'], timeout, retries) + + def Remount(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + """Remounts the /system partition on the device read-write.""" + self._RunDeviceAdbCmd(['remount'], timeout, retries) + + def Reboot(self, to_bootloader=False, timeout=60 * 5, + retries=_DEFAULT_RETRIES): + """Reboots the device. + + Args: + to_bootloader: (optional) If set reboots to the bootloader. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + if to_bootloader: + cmd = ['reboot-bootloader'] + else: + cmd = ['reboot'] + self._RunDeviceAdbCmd(cmd, timeout, retries) + + def Root(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + """Restarts the adbd daemon with root permissions, if possible. + + Args: + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + """ + output = self._RunDeviceAdbCmd(['root'], timeout, retries) + if 'cannot' in output: + raise device_errors.AdbCommandFailedError( + ['root'], output, device_serial=self._device_serial) + + def Emu(self, cmd, timeout=_DEFAULT_TIMEOUT, + retries=_DEFAULT_RETRIES): + """Runs an emulator console command. + + See http://developer.android.com/tools/devices/emulator.html#console + + Args: + cmd: The command to run on the emulator console. + timeout: (optional) Timeout per try in seconds. + retries: (optional) Number of retries to attempt. + + Returns: + The output of the emulator console command. + """ + if isinstance(cmd, basestring): + cmd = [cmd] + return self._RunDeviceAdbCmd(['emu'] + cmd, timeout, retries) + + @property + def is_emulator(self): + return _EMULATOR_RE.match(self._device_serial) + + @property + def is_ready(self): + try: + return self.GetState() == _READY_STATE + except device_errors.CommandFailedError: + return False diff --git a/catapult/devil/devil/android/sdk/adb_wrapper_devicetest.py b/catapult/devil/devil/android/sdk/adb_wrapper_devicetest.py new file mode 100644 index 00000000..59755c00 --- /dev/null +++ b/catapult/devil/devil/android/sdk/adb_wrapper_devicetest.py @@ -0,0 +1,96 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for the AdbWrapper class.""" + +import os +import tempfile +import time +import unittest + +from devil.android import device_errors +from devil.android.sdk import adb_wrapper + + +class TestAdbWrapper(unittest.TestCase): + + def setUp(self): + devices = adb_wrapper.AdbWrapper.Devices() + assert devices, 'A device must be attached' + self._adb = devices[0] + self._adb.WaitForDevice() + + @staticmethod + def _MakeTempFile(contents): + """Make a temporary file with the given contents. + + Args: + contents: string to write to the temporary file. + + Returns: + The absolute path to the file. + """ + fi, path = tempfile.mkstemp() + with os.fdopen(fi, 'wb') as f: + f.write(contents) + return path + + def testShell(self): + output = self._adb.Shell('echo test', expect_status=0) + self.assertEqual(output.strip(), 'test') + output = self._adb.Shell('echo test') + self.assertEqual(output.strip(), 'test') + with self.assertRaises(device_errors.AdbCommandFailedError): + self._adb.Shell('echo test', expect_status=1) + + def testPushLsPull(self): + path = self._MakeTempFile('foo') + device_path = '/data/local/tmp/testfile.txt' + local_tmpdir = os.path.dirname(path) + self._adb.Push(path, device_path) + files = dict(self._adb.Ls('/data/local/tmp')) + self.assertTrue('testfile.txt' in files) + self.assertEquals(3, files['testfile.txt'].st_size) + self.assertEqual(self._adb.Shell('cat %s' % device_path), 'foo') + self._adb.Pull(device_path, local_tmpdir) + with open(os.path.join(local_tmpdir, 'testfile.txt'), 'r') as f: + self.assertEqual(f.read(), 'foo') + + def testInstall(self): + path = self._MakeTempFile('foo') + with self.assertRaises(device_errors.AdbCommandFailedError): + self._adb.Install(path) + + def testForward(self): + with self.assertRaises(device_errors.AdbCommandFailedError): + self._adb.Forward(0, 0) + + def testUninstall(self): + with self.assertRaises(device_errors.AdbCommandFailedError): + self._adb.Uninstall('some.nonexistant.package') + + def testRebootWaitForDevice(self): + self._adb.Reboot() + print 'waiting for device to reboot...' + while self._adb.GetState() == 'device': + time.sleep(1) + self._adb.WaitForDevice() + self.assertEqual(self._adb.GetState(), 'device') + print 'waiting for package manager...' + while 'package:' not in self._adb.Shell('pm path android'): + time.sleep(1) + + def testRootRemount(self): + self._adb.Root() + while True: + try: + self._adb.Shell('start') + break + except device_errors.AdbCommandFailedError: + time.sleep(1) + self._adb.Remount() + + +if __name__ == '__main__': + unittest.main() diff --git a/catapult/devil/devil/android/sdk/build_tools.py b/catapult/devil/devil/android/sdk/build_tools.py new file mode 100644 index 00000000..99083d99 --- /dev/null +++ b/catapult/devil/devil/android/sdk/build_tools.py @@ -0,0 +1,51 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os + +from devil import devil_env +from devil.utils import lazy + +with devil_env.SysPath(devil_env.DEPENDENCY_MANAGER_PATH): + import dependency_manager # pylint: disable=import-error + + +def GetPath(build_tool): + try: + return devil_env.config.LocalPath(build_tool) + except dependency_manager.NoPathFoundError: + pass + + try: + return _PathInLocalSdk(build_tool) + except dependency_manager.NoPathFoundError: + pass + + return devil_env.config.FetchPath(build_tool) + + +def _PathInLocalSdk(build_tool): + build_tools_path = _build_tools_path.read() + return (os.path.join(build_tools_path, build_tool) if build_tools_path + else None) + + +def _FindBuildTools(): + android_sdk_path = devil_env.config.LocalPath('android_sdk') + if not android_sdk_path: + return None + + build_tools_contents = os.listdir( + os.path.join(android_sdk_path, 'build-tools')) + + if not build_tools_contents: + return None + else: + if len(build_tools_contents) > 1: + build_tools_contents.sort() + return os.path.join(android_sdk_path, 'build-tools', + build_tools_contents[-1]) + + +_build_tools_path = lazy.WeakConstant(_FindBuildTools) diff --git a/catapult/devil/devil/android/sdk/dexdump.py b/catapult/devil/devil/android/sdk/dexdump.py new file mode 100644 index 00000000..992366e8 --- /dev/null +++ b/catapult/devil/devil/android/sdk/dexdump.py @@ -0,0 +1,31 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from devil.android.sdk import build_tools +from devil.utils import cmd_helper +from devil.utils import lazy + + +_dexdump_path = lazy.WeakConstant(lambda: build_tools.GetPath('dexdump')) + + +def DexDump(dexfiles, file_summary=False): + """A wrapper around the Android SDK's dexdump tool. + + Args: + dexfiles: The dexfile or list of dex files to dump. + file_summary: Display summary information from the file header. (-f) + + Returns: + An iterable over the output lines. + """ + # TODO(jbudorick): Add support for more options as necessary. + if isinstance(dexfiles, basestring): + dexfiles = [dexfiles] + args = [_dexdump_path.read()] + dexfiles + if file_summary: + args.append('-f') + + return cmd_helper.IterCmdOutputLines(args) + diff --git a/catapult/devil/devil/android/sdk/fastboot.py b/catapult/devil/devil/android/sdk/fastboot.py new file mode 100644 index 00000000..d9fa653b --- /dev/null +++ b/catapult/devil/devil/android/sdk/fastboot.py @@ -0,0 +1,101 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This module wraps Android's fastboot tool. + +This is a thin wrapper around the fastboot interface. Any additional complexity +should be delegated to a higher level (ex. FastbootUtils). +""" +# pylint: disable=unused-argument + +import os + +from devil import devil_env +from devil.android import decorators +from devil.android import device_errors +from devil.utils import cmd_helper +from devil.utils import lazy + +_DEFAULT_TIMEOUT = 30 +_DEFAULT_RETRIES = 3 +_FLASH_TIMEOUT = _DEFAULT_TIMEOUT * 10 + + +class Fastboot(object): + + _fastboot_path = lazy.WeakConstant(lambda: os.path.join( + devil_env.config.LocalPath('android_sdk'), 'platform-tools', 'adb')) + + def __init__(self, device_serial, default_timeout=_DEFAULT_TIMEOUT, + default_retries=_DEFAULT_RETRIES): + """Initializes the FastbootWrapper. + + Args: + device_serial: The device serial number as a string. + """ + if not device_serial: + raise ValueError('A device serial must be specified') + self._device_serial = str(device_serial) + self._default_timeout = default_timeout + self._default_retries = default_retries + + @decorators.WithTimeoutAndRetriesFromInstance() + def _RunFastbootCommand(self, cmd, timeout=None, retries=None): + """Run a command line command using the fastboot android tool. + + Args: + cmd: Command to run. Must be list of args, the first one being the command + + Returns: + output of command. + + Raises: + TypeError: If cmd is not of type list. + """ + if type(cmd) == list: + cmd = [self._fastboot_path.read(), '-s', self._device_serial] + cmd + else: + raise TypeError( + 'Command for _RunFastbootCommand must be a list.') + status, output = cmd_helper.GetCmdStatusAndOutput(cmd) + if int(status) != 0: + raise device_errors.FastbootCommandFailedError( + cmd, output, status, self._device_serial) + return output + + @decorators.WithTimeoutAndRetriesDefaults(_FLASH_TIMEOUT, 0) + def Flash(self, partition, image, timeout=None, retries=None): + """Flash partition with img. + + Args: + partition: Partition to be flashed. + image: location of image to flash with. + """ + self._RunFastbootCommand(['flash', partition, image]) + + @decorators.WithTimeoutAndRetriesFromInstance() + def Devices(self, timeout=None, retries=None): + """Outputs list of devices in fastboot mode.""" + output = self._RunFastbootCommand(['devices']) + return [line.split()[0] for line in output.splitlines()] + + @decorators.WithTimeoutAndRetriesFromInstance() + def RebootBootloader(self, timeout=None, retries=None): + """Reboot from fastboot, into fastboot.""" + self._RunFastbootCommand(['reboot-bootloader']) + + @decorators.WithTimeoutAndRetriesDefaults(_FLASH_TIMEOUT, 0) + def Reboot(self, timeout=None, retries=None): + """Reboot from fastboot to normal usage""" + self._RunFastbootCommand(['reboot']) + + @decorators.WithTimeoutAndRetriesFromInstance() + def SetOemOffModeCharge(self, value, timeout=None, retries=None): + """Sets off mode charging + + Args: + value: boolean value to set off-mode-charging on or off. + """ + self._RunFastbootCommand( + ['oem', 'off-mode-charge', str(int(value))]) diff --git a/catapult/devil/devil/android/sdk/gce_adb_wrapper.py b/catapult/devil/devil/android/sdk/gce_adb_wrapper.py new file mode 100644 index 00000000..5ee7959c --- /dev/null +++ b/catapult/devil/devil/android/sdk/gce_adb_wrapper.py @@ -0,0 +1,146 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Provides a work around for various adb commands on android gce instances. + +Some adb commands don't work well when the device is a cloud vm, namely +'push' and 'pull'. With gce instances, moving files through adb can be +painfully slow and hit timeouts, so the methods here just use scp instead. +""" +# pylint: disable=unused-argument + +import logging +import os +import subprocess + +from devil.android import device_errors +from devil.android.sdk import adb_wrapper +from devil.utils import cmd_helper + + +# SSH key file for accessing the instances. The keys are created at +# startup and removed & revoked at teardown. +_SSH_KEY_FILE = '/tmp/ssh_android_gce_instance' + + +class GceAdbWrapper(adb_wrapper.AdbWrapper): + + def __init__(self, device_serial): + super(GceAdbWrapper, self).__init__(device_serial) + self._instance_ip = self.Shell('getprop net.gce.ip_address').strip() + + # override + def Push(self, local, remote, **kwargs): + """Pushes an object from the host to the gce instance. + + Args: + local: Path on the host filesystem. + remote: Path on the instance filesystem. + """ + adb_wrapper.VerifyLocalFileExists(_SSH_KEY_FILE) + adb_wrapper.VerifyLocalFileExists(local) + if os.path.isdir(local): + self.Shell('mkdir -p %s' % cmd_helper.SingleQuote(remote)) + + # When the object to be pushed is a directory, adb merges the source dir + # with the destination dir. So if local is a dir, just scp its contents. + for f in os.listdir(local): + self._PushObject(os.path.join(local, f), os.path.join(remote, f)) + self.Shell('chmod 777 %s' % + cmd_helper.SingleQuote(os.path.join(remote, f))) + else: + parent_dir = remote[0:remote.rfind('/')] + if parent_dir: + self.Shell('mkdir -p %s' % cmd_helper.SingleQuote(parent_dir)) + self._PushObject(local, remote) + self.Shell('chmod 777 %s' % cmd_helper.SingleQuote(remote)) + + def _PushObject(self, local, remote): + """Copies an object from the host to the gce instance using scp. + + Args: + local: Path on the host filesystem. + remote: Path on the instance filesystem. + """ + cmd = [ + 'scp', + '-r', + '-i', _SSH_KEY_FILE, + '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'StrictHostKeyChecking=no', + local, + 'root@%s:%s' % (self._instance_ip, remote) + ] + status, _ = cmd_helper.GetCmdStatusAndOutput(cmd) + if status: + raise device_errors.AdbCommandFailedError( + cmd, 'File not reachable on host: %s' % local, + device_serial=str(self)) + + # override + def Pull(self, remote, local, **kwargs): + """Pulls a file from the gce instance to the host. + + Args: + remote: Path on the instance filesystem. + local: Path on the host filesystem. + """ + adb_wrapper.VerifyLocalFileExists(_SSH_KEY_FILE) + cmd = [ + 'scp', + '-p', + '-r', + '-i', _SSH_KEY_FILE, + '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'StrictHostKeyChecking=no', + 'root@%s:%s' % (self._instance_ip, remote), + local, + ] + status, _ = cmd_helper.GetCmdStatusAndOutput(cmd) + if status: + raise device_errors.AdbCommandFailedError( + cmd, 'File not reachable on host: %s' % local, + device_serial=str(self)) + + try: + adb_wrapper.VerifyLocalFileExists(local) + except (subprocess.CalledProcessError, IOError): + logging.exception('Error when pulling files from android instance.') + raise device_errors.AdbCommandFailedError( + cmd, 'File not reachable on host: %s' % local, + device_serial=str(self)) + + # override + def Install(self, apk_path, forward_lock=False, reinstall=False, + sd_card=False, **kwargs): + """Installs an apk on the gce instance + + Args: + apk_path: Host path to the APK file. + forward_lock: (optional) If set forward-locks the app. + reinstall: (optional) If set reinstalls the app, keeping its data. + sd_card: (optional) If set installs on the SD card. + """ + adb_wrapper.VerifyLocalFileExists(_SSH_KEY_FILE) + adb_wrapper.VerifyLocalFileExists(apk_path) + cmd = ['install'] + if forward_lock: + cmd.append('-l') + if reinstall: + cmd.append('-r') + if sd_card: + cmd.append('-s') + self.Push(apk_path, '/data/local/tmp/tmp.apk') + cmd = ['pm'] + cmd + cmd.append('/data/local/tmp/tmp.apk') + output = self.Shell(' '.join(cmd)) + self.Shell('rm /data/local/tmp/tmp.apk') + if 'Success' not in output: + raise device_errors.AdbCommandFailedError( + cmd, output, device_serial=self._device_serial) + + # override + @property + def is_emulator(self): + return True diff --git a/catapult/devil/devil/android/sdk/intent.py b/catapult/devil/devil/android/sdk/intent.py new file mode 100644 index 00000000..e612f76b --- /dev/null +++ b/catapult/devil/devil/android/sdk/intent.py @@ -0,0 +1,114 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Manages intents and associated information. + +This is generally intended to be used with functions that calls Android's +Am command. +""" + + +class Intent(object): + + def __init__(self, action='android.intent.action.VIEW', activity=None, + category=None, component=None, data=None, extras=None, + flags=None, package=None): + """Creates an Intent. + + Args: + action: A string containing the action. + activity: A string that, with |package|, can be used to specify the + component. + category: A string or list containing any categories. + component: A string that specifies the component to send the intent to. + data: A string containing a data URI. + extras: A dict containing extra parameters to be passed along with the + intent. + flags: A string containing flags to pass. + package: A string that, with activity, can be used to specify the + component. + """ + self._action = action + self._activity = activity + if isinstance(category, list) or category is None: + self._category = category + else: + self._category = [category] + self._component = component + self._data = data + self._extras = extras + self._flags = flags + self._package = package + + if self._component and '/' in component: + self._package, self._activity = component.split('/', 1) + elif self._package and self._activity: + self._component = '%s/%s' % (package, activity) + + @property + def action(self): + return self._action + + @property + def activity(self): + return self._activity + + @property + def category(self): + return self._category + + @property + def component(self): + return self._component + + @property + def data(self): + return self._data + + @property + def extras(self): + return self._extras + + @property + def flags(self): + return self._flags + + @property + def package(self): + return self._package + + @property + def am_args(self): + """Returns the intent as a list of arguments for the activity manager. + + For details refer to the specification at: + - http://developer.android.com/tools/help/adb.html#IntentSpec + """ + args = [] + if self.action: + args.extend(['-a', self.action]) + if self.data: + args.extend(['-d', self.data]) + if self.category: + args.extend(arg for cat in self.category for arg in ('-c', cat)) + if self.component: + args.extend(['-n', self.component]) + if self.flags: + args.extend(['-f', self.flags]) + if self.extras: + for key, value in self.extras.iteritems(): + if value is None: + args.extend(['--esn', key]) + elif isinstance(value, str): + args.extend(['--es', key, value]) + elif isinstance(value, bool): + args.extend(['--ez', key, str(value)]) + elif isinstance(value, int): + args.extend(['--ei', key, str(value)]) + elif isinstance(value, float): + args.extend(['--ef', key, str(value)]) + else: + raise NotImplementedError( + 'Intent does not know how to pass %s extras' % type(value)) + return args diff --git a/catapult/devil/devil/android/sdk/keyevent.py b/catapult/devil/devil/android/sdk/keyevent.py new file mode 100644 index 00000000..732a7dc9 --- /dev/null +++ b/catapult/devil/devil/android/sdk/keyevent.py @@ -0,0 +1,14 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Android KeyEvent constants. + +http://developer.android.com/reference/android/view/KeyEvent.html +""" + +KEYCODE_BACK = 4 +KEYCODE_DPAD_RIGHT = 22 +KEYCODE_ENTER = 66 +KEYCODE_MENU = 82 +KEYCODE_APP_SWITCH = 187 diff --git a/catapult/devil/devil/android/sdk/shared_prefs.py b/catapult/devil/devil/android/sdk/shared_prefs.py new file mode 100644 index 00000000..50ff5c62 --- /dev/null +++ b/catapult/devil/devil/android/sdk/shared_prefs.py @@ -0,0 +1,391 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Helper object to read and modify Shared Preferences from Android apps. + +See e.g.: + http://developer.android.com/reference/android/content/SharedPreferences.html +""" + +import logging +import posixpath + +from xml.etree import ElementTree + + +_XML_DECLARATION = "<?xml version='1.0' encoding='utf-8' standalone='yes' ?>\n" + + +class BasePref(object): + """Base class for getting/setting the value of a specific preference type. + + Should not be instantiated directly. The SharedPrefs collection will + instantiate the appropriate subclasses, which directly manipulate the + underlying xml document, to parse and serialize values according to their + type. + + Args: + elem: An xml ElementTree object holding the preference data. + + Properties: + tag_name: A string with the tag that must be used for this preference type. + """ + tag_name = None + + def __init__(self, elem): + if elem.tag != type(self).tag_name: + raise TypeError('Property %r has type %r, but trying to access as %r' % + (elem.get('name'), elem.tag, type(self).tag_name)) + self._elem = elem + + def __str__(self): + """Get the underlying xml element as a string.""" + return ElementTree.tostring(self._elem) + + def get(self): + """Get the value of this preference.""" + return self._elem.get('value') + + def set(self, value): + """Set from a value casted as a string.""" + self._elem.set('value', str(value)) + + @property + def has_value(self): + """Check whether the element has a value.""" + return self._elem.get('value') is not None + + +class BooleanPref(BasePref): + """Class for getting/setting a preference with a boolean value. + + The underlying xml element has the form, e.g.: + <boolean name="featureEnabled" value="false" /> + """ + tag_name = 'boolean' + VALUES = {'true': True, 'false': False} + + def get(self): + """Get the value as a Python bool.""" + return type(self).VALUES[super(BooleanPref, self).get()] + + def set(self, value): + """Set from a value casted as a bool.""" + super(BooleanPref, self).set('true' if value else 'false') + + +class FloatPref(BasePref): + """Class for getting/setting a preference with a float value. + + The underlying xml element has the form, e.g.: + <float name="someMetric" value="4.7" /> + """ + tag_name = 'float' + + def get(self): + """Get the value as a Python float.""" + return float(super(FloatPref, self).get()) + + +class IntPref(BasePref): + """Class for getting/setting a preference with an int value. + + The underlying xml element has the form, e.g.: + <int name="aCounter" value="1234" /> + """ + tag_name = 'int' + + def get(self): + """Get the value as a Python int.""" + return int(super(IntPref, self).get()) + + +class LongPref(IntPref): + """Class for getting/setting a preference with a long value. + + The underlying xml element has the form, e.g.: + <long name="aLongCounter" value="1234" /> + + We use the same implementation from IntPref. + """ + tag_name = 'long' + + +class StringPref(BasePref): + """Class for getting/setting a preference with a string value. + + The underlying xml element has the form, e.g.: + <string name="someHashValue">249b3e5af13d4db2</string> + """ + tag_name = 'string' + + def get(self): + """Get the value as a Python string.""" + return self._elem.text + + def set(self, value): + """Set from a value casted as a string.""" + self._elem.text = str(value) + + +class StringSetPref(StringPref): + """Class for getting/setting a preference with a set of string values. + + The underlying xml element has the form, e.g.: + <set name="managed_apps"> + <string>com.mine.app1</string> + <string>com.mine.app2</string> + <string>com.mine.app3</string> + </set> + """ + tag_name = 'set' + + def get(self): + """Get a list with the string values contained.""" + value = [] + for child in self._elem: + assert child.tag == 'string' + value.append(child.text) + return value + + def set(self, value): + """Set from a sequence of values, each casted as a string.""" + for child in list(self._elem): + self._elem.remove(child) + for item in value: + ElementTree.SubElement(self._elem, 'string').text = str(item) + + +_PREF_TYPES = {c.tag_name: c for c in [BooleanPref, FloatPref, IntPref, + LongPref, StringPref, StringSetPref]} + + +class SharedPrefs(object): + + def __init__(self, device, package, filename): + """Helper object to read and update "Shared Prefs" of Android apps. + + Such files typically look like, e.g.: + + <?xml version='1.0' encoding='utf-8' standalone='yes' ?> + <map> + <int name="databaseVersion" value="107" /> + <boolean name="featureEnabled" value="false" /> + <string name="someHashValue">249b3e5af13d4db2</string> + </map> + + Example usage: + + prefs = shared_prefs.SharedPrefs(device, 'com.my.app', 'my_prefs.xml') + prefs.Load() + prefs.GetString('someHashValue') # => '249b3e5af13d4db2' + prefs.SetInt('databaseVersion', 42) + prefs.Remove('featureEnabled') + prefs.Commit() + + The object may also be used as a context manager to automatically load and + commit, respectively, upon entering and leaving the context. + + Args: + device: A DeviceUtils object. + package: A string with the package name of the app that owns the shared + preferences file. + filename: A string with the name of the preferences file to read/write. + """ + self._device = device + self._xml = None + self._package = package + self._filename = filename + self._path = '/data/data/%s/shared_prefs/%s' % (package, filename) + self._changed = False + + def __repr__(self): + """Get a useful printable representation of the object.""" + return '<{cls} file {filename} for {package} on {device}>'.format( + cls=type(self).__name__, filename=self.filename, package=self.package, + device=str(self._device)) + + def __str__(self): + """Get the underlying xml document as a string.""" + return _XML_DECLARATION + ElementTree.tostring(self.xml) + + @property + def package(self): + """Get the package name of the app that owns the shared preferences.""" + return self._package + + @property + def filename(self): + """Get the filename of the shared preferences file.""" + return self._filename + + @property + def path(self): + """Get the full path to the shared preferences file on the device.""" + return self._path + + @property + def changed(self): + """True if properties have changed and a commit would be needed.""" + return self._changed + + @property + def xml(self): + """Get the underlying xml document as an ElementTree object.""" + if self._xml is None: + self._xml = ElementTree.Element('map') + return self._xml + + def Load(self): + """Load the shared preferences file from the device. + + A empty xml document, which may be modified and saved on |commit|, is + created if the file does not already exist. + """ + if self._device.FileExists(self.path): + self._xml = ElementTree.fromstring( + self._device.ReadFile(self.path, as_root=True)) + assert self._xml.tag == 'map' + else: + self._xml = None + self._changed = False + + def Clear(self): + """Clear all of the preferences contained in this object.""" + if self._xml is not None and len(self): # only clear if not already empty + self._xml = None + self._changed = True + + def Commit(self): + """Save the current set of preferences to the device. + + Only actually saves if some preferences have been modified. + """ + if not self.changed: + return + self._device.RunShellCommand( + ['mkdir', '-p', posixpath.dirname(self.path)], + as_root=True, check_return=True) + self._device.WriteFile(self.path, str(self), as_root=True) + self._device.KillAll(self.package, exact=True, as_root=True, quiet=True) + self._changed = False + + def __len__(self): + """Get the number of preferences in this collection.""" + return len(self.xml) + + def PropertyType(self, key): + """Get the type (i.e. tag name) of a property in the collection.""" + return self._GetChild(key).tag + + def HasProperty(self, key): + try: + self._GetChild(key) + return True + except KeyError: + return False + + def GetBoolean(self, key): + """Get a boolean property.""" + return BooleanPref(self._GetChild(key)).get() + + def SetBoolean(self, key, value): + """Set a boolean property.""" + self._SetPrefValue(key, value, BooleanPref) + + def GetFloat(self, key): + """Get a float property.""" + return FloatPref(self._GetChild(key)).get() + + def SetFloat(self, key, value): + """Set a float property.""" + self._SetPrefValue(key, value, FloatPref) + + def GetInt(self, key): + """Get an int property.""" + return IntPref(self._GetChild(key)).get() + + def SetInt(self, key, value): + """Set an int property.""" + self._SetPrefValue(key, value, IntPref) + + def GetLong(self, key): + """Get a long property.""" + return LongPref(self._GetChild(key)).get() + + def SetLong(self, key, value): + """Set a long property.""" + self._SetPrefValue(key, value, LongPref) + + def GetString(self, key): + """Get a string property.""" + return StringPref(self._GetChild(key)).get() + + def SetString(self, key, value): + """Set a string property.""" + self._SetPrefValue(key, value, StringPref) + + def GetStringSet(self, key): + """Get a string set property.""" + return StringSetPref(self._GetChild(key)).get() + + def SetStringSet(self, key, value): + """Set a string set property.""" + self._SetPrefValue(key, value, StringSetPref) + + def Remove(self, key): + """Remove a preference from the collection.""" + self.xml.remove(self._GetChild(key)) + + def AsDict(self): + """Return the properties and their values as a dictionary.""" + d = {} + for child in self.xml: + pref = _PREF_TYPES[child.tag](child) + d[child.get('name')] = pref.get() + return d + + def __enter__(self): + """Load preferences file from the device when entering a context.""" + self.Load() + return self + + def __exit__(self, exc_type, _exc_value, _traceback): + """Save preferences file to the device when leaving a context.""" + if not exc_type: + self.Commit() + + def _GetChild(self, key): + """Get the underlying xml node that holds the property of a given key. + + Raises: + KeyError when the key is not found in the collection. + """ + for child in self.xml: + if child.get('name') == key: + return child + raise KeyError(key) + + def _SetPrefValue(self, key, value, pref_cls): + """Set the value of a property. + + Args: + key: The key of the property to set. + value: The new value of the property. + pref_cls: A subclass of BasePref used to access the property. + + Raises: + TypeError when the key already exists but with a different type. + """ + try: + pref = pref_cls(self._GetChild(key)) + old_value = pref.get() + except KeyError: + pref = pref_cls(ElementTree.SubElement( + self.xml, pref_cls.tag_name, {'name': key})) + old_value = None + if old_value != value: + pref.set(value) + self._changed = True + logging.info('Setting property: %s', pref) diff --git a/catapult/devil/devil/android/sdk/shared_prefs_test.py b/catapult/devil/devil/android/sdk/shared_prefs_test.py new file mode 100755 index 00000000..ff3b9a13 --- /dev/null +++ b/catapult/devil/devil/android/sdk/shared_prefs_test.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Unit tests for the contents of shared_prefs.py (mostly SharedPrefs). +""" + +import logging +import unittest + +from devil import devil_env +from devil.android import device_utils +from devil.android.sdk import shared_prefs + +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + + +def MockDeviceWithFiles(files=None): + if files is None: + files = {} + + def file_exists(path): + return path in files + + def write_file(path, contents, **_kwargs): + files[path] = contents + + def read_file(path, **_kwargs): + return files[path] + + device = mock.MagicMock(spec=device_utils.DeviceUtils) + device.FileExists = mock.Mock(side_effect=file_exists) + device.WriteFile = mock.Mock(side_effect=write_file) + device.ReadFile = mock.Mock(side_effect=read_file) + return device + + +class SharedPrefsTest(unittest.TestCase): + + def setUp(self): + self.device = MockDeviceWithFiles({ + '/data/data/com.some.package/shared_prefs/prefs.xml': + "<?xml version='1.0' encoding='utf-8' standalone='yes' ?>\n" + '<map>\n' + ' <int name="databaseVersion" value="107" />\n' + ' <boolean name="featureEnabled" value="false" />\n' + ' <string name="someHashValue">249b3e5af13d4db2</string>\n' + '</map>'}) + self.expected_data = {'databaseVersion': 107, + 'featureEnabled': False, + 'someHashValue': '249b3e5af13d4db2'} + + def testPropertyLifetime(self): + prefs = shared_prefs.SharedPrefs( + self.device, 'com.some.package', 'prefs.xml') + self.assertEquals(len(prefs), 0) # collection is empty before loading + prefs.SetInt('myValue', 444) + self.assertEquals(len(prefs), 1) + self.assertEquals(prefs.GetInt('myValue'), 444) + self.assertTrue(prefs.HasProperty('myValue')) + prefs.Remove('myValue') + self.assertEquals(len(prefs), 0) + self.assertFalse(prefs.HasProperty('myValue')) + with self.assertRaises(KeyError): + prefs.GetInt('myValue') + + def testPropertyType(self): + prefs = shared_prefs.SharedPrefs( + self.device, 'com.some.package', 'prefs.xml') + prefs.SetInt('myValue', 444) + self.assertEquals(prefs.PropertyType('myValue'), 'int') + with self.assertRaises(TypeError): + prefs.GetString('myValue') + with self.assertRaises(TypeError): + prefs.SetString('myValue', 'hello') + + def testLoad(self): + prefs = shared_prefs.SharedPrefs( + self.device, 'com.some.package', 'prefs.xml') + self.assertEquals(len(prefs), 0) # collection is empty before loading + prefs.Load() + self.assertEquals(len(prefs), len(self.expected_data)) + self.assertEquals(prefs.AsDict(), self.expected_data) + self.assertFalse(prefs.changed) + + def testClear(self): + prefs = shared_prefs.SharedPrefs( + self.device, 'com.some.package', 'prefs.xml') + prefs.Load() + self.assertEquals(prefs.AsDict(), self.expected_data) + self.assertFalse(prefs.changed) + prefs.Clear() + self.assertEquals(len(prefs), 0) # collection is empty now + self.assertTrue(prefs.changed) + + def testCommit(self): + prefs = shared_prefs.SharedPrefs( + self.device, 'com.some.package', 'other_prefs.xml') + self.assertFalse(self.device.FileExists(prefs.path)) # file does not exist + prefs.Load() + self.assertEquals(len(prefs), 0) # file did not exist, collection is empty + prefs.SetInt('magicNumber', 42) + prefs.SetFloat('myMetric', 3.14) + prefs.SetLong('bigNumner', 6000000000) + prefs.SetStringSet('apps', ['gmail', 'chrome', 'music']) + self.assertFalse(self.device.FileExists(prefs.path)) # still does not exist + self.assertTrue(prefs.changed) + prefs.Commit() + self.assertTrue(self.device.FileExists(prefs.path)) # should exist now + self.device.KillAll.assert_called_once_with(prefs.package, exact=True, + as_root=True, quiet=True) + self.assertFalse(prefs.changed) + + prefs = shared_prefs.SharedPrefs( + self.device, 'com.some.package', 'other_prefs.xml') + self.assertEquals(len(prefs), 0) # collection is empty before loading + prefs.Load() + self.assertEquals(prefs.AsDict(), { + 'magicNumber': 42, + 'myMetric': 3.14, + 'bigNumner': 6000000000, + 'apps': ['gmail', 'chrome', 'music']}) # data survived roundtrip + + def testAsContextManager_onlyReads(self): + with shared_prefs.SharedPrefs( + self.device, 'com.some.package', 'prefs.xml') as prefs: + self.assertEquals(prefs.AsDict(), self.expected_data) # loaded and ready + self.assertEquals(self.device.WriteFile.call_args_list, []) # did not write + + def testAsContextManager_readAndWrite(self): + with shared_prefs.SharedPrefs( + self.device, 'com.some.package', 'prefs.xml') as prefs: + prefs.SetBoolean('featureEnabled', True) + prefs.Remove('someHashValue') + prefs.SetString('newString', 'hello') + + self.assertTrue(self.device.WriteFile.called) # did write + with shared_prefs.SharedPrefs( + self.device, 'com.some.package', 'prefs.xml') as prefs: + # changes persisted + self.assertTrue(prefs.GetBoolean('featureEnabled')) + self.assertFalse(prefs.HasProperty('someHashValue')) + self.assertEquals(prefs.GetString('newString'), 'hello') + self.assertTrue(prefs.HasProperty('databaseVersion')) # still there + + def testAsContextManager_commitAborted(self): + with self.assertRaises(TypeError): + with shared_prefs.SharedPrefs( + self.device, 'com.some.package', 'prefs.xml') as prefs: + prefs.SetBoolean('featureEnabled', True) + prefs.Remove('someHashValue') + prefs.SetString('newString', 'hello') + prefs.SetInt('newString', 123) # oops! + + self.assertEquals(self.device.WriteFile.call_args_list, []) # did not write + with shared_prefs.SharedPrefs( + self.device, 'com.some.package', 'prefs.xml') as prefs: + # contents were not modified + self.assertEquals(prefs.AsDict(), self.expected_data) + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.DEBUG) + unittest.main(verbosity=2) diff --git a/catapult/devil/devil/android/sdk/split_select.py b/catapult/devil/devil/android/sdk/split_select.py new file mode 100644 index 00000000..6c3d231a --- /dev/null +++ b/catapult/devil/devil/android/sdk/split_select.py @@ -0,0 +1,63 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""This module wraps Android's split-select tool.""" + +from devil.android.sdk import build_tools +from devil.utils import cmd_helper +from devil.utils import lazy + + +_split_select_path = lazy.WeakConstant( + lambda: build_tools.GetPath('split-select')) + + +def _RunSplitSelectCmd(args): + """Runs a split-select command. + + Args: + args: A list of arguments for split-select. + + Returns: + The output of the command. + """ + cmd = [_split_select_path.read()] + args + status, output = cmd_helper.GetCmdStatusAndOutput(cmd) + if status != 0: + raise Exception('Failed running command "%s" with output "%s".' % + (' '.join(cmd), output)) + return output + + +def _SplitConfig(device, allow_cached_props=False): + """Returns a config specifying which APK splits are required by the device. + + Args: + device: A DeviceUtils object. + allow_cached_props: Whether to use cached values for device properties. + """ + return ('%s-r%s-%s:%s' % + (device.GetLanguage(cache=allow_cached_props), + device.GetCountry(cache=allow_cached_props), + device.screen_density, + device.product_cpu_abi)) + + +def SelectSplits(device, base_apk, split_apks, allow_cached_props=False): + """Determines which APK splits the device requires. + + Args: + device: A DeviceUtils object. + base_apk: The path of the base APK. + split_apks: A list of paths of APK splits. + allow_cached_props: Whether to use cached values for device properties. + + Returns: + The list of APK splits that the device requires. + """ + config = _SplitConfig(device, allow_cached_props=allow_cached_props) + args = ['--target', config, '--base', base_apk] + for split in split_apks: + args.extend(['--split', split]) + return _RunSplitSelectCmd(args).splitlines() diff --git a/catapult/devil/devil/android/sdk/version_codes.py b/catapult/devil/devil/android/sdk/version_codes.py new file mode 100644 index 00000000..410379b9 --- /dev/null +++ b/catapult/devil/devil/android/sdk/version_codes.py @@ -0,0 +1,18 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Android SDK version codes. + +http://developer.android.com/reference/android/os/Build.VERSION_CODES.html +""" + +JELLY_BEAN = 16 +JELLY_BEAN_MR1 = 17 +JELLY_BEAN_MR2 = 18 +KITKAT = 19 +KITKAT_WATCH = 20 +LOLLIPOP = 21 +LOLLIPOP_MR1 = 22 +MARSHMALLOW = 23 + diff --git a/catapult/devil/devil/android/tools/__init__.py b/catapult/devil/devil/android/tools/__init__.py new file mode 100644 index 00000000..50b23dff --- /dev/null +++ b/catapult/devil/devil/android/tools/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/catapult/devil/devil/android/tools/adb_run_shell_cmd.py b/catapult/devil/devil/android/tools/adb_run_shell_cmd.py new file mode 100755 index 00000000..f995d272 --- /dev/null +++ b/catapult/devil/devil/android/tools/adb_run_shell_cmd.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import json +import logging +import sys + +from devil.android import device_blacklist +from devil.android import device_errors +from devil.android import device_utils +from devil.utils import run_tests_helper + + +def main(): + parser = argparse.ArgumentParser( + 'Run an adb shell command on selected devices') + parser.add_argument('cmd', help='Adb shell command to run.', nargs="+") + parser.add_argument('-d', '--device', action='append', dest='devices', + help='Device to run cmd on. Runs on all devices if not ' + 'specified. Set multiple times for multiple devices') + parser.add_argument('-v', '--verbose', default=0, action='count', + help='Verbose level (multiple times for more)') + parser.add_argument('--blacklist-file', help='Device blacklist file.') + parser.add_argument('--as-root', action='store_true', help='Run as root.') + parser.add_argument('--json-output', + help='File to dump json output to.') + args = parser.parse_args() + run_tests_helper.SetLogLevel(args.verbose) + + args.blacklist_file = device_blacklist.Blacklist( + args.blacklist_file) if args.blacklist_file else None + attached_devices = device_utils.DeviceUtils.HealthyDevices( + blacklist=args.blacklist_file) + + if args.devices: + selected_devices = [] + attached_devices = {str(d): d for d in attached_devices} + for serial in args.devices: + if serial in attached_devices: + selected_devices.append(attached_devices[serial]) + else: + logging.warning('Specified device %s not found.', serial) + else: + selected_devices = attached_devices + + if not selected_devices: + raise device_errors.NoDevicesError + + p_out = (device_utils.DeviceUtils.parallel(selected_devices).RunShellCommand( + args.cmd, large_output=True, as_root=args.as_root, check_return=True) + .pGet(None)) + + data = {} + for device, output in zip(selected_devices, p_out): + for line in output: + print '%s: %s' % (device, line) + data[str(device)] = output + + if args.json_output: + with open(args.json_output, 'w') as f: + json.dump(data, f) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/catapult/devil/devil/android/tools/flash_device.py b/catapult/devil/devil/android/tools/flash_device.py new file mode 100755 index 00000000..50ed6961 --- /dev/null +++ b/catapult/devil/devil/android/tools/flash_device.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import logging +import os +import sys + +if __name__ == '__main__': + sys.path.append(os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', '..', '..'))) +from devil.android import device_blacklist +from devil.android import device_utils +from devil.android import fastboot_utils +from devil.android.tools import script_common +from devil.constants import exit_codes +from devil.utils import run_tests_helper + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('build_path', help='Path to android build.') + parser.add_argument('-d', '--device', dest='devices', action='append', + help='Device(s) to flash.') + parser.add_argument('-v', '--verbose', default=0, action='count', + help='Verbose level (multiple times for more)') + parser.add_argument('-w', '--wipe', action='store_true', + help='If set, wipes user data') + parser.add_argument('--blacklist-file', help='Device blacklist file.') + args = parser.parse_args() + run_tests_helper.SetLogLevel(args.verbose) + + if args.blacklist_file: + blacklist = device_blacklist.Blacklist(args.blacklist_file).Read() + if blacklist: + logging.critical('Device(s) in blacklist, not flashing devices:') + for key in blacklist: + logging.critical(' %s', key) + return exit_codes.INFRA + + flashed_devices = [] + failed_devices = [] + + def flash(device): + fastboot = fastboot_utils.FastbootUtils(device) + try: + fastboot.FlashDevice(args.build_path, wipe=args.wipe) + flashed_devices.append(device) + except Exception: # pylint: disable=broad-except + logging.exception('Device %s failed to flash.', str(device)) + failed_devices.append(device) + + devices = script_common.GetDevices(args.devices, args.blacklist_file) + device_utils.DeviceUtils.parallel(devices).pMap(flash) + + if flashed_devices: + logging.info('The following devices were flashed:') + logging.info(' %s', ' '.join(str(d) for d in flashed_devices)) + if failed_devices: + logging.critical('The following devices failed to flash:') + logging.critical(' %s', ' '.join(str(d) for d in failed_devices)) + return exit_codes.INFRA + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/catapult/devil/devil/android/tools/screenshot.py b/catapult/devil/devil/android/tools/screenshot.py new file mode 100755 index 00000000..326bb162 --- /dev/null +++ b/catapult/devil/devil/android/tools/screenshot.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Takes a screenshot from an Android device.""" + +import argparse +import logging +import os +import sys + +if __name__ == '__main__': + sys.path.append(os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', '..', '..'))) +from devil.android import device_utils +from devil.android.tools import script_common + + +def main(): + # Parse options. + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('-d', '--device', dest='devices', action='append', + help='Serial number of Android device to use.') + parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') + parser.add_argument('-f', '--file', metavar='FILE', + help='Save result to file instead of generating a ' + 'timestamped file name.') + parser.add_argument('-v', '--verbose', action='store_true', + help='Verbose logging.') + parser.add_argument('host_file', nargs='?', + help='File to which the screenshot will be saved.') + + args = parser.parse_args() + + host_file = args.host_file or args.file + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + devices = script_common.GetDevices(args.devices, args.blacklist_file) + + def screenshot(device): + f = None + if host_file: + root, ext = os.path.splitext(host_file) + f = '%s_%s%s' % (root, str(device), ext) + f = device.TakeScreenshot(f) + print 'Screenshot for device %s written to %s' % ( + str(device), os.path.abspath(f)) + + device_utils.DeviceUtils.parallel(devices).pMap(screenshot) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/catapult/devil/devil/android/tools/script_common.py b/catapult/devil/devil/android/tools/script_common.py new file mode 100644 index 00000000..eb91cdcb --- /dev/null +++ b/catapult/devil/devil/android/tools/script_common.py @@ -0,0 +1,28 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from devil.android import device_blacklist +from devil.android import device_errors +from devil.android import device_utils + + +def GetDevices(requested_devices, blacklist_file): + blacklist = (device_blacklist.Blacklist(blacklist_file) + if blacklist_file + else None) + + devices = device_utils.DeviceUtils.HealthyDevices(blacklist) + if not devices: + raise device_errors.NoDevicesError() + elif requested_devices: + requested = set(requested_devices) + available = set(str(d) for d in devices) + missing = requested.difference(available) + if missing: + raise device_errors.DeviceUnreachableError(next(iter(missing))) + return sorted(device_utils.DeviceUtils(d) + for d in available.intersection(requested)) + else: + return devices + diff --git a/catapult/devil/devil/android/tools/script_common_test.py b/catapult/devil/devil/android/tools/script_common_test.py new file mode 100755 index 00000000..a2267645 --- /dev/null +++ b/catapult/devil/devil/android/tools/script_common_test.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +import sys +import unittest + +from devil import devil_env +from devil.android import device_errors +from devil.android import device_utils +from devil.android.tools import script_common + +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + + +class ScriptCommonTest(unittest.TestCase): + + def testGetDevices_noSpecs(self): + devices = [ + device_utils.DeviceUtils('123'), + device_utils.DeviceUtils('456'), + ] + with mock.patch('devil.android.device_utils.DeviceUtils.HealthyDevices', + return_value=devices): + self.assertEquals( + devices, + script_common.GetDevices(None, None)) + + def testGetDevices_withDevices(self): + devices = [ + device_utils.DeviceUtils('123'), + device_utils.DeviceUtils('456'), + ] + with mock.patch('devil.android.device_utils.DeviceUtils.HealthyDevices', + return_value=devices): + self.assertEquals( + [device_utils.DeviceUtils('456')], + script_common.GetDevices(['456'], None)) + + def testGetDevices_missingDevice(self): + with mock.patch('devil.android.device_utils.DeviceUtils.HealthyDevices', + return_value=[device_utils.DeviceUtils('123')]): + with self.assertRaises(device_errors.DeviceUnreachableError): + script_common.GetDevices(['456'], None) + + def testGetDevices_noDevices(self): + with mock.patch('devil.android.device_utils.DeviceUtils.HealthyDevices', + return_value=[]): + with self.assertRaises(device_errors.NoDevicesError): + script_common.GetDevices(None, None) + + +if __name__ == '__main__': + sys.exit(unittest.main()) + diff --git a/catapult/devil/devil/android/tools/video_recorder.py b/catapult/devil/devil/android/tools/video_recorder.py new file mode 100755 index 00000000..bcc9a75e --- /dev/null +++ b/catapult/devil/devil/android/tools/video_recorder.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Captures a video from an Android device.""" + +import argparse +import logging +import os +import threading +import time +import sys + +if __name__ == '__main__': + sys.path.append(os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', '..', '..'))) +from devil.android import device_signal +from devil.android import device_utils +from devil.android.tools import script_common +from devil.utils import cmd_helper +from devil.utils import reraiser_thread +from devil.utils import timeout_retry + + +class VideoRecorder(object): + """Records a screen capture video from an Android Device (KitKat or newer).""" + + def __init__(self, device, megabits_per_second=4, size=None, + rotate=False): + """Creates a VideoRecorder instance. + + Args: + device: DeviceUtils instance. + host_file: Path to the video file to store on the host. + megabits_per_second: Video bitrate in megabits per second. Allowed range + from 0.1 to 100 mbps. + size: Video frame size tuple (width, height) or None to use the device + default. + rotate: If True, the video will be rotated 90 degrees. + """ + self._bit_rate = megabits_per_second * 1000 * 1000 + self._device = device + self._device_file = ( + '%s/screen-recording.mp4' % device.GetExternalStoragePath()) + self._recorder_thread = None + self._rotate = rotate + self._size = size + self._started = threading.Event() + + def __enter__(self): + self.Start() + + def Start(self, timeout=None): + """Start recording video.""" + def screenrecord_started(): + return bool(self._device.GetPids('screenrecord')) + + if screenrecord_started(): + raise Exception("Can't run multiple concurrent video captures.") + + self._started.clear() + self._recorder_thread = reraiser_thread.ReraiserThread(self._Record) + self._recorder_thread.start() + timeout_retry.WaitFor( + screenrecord_started, wait_period=1, max_tries=timeout) + self._started.wait(timeout) + + def _Record(self): + cmd = ['screenrecord', '--verbose', '--bit-rate', str(self._bit_rate)] + if self._rotate: + cmd += ['--rotate'] + if self._size: + cmd += ['--size', '%dx%d' % self._size] + cmd += [self._device_file] + for line in self._device.adb.IterShell( + ' '.join(cmd_helper.SingleQuote(i) for i in cmd), None): + if line.startswith('Content area is '): + self._started.set() + + def __exit__(self, _exc_type, _exc_value, _traceback): + self.Stop() + + def Stop(self): + """Stop recording video.""" + if not self._device.KillAll('screenrecord', signum=device_signal.SIGINT, + quiet=True): + logging.warning('Nothing to kill: screenrecord was not running') + self._recorder_thread.join() + + def Pull(self, host_file=None): + """Pull resulting video file from the device. + + Args: + host_file: Path to the video file to store on the host. + Returns: + Output video file name on the host. + """ + # TODO(jbudorick): Merge filename generation with the logic for doing so in + # DeviceUtils. + host_file_name = ( + host_file + or 'screen-recording-%s-%s.mp4' % ( + str(self._device), + time.strftime('%Y%m%dT%H%M%S', time.localtime()))) + host_file_name = os.path.abspath(host_file_name) + self._device.PullFile(self._device_file, host_file_name) + self._device.RunShellCommand('rm -f "%s"' % self._device_file) + return host_file_name + + +def main(): + # Parse options. + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('-d', '--device', dest='devices', action='append', + help='Serial number of Android device to use.') + parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') + parser.add_argument('-f', '--file', metavar='FILE', + help='Save result to file instead of generating a ' + 'timestamped file name.') + parser.add_argument('-v', '--verbose', action='store_true', + help='Verbose logging.') + parser.add_argument('-b', '--bitrate', default=4, type=float, + help='Bitrate in megabits/s, from 0.1 to 100 mbps, ' + '%default mbps by default.') + parser.add_argument('-r', '--rotate', action='store_true', + help='Rotate video by 90 degrees.') + parser.add_argument('-s', '--size', metavar='WIDTHxHEIGHT', + help='Frame size to use instead of the device ' + 'screen size.') + parser.add_argument('host_file', nargs='?', + help='File to which the video capture will be written.') + + args = parser.parse_args() + + host_file = args.host_file or args.file + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + size = (tuple(int(i) for i in args.size.split('x')) + if args.size + else None) + + def record_video(device, stop_recording): + recorder = VideoRecorder( + device, megabits_per_second=args.bitrate, size=size, rotate=args.rotate) + with recorder: + stop_recording.wait() + + f = None + if host_file: + root, ext = os.path.splitext(host_file) + f = '%s_%s%s' % (root, str(device), ext) + f = recorder.Pull(f) + print 'Video written to %s' % os.path.abspath(f) + + parallel_devices = device_utils.DeviceUtils.parallel( + script_common.GetDevices(args.devices, args.blacklist_file), + async=True) + stop_recording = threading.Event() + running_recording = parallel_devices.pMap(record_video, stop_recording) + print 'Recording. Press Enter to stop.', + sys.stdout.flush() + raw_input() + stop_recording.set() + + running_recording.pGet(None) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/catapult/devil/devil/android/valgrind_tools/__init__.py b/catapult/devil/devil/android/valgrind_tools/__init__.py new file mode 100644 index 00000000..0182d4c1 --- /dev/null +++ b/catapult/devil/devil/android/valgrind_tools/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +""" +Classes in this package define additional actions that need to be taken to run a +test under some kind of runtime error detection tool. + +The interface is intended to be used as follows. + +1. For tests that simply run a native process (i.e. no activity is spawned): + +Call tool.CopyFiles(device). +Prepend test command line with tool.GetTestWrapper(). + +2. For tests that spawn an activity: + +Call tool.CopyFiles(device). +Call tool.SetupEnvironment(). +Run the test as usual. +Call tool.CleanUpEnvironment(). +""" diff --git a/catapult/devil/devil/android/valgrind_tools/base_tool.py b/catapult/devil/devil/android/valgrind_tools/base_tool.py new file mode 100644 index 00000000..2e6e9af3 --- /dev/null +++ b/catapult/devil/devil/android/valgrind_tools/base_tool.py @@ -0,0 +1,53 @@ +# Copyright (c) 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +class BaseTool(object): + """A tool that does nothing.""" + # pylint: disable=R0201 + + def __init__(self): + """Does nothing.""" + pass + + def GetTestWrapper(self): + """Returns a string that is to be prepended to the test command line.""" + return '' + + def GetUtilWrapper(self): + """Returns the wrapper name for the utilities. + + Returns: + A string that is to be prepended to the command line of utility + processes (forwarder, etc.). + """ + return '' + + @classmethod + def CopyFiles(cls, device): + """Copies tool-specific files to the device, create directories, etc.""" + pass + + def SetupEnvironment(self): + """Sets up the system environment for a test. + + This is a good place to set system properties. + """ + pass + + def CleanUpEnvironment(self): + """Cleans up environment.""" + pass + + def GetTimeoutScale(self): + """Returns a multiplier that should be applied to timeout values.""" + return 1.0 + + def NeedsDebugInfo(self): + """Whether this tool requires debug info. + + Returns: + True if this tool can not work with stripped binaries. + """ + return False diff --git a/catapult/devil/devil/base_error.py b/catapult/devil/devil/base_error.py new file mode 100644 index 00000000..dadf4da2 --- /dev/null +++ b/catapult/devil/devil/base_error.py @@ -0,0 +1,17 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +class BaseError(Exception): + """Base error for all test runner errors.""" + + def __init__(self, message, is_infra_error=False): + super(BaseError, self).__init__(message) + self._is_infra_error = is_infra_error + + @property + def is_infra_error(self): + """Property to indicate if error was caused by an infrastructure issue.""" + return self._is_infra_error + diff --git a/catapult/devil/devil/constants/__init__.py b/catapult/devil/devil/constants/__init__.py new file mode 100644 index 00000000..50b23dff --- /dev/null +++ b/catapult/devil/devil/constants/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/catapult/devil/devil/constants/exit_codes.py b/catapult/devil/devil/constants/exit_codes.py new file mode 100644 index 00000000..aaeca4a8 --- /dev/null +++ b/catapult/devil/devil/constants/exit_codes.py @@ -0,0 +1,9 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Common exit codes used by devil.""" + +ERROR = 1 +INFRA = 87 +WARNING = 88 diff --git a/catapult/devil/devil/devil_dependencies.json b/catapult/devil/devil/devil_dependencies.json new file mode 100644 index 00000000..4c185c0e --- /dev/null +++ b/catapult/devil/devil/devil_dependencies.json @@ -0,0 +1,117 @@ +{ + "config_type": "BaseConfig", + "dependencies": { + "aapt": { + "cloud_storage_bucket": "chromium-telemetry", + "cloud_storage_base_folder": "binary_dependencies", + "file_info": { + "linux2_x86_64": { + "cloud_storage_hash": "7448de3cb5e834afdedeaad8b40ba63ac53f3dc4", + "download_path": "../bin/deps/linux2/x86_64/bin/aapt" + } + } + }, + "adb": { + "cloud_storage_bucket": "chromium-telemetry", + "cloud_storage_base_folder": "binary_dependencies", + "file_info": { + "linux2_x86_64": { + "cloud_storage_hash": "0c2043552619c8ec8bb5d986ba75703a598611fc", + "download_path": "../bin/deps/linux2/x86_64/bin/adb" + } + } + }, + "android_build_tools_libc++": { + "cloud_storage_bucket": "chromium-telemetry", + "cloud_storage_base_folder": "binary_dependencies", + "file_info": { + "linux2_x86_64": { + "cloud_storage_hash": "52d150a7ccde835f38b4337392152f3013d5f303", + "download_path": "../bin/deps/linux2/x86_64/lib/libc++.so" + } + } + }, + "chromium_commands": { + "cloud_storage_bucket": "chromium-telemetry", + "cloud_storage_base_folder": "binary_dependencies", + "file_info": { + "linux2_x86_64": { + "cloud_storage_hash": "049f482f29bc34e2ed844e2e47b7609f8ffbeb4f", + "download_path": "../bin/deps/linux2/x86_64/lib.java/chromium_commands.dex.jar" + } + } + }, + "dexdump": { + "cloud_storage_bucket": "chromium-telemetry", + "cloud_storage_base_folder": "binary_dependencies", + "file_info": { + "linux2_x86_64": { + "cloud_storage_hash": "38765b5b358c29003e56b1d214606ea13467b6fe", + "download_path": "../bin/deps/linux2/x86_64/bin/dexdump" + } + } + }, + "forwarder_device": { + "cloud_storage_bucket": "chromium-telemetry", + "cloud_storage_base_folder": "binary_dependencies", + "file_info": { + "android_armeabi-v7a": { + "cloud_storage_hash": "4858c9e41da72ad8ff24414731feae2137229361", + "download_path": "../bin/deps/android/armeabi-v7a/bin/forwarder_device" + }, + "android_arm64-v8a": { + "cloud_storage_hash": "8cbd1ac2079ee82ce5f1cf4d3e85fc1e53a8f018", + "download_path": "../bin/deps/android/arm64-v8a/bin/forwarder_device" + } + } + }, + "forwarder_host": { + "cloud_storage_bucket": "chromium-telemetry", + "cloud_storage_base_folder": "binary_dependencies", + "file_info": { + "linux2_x86_64": { + "cloud_storage_hash": "b3dda9fbdd4a3fb933b64111c11070aa809c7ed4", + "download_path": "../bin/deps/linux2/x86_64/forwarder_host" + } + } + }, + "md5sum_device": { + "cloud_storage_bucket": "chromium-telemetry", + "cloud_storage_base_folder": "binary_dependencies", + "file_info": { + "android_armeabi-v7a": { + "cloud_storage_hash": "c8894480be71d5e49118483d83ba7a6e0097cba6", + "download_path": "../bin/deps/android/armeabi-v7a/bin/md5sum_device" + }, + "android_arm64-v8a": { + "cloud_storage_hash": "bbe410e2ffb48367ac4ca0874598d4f85fd16d9d", + "download_path": "../bin/deps/andorid/arm64-v8a/bin/md5sum_device" + }, + "android_x86": { + "cloud_storage_hash": "b578a5c2c400ce39761e2558cdf2237567a57257", + "download_path": "../bin/deps/android/x86/bin/md5sum_device" + } + } + }, + "md5sum_host": { + "cloud_storage_bucket": "chromium-telemetry", + "cloud_storage_base_folder": "binary_dependencies", + "file_info": { + "linux2_x86_64": { + "cloud_storage_hash": "49e36c9c4246cfebef26cbd07436c1a8343254aa", + "download_path": "../bin/deps/linux2/x86_64/bin/md5sum_host" + } + } + }, + "split-select": { + "cloud_storage_bucket": "chromium-telemetry", + "cloud_storage_base_folder": "binary_dependencies", + "file_info": { + "linux2_x86_64": { + "cloud_storage_hash": "3327881fa3951a503b9467425ea8e781cdffeb9f", + "download_path": "../bin/deps/linux2/x86_64/bin/split-select" + } + } + } + } +} diff --git a/catapult/devil/devil/devil_env.py b/catapult/devil/devil/devil_env.py new file mode 100644 index 00000000..b54e6f5b --- /dev/null +++ b/catapult/devil/devil/devil_env.py @@ -0,0 +1,146 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import contextlib +import json +import os +import platform +import sys +import tempfile +import threading + +CATAPULT_ROOT_PATH = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', '..')) +DEPENDENCY_MANAGER_PATH = os.path.join( + CATAPULT_ROOT_PATH, 'dependency_manager') +PYMOCK_PATH = os.path.join( + CATAPULT_ROOT_PATH, 'third_party', 'mock') + + +@contextlib.contextmanager +def SysPath(path): + sys.path.append(path) + yield + if sys.path[-1] != path: + sys.path.remove(path) + else: + sys.path.pop() + +with SysPath(DEPENDENCY_MANAGER_PATH): + import dependency_manager # pylint: disable=import-error + +_ANDROID_BUILD_TOOLS = {'aapt', 'dexdump', 'split-select'} + +_DEVIL_DEFAULT_CONFIG = os.path.abspath(os.path.join( + os.path.dirname(__file__), 'devil_dependencies.json')) + +_LEGACY_ENVIRONMENT_VARIABLES = { + 'ADB_PATH': { + 'dependency_name': 'adb', + 'platform': 'linux2_x86_64', + }, + 'ANDROID_SDK_ROOT': { + 'dependency_name': 'android_sdk', + 'platform': 'linux2_x86_64', + }, +} + + +def _GetEnvironmentVariableConfig(): + path_config = ( + (os.environ.get(k), v) + for k, v in _LEGACY_ENVIRONMENT_VARIABLES.iteritems()) + return { + 'config_type': 'BaseConfig', + 'dependencies': { + c['dependency_name']: { + 'file_info': { + c['platform']: { + 'local_paths': [p], + }, + }, + } for p, c in path_config if p + }, + } + + +class _Environment(object): + + def __init__(self): + self._dm_init_lock = threading.Lock() + self._dm = None + + def Initialize(self, configs=None, config_files=None): + """Initialize devil's environment from configuration files. + + This uses all configurations provided via |configs| and |config_files| + to determine the locations of devil's dependencies. Configurations should + all take the form described by catapult_base.dependency_manager.BaseConfig. + If no configurations are provided, a default one will be used if available. + + Args: + configs: An optional list of dict configurations. + config_files: An optional list of files to load + """ + + # Make sure we only initialize self._dm once. + with self._dm_init_lock: + if self._dm is None: + if configs is None: + configs = [] + + env_config = _GetEnvironmentVariableConfig() + if env_config: + configs.insert(0, env_config) + self._InitializeRecursive( + configs=configs, + config_files=config_files) + assert self._dm is not None, 'Failed to create dependency manager.' + + def _InitializeRecursive(self, configs=None, config_files=None): + # This recurses through configs to create temporary files for each and + # take advantage of context managers to appropriately close those files. + # TODO(jbudorick): Remove this recursion if/when dependency_manager + # supports loading configurations directly from a dict. + if configs: + with tempfile.NamedTemporaryFile(delete=False) as next_config_file: + try: + next_config_file.write(json.dumps(configs[0])) + next_config_file.close() + self._InitializeRecursive( + configs=configs[1:], + config_files=[next_config_file.name] + (config_files or [])) + finally: + if os.path.exists(next_config_file.name): + os.remove(next_config_file.name) + else: + config_files = config_files or [] + if 'DEVIL_ENV_CONFIG' in os.environ: + config_files.append(os.environ.get('DEVIL_ENV_CONFIG')) + config_files.append(_DEVIL_DEFAULT_CONFIG) + + self._dm = dependency_manager.DependencyManager( + [dependency_manager.BaseConfig(c) for c in config_files]) + + def FetchPath(self, dependency, arch=None, device=None): + if self._dm is None: + self.Initialize() + if dependency in _ANDROID_BUILD_TOOLS: + self.FetchPath('android_build_tools_libc++', arch=arch, device=device) + return self._dm.FetchPath(dependency, GetPlatform(arch, device)) + + def LocalPath(self, dependency, arch=None, device=None): + if self._dm is None: + self.Initialize() + return self._dm.LocalPath(dependency, GetPlatform(arch, device)) + + +def GetPlatform(arch=None, device=None): + if device: + return 'android_%s' % (arch or device.product_cpu_abi) + return '%s_%s' % (sys.platform, platform.machine()) + + +config = _Environment() + diff --git a/catapult/devil/devil/devil_env_test.py b/catapult/devil/devil/devil_env_test.py new file mode 100755 index 00000000..e78221a0 --- /dev/null +++ b/catapult/devil/devil/devil_env_test.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# pylint: disable=protected-access + +import logging +import sys +import unittest + +from devil import devil_env + +_sys_path_before = list(sys.path) +with devil_env.SysPath(devil_env.PYMOCK_PATH): + _sys_path_with_pymock = list(sys.path) + import mock # pylint: disable=import-error +_sys_path_after = list(sys.path) + + +class DevilEnvTest(unittest.TestCase): + + def testSysPath(self): + self.assertEquals(_sys_path_before, _sys_path_after) + self.assertEquals( + _sys_path_before + [devil_env.PYMOCK_PATH], + _sys_path_with_pymock) + + def testGetEnvironmentVariableConfig_configType(self): + with mock.patch('os.environ.get', + mock.Mock(side_effect=lambda _env_var: None)): + env_config = devil_env._GetEnvironmentVariableConfig() + self.assertEquals('BaseConfig', env_config.get('config_type')) + + def testGetEnvironmentVariableConfig_noEnv(self): + with mock.patch('os.environ.get', + mock.Mock(side_effect=lambda _env_var: None)): + env_config = devil_env._GetEnvironmentVariableConfig() + self.assertEquals({}, env_config.get('dependencies')) + + def testGetEnvironmentVariableConfig_adbPath(self): + def mock_environment(env_var): + return '/my/fake/adb/path' if env_var == 'ADB_PATH' else None + + with mock.patch('os.environ.get', + mock.Mock(side_effect=mock_environment)): + env_config = devil_env._GetEnvironmentVariableConfig() + self.assertEquals( + { + 'adb': { + 'file_info': { + 'linux2_x86_64': { + 'local_paths': ['/my/fake/adb/path'], + }, + }, + }, + }, + env_config.get('dependencies')) + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.DEBUG) + unittest.main(verbosity=2) diff --git a/catapult/devil/devil/utils/__init__.py b/catapult/devil/devil/utils/__init__.py new file mode 100644 index 00000000..50b23dff --- /dev/null +++ b/catapult/devil/devil/utils/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/catapult/devil/devil/utils/cmd_helper.py b/catapult/devil/devil/utils/cmd_helper.py new file mode 100644 index 00000000..b6237757 --- /dev/null +++ b/catapult/devil/devil/utils/cmd_helper.py @@ -0,0 +1,314 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A wrapper for subprocess to make calling shell commands easier.""" + +import logging +import os +import pipes +import select +import signal +import string +import StringIO +import subprocess +import time + +# fcntl is not available on Windows. +try: + import fcntl +except ImportError: + fcntl = None + +_SafeShellChars = frozenset(string.ascii_letters + string.digits + '@%_-+=:,./') + + +def SingleQuote(s): + """Return an shell-escaped version of the string using single quotes. + + Reliably quote a string which may contain unsafe characters (e.g. space, + quote, or other special characters such as '$'). + + The returned value can be used in a shell command line as one token that gets + to be interpreted literally. + + Args: + s: The string to quote. + + Return: + The string quoted using single quotes. + """ + return pipes.quote(s) + + +def DoubleQuote(s): + """Return an shell-escaped version of the string using double quotes. + + Reliably quote a string which may contain unsafe characters (e.g. space + or quote characters), while retaining some shell features such as variable + interpolation. + + The returned value can be used in a shell command line as one token that gets + to be further interpreted by the shell. + + The set of characters that retain their special meaning may depend on the + shell implementation. This set usually includes: '$', '`', '\', '!', '*', + and '@'. + + Args: + s: The string to quote. + + Return: + The string quoted using double quotes. + """ + if not s: + return '""' + elif all(c in _SafeShellChars for c in s): + return s + else: + return '"' + s.replace('"', '\\"') + '"' + + +def ShrinkToSnippet(cmd_parts, var_name, var_value): + """Constructs a shell snippet for a command using a variable to shrink it. + + Takes into account all quoting that needs to happen. + + Args: + cmd_parts: A list of command arguments. + var_name: The variable that holds var_value. + var_value: The string to replace in cmd_parts with $var_name + + Returns: + A shell snippet that does not include setting the variable. + """ + def shrink(value): + parts = (x and SingleQuote(x) for x in value.split(var_value)) + with_substitutions = ('"$%s"' % var_name).join(parts) + return with_substitutions or "''" + + return ' '.join(shrink(part) for part in cmd_parts) + + +def Popen(args, stdout=None, stderr=None, shell=None, cwd=None, env=None): + return subprocess.Popen( + args=args, cwd=cwd, stdout=stdout, stderr=stderr, + shell=shell, close_fds=True, env=env, + preexec_fn=lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL)) + + +def Call(args, stdout=None, stderr=None, shell=None, cwd=None, env=None): + pipe = Popen(args, stdout=stdout, stderr=stderr, shell=shell, cwd=cwd, + env=env) + pipe.communicate() + return pipe.wait() + + +def RunCmd(args, cwd=None): + """Opens a subprocess to execute a program and returns its return value. + + Args: + args: A string or a sequence of program arguments. The program to execute is + the string or the first item in the args sequence. + cwd: If not None, the subprocess's current directory will be changed to + |cwd| before it's executed. + + Returns: + Return code from the command execution. + """ + logging.info(str(args) + ' ' + (cwd or '')) + return Call(args, cwd=cwd) + + +def GetCmdOutput(args, cwd=None, shell=False): + """Open a subprocess to execute a program and returns its output. + + Args: + args: A string or a sequence of program arguments. The program to execute is + the string or the first item in the args sequence. + cwd: If not None, the subprocess's current directory will be changed to + |cwd| before it's executed. + shell: Whether to execute args as a shell command. + + Returns: + Captures and returns the command's stdout. + Prints the command's stderr to logger (which defaults to stdout). + """ + (_, output) = GetCmdStatusAndOutput(args, cwd, shell) + return output + + +def _ValidateAndLogCommand(args, cwd, shell): + if isinstance(args, basestring): + if not shell: + raise Exception('string args must be run with shell=True') + else: + if shell: + raise Exception('array args must be run with shell=False') + args = ' '.join(SingleQuote(c) for c in args) + if cwd is None: + cwd = '' + else: + cwd = ':' + cwd + logging.info('[host]%s> %s', cwd, args) + return args + + +def GetCmdStatusAndOutput(args, cwd=None, shell=False): + """Executes a subprocess and returns its exit code and output. + + Args: + args: A string or a sequence of program arguments. The program to execute is + the string or the first item in the args sequence. + cwd: If not None, the subprocess's current directory will be changed to + |cwd| before it's executed. + shell: Whether to execute args as a shell command. Must be True if args + is a string and False if args is a sequence. + + Returns: + The 2-tuple (exit code, output). + """ + status, stdout, stderr = GetCmdStatusOutputAndError( + args, cwd=cwd, shell=shell) + + if stderr: + logging.critical(stderr) + if len(stdout) > 4096: + logging.debug('Truncated output:') + logging.debug(stdout[:4096]) + return (status, stdout) + + +def GetCmdStatusOutputAndError(args, cwd=None, shell=False): + """Executes a subprocess and returns its exit code, output, and errors. + + Args: + args: A string or a sequence of program arguments. The program to execute is + the string or the first item in the args sequence. + cwd: If not None, the subprocess's current directory will be changed to + |cwd| before it's executed. + shell: Whether to execute args as a shell command. Must be True if args + is a string and False if args is a sequence. + + Returns: + The 2-tuple (exit code, output). + """ + _ValidateAndLogCommand(args, cwd, shell) + pipe = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=shell, cwd=cwd) + stdout, stderr = pipe.communicate() + return (pipe.returncode, stdout, stderr) + + +class TimeoutError(Exception): + """Module-specific timeout exception.""" + + def __init__(self, output=None): + super(TimeoutError, self).__init__() + self._output = output + + @property + def output(self): + return self._output + + +def _IterProcessStdout(process, timeout=None, buffer_size=4096, + poll_interval=1): + assert fcntl, 'fcntl module is required' + try: + # Enable non-blocking reads from the child's stdout. + child_fd = process.stdout.fileno() + fl = fcntl.fcntl(child_fd, fcntl.F_GETFL) + fcntl.fcntl(child_fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) + + end_time = (time.time() + timeout) if timeout else None + while True: + if end_time and time.time() > end_time: + raise TimeoutError() + read_fds, _, _ = select.select([child_fd], [], [], poll_interval) + if child_fd in read_fds: + data = os.read(child_fd, buffer_size) + if not data: + break + yield data + if process.poll() is not None: + break + finally: + try: + # Make sure the process doesn't stick around if we fail with an + # exception. + process.kill() + except OSError: + pass + process.wait() + + +def GetCmdStatusAndOutputWithTimeout(args, timeout, cwd=None, shell=False, + logfile=None): + """Executes a subprocess with a timeout. + + Args: + args: List of arguments to the program, the program to execute is the first + element. + timeout: the timeout in seconds or None to wait forever. + cwd: If not None, the subprocess's current directory will be changed to + |cwd| before it's executed. + shell: Whether to execute args as a shell command. Must be True if args + is a string and False if args is a sequence. + logfile: Optional file-like object that will receive output from the + command as it is running. + + Returns: + The 2-tuple (exit code, output). + """ + _ValidateAndLogCommand(args, cwd, shell) + output = StringIO.StringIO() + process = Popen(args, cwd=cwd, shell=shell, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + try: + for data in _IterProcessStdout(process, timeout=timeout): + if logfile: + logfile.write(data) + output.write(data) + except TimeoutError: + raise TimeoutError(output.getvalue()) + + return process.returncode, output.getvalue() + + +def IterCmdOutputLines(args, timeout=None, cwd=None, shell=False, + check_status=True): + """Executes a subprocess and continuously yields lines from its output. + + Args: + args: List of arguments to the program, the program to execute is the first + element. + cwd: If not None, the subprocess's current directory will be changed to + |cwd| before it's executed. + shell: Whether to execute args as a shell command. Must be True if args + is a string and False if args is a sequence. + check_status: A boolean indicating whether to check the exit status of the + process after all output has been read. + + Yields: + The output of the subprocess, line by line. + + Raises: + CalledProcessError if check_status is True and the process exited with a + non-zero exit status. + """ + cmd = _ValidateAndLogCommand(args, cwd, shell) + process = Popen(args, cwd=cwd, shell=shell, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + buffer_output = '' + for data in _IterProcessStdout(process, timeout=timeout): + buffer_output += data + has_incomplete_line = buffer_output[-1] not in '\r\n' + lines = buffer_output.splitlines() + buffer_output = lines.pop() if has_incomplete_line else '' + for line in lines: + yield line + if buffer_output: + yield buffer_output + if check_status and process.returncode: + raise subprocess.CalledProcessError(process.returncode, cmd) diff --git a/catapult/devil/devil/utils/cmd_helper_test.py b/catapult/devil/devil/utils/cmd_helper_test.py new file mode 100755 index 00000000..a04f1adf --- /dev/null +++ b/catapult/devil/devil/utils/cmd_helper_test.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for the cmd_helper module.""" + +import unittest +import subprocess + +from devil.utils import cmd_helper + + +class CmdHelperSingleQuoteTest(unittest.TestCase): + + def testSingleQuote_basic(self): + self.assertEquals('hello', + cmd_helper.SingleQuote('hello')) + + def testSingleQuote_withSpaces(self): + self.assertEquals("'hello world'", + cmd_helper.SingleQuote('hello world')) + + def testSingleQuote_withUnsafeChars(self): + self.assertEquals("""'hello'"'"'; rm -rf /'""", + cmd_helper.SingleQuote("hello'; rm -rf /")) + + def testSingleQuote_dontExpand(self): + test_string = 'hello $TEST_VAR' + cmd = 'TEST_VAR=world; echo %s' % cmd_helper.SingleQuote(test_string) + self.assertEquals(test_string, + cmd_helper.GetCmdOutput(cmd, shell=True).rstrip()) + + +class CmdHelperDoubleQuoteTest(unittest.TestCase): + + def testDoubleQuote_basic(self): + self.assertEquals('hello', + cmd_helper.DoubleQuote('hello')) + + def testDoubleQuote_withSpaces(self): + self.assertEquals('"hello world"', + cmd_helper.DoubleQuote('hello world')) + + def testDoubleQuote_withUnsafeChars(self): + self.assertEquals('''"hello\\"; rm -rf /"''', + cmd_helper.DoubleQuote('hello"; rm -rf /')) + + def testSingleQuote_doExpand(self): + test_string = 'hello $TEST_VAR' + cmd = 'TEST_VAR=world; echo %s' % cmd_helper.DoubleQuote(test_string) + self.assertEquals('hello world', + cmd_helper.GetCmdOutput(cmd, shell=True).rstrip()) + + +class CmdHelperShinkToSnippetTest(unittest.TestCase): + + def testShrinkToSnippet_noArgs(self): + self.assertEquals('foo', + cmd_helper.ShrinkToSnippet(['foo'], 'a', 'bar')) + self.assertEquals("'foo foo'", + cmd_helper.ShrinkToSnippet(['foo foo'], 'a', 'bar')) + self.assertEquals('"$a"\' bar\'', + cmd_helper.ShrinkToSnippet(['foo bar'], 'a', 'foo')) + self.assertEquals('\'foo \'"$a"', + cmd_helper.ShrinkToSnippet(['foo bar'], 'a', 'bar')) + self.assertEquals('foo"$a"', + cmd_helper.ShrinkToSnippet(['foobar'], 'a', 'bar')) + + def testShrinkToSnippet_singleArg(self): + self.assertEquals("foo ''", + cmd_helper.ShrinkToSnippet(['foo', ''], 'a', 'bar')) + self.assertEquals("foo foo", + cmd_helper.ShrinkToSnippet(['foo', 'foo'], 'a', 'bar')) + self.assertEquals('"$a" "$a"', + cmd_helper.ShrinkToSnippet(['foo', 'foo'], 'a', 'foo')) + self.assertEquals('foo "$a""$a"', + cmd_helper.ShrinkToSnippet(['foo', 'barbar'], 'a', 'bar')) + self.assertEquals('foo "$a"\' \'"$a"', + cmd_helper.ShrinkToSnippet(['foo', 'bar bar'], 'a', 'bar')) + self.assertEquals('foo "$a""$a"\' \'', + cmd_helper.ShrinkToSnippet(['foo', 'barbar '], 'a', 'bar')) + self.assertEquals('foo \' \'"$a""$a"\' \'', + cmd_helper.ShrinkToSnippet(['foo', ' barbar '], 'a', 'bar')) + + +class CmdHelperIterCmdOutputLinesTest(unittest.TestCase): + """Test IterCmdOutputLines with some calls to the unix 'seq' command.""" + + def testIterCmdOutputLines_success(self): + for num, line in enumerate( + cmd_helper.IterCmdOutputLines(['seq', '10']), 1): + self.assertEquals(num, int(line)) + + def testIterCmdOutputLines_exitStatusFail(self): + with self.assertRaises(subprocess.CalledProcessError): + for num, line in enumerate( + cmd_helper.IterCmdOutputLines('seq 10 && false', shell=True), 1): + self.assertEquals(num, int(line)) + # after reading all the output we get an exit status of 1 + + def testIterCmdOutputLines_exitStatusIgnored(self): + for num, line in enumerate( + cmd_helper.IterCmdOutputLines('seq 10 && false', shell=True, + check_status=False), 1): + self.assertEquals(num, int(line)) + + def testIterCmdOutputLines_exitStatusSkipped(self): + for num, line in enumerate( + cmd_helper.IterCmdOutputLines('seq 10 && false', shell=True), 1): + self.assertEquals(num, int(line)) + # no exception will be raised because we don't attempt to read past + # the end of the output and, thus, the status never gets checked + if num == 10: + break + + +if __name__ == '__main__': + unittest.main() diff --git a/catapult/devil/devil/utils/file_utils.py b/catapult/devil/devil/utils/file_utils.py new file mode 100644 index 00000000..dc5a9efc --- /dev/null +++ b/catapult/devil/devil/utils/file_utils.py @@ -0,0 +1,31 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os + + +def MergeFiles(dest_file, source_files): + """Merge list of files into single destination file. + + Args: + dest_file: File to be written to. + source_files: List of files to be merged. Will be merged in the order they + appear in the list. + """ + if not os.path.exists(os.path.dirname(dest_file)): + os.makedirs(os.path.dirname(dest_file)) + try: + with open(dest_file, 'w') as dest_f: + for source_file in source_files: + with open(source_file, 'r') as source_f: + dest_f.write(source_f.read()) + except Exception as e: # pylint: disable=broad-except + # Something went wrong when creating dest_file. Cleaning up. + try: + os.remove(dest_file) + except OSError: + pass + raise e + + diff --git a/catapult/devil/devil/utils/find_usb_devices.py b/catapult/devil/devil/utils/find_usb_devices.py new file mode 100755 index 00000000..4982e46a --- /dev/null +++ b/catapult/devil/devil/utils/find_usb_devices.py @@ -0,0 +1,628 @@ +#!/usr/bin/python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import re +import sys +import argparse + +from devil.utils import cmd_helper +from devil.utils import lsusb + +# Note: In the documentation below, "virtual port" refers to the port number +# as observed by the system (e.g. by usb-devices) and "physical port" refers +# to the physical numerical label on the physical port e.g. on a USB hub. +# The mapping between virtual and physical ports is not always the identity +# (e.g. the port labeled "1" on a USB hub does not always show up as "port 1" +# when you plug something into it) but, as far as we are aware, the mapping +# between virtual and physical ports is always the same for a given +# model of USB hub. When "port number" is referenced without specifying, it +# means the virtual port number. + + +# Wrapper functions for system commands to get output. These are in wrapper +# functions so that they can be more easily mocked-out for tests. +def _GetParsedLSUSBOutput(): + return lsusb.lsusb() + + +def _GetUSBDevicesOutput(): + return cmd_helper.GetCmdOutput(['usb-devices']) + + +def _GetTtyUSBInfo(tty_string): + cmd = ['udevadm', 'info', '--name=/dev/' + tty_string, '--attribute-walk'] + return cmd_helper.GetCmdOutput(cmd) + + +def _GetCommList(): + return cmd_helper.GetCmdOutput('ls /dev', shell=True) + + +def GetTTYList(): + return [x for x in _GetCommList().splitlines() if 'ttyUSB' in x] + + +def GetBattorList(device_tree_map): + return [x for x in GetTTYList() if IsBattor(x, device_tree_map)] + + +def IsBattor(tty_string, device_tree_map): + (bus, device) = GetBusDeviceFromTTY(tty_string) + node = device_tree_map[bus].FindDeviceNumber(device) + return 'Future Technology Devices International' in node.desc + + +# Class to identify nodes in the USB topology. USB topology is organized as +# a tree. +class USBNode(object): + def __init__(self): + self._port_to_node = {} + + @property + def desc(self): + raise NotImplementedError + + @property + def info(self): + raise NotImplementedError + + @property + def device_num(self): + raise NotImplementedError + + @property + def bus_num(self): + raise NotImplementedError + + def HasPort(self, port): + """Determines if this device has a device connected to the given port.""" + return port in self._port_to_node + + def PortToDevice(self, port): + """Gets the device connected to the given port on this device.""" + return self._port_to_node[port] + + def Display(self, port_chain='', info=False): + """Displays information about this node and its descendants. + + Output format is, e.g. 1:3:3:Device 42 (ID 1234:5678 Some Device) + meaning that from the bus, if you look at the device connected + to port 1, then the device connected to port 3 of that, + then the device connected to port 3 of that, you get the device + assigned device number 42, which is Some Device. Note that device + numbers will be reassigned whenever a connected device is powercycled + or reinserted, but port numbers stay the same as long as the device + is reinserted back into the same physical port. + + Args: + port_chain: [string] Chain of ports from bus to this node (e.g. '2:4:') + info: [bool] Whether to display detailed info as well. + """ + raise NotImplementedError + + def AddChild(self, port, device): + """Adds child to the device tree. + + Args: + port: [int] Port number of the device. + device: [USBDeviceNode] Device to add. + + Raises: + ValueError: If device already has a child at the given port. + """ + if self.HasPort(port): + raise ValueError('Duplicate port number') + else: + self._port_to_node[port] = device + + def AllNodes(self): + """Generator that yields this node and all of its descendants. + + Yields: + [USBNode] First this node, then each of its descendants (recursively) + """ + yield self + for child_node in self._port_to_node.values(): + for descendant_node in child_node.AllNodes(): + yield descendant_node + + def FindDeviceNumber(self, findnum): + """Find device with given number in tree + + Searches the portion of the device tree rooted at this node for + a device with the given device number. + + Args: + findnum: [int] Device number to search for. + + Returns: + [USBDeviceNode] Node that is found. + """ + for node in self.AllNodes(): + if node.device_num == findnum: + return node + return None + + +class USBDeviceNode(USBNode): + def __init__(self, bus_num=0, device_num=0, serial=None, info=None): + """Class that represents a device in USB tree. + + Args: + bus_num: [int] Bus number that this node is attached to. + device_num: [int] Device number of this device (or 0, if this is a bus) + serial: [string] Serial number. + info: [dict] Map giving detailed device info. + """ + super(USBDeviceNode, self).__init__() + self._bus_num = bus_num + self._device_num = device_num + self._serial = serial + self._info = {} if info is None else info + + #override + @property + def desc(self): + return self._info.get('desc') + + #override + @property + def info(self): + return self._info + + #override + @property + def device_num(self): + return self._device_num + + #override + @property + def bus_num(self): + return self._bus_num + + @property + def serial(self): + return self._serial + + @serial.setter + def serial(self, serial): + self._serial = serial + + #override + def Display(self, port_chain='', info=False): + print '%s Device %d (%s)' % (port_chain, self.device_num, self.desc) + if info: + print self.info + for (port, device) in self._port_to_node.iteritems(): + device.Display('%s%d:' % (port_chain, port), info=info) + + +class USBBusNode(USBNode): + def __init__(self, bus_num=0): + """Class that represents a node (either a bus or device) in USB tree. + + Args: + is_bus: [bool] If true, node is bus; if not, node is device. + bus_num: [int] Bus number that this node is attached to. + device_num: [int] Device number of this device (or 0, if this is a bus) + desc: [string] Short description of device. + serial: [string] Serial number. + info: [dict] Map giving detailed device info. + port_to_dev: [dict(int:USBDeviceNode)] + Maps port # to device connected to port. + """ + super(USBBusNode, self).__init__() + self._bus_num = bus_num + + #override + @property + def desc(self): + return 'BUS %d' % self._bus_num + + #override + @property + def info(self): + return {} + + #override + @property + def device_num(self): + return -1 + + #override + @property + def bus_num(self): + return self._bus_num + + #override + def Display(self, port_chain='', info=False): + print "=== %s ===" % self.desc + for (port, device) in self._port_to_node.iteritems(): + device.Display('%s%d:' % (port_chain, port), info=info) + + +_T_LINE_REGEX = re.compile(r'T: Bus=(?P<bus>\d{2}) Lev=(?P<lev>\d{2}) ' + r'Prnt=(?P<prnt>\d{2,3}) Port=(?P<port>\d{2}) ' + r'Cnt=(?P<cnt>\d{2}) Dev#=(?P<dev>.{3}) .*') + +_S_LINE_REGEX = re.compile(r'S: SerialNumber=(?P<serial>.*)') +_LSUSB_BUS_DEVICE_RE = re.compile(r'^Bus (\d{3}) Device (\d{3}): (.*)') + + +def GetBusNumberToDeviceTreeMap(fast=False): + """Gets devices currently attached. + + Args: + fast [bool]: whether to do it fast (only get description, not + the whole dictionary, from lsusb) + + Returns: + map of {bus number: bus object} + where the bus object has all the devices attached to it in a tree. + """ + if fast: + info_map = {} + for line in lsusb.raw_lsusb().splitlines(): + match = _LSUSB_BUS_DEVICE_RE.match(line) + if match: + info_map[(int(match.group(1)), int(match.group(2)))] = ( + {'desc':match.group(3)}) + else: + info_map = {((int(line['bus']), int(line['device']))): line + for line in _GetParsedLSUSBOutput()} + + + tree = {} + bus_num = -1 + for line in _GetUSBDevicesOutput().splitlines(): + match = _T_LINE_REGEX.match(line) + if match: + bus_num = int(match.group('bus')) + parent_num = int(match.group('prnt')) + # usb-devices starts counting ports from 0, so add 1 + port_num = int(match.group('port')) + 1 + device_num = int(match.group('dev')) + + # create new bus if necessary + if bus_num not in tree: + tree[bus_num] = USBBusNode(bus_num=bus_num) + + # create the new device + new_device = USBDeviceNode(bus_num=bus_num, + device_num=device_num, + info=info_map[(bus_num, device_num)]) + + # add device to bus + if parent_num != 0: + tree[bus_num].FindDeviceNumber(parent_num).AddChild( + port_num, new_device) + else: + tree[bus_num].AddChild(port_num, new_device) + + match = _S_LINE_REGEX.match(line) + if match: + if bus_num == -1: + raise ValueError('S line appears before T line in input file') + # put the serial number in the device + tree[bus_num].FindDeviceNumber(device_num).serial = match.group('serial') + + return tree + + +class HubType(object): + def __init__(self, id_func, port_mapping): + """Defines a type of hub. + + Args: + id_func: [USBNode -> bool] is a function that can be run on a node + to determine if the node represents this type of hub. + port_mapping: [dict(int:(int|dict))] maps virtual to physical port + numbers. For instance, {3:1, 1:2, 2:3} means that virtual port 3 + corresponds to physical port 1, virtual port 1 corresponds to physical + port 2, and virtual port 2 corresponds to physical port 3. In the + case of hubs with "internal" topology, this is represented by nested + maps. For instance, {1:{1:1,2:2},2:{1:3,2:4}} means, e.g. that the + device plugged into physical port 3 will show up as being connected + to port 1, on a device which is connected to port 2 on the hub. + """ + self._id_func = id_func + # v2p = "virtual to physical" ports + self._v2p_port = port_mapping + + def IsType(self, node): + """Determines if the given Node is a hub of this type. + + Args: + node: [USBNode] Node to check. + """ + return self._id_func(node) + + def GetPhysicalPortToNodeTuples(self, node): + """Gets devices connected to the physical ports on a hub of this type. + + Args: + node: [USBNode] Node representing a hub of this type. + + Yields: + A series of (int, USBNode) tuples giving a physical port + and the USBNode connected to it. + + Raises: + ValueError: If the given node isn't a hub of this type. + """ + if self.IsType(node): + for res in self._GppHelper(node, self._v2p_port): + yield res + else: + raise ValueError('Node must be a hub of this type') + + def _GppHelper(self, node, mapping): + """Helper function for GetPhysicalPortToNodeMap. + + Gets devices connected to physical ports, based on device tree + rooted at the given node and the mapping between virtual and physical + ports. + + Args: + node: [USBNode] Root of tree to search for devices. + mapping: [dict] Mapping between virtual and physical ports. + + Yields: + A series of (int, USBNode) tuples giving a physical port + and the Node connected to it. + """ + for (virtual, physical) in mapping.iteritems(): + if node.HasPort(virtual): + if isinstance(physical, dict): + for res in self._GppHelper(node.PortToDevice(virtual), physical): + yield res + else: + yield (physical, node.PortToDevice(virtual)) + + +def GetHubsOnBus(bus, hub_types): + """Scans for all hubs on a bus of given hub types. + + Args: + bus: [USBNode] Bus object. + hub_types: [iterable(HubType)] Possible types of hubs. + + Yields: + Sequence of tuples representing (hub, type of hub) + """ + for device in bus.AllNodes(): + for hub_type in hub_types: + if hub_type.IsType(device): + yield (device, hub_type) + + +def GetPhysicalPortToNodeMap(hub, hub_type): + """Gets physical-port:node mapping for a given hub. + Args: + hub: [USBNode] Hub to get map for. + hub_type: [HubType] Which type of hub it is. + + Returns: + Dict of {physical port: node} + """ + port_device = hub_type.GetPhysicalPortToNodeTuples(hub) + return {port: device for (port, device) in port_device} + + +def GetPhysicalPortToBusDeviceMap(hub, hub_type): + """Gets physical-port:(bus#, device#) mapping for a given hub. + Args: + hub: [USBNode] Hub to get map for. + hub_type: [HubType] Which type of hub it is. + + Returns: + Dict of {physical port: (bus number, device number)} + """ + port_device = hub_type.GetPhysicalPortToNodeTuples(hub) + return {port: (device.bus_num, device.device_num) + for (port, device) in port_device} + + +def GetPhysicalPortToSerialMap(hub, hub_type): + """Gets physical-port:serial# mapping for a given hub. + Args: + hub: [USBNode] Hub to get map for. + hub_type: [HubType] Which type of hub it is. + + Returns: + Dict of {physical port: serial number)} + """ + port_device = hub_type.GetPhysicalPortToNodeTuples(hub) + return {port: device.serial + for (port, device) in port_device + if device.serial} + + +def GetPhysicalPortToTTYMap(device, hub_type): + """Gets physical-port:tty-string mapping for a given hub. + Args: + hub: [USBNode] Hub to get map for. + hub_type: [HubType] Which type of hub it is. + + Returns: + Dict of {physical port: tty-string)} + """ + port_device = hub_type.GetPhysicalPortToNodeTuples(device) + bus_device_to_tty = GetBusDeviceToTTYMap() + return {port: bus_device_to_tty[(device.bus_num, device.device_num)] + for (port, device) in port_device + if (device.bus_num, device.device_num) in bus_device_to_tty} + + +def CollectHubMaps(hub_types, map_func, device_tree_map=None, fast=False): + """Runs a function on all hubs in the system and collects their output. + + Args: + hub_types: [HubType] List of possible hub types. + map_func: [string] Function to run on each hub. + device_tree: Previously constructed device tree map, if any. + fast: Whether to construct device tree fast, if not already provided + + Yields: + Sequence of dicts of {physical port: device} where the type of + device depends on the ident keyword. Each dict is a separate hub. + """ + if device_tree_map is None: + device_tree_map = GetBusNumberToDeviceTreeMap(fast=fast) + for bus in device_tree_map.values(): + for (hub, hub_type) in GetHubsOnBus(bus, hub_types): + yield map_func(hub, hub_type) + + +def GetAllPhysicalPortToNodeMaps(hub_types, **kwargs): + return CollectHubMaps(hub_types, GetPhysicalPortToNodeMap, **kwargs) + + +def GetAllPhysicalPortToBusDeviceMaps(hub_types, **kwargs): + return CollectHubMaps(hub_types, GetPhysicalPortToBusDeviceMap, **kwargs) + + +def GetAllPhysicalPortToSerialMaps(hub_types, **kwargs): + return CollectHubMaps(hub_types, GetPhysicalPortToSerialMap, **kwargs) + + +def GetAllPhysicalPortToTTYMaps(hub_types, **kwargs): + return CollectHubMaps(hub_types, GetPhysicalPortToTTYMap, **kwargs) + + +_BUS_NUM_REGEX = re.compile(r'.*ATTRS{busnum}=="(\d*)".*') +_DEVICE_NUM_REGEX = re.compile(r'.*ATTRS{devnum}=="(\d*)".*') + + +def GetBusDeviceFromTTY(tty_string): + """Gets bus and device number connected to a ttyUSB port. + + Args: + tty_string: [String] Identifier for ttyUSB (e.g. 'ttyUSB0') + + Returns: + Tuple (bus, device) giving device connected to that ttyUSB. + + Raises: + ValueError: If bus and device information could not be found. + """ + bus_num = None + device_num = None + # Expected output of GetCmdOutput should be something like: + # looking at device /devices/something/.../.../... + # KERNELS="ttyUSB0" + # SUBSYSTEMS=... + # DRIVERS=... + # ATTRS{foo}=... + # ATTRS{bar}=... + # ... + for line in _GetTtyUSBInfo(tty_string).splitlines(): + bus_match = _BUS_NUM_REGEX.match(line) + device_match = _DEVICE_NUM_REGEX.match(line) + if bus_match and bus_num == None: + bus_num = int(bus_match.group(1)) + if device_match and device_num == None: + device_num = int(device_match.group(1)) + if bus_num is None or device_num is None: + raise ValueError('Info not found') + return (bus_num, device_num) + + +def GetBusDeviceToTTYMap(): + """Gets all mappings from (bus, device) to ttyUSB string. + + Gets mapping from (bus, device) to ttyUSB string (e.g. 'ttyUSB0'), + for all ttyUSB strings currently active. + + Returns: + [dict] Dict that maps (bus, device) to ttyUSB string + """ + result = {} + for tty in GetTTYList(): + result[GetBusDeviceFromTTY(tty)] = tty + return result + + +# This dictionary described the mapping between physical and +# virtual ports on a Plugable 7-Port Hub (model USB2-HUB7BC). +# Keys are the virtual ports, values are the physical port. +# The entry 4:{1:4, 2:3, 3:2, 4:1} indicates that virtual port +# 4 connects to another 'virtual' hub that itself has the +# virtual-to-physical port mapping {1:4, 2:3, 3:2, 4:1}. + +PLUGABLE_7PORT_LAYOUT = {1:7, + 2:6, + 3:5, + 4:{1:4, 2:3, 3:2, 4:1}} + +def TestUSBTopologyScript(): + """Test display and hub identification.""" + # Identification criteria for Plugable 7-Port Hub + def _is_plugable_7port_hub(node): + """Check if a node is a Plugable 7-Port Hub + (Model USB2-HUB7BC) + The topology of this device is a 4-port hub, + with another 4-port hub connected on port 4. + """ + if not isinstance(node, USBDeviceNode): + return False + if '4-Port HUB' not in node.desc: + return False + if not node.HasPort(4): + return False + return '4-Port HUB' in node.PortToDevice(4).desc + + plugable_7port = HubType(_is_plugable_7port_hub, + PLUGABLE_7PORT_LAYOUT) + print '==== USB TOPOLOGY SCRIPT TEST ====' + + # Display devices + print '==== DEVICE DISPLAY ====' + device_trees = GetBusNumberToDeviceTreeMap(fast=True) + for device_tree in device_trees.values(): + device_tree.Display() + print + + # Display TTY information about devices plugged into hubs. + print '==== TTY INFORMATION ====' + for port_map in GetAllPhysicalPortToTTYMaps([plugable_7port], + device_tree_map=device_trees): + print port_map + print + + # Display serial number information about devices plugged into hubs. + print '==== SERIAL NUMBER INFORMATION ====' + for port_map in GetAllPhysicalPortToSerialMaps([plugable_7port], + device_tree_map=device_trees): + print port_map + print '' + return 0 + +def parse_options(argv): + """Parses and checks the command-line options. + + Returns: + A tuple containing the options structure and a list of categories to + be traced. + """ + USAGE = '''./find_usb_devices [--help] + This script shows the mapping between USB devices and port numbers. + Clients are not intended to call this script from the command line. + Clients are intended to call the functions in this script directly. + For instance, GetAllPhysicalPortToSerialMaps(...) + Running this script with --help will display this message. + Running this script without --help will display information about + devices attached, TTY mapping, and serial number mapping, + for testing purposes. See design document for API documentation. + ''' + parser = argparse.ArgumentParser(usage=USAGE) + return parser.parse_args(argv[1:]) + +def main(): + parse_options(sys.argv) + TestUSBTopologyScript() + +if __name__ == "__main__": + sys.exit(main()) diff --git a/catapult/devil/devil/utils/find_usb_devices_test.py b/catapult/devil/devil/utils/find_usb_devices_test.py new file mode 100755 index 00000000..2e94dcd2 --- /dev/null +++ b/catapult/devil/devil/utils/find_usb_devices_test.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# pylint: disable=protected-access + +""" +Unit tests for the contents of find_usb_devices.py. + +Device tree for these tests is as follows: +Bus 001: +1: Device 011 "foo" +2: Device 012 "bar" +3: Device 013 "baz" + +Bus 002: +1: Device 011 "quux" +2: Device 020 "My Test HUB" #hub 1 +2:1: Device 021 "battor_p7_h1_t0" #physical port 7 on hub 1, on ttyUSB0 +2:3: Device 022 "battor_p5_h1_t1" #physical port 5 on hub 1, on ttyUSB1 +2:4: Device 023 "My Test Internal HUB" #internal section of hub 1 +2:4:2: Device 024 "battor_p3_h1_t2" #physical port 3 on hub 1, on ttyUSB2 +2:4:3: Device 026 "Not a Battery Monitor" #physical port 1 on hub 1, on ttyUSB3 +2:4:4: Device 025 "battor_p1_h1_t3" #physical port 1 on hub 1, on ttyUSB3 +3: Device 100 "My Test HUB" #hub 2 +3:4: Device 101 "My Test Internal HUB" #internal section of hub 2 +3:4:4: Device 102 "battor_p1_h2_t4" #physical port 1 on hub 2, on ttyusb4 +""" + +import logging +import unittest + +from devil import devil_env +from devil.utils import find_usb_devices +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + +# Output of lsusb.lsusb(). +# We just test that the dictionary is working by creating an +# "ID number" equal to (bus_num*1000)+device_num and seeing if +# it is picked up correctly. Also we test the description + +DEVLIST = [(1, 11, 'foo'), + (1, 12, 'bar'), + (1, 13, 'baz'), + (2, 11, 'quux'), + (2, 20, 'My Test HUB'), + (2, 21, 'Future Technology Devices International battor_p7_h1_t0'), + (2, 22, 'Future Technology Devices International battor_p5_h1_t1'), + (2, 23, 'My Test Internal HUB'), + (2, 24, 'Future Technology Devices International battor_p3_h1_t2'), + (2, 25, 'Future Technology Devices International battor_p1_h1_t3'), + (2, 26, 'Not a Battery Monitor'), + (2, 100, 'My Test HUB'), + (2, 101, 'My Test Internal HUB'), + (2, 102, 'Future Technology Devices International battor_p1_h1_t4')] + +LSUSB_OUTPUT = [ + {'bus': b, 'device': d, 'desc': t, 'id': (1000*b)+d} + for (b, d, t) in DEVLIST] + + +# Note: "Lev", "Cnt", "Spd", and "MxCh" are not used by parser, +# so we just leave them as zeros here. Also note that the port +# numbers reported here start at 0, so they're 1 less than the +# port numbers reported elsewhere. +USB_DEVICES_OUTPUT = ''' +T: Bus=01 Lev=00 Prnt=00 Port=00 Cnt=00 Dev#= 11 Spd=000 MxCh=00 +S: SerialNumber=FooSerial +T: Bus=01 Lev=00 Prnt=00 Port=01 Cnt=00 Dev#= 12 Spd=000 MxCh=00 +S: SerialNumber=BarSerial +T: Bus=01 Lev=00 Prnt=00 Port=02 Cnt=00 Dev#= 13 Spd=000 MxCh=00 +S: SerialNumber=BazSerial + +T: Bus=02 Lev=00 Prnt=00 Port=00 Cnt=00 Dev#= 11 Spd=000 MxCh=00 + +T: Bus=02 Lev=00 Prnt=00 Port=01 Cnt=00 Dev#= 20 Spd=000 MxCh=00 +T: Bus=02 Lev=00 Prnt=20 Port=00 Cnt=00 Dev#= 21 Spd=000 MxCh=00 +S: SerialNumber=Battor0 +T: Bus=02 Lev=00 Prnt=20 Port=02 Cnt=00 Dev#= 22 Spd=000 MxCh=00 +S: SerialNumber=Battor1 +T: Bus=02 Lev=00 Prnt=20 Port=03 Cnt=00 Dev#= 23 Spd=000 MxCh=00 +T: Bus=02 Lev=00 Prnt=23 Port=01 Cnt=00 Dev#= 24 Spd=000 MxCh=00 +S: SerialNumber=Battor2 +T: Bus=02 Lev=00 Prnt=23 Port=03 Cnt=00 Dev#= 25 Spd=000 MxCh=00 +S: SerialNumber=Battor3 +T: Bus=02 Lev=00 Prnt=23 Port=02 Cnt=00 Dev#= 26 Spd=000 MxCh=00 + +T: Bus=02 Lev=00 Prnt=00 Port=02 Cnt=00 Dev#=100 Spd=000 MxCh=00 +T: Bus=02 Lev=00 Prnt=100 Port=03 Cnt=00 Dev#=101 Spd=000 MxCh=00 +T: Bus=02 Lev=00 Prnt=101 Port=03 Cnt=00 Dev#=102 Spd=000 MxCh=00 +''' + +LIST_TTY_OUTPUT = ''' +ttyUSB0 +Something-else-0 +ttyUSB1 +ttyUSB2 +Something-else-1 +ttyUSB3 +ttyUSB4 +Something-else-2 +ttyUSB5 +''' + +# Note: The real output will have multiple lines with +# ATTRS{busnum} and ATTRS{devnum}, but only the first +# one counts. Thus the test output duplicates this. +UDEVADM_USBTTY0_OUTPUT = ''' +ATTRS{busnum}=="2" +ATTRS{devnum}=="21" +ATTRS{busnum}=="0" +ATTRS{devnum}=="0" +''' + +UDEVADM_USBTTY1_OUTPUT = ''' +ATTRS{busnum}=="2" +ATTRS{devnum}=="22" +ATTRS{busnum}=="0" +ATTRS{devnum}=="0" +''' + +UDEVADM_USBTTY2_OUTPUT = ''' +ATTRS{busnum}=="2" +ATTRS{devnum}=="24" +ATTRS{busnum}=="0" +ATTRS{devnum}=="0" +''' + +UDEVADM_USBTTY3_OUTPUT = ''' +ATTRS{busnum}=="2" +ATTRS{devnum}=="25" +ATTRS{busnum}=="0" +ATTRS{devnum}=="0" +''' + +UDEVADM_USBTTY4_OUTPUT = ''' +ATTRS{busnum}=="2" +ATTRS{devnum}=="102" +ATTRS{busnum}=="0" +ATTRS{devnum}=="0" +''' + +UDEVADM_USBTTY5_OUTPUT = ''' +ATTRS{busnum}=="2" +ATTRS{devnum}=="26" +ATTRS{busnum}=="0" +ATTRS{devnum}=="0" +''' + +UDEVADM_OUTPUT_DICT = { + 'ttyUSB0': UDEVADM_USBTTY0_OUTPUT, + 'ttyUSB1': UDEVADM_USBTTY1_OUTPUT, + 'ttyUSB2': UDEVADM_USBTTY2_OUTPUT, + 'ttyUSB3': UDEVADM_USBTTY3_OUTPUT, + 'ttyUSB4': UDEVADM_USBTTY4_OUTPUT, + 'ttyUSB5': UDEVADM_USBTTY5_OUTPUT} + +# Identification criteria for Plugable 7-Port Hub +def isTestHub(node): + """Check if a node is a Plugable 7-Port Hub + (Model USB2-HUB7BC) + The topology of this device is a 4-port hub, + with another 4-port hub connected on port 4. + """ + if not isinstance(node, find_usb_devices.USBDeviceNode): + return False + if 'Test HUB' not in node.desc: + return False + if not node.HasPort(4): + return False + return 'Test Internal HUB' in node.PortToDevice(4).desc + +TEST_HUB = find_usb_devices.HubType(isTestHub, + {1:7, + 2:6, + 3:5, + 4:{1:4, 2:3, 3:2, 4:1}}) + +class USBScriptTest(unittest.TestCase): + def setUp(self): + find_usb_devices._GetTtyUSBInfo = mock.Mock( + side_effect=lambda x: UDEVADM_OUTPUT_DICT[x]) + find_usb_devices._GetParsedLSUSBOutput = mock.Mock( + return_value=LSUSB_OUTPUT) + find_usb_devices._GetUSBDevicesOutput = mock.Mock( + return_value=USB_DEVICES_OUTPUT) + find_usb_devices._GetCommList = mock.Mock( + return_value=LIST_TTY_OUTPUT) + + def testIsBattor(self): + bd = find_usb_devices.GetBusNumberToDeviceTreeMap() + self.assertTrue(find_usb_devices.IsBattor('ttyUSB3', bd)) + self.assertFalse(find_usb_devices.IsBattor('ttyUSB5', bd)) + + def testGetBattors(self): + bd = find_usb_devices.GetBusNumberToDeviceTreeMap() + self.assertEquals(find_usb_devices.GetBattorList(bd), + ['ttyUSB0', 'ttyUSB1', 'ttyUSB2', + 'ttyUSB3', 'ttyUSB4']) + + def testGetTTYDevices(self): + pp = find_usb_devices.GetAllPhysicalPortToTTYMaps([TEST_HUB]) + result = list(pp) + self.assertEquals(result[0], {7:'ttyUSB0', + 5:'ttyUSB1', + 3:'ttyUSB2', + 2:'ttyUSB5', + 1:'ttyUSB3'}) + self.assertEquals(result[1], {1:'ttyUSB4'}) + + def testGetPortDeviceMapping(self): + pp = find_usb_devices.GetAllPhysicalPortToBusDeviceMaps([TEST_HUB]) + result = list(pp) + self.assertEquals(result[0], {7:(2, 21), + 5:(2, 22), + 3:(2, 24), + 2:(2, 26), + 1:(2, 25)}) + self.assertEquals(result[1], {1:(2, 102)}) + + def testGetSerialMapping(self): + pp = find_usb_devices.GetAllPhysicalPortToSerialMaps([TEST_HUB]) + result = list(pp) + self.assertEquals(result[0], {7:'Battor0', + 5:'Battor1', + 3:'Battor2', + 1:'Battor3'}) + self.assertEquals(result[1], {}) + + def testDeviceDescriptions(self): + bd = find_usb_devices.GetBusNumberToDeviceTreeMap() + dev_foo = bd[1].FindDeviceNumber(11) + dev_bar = bd[1].FindDeviceNumber(12) + dev_battor_p7_h1_t0 = bd[2].FindDeviceNumber(21) + self.assertEquals(dev_foo.desc, 'foo') + self.assertEquals(dev_bar.desc, 'bar') + self.assertEquals(dev_battor_p7_h1_t0.desc, + 'Future Technology Devices International battor_p7_h1_t0') + + def testDeviceInformation(self): + bd = find_usb_devices.GetBusNumberToDeviceTreeMap() + dev_foo = bd[1].FindDeviceNumber(11) + dev_bar = bd[1].FindDeviceNumber(12) + dev_battor_p7_h1_t0 = bd[2].FindDeviceNumber(21) + self.assertEquals(dev_foo.info['id'], 1011) + self.assertEquals(dev_bar.info['id'], 1012) + self.assertEquals(dev_battor_p7_h1_t0.info['id'], 2021) + + def testSerialNumber(self): + bd = find_usb_devices.GetBusNumberToDeviceTreeMap() + dev_foo = bd[1].FindDeviceNumber(11) + dev_bar = bd[1].FindDeviceNumber(12) + dev_battor_p7_h1_t0 = bd[2].FindDeviceNumber(21) + self.assertEquals(dev_foo.serial, 'FooSerial') + self.assertEquals(dev_bar.serial, 'BarSerial') + self.assertEquals(dev_battor_p7_h1_t0.serial, 'Battor0') + +if __name__ == "__main__": + logging.getLogger().setLevel(logging.DEBUG) + unittest.main(verbosity=2) diff --git a/catapult/devil/devil/utils/geometry.py b/catapult/devil/devil/utils/geometry.py new file mode 100644 index 00000000..da21770b --- /dev/null +++ b/catapult/devil/devil/utils/geometry.py @@ -0,0 +1,75 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Objects for convenient manipulation of points and other surface areas.""" + +import collections + + +class Point(collections.namedtuple('Point', ['x', 'y'])): + """Object to represent an (x, y) point on a surface. + + Args: + x, y: Two numeric coordinates that define the point. + """ + __slots__ = () + + def __str__(self): + """Get a useful string representation of the object.""" + return '(%s, %s)' % (self.x, self.y) + + def __add__(self, other): + """Sum of two points, e.g. p + q.""" + if isinstance(other, Point): + return Point(self.x + other.x, self.y + other.y) + else: + return NotImplemented + + def __mul__(self, factor): + """Multiplication on the right is not implemented.""" + # This overrides the default behaviour of a tuple multiplied by a constant + # on the right, which does not make sense for a Point. + return NotImplemented + + def __rmul__(self, factor): + """Multiply a point by a scalar factor on the left, e.g. 2 * p.""" + return Point(factor * self.x, factor * self.y) + + +class Rectangle( + collections.namedtuple('Rectangle', ['top_left', 'bottom_right'])): + """Object to represent a rectangle on a surface. + + Args: + top_left: A pair of (left, top) coordinates. Might be given as a Point + or as a two-element sequence (list, tuple, etc.). + bottom_right: A pair (right, bottom) coordinates. + """ + __slots__ = () + + def __new__(cls, top_left, bottom_right): + if not isinstance(top_left, Point): + top_left = Point(*top_left) + if not isinstance(bottom_right, Point): + bottom_right = Point(*bottom_right) + return super(Rectangle, cls).__new__(cls, top_left, bottom_right) + + def __str__(self): + """Get a useful string representation of the object.""" + return '[%s, %s]' % (self.top_left, self.bottom_right) + + @property + def center(self): + """Get the point at the center of the rectangle.""" + return 0.5 * (self.top_left + self.bottom_right) + + @classmethod + def FromDict(cls, d): + """Create a rectangle object from a dictionary. + + Args: + d: A dictionary (or mapping) of the form, e.g., {'top': 0, 'left': 0, + 'bottom': 1, 'right': 1}. + """ + return cls(Point(d['left'], d['top']), Point(d['right'], d['bottom'])) diff --git a/catapult/devil/devil/utils/geometry_test.py b/catapult/devil/devil/utils/geometry_test.py new file mode 100644 index 00000000..af694429 --- /dev/null +++ b/catapult/devil/devil/utils/geometry_test.py @@ -0,0 +1,61 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for the geometry module.""" + +import unittest + +from devil.utils import geometry as g + + +class PointTest(unittest.TestCase): + + def testStr(self): + p = g.Point(1, 2) + self.assertEquals(str(p), '(1, 2)') + + def testAdd(self): + p = g.Point(1, 2) + q = g.Point(3, 4) + r = g.Point(4, 6) + self.assertEquals(p + q, r) + + def testAdd_TypeErrorWithInvalidOperands(self): + # pylint: disable=pointless-statement + p = g.Point(1, 2) + with self.assertRaises(TypeError): + p + 4 # Can't add point and scalar. + with self.assertRaises(TypeError): + 4 + p # Can't add scalar and point. + + def testMult(self): + p = g.Point(1, 2) + r = g.Point(2, 4) + self.assertEquals(2 * p, r) # Multiply by scalar on the left. + + def testMult_TypeErrorWithInvalidOperands(self): + # pylint: disable=pointless-statement + p = g.Point(1, 2) + q = g.Point(2, 4) + with self.assertRaises(TypeError): + p * q # Can't multiply points. + with self.assertRaises(TypeError): + p * 4 # Can't multiply by a scalar on the right. + + +class RectangleTest(unittest.TestCase): + + def testStr(self): + r = g.Rectangle(g.Point(0, 1), g.Point(2, 3)) + self.assertEquals(str(r), '[(0, 1), (2, 3)]') + + def testCenter(self): + r = g.Rectangle(g.Point(0, 1), g.Point(2, 3)) + c = g.Point(1, 2) + self.assertEquals(r.center, c) + + def testFromJson(self): + r1 = g.Rectangle(g.Point(0, 1), g.Point(2, 3)) + r2 = g.Rectangle.FromDict({'top': 1, 'left': 0, 'bottom': 3, 'right': 2}) + self.assertEquals(r1, r2) diff --git a/catapult/devil/devil/utils/host_utils.py b/catapult/devil/devil/utils/host_utils.py new file mode 100644 index 00000000..580721f1 --- /dev/null +++ b/catapult/devil/devil/utils/host_utils.py @@ -0,0 +1,16 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os + + +def GetRecursiveDiskUsage(path): + """Returns the disk usage in bytes of |path|. Similar to `du -sb |path|`.""" + running_size = os.path.getsize(path) + if os.path.isdir(path): + for root, dirs, files in os.walk(path): + running_size += sum([os.path.getsize(os.path.join(root, f)) + for f in files + dirs]) + return running_size + diff --git a/catapult/devil/devil/utils/lazy/__init__.py b/catapult/devil/devil/utils/lazy/__init__.py new file mode 100644 index 00000000..3cc56c0a --- /dev/null +++ b/catapult/devil/devil/utils/lazy/__init__.py @@ -0,0 +1,5 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from devil.utils.lazy.weak_constant import WeakConstant diff --git a/catapult/devil/devil/utils/lazy/weak_constant.py b/catapult/devil/devil/utils/lazy/weak_constant.py new file mode 100644 index 00000000..3558f29a --- /dev/null +++ b/catapult/devil/devil/utils/lazy/weak_constant.py @@ -0,0 +1,29 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import threading + + +class WeakConstant(object): + """A thread-safe, lazily initialized object. + + This does not support modification after initialization. The intended + constant nature of the object is not enforced, though, hence the "weak". + """ + + def __init__(self, initializer): + self._initialized = False + self._initializer = initializer + self._lock = threading.Lock() + self._val = None + + def read(self): + """Get the object, creating it if necessary.""" + if self._initialized: + return self._val + with self._lock: + if not self._initialized: + self._val = self._initializer() + self._initialized = True + return self._val diff --git a/catapult/devil/devil/utils/lsusb.py b/catapult/devil/devil/utils/lsusb.py new file mode 100644 index 00000000..d6306dfd --- /dev/null +++ b/catapult/devil/devil/utils/lsusb.py @@ -0,0 +1,109 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import logging +import re + +from devil.utils import cmd_helper + +_COULDNT_OPEN_ERROR_RE = re.compile(r'Couldn\'t open device.*') +_INDENTATION_RE = re.compile(r'^( *)') +_LSUSB_BUS_DEVICE_RE = re.compile(r'^Bus (\d{3}) Device (\d{3}): (.*)') +_LSUSB_ENTRY_RE = re.compile(r'^ *([^ ]+) +([^ ]+) *([^ ].*)?$') +_LSUSB_GROUP_RE = re.compile(r'^ *([^ ]+.*):$') + + +def _lsusbv_on_device(bus_id, dev_id): + """Calls lsusb -v on device.""" + _, raw_output = cmd_helper.GetCmdStatusAndOutputWithTimeout( + ['lsusb', '-v', '-s', '%s:%s' % (bus_id, dev_id)], timeout=10) + + device = {'bus': bus_id, 'device': dev_id} + depth_stack = [device] + + # TODO(jbudorick): Add documentation for parsing. + for line in raw_output.splitlines(): + # Ignore blank lines. + if not line: + continue + # Filter out error mesage about opening device. + if _COULDNT_OPEN_ERROR_RE.match(line): + continue + # Find start of device information. + m = _LSUSB_BUS_DEVICE_RE.match(line) + if m: + if m.group(1) != bus_id: + logging.warning( + 'Expected bus_id value: %r, seen %r', bus_id, m.group(1)) + if m.group(2) != dev_id: + logging.warning( + 'Expected dev_id value: %r, seen %r', dev_id, m.group(2)) + device['desc'] = m.group(3) + continue + + indent_match = _INDENTATION_RE.match(line) + if not indent_match: + continue + + depth = 1 + len(indent_match.group(1)) / 2 + if depth > len(depth_stack): + logging.error( + 'lsusb parsing error: unexpected indentation: "%s"', line) + continue + + while depth < len(depth_stack): + depth_stack.pop() + + cur = depth_stack[-1] + + m = _LSUSB_GROUP_RE.match(line) + if m: + new_group = {} + cur[m.group(1)] = new_group + depth_stack.append(new_group) + continue + + m = _LSUSB_ENTRY_RE.match(line) + if m: + new_entry = { + '_value': m.group(2), + '_desc': m.group(3), + } + cur[m.group(1)] = new_entry + depth_stack.append(new_entry) + continue + + logging.error('lsusb parsing error: unrecognized line: "%s"', line) + + return device + +def lsusb(): + """Call lsusb and return the parsed output.""" + _, lsusb_list_output = cmd_helper.GetCmdStatusAndOutputWithTimeout( + ['lsusb'], timeout=10) + devices = [] + for line in lsusb_list_output.splitlines(): + m = _LSUSB_BUS_DEVICE_RE.match(line) + if m: + bus_num = m.group(1) + dev_num = m.group(2) + try: + devices.append(_lsusbv_on_device(bus_num, dev_num)) + except cmd_helper.TimeoutError: + # Will be blacklisted if it is in expected device file, but times out. + logging.info('lsusb -v %s:%s timed out.', bus_num, dev_num) + return devices + +def raw_lsusb(): + return cmd_helper.GetCmdOutput(['lsusb']) + +def get_lsusb_serial(device): + try: + return device['Device Descriptor']['iSerial']['_desc'] + except KeyError: + return None + +def get_android_devices(): + return [serial for serial in (get_lsusb_serial(d) for d in lsusb()) + if serial] diff --git a/catapult/devil/devil/utils/lsusb_test.py b/catapult/devil/devil/utils/lsusb_test.py new file mode 100755 index 00000000..f381e72f --- /dev/null +++ b/catapult/devil/devil/utils/lsusb_test.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for the cmd_helper module.""" + +import unittest + +from devil import devil_env +from devil.utils import lsusb +from devil.utils import mock_calls + +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + +RAW_OUTPUT = """ +Bus 003 Device 007: ID 18d1:4ee2 Google Inc. Nexus 4 (debug) +Device Descriptor: + bLength 18 + bDescriptorType 1 + bcdUSB 2.00 + bDeviceClass 0 (Defined at Interface level) + bDeviceSubClass 0 + bDeviceProtocol 0 + bMaxPacketSize0 64 + idVendor 0x18d1 Google Inc. + idProduct 0x4ee2 Nexus 4 (debug) + bcdDevice 2.28 + iManufacturer 1 LGE + iProduct 2 Nexus 4 + iSerial 3 01d2450ea194a93b + bNumConfigurations 1 + Configuration Descriptor: + bLength 9 + bDescriptorType 2 + wTotalLength 62 + bNumInterfaces 2 + bConfigurationValue 1 + iConfiguration 0 + bmAttributes 0x80 + (Bus Powered) + MaxPower 500mA + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 0 + bAlternateSetting 0 + bNumEndpoints 3 + bInterfaceClass 255 Vendor Specific Class + bInterfaceSubClass 255 Vendor Specific Subclass + bInterfaceProtocol 0 + iInterface 4 MTP + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x81 EP 1 IN + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0040 1x 64 bytes + bInterval 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x01 EP 1 OUT + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0040 1x 64 bytes + bInterval 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x82 EP 2 IN + bmAttributes 3 + Transfer Type Interrupt + Synch Type None + Usage Type Data + wMaxPacketSize 0x001c 1x 28 bytes + bInterval 6 + Interface Descriptor: + bLength 9 + bDescriptorType 4 + bInterfaceNumber 1 + bAlternateSetting 0 + bNumEndpoints 2 + bInterfaceClass 255 Vendor Specific Class + bInterfaceSubClass 66 + bInterfaceProtocol 1 + iInterface 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x83 EP 3 IN + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0040 1x 64 bytes + bInterval 0 + Endpoint Descriptor: + bLength 7 + bDescriptorType 5 + bEndpointAddress 0x02 EP 2 OUT + bmAttributes 2 + Transfer Type Bulk + Synch Type None + Usage Type Data + wMaxPacketSize 0x0040 1x 64 bytes + bInterval 0 +Device Qualifier (for other device speed): + bLength 10 + bDescriptorType 6 + bcdUSB 2.00 + bDeviceClass 0 (Defined at Interface level) + bDeviceSubClass 0 + bDeviceProtocol 0 + bMaxPacketSize0 64 + bNumConfigurations 1 +Device Status: 0x0000 + (Bus Powered) +""" +DEVICE_LIST = 'Bus 003 Device 007: ID 18d1:4ee2 Google Inc. Nexus 4 (debug)' + +EXPECTED_RESULT = { + 'device': '007', + 'bus': '003', + 'desc': 'ID 18d1:4ee2 Google Inc. Nexus 4 (debug)', + 'Device': { + '_value': 'Status:', + '_desc': '0x0000', + '(Bus': { + '_value': 'Powered)', + '_desc': None + } + }, + 'Device Descriptor': { + 'bLength': {'_value': '18', '_desc': None}, + 'bcdDevice': {'_value': '2.28', '_desc': None}, + 'bDeviceSubClass': {'_value': '0', '_desc': None}, + 'idVendor': {'_value': '0x18d1', '_desc': 'Google Inc.'}, + 'bcdUSB': {'_value': '2.00', '_desc': None}, + 'bDeviceProtocol': {'_value': '0', '_desc': None}, + 'bDescriptorType': {'_value': '1', '_desc': None}, + 'Configuration Descriptor': { + 'bLength': {'_value': '9', '_desc': None}, + 'wTotalLength': {'_value': '62', '_desc': None}, + 'bConfigurationValue': {'_value': '1', '_desc': None}, + 'Interface Descriptor': { + 'bLength': {'_value': '9', '_desc': None}, + 'bAlternateSetting': {'_value': '0', '_desc': None}, + 'bInterfaceNumber': {'_value': '1', '_desc': None}, + 'bNumEndpoints': {'_value': '2', '_desc': None}, + 'bDescriptorType': {'_value': '4', '_desc': None}, + 'bInterfaceSubClass': {'_value': '66', '_desc': None}, + 'bInterfaceClass': { + '_value': '255', + '_desc': 'Vendor Specific Class' + }, + 'bInterfaceProtocol': {'_value': '1', '_desc': None}, + 'Endpoint Descriptor': { + 'bLength': {'_value': '7', '_desc': None}, + 'bEndpointAddress': {'_value': '0x02', '_desc': 'EP 2 OUT'}, + 'bInterval': {'_value': '0', '_desc': None}, + 'bDescriptorType': {'_value': '5', '_desc': None}, + 'bmAttributes': { + '_value': '2', + 'Transfer': {'_value': 'Type', '_desc': 'Bulk'}, + 'Usage': {'_value': 'Type', '_desc': 'Data'}, + '_desc': None, + 'Synch': {'_value': 'Type', '_desc': 'None'} + }, + 'wMaxPacketSize': { + '_value': '0x0040', + '_desc': '1x 64 bytes' + } + }, + 'iInterface': {'_value': '0', '_desc': None} + }, + 'bDescriptorType': {'_value': '2', '_desc': None}, + 'iConfiguration': {'_value': '0', '_desc': None}, + 'bmAttributes': { + '_value': '0x80', + '_desc': None, + '(Bus': {'_value': 'Powered)', '_desc': None} + }, + 'bNumInterfaces': {'_value': '2', '_desc': None}, + 'MaxPower': {'_value': '500mA', '_desc': None} + }, + 'iSerial': {'_value': '3', '_desc': '01d2450ea194a93b'}, + 'idProduct': {'_value': '0x4ee2', '_desc': 'Nexus 4 (debug)'}, + 'iManufacturer': {'_value': '1', '_desc': 'LGE'}, + 'bDeviceClass': { + '_value': '0', + '_desc': '(Defined at Interface level)' + }, + 'iProduct': {'_value': '2', '_desc': 'Nexus 4'}, + 'bMaxPacketSize0': {'_value': '64', '_desc': None}, + 'bNumConfigurations': {'_value': '1', '_desc': None} + }, + 'Device Qualifier (for other device speed)': { + 'bLength': {'_value': '10', '_desc': None}, + 'bNumConfigurations': {'_value': '1', '_desc': None}, + 'bDeviceSubClass': {'_value': '0', '_desc': None}, + 'bcdUSB': {'_value': '2.00', '_desc': None}, + 'bDeviceProtocol': {'_value': '0', '_desc': None}, + 'bDescriptorType': {'_value': '6', '_desc': None}, + 'bDeviceClass': { + '_value': '0', + '_desc': '(Defined at Interface level)' + }, + 'bMaxPacketSize0': {'_value': '64', '_desc': None} + } +} + + +class LsusbTest(mock_calls.TestCase): + """Test Lsusb parsing.""" + + def testLsusb(self): + with self.assertCalls( + (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutputWithTimeout( + ['lsusb'], timeout=10), (None, DEVICE_LIST)), + (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutputWithTimeout( + ['lsusb', '-v', '-s', '003:007'], timeout=10), (None, RAW_OUTPUT))): + self.assertDictEqual(lsusb.lsusb().pop(), EXPECTED_RESULT) + + def testGetSerial(self): + with self.assertCalls( + (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutputWithTimeout( + ['lsusb'], timeout=10), (None, DEVICE_LIST)), + (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutputWithTimeout( + ['lsusb', '-v', '-s', '003:007'], timeout=10), (None, RAW_OUTPUT))): + self.assertEqual(lsusb.get_android_devices(), ['01d2450ea194a93b']) + + def testGetLsusbSerial(self): + with self.assertCalls( + (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutputWithTimeout( + ['lsusb'], timeout=10), (None, DEVICE_LIST)), + (mock.call.devil.utils.cmd_helper.GetCmdStatusAndOutputWithTimeout( + ['lsusb', '-v', '-s', '003:007'], timeout=10), (None, RAW_OUTPUT))): + out = lsusb.lsusb().pop() + self.assertEqual(lsusb.get_lsusb_serial(out), '01d2450ea194a93b') + + +if __name__ == '__main__': + unittest.main() diff --git a/catapult/devil/devil/utils/mock_calls.py b/catapult/devil/devil/utils/mock_calls.py new file mode 100644 index 00000000..5ae951e3 --- /dev/null +++ b/catapult/devil/devil/utils/mock_calls.py @@ -0,0 +1,180 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +A test facility to assert call sequences while mocking their behavior. +""" + +import unittest + +from devil import devil_env + +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + + +class TestCase(unittest.TestCase): + """Adds assertCalls to TestCase objects.""" + class _AssertCalls(object): + + def __init__(self, test_case, expected_calls, watched): + def call_action(pair): + if isinstance(pair, type(mock.call)): + return (pair, None) + else: + return pair + + def do_check(call): + def side_effect(*args, **kwargs): + received_call = call(*args, **kwargs) + self._test_case.assertTrue( + self._expected_calls, + msg=('Unexpected call: %s' % str(received_call))) + expected_call, action = self._expected_calls.pop(0) + self._test_case.assertTrue( + received_call == expected_call, + msg=('Expected call mismatch:\n' + ' expected: %s\n' + ' received: %s\n' + % (str(expected_call), str(received_call)))) + if callable(action): + return action(*args, **kwargs) + else: + return action + return side_effect + + self._test_case = test_case + self._expected_calls = [call_action(pair) for pair in expected_calls] + watched = watched.copy() # do not pollute the caller's dict + watched.update((call.parent.name, call.parent) + for call, _ in self._expected_calls) + self._patched = [test_case.patch_call(call, side_effect=do_check(call)) + for call in watched.itervalues()] + + def __enter__(self): + for patch in self._patched: + patch.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for patch in self._patched: + patch.__exit__(exc_type, exc_val, exc_tb) + if exc_type is None: + missing = ''.join(' expected: %s\n' % str(call) + for call, _ in self._expected_calls) + self._test_case.assertFalse( + missing, + msg='Expected calls not found:\n' + missing) + + def __init__(self, *args, **kwargs): + super(TestCase, self).__init__(*args, **kwargs) + self.call = mock.call.self + self._watched = {} + + def call_target(self, call): + """Resolve a self.call instance to the target it represents. + + Args: + call: a self.call instance, e.g. self.call.adb.Shell + + Returns: + The target object represented by the call, e.g. self.adb.Shell + + Raises: + ValueError if the path of the call does not start with "self", i.e. the + target of the call is external to the self object. + AttributeError if the path of the call does not specify a valid + chain of attributes (without any calls) starting from "self". + """ + path = call.name.split('.') + if path.pop(0) != 'self': + raise ValueError("Target %r outside of 'self' object" % call.name) + target = self + for attr in path: + target = getattr(target, attr) + return target + + def patch_call(self, call, **kwargs): + """Patch the target of a mock.call instance. + + Args: + call: a mock.call instance identifying a target to patch + Extra keyword arguments are processed by mock.patch + + Returns: + A context manager to mock/unmock the target of the call + """ + if call.name.startswith('self.'): + target = self.call_target(call.parent) + _, attribute = call.name.rsplit('.', 1) + if (hasattr(type(target), attribute) + and isinstance(getattr(type(target), attribute), property)): + return mock.patch.object( + type(target), attribute, new_callable=mock.PropertyMock, **kwargs) + else: + return mock.patch.object(target, attribute, **kwargs) + else: + return mock.patch(call.name, **kwargs) + + def watchCalls(self, calls): + """Add calls to the set of watched calls. + + Args: + calls: a sequence of mock.call instances identifying targets to watch + """ + self._watched.update((call.name, call) for call in calls) + + def watchMethodCalls(self, call, ignore=None): + """Watch all public methods of the target identified by a self.call. + + Args: + call: a self.call instance indetifying an object + ignore: a list of public methods to ignore when watching for calls + """ + target = self.call_target(call) + if ignore is None: + ignore = [] + self.watchCalls(getattr(call, method) + for method in dir(target.__class__) + if not method.startswith('_') and not method in ignore) + + def clearWatched(self): + """Clear the set of watched calls.""" + self._watched = {} + + def assertCalls(self, *calls): + """A context manager to assert that a sequence of calls is made. + + During the assertion, a number of functions and methods will be "watched", + and any calls made to them is expected to appear---in the exact same order, + and with the exact same arguments---as specified by the argument |calls|. + + By default, the targets of all expected calls are watched. Further targets + to watch may be added using watchCalls and watchMethodCalls. + + Optionaly, each call may be accompanied by an action. If the action is a + (non-callable) value, this value will be used as the return value given to + the caller when the matching call is found. Alternatively, if the action is + a callable, the action will be then called with the same arguments as the + intercepted call, so that it can provide a return value or perform other + side effects. If the action is missing, a return value of None is assumed. + + Note that mock.Mock objects are often convenient to use as a callable + action, e.g. to raise exceptions or return other objects which are + themselves callable. + + Args: + calls: each argument is either a pair (expected_call, action) or just an + expected_call, where expected_call is a mock.call instance. + + Raises: + AssertionError if the watched targets do not receive the exact sequence + of calls specified. Missing calls, extra calls, and calls with + mismatching arguments, all cause the assertion to fail. + """ + return self._AssertCalls(self, calls, self._watched) + + def assertCall(self, call, action=None): + return self.assertCalls((call, action)) + diff --git a/catapult/devil/devil/utils/mock_calls_test.py b/catapult/devil/devil/utils/mock_calls_test.py new file mode 100755 index 00000000..8eb4fc9d --- /dev/null +++ b/catapult/devil/devil/utils/mock_calls_test.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Unit tests for the contents of mock_calls.py. +""" + +import logging +import os +import unittest + +from devil import devil_env +from devil.android.sdk import version_codes +from devil.utils import mock_calls + +with devil_env.SysPath(devil_env.PYMOCK_PATH): + import mock # pylint: disable=import-error + + +class _DummyAdb(object): + + def __str__(self): + return '0123456789abcdef' + + def Push(self, host_path, device_path): + logging.debug('(device %s) pushing %r to %r', self, host_path, device_path) + + def IsOnline(self): + logging.debug('(device %s) checking device online', self) + return True + + def Shell(self, cmd): + logging.debug('(device %s) running command %r', self, cmd) + return "nice output\n" + + def Reboot(self): + logging.debug('(device %s) rebooted!', self) + + @property + def build_version_sdk(self): + logging.debug('(device %s) getting build_version_sdk', self) + return version_codes.LOLLIPOP + + +class TestCaseWithAssertCallsTest(mock_calls.TestCase): + + def setUp(self): + self.adb = _DummyAdb() + + def ShellError(self): + def action(cmd): + raise ValueError('(device %s) command %r is not nice' % (self.adb, cmd)) + return action + + def get_answer(self): + logging.debug("called 'get_answer' of %r object", self) + return 42 + + def echo(self, thing): + logging.debug("called 'echo' of %r object", self) + return thing + + def testCallTarget_succeds(self): + self.assertEquals(self.adb.Shell, + self.call_target(self.call.adb.Shell)) + + def testCallTarget_failsExternal(self): + with self.assertRaises(ValueError): + self.call_target(mock.call.sys.getcwd) + + def testCallTarget_failsUnknownAttribute(self): + with self.assertRaises(AttributeError): + self.call_target(self.call.adb.Run) + + def testCallTarget_failsIntermediateCalls(self): + with self.assertRaises(AttributeError): + self.call_target(self.call.adb.RunShell('cmd').append) + + def testPatchCall_method(self): + self.assertEquals(42, self.get_answer()) + with self.patch_call(self.call.get_answer, return_value=123): + self.assertEquals(123, self.get_answer()) + self.assertEquals(42, self.get_answer()) + + def testPatchCall_attribute_method(self): + with self.patch_call(self.call.adb.Shell, return_value='hello'): + self.assertEquals('hello', self.adb.Shell('echo hello')) + + def testPatchCall_global(self): + with self.patch_call(mock.call.os.getcwd, return_value='/some/path'): + self.assertEquals('/some/path', os.getcwd()) + + def testPatchCall_withSideEffect(self): + with self.patch_call(self.call.adb.Shell, side_effect=ValueError): + with self.assertRaises(ValueError): + self.adb.Shell('echo hello') + + def testPatchCall_property(self): + self.assertEquals(version_codes.LOLLIPOP, self.adb.build_version_sdk) + with self.patch_call( + self.call.adb.build_version_sdk, + return_value=version_codes.KITKAT): + self.assertEquals(version_codes.KITKAT, self.adb.build_version_sdk) + self.assertEquals(version_codes.LOLLIPOP, self.adb.build_version_sdk) + + def testAssertCalls_succeeds_simple(self): + self.assertEquals(42, self.get_answer()) + with self.assertCall(self.call.get_answer(), 123): + self.assertEquals(123, self.get_answer()) + self.assertEquals(42, self.get_answer()) + + def testAssertCalls_succeeds_multiple(self): + with self.assertCalls( + (mock.call.os.getcwd(), '/some/path'), + (self.call.echo('hello'), 'hello'), + (self.call.get_answer(), 11), + self.call.adb.Push('this_file', 'that_file'), + (self.call.get_answer(), 12)): + self.assertEquals(os.getcwd(), '/some/path') + self.assertEquals('hello', self.echo('hello')) + self.assertEquals(11, self.get_answer()) + self.adb.Push('this_file', 'that_file') + self.assertEquals(12, self.get_answer()) + + def testAsserCalls_succeeds_withAction(self): + with self.assertCall( + self.call.adb.Shell('echo hello'), self.ShellError()): + with self.assertRaises(ValueError): + self.adb.Shell('echo hello') + + def testAssertCalls_fails_tooManyCalls(self): + with self.assertRaises(AssertionError): + with self.assertCalls(self.call.adb.IsOnline()): + self.adb.IsOnline() + self.adb.IsOnline() + + def testAssertCalls_fails_tooFewCalls(self): + with self.assertRaises(AssertionError): + with self.assertCalls(self.call.adb.IsOnline()): + pass + + def testAssertCalls_succeeds_extraCalls(self): + # we are not watching Reboot, so the assertion succeeds + with self.assertCalls(self.call.adb.IsOnline()): + self.adb.IsOnline() + self.adb.Reboot() + + def testAssertCalls_fails_extraCalls(self): + self.watchCalls([self.call.adb.Reboot]) + # this time we are also watching Reboot, so the assertion fails + with self.assertRaises(AssertionError): + with self.assertCalls(self.call.adb.IsOnline()): + self.adb.IsOnline() + self.adb.Reboot() + + def testAssertCalls_succeeds_NoCalls(self): + self.watchMethodCalls(self.call.adb) # we are watching all adb methods + with self.assertCalls(): + pass + + def testAssertCalls_fails_NoCalls(self): + self.watchMethodCalls(self.call.adb) + with self.assertRaises(AssertionError): + with self.assertCalls(): + self.adb.IsOnline() + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.DEBUG) + unittest.main(verbosity=2) + diff --git a/catapult/devil/devil/utils/parallelizer.py b/catapult/devil/devil/utils/parallelizer.py new file mode 100644 index 00000000..b8a2824e --- /dev/null +++ b/catapult/devil/devil/utils/parallelizer.py @@ -0,0 +1,242 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" Wrapper that allows method execution in parallel. + +This class wraps a list of objects of the same type, emulates their +interface, and executes any functions called on the objects in parallel +in ReraiserThreads. + +This means that, given a list of objects: + + class Foo: + def __init__(self): + self.baz = Baz() + + def bar(self, my_param): + // do something + + list_of_foos = [Foo(1), Foo(2), Foo(3)] + +we can take a sequential operation on that list of objects: + + for f in list_of_foos: + f.bar('Hello') + +and run it in parallel across all of the objects: + + Parallelizer(list_of_foos).bar('Hello') + +It can also handle (non-method) attributes of objects, so that this: + + for f in list_of_foos: + f.baz.myBazMethod() + +can be run in parallel with: + + Parallelizer(list_of_foos).baz.myBazMethod() + +Because it emulates the interface of the wrapped objects, a Parallelizer +can be passed to a method or function that takes objects of that type: + + def DoesSomethingWithFoo(the_foo): + the_foo.bar('Hello') + the_foo.bar('world') + the_foo.baz.myBazMethod + + DoesSomethingWithFoo(Parallelizer(list_of_foos)) + +Note that this class spins up a thread for each object. Using this class +to parallelize operations that are already fast will incur a net performance +penalty. + +""" +# pylint: disable=protected-access + +from devil.utils import reraiser_thread +from devil.utils import watchdog_timer + +_DEFAULT_TIMEOUT = 30 +_DEFAULT_RETRIES = 3 + + +class Parallelizer(object): + """Allows parallel execution of method calls across a group of objects.""" + + def __init__(self, objs): + assert (objs is not None and len(objs) > 0), ( + "Passed empty list to 'Parallelizer'") + self._orig_objs = objs + self._objs = objs + + def __getattr__(self, name): + """Emulate getting the |name| attribute of |self|. + + Args: + name: The name of the attribute to retrieve. + Returns: + A Parallelizer emulating the |name| attribute of |self|. + """ + self.pGet(None) + + r = type(self)(self._orig_objs) + r._objs = [getattr(o, name) for o in self._objs] + return r + + def __getitem__(self, index): + """Emulate getting the value of |self| at |index|. + + Returns: + A Parallelizer emulating the value of |self| at |index|. + """ + self.pGet(None) + + r = type(self)(self._orig_objs) + r._objs = [o[index] for o in self._objs] + return r + + def __call__(self, *args, **kwargs): + """Emulate calling |self| with |args| and |kwargs|. + + Note that this call is asynchronous. Call pFinish on the return value to + block until the call finishes. + + Returns: + A Parallelizer wrapping the ReraiserThreadGroup running the call in + parallel. + Raises: + AttributeError if the wrapped objects aren't callable. + """ + self.pGet(None) + + if not self._objs: + raise AttributeError('Nothing to call.') + for o in self._objs: + if not callable(o): + raise AttributeError("'%s' is not callable" % o.__name__) + + r = type(self)(self._orig_objs) + r._objs = reraiser_thread.ReraiserThreadGroup( + [reraiser_thread.ReraiserThread( + o, args=args, kwargs=kwargs, + name='%s.%s' % (str(d), o.__name__)) + for d, o in zip(self._orig_objs, self._objs)]) + r._objs.StartAll() # pylint: disable=W0212 + return r + + def pFinish(self, timeout): + """Finish any outstanding asynchronous operations. + + Args: + timeout: The maximum number of seconds to wait for an individual + result to return, or None to wait forever. + Returns: + self, now emulating the return values. + """ + self._assertNoShadow('pFinish') + if isinstance(self._objs, reraiser_thread.ReraiserThreadGroup): + self._objs.JoinAll() + self._objs = self._objs.GetAllReturnValues( + watchdog_timer.WatchdogTimer(timeout)) + return self + + def pGet(self, timeout): + """Get the current wrapped objects. + + Args: + timeout: Same as |pFinish|. + Returns: + A list of the results, in order of the provided devices. + Raises: + Any exception raised by any of the called functions. + """ + self._assertNoShadow('pGet') + self.pFinish(timeout) + return self._objs + + def pMap(self, f, *args, **kwargs): + """Map a function across the current wrapped objects in parallel. + + This calls f(o, *args, **kwargs) for each o in the set of wrapped objects. + + Note that this call is asynchronous. Call pFinish on the return value to + block until the call finishes. + + Args: + f: The function to call. + args: The positional args to pass to f. + kwargs: The keyword args to pass to f. + Returns: + A Parallelizer wrapping the ReraiserThreadGroup running the map in + parallel. + """ + self._assertNoShadow('pMap') + r = type(self)(self._orig_objs) + r._objs = reraiser_thread.ReraiserThreadGroup( + [reraiser_thread.ReraiserThread( + f, args=tuple([o] + list(args)), kwargs=kwargs, + name='%s(%s)' % (f.__name__, d)) + for d, o in zip(self._orig_objs, self._objs)]) + r._objs.StartAll() # pylint: disable=W0212 + return r + + def _assertNoShadow(self, attr_name): + """Ensures that |attr_name| isn't shadowing part of the wrapped obejcts. + + If the wrapped objects _do_ have an |attr_name| attribute, it will be + inaccessible to clients. + + Args: + attr_name: The attribute to check. + Raises: + AssertionError if the wrapped objects have an attribute named 'attr_name' + or '_assertNoShadow'. + """ + if isinstance(self._objs, reraiser_thread.ReraiserThreadGroup): + assert not hasattr(self._objs, '_assertNoShadow') + assert not hasattr(self._objs, attr_name) + else: + assert not any(hasattr(o, '_assertNoShadow') for o in self._objs) + assert not any(hasattr(o, attr_name) for o in self._objs) + + +class SyncParallelizer(Parallelizer): + """A Parallelizer that blocks on function calls.""" + + # override + def __call__(self, *args, **kwargs): + """Emulate calling |self| with |args| and |kwargs|. + + Note that this call is synchronous. + + Returns: + A Parallelizer emulating the value returned from calling |self| with + |args| and |kwargs|. + Raises: + AttributeError if the wrapped objects aren't callable. + """ + r = super(SyncParallelizer, self).__call__(*args, **kwargs) + r.pFinish(None) + return r + + # override + def pMap(self, f, *args, **kwargs): + """Map a function across the current wrapped objects in parallel. + + This calls f(o, *args, **kwargs) for each o in the set of wrapped objects. + + Note that this call is synchronous. + + Args: + f: The function to call. + args: The positional args to pass to f. + kwargs: The keyword args to pass to f. + Returns: + A Parallelizer wrapping the ReraiserThreadGroup running the map in + parallel. + """ + r = super(SyncParallelizer, self).pMap(f, *args, **kwargs) + r.pFinish(None) + return r + diff --git a/catapult/devil/devil/utils/parallelizer_test.py b/catapult/devil/devil/utils/parallelizer_test.py new file mode 100644 index 00000000..3162a4f5 --- /dev/null +++ b/catapult/devil/devil/utils/parallelizer_test.py @@ -0,0 +1,166 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unit tests for the contents of parallelizer.py.""" + +# pylint: disable=W0212 +# pylint: disable=W0613 + +import os +import tempfile +import time +import unittest + +from devil.utils import parallelizer + + +class ParallelizerTestObject(object): + """Class used to test parallelizer.Parallelizer.""" + + parallel = parallelizer.Parallelizer + + def __init__(self, thing, completion_file_name=None): + self._thing = thing + self._completion_file_name = completion_file_name + self.helper = ParallelizerTestObjectHelper(thing) + + @staticmethod + def doReturn(what): + return what + + @classmethod + def doRaise(cls, what): + raise what + + def doSetTheThing(self, new_thing): + self._thing = new_thing + + def doReturnTheThing(self): + return self._thing + + def doRaiseTheThing(self): + raise self._thing + + def doRaiseIfExceptionElseSleepFor(self, sleep_duration): + if isinstance(self._thing, Exception): + raise self._thing + time.sleep(sleep_duration) + self._write_completion_file() + return self._thing + + def _write_completion_file(self): + if self._completion_file_name and len(self._completion_file_name): + with open(self._completion_file_name, 'w+b') as completion_file: + completion_file.write('complete') + + def __getitem__(self, index): + return self._thing[index] + + def __str__(self): + return type(self).__name__ + + +class ParallelizerTestObjectHelper(object): + + def __init__(self, thing): + self._thing = thing + + def doReturnStringThing(self): + return str(self._thing) + + +class ParallelizerTest(unittest.TestCase): + + def testInitWithNone(self): + with self.assertRaises(AssertionError): + parallelizer.Parallelizer(None) + + def testInitEmptyList(self): + with self.assertRaises(AssertionError): + parallelizer.Parallelizer([]) + + def testMethodCall(self): + test_data = ['abc_foo', 'def_foo', 'ghi_foo'] + expected = ['abc_bar', 'def_bar', 'ghi_bar'] + r = parallelizer.Parallelizer(test_data).replace('_foo', '_bar').pGet(0.1) + self.assertEquals(expected, r) + + def testMutate(self): + devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + self.assertTrue(all(d.doReturnTheThing() for d in devices)) + ParallelizerTestObject.parallel(devices).doSetTheThing(False).pFinish(1) + self.assertTrue(not any(d.doReturnTheThing() for d in devices)) + + def testAllReturn(self): + devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + results = ParallelizerTestObject.parallel( + devices).doReturnTheThing().pGet(1) + self.assertTrue(isinstance(results, list)) + self.assertEquals(10, len(results)) + self.assertTrue(all(results)) + + def testAllRaise(self): + devices = [ParallelizerTestObject(Exception('thing %d' % i)) + for i in xrange(0, 10)] + p = ParallelizerTestObject.parallel(devices).doRaiseTheThing() + with self.assertRaises(Exception): + p.pGet(1) + + def testOneFailOthersComplete(self): + parallel_device_count = 10 + exception_index = 7 + exception_msg = 'thing %d' % exception_index + + try: + completion_files = [tempfile.NamedTemporaryFile(delete=False) + for _ in xrange(0, parallel_device_count)] + devices = [ + ParallelizerTestObject( + i if i != exception_index else Exception(exception_msg), + completion_files[i].name) + for i in xrange(0, parallel_device_count)] + for f in completion_files: + f.close() + p = ParallelizerTestObject.parallel(devices) + with self.assertRaises(Exception) as e: + p.doRaiseIfExceptionElseSleepFor(2).pGet(3) + self.assertTrue(exception_msg in str(e.exception)) + for i in xrange(0, parallel_device_count): + with open(completion_files[i].name) as f: + if i == exception_index: + self.assertEquals('', f.read()) + else: + self.assertEquals('complete', f.read()) + finally: + for f in completion_files: + os.remove(f.name) + + def testReusable(self): + devices = [ParallelizerTestObject(True) for _ in xrange(0, 10)] + p = ParallelizerTestObject.parallel(devices) + results = p.doReturn(True).pGet(1) + self.assertTrue(all(results)) + results = p.doReturn(True).pGet(1) + self.assertTrue(all(results)) + with self.assertRaises(Exception): + results = p.doRaise(Exception('reusableTest')).pGet(1) + + def testContained(self): + devices = [ParallelizerTestObject(i) for i in xrange(0, 10)] + results = (ParallelizerTestObject.parallel(devices).helper + .doReturnStringThing().pGet(1)) + self.assertTrue(isinstance(results, list)) + self.assertEquals(10, len(results)) + for i in xrange(0, 10): + self.assertEquals(str(i), results[i]) + + def testGetItem(self): + devices = [ParallelizerTestObject(range(i, i + 10)) for i in xrange(0, 10)] + results = ParallelizerTestObject.parallel(devices)[9].pGet(1) + self.assertEquals(range(9, 19), results) + + +if __name__ == '__main__': + unittest.main(verbosity=2) + diff --git a/catapult/devil/devil/utils/reraiser_thread.py b/catapult/devil/devil/utils/reraiser_thread.py new file mode 100644 index 00000000..56d95f39 --- /dev/null +++ b/catapult/devil/devil/utils/reraiser_thread.py @@ -0,0 +1,228 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Thread and ThreadGroup that reraise exceptions on the main thread.""" +# pylint: disable=W0212 + +import logging +import sys +import threading +import time +import traceback + +from devil.utils import watchdog_timer + + +class TimeoutError(Exception): + """Module-specific timeout exception.""" + pass + + +def LogThreadStack(thread, error_log_func=logging.critical): + """Log the stack for the given thread. + + Args: + thread: a threading.Thread instance. + error_log_func: Logging function when logging errors. + """ + stack = sys._current_frames()[thread.ident] + error_log_func('*' * 80) + error_log_func('Stack dump for thread %r', thread.name) + error_log_func('*' * 80) + for filename, lineno, name, line in traceback.extract_stack(stack): + error_log_func('File: "%s", line %d, in %s', filename, lineno, name) + if line: + error_log_func(' %s', line.strip()) + error_log_func('*' * 80) + + +class ReraiserThread(threading.Thread): + """Thread class that can reraise exceptions.""" + + def __init__(self, func, args=None, kwargs=None, name=None): + """Initialize thread. + + Args: + func: callable to call on a new thread. + args: list of positional arguments for callable, defaults to empty. + kwargs: dictionary of keyword arguments for callable, defaults to empty. + name: thread name, defaults to Thread-N. + """ + if not name and func.__name__ != '<lambda>': + name = func.__name__ + super(ReraiserThread, self).__init__(name=name) + if not args: + args = [] + if not kwargs: + kwargs = {} + self.daemon = True + self._func = func + self._args = args + self._kwargs = kwargs + self._ret = None + self._exc_info = None + self._thread_group = None + + def ReraiseIfException(self): + """Reraise exception if an exception was raised in the thread.""" + if self._exc_info: + raise self._exc_info[0], self._exc_info[1], self._exc_info[2] + + def GetReturnValue(self): + """Reraise exception if present, otherwise get the return value.""" + self.ReraiseIfException() + return self._ret + + # override + def run(self): + """Overrides Thread.run() to add support for reraising exceptions.""" + try: + self._ret = self._func(*self._args, **self._kwargs) + except: # pylint: disable=W0702 + self._exc_info = sys.exc_info() + + +class ReraiserThreadGroup(object): + """A group of ReraiserThread objects.""" + + def __init__(self, threads=None): + """Initialize thread group. + + Args: + threads: a list of ReraiserThread objects; defaults to empty. + """ + self._threads = [] + # Set when a thread from one group has called JoinAll on another. It is used + # to detect when a there is a TimeoutRetryThread active that links to the + # current thread. + self.blocked_parent_thread_group = None + if threads: + for thread in threads: + self.Add(thread) + + def Add(self, thread): + """Add a thread to the group. + + Args: + thread: a ReraiserThread object. + """ + assert thread._thread_group is None + thread._thread_group = self + self._threads.append(thread) + + def StartAll(self, will_block=False): + """Start all threads. + + Args: + will_block: Whether the calling thread will subsequently block on this + thread group. Causes the active ReraiserThreadGroup (if there is one) + to be marked as blocking on this thread group. + """ + if will_block: + # Multiple threads blocking on the same outer thread should not happen in + # practice. + assert not self.blocked_parent_thread_group + self.blocked_parent_thread_group = CurrentThreadGroup() + for thread in self._threads: + thread.start() + + def _JoinAll(self, watcher=None, timeout=None): + """Join all threads without stack dumps. + + Reraises exceptions raised by the child threads and supports breaking + immediately on exceptions raised on the main thread. + + Args: + watcher: Watchdog object providing the thread timeout. If none is + provided, the thread will never be timed out. + timeout: An optional number of seconds to wait before timing out the join + operation. This will not time out the threads. + """ + if watcher is None: + watcher = watchdog_timer.WatchdogTimer(None) + alive_threads = self._threads[:] + end_time = (time.time() + timeout) if timeout else None + try: + while alive_threads and (end_time is None or end_time > time.time()): + for thread in alive_threads[:]: + if watcher.IsTimedOut(): + raise TimeoutError('Timed out waiting for %d of %d threads.' % + (len(alive_threads), len(self._threads))) + # Allow the main thread to periodically check for interrupts. + thread.join(0.1) + if not thread.isAlive(): + alive_threads.remove(thread) + # All threads are allowed to complete before reraising exceptions. + for thread in self._threads: + thread.ReraiseIfException() + finally: + self.blocked_parent_thread_group = None + + def IsAlive(self): + """Check whether any of the threads are still alive. + + Returns: + Whether any of the threads are still alive. + """ + return any(t.isAlive() for t in self._threads) + + def JoinAll(self, watcher=None, timeout=None, + error_log_func=logging.critical): + """Join all threads. + + Reraises exceptions raised by the child threads and supports breaking + immediately on exceptions raised on the main thread. Unfinished threads' + stacks will be logged on watchdog timeout. + + Args: + watcher: Watchdog object providing the thread timeout. If none is + provided, the thread will never be timed out. + timeout: An optional number of seconds to wait before timing out the join + operation. This will not time out the threads. + error_log_func: Logging function when logging errors. + """ + try: + self._JoinAll(watcher, timeout) + except TimeoutError: + error_log_func('Timed out. Dumping threads.') + for thread in (t for t in self._threads if t.isAlive()): + LogThreadStack(thread, error_log_func=error_log_func) + raise + + def GetAllReturnValues(self, watcher=None): + """Get all return values, joining all threads if necessary. + + Args: + watcher: same as in |JoinAll|. Only used if threads are alive. + """ + if any([t.isAlive() for t in self._threads]): + self.JoinAll(watcher) + return [t.GetReturnValue() for t in self._threads] + + +def CurrentThreadGroup(): + """Returns the ReraiserThreadGroup that owns the running thread. + + Returns: + The current thread group, otherwise None. + """ + current_thread = threading.current_thread() + if isinstance(current_thread, ReraiserThread): + return current_thread._thread_group # pylint: disable=no-member + return None + + +def RunAsync(funcs, watcher=None): + """Executes the given functions in parallel and returns their results. + + Args: + funcs: List of functions to perform on their own threads. + watcher: Watchdog object providing timeout, by default waits forever. + + Returns: + A list of return values in the order of the given functions. + """ + thread_group = ReraiserThreadGroup(ReraiserThread(f) for f in funcs) + thread_group.StartAll(will_block=True) + return thread_group.GetAllReturnValues(watcher=watcher) diff --git a/catapult/devil/devil/utils/reraiser_thread_unittest.py b/catapult/devil/devil/utils/reraiser_thread_unittest.py new file mode 100644 index 00000000..e3c4e6be --- /dev/null +++ b/catapult/devil/devil/utils/reraiser_thread_unittest.py @@ -0,0 +1,117 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unittests for reraiser_thread.py.""" + +import threading +import unittest + +from devil.utils import reraiser_thread +from devil.utils import watchdog_timer + + +class TestException(Exception): + pass + + +class TestReraiserThread(unittest.TestCase): + """Tests for reraiser_thread.ReraiserThread.""" + + def testNominal(self): + result = [None, None] + + def f(a, b=None): + result[0] = a + result[1] = b + + thread = reraiser_thread.ReraiserThread(f, [1], {'b': 2}) + thread.start() + thread.join() + self.assertEqual(result[0], 1) + self.assertEqual(result[1], 2) + + def testRaise(self): + def f(): + raise TestException + + thread = reraiser_thread.ReraiserThread(f) + thread.start() + thread.join() + with self.assertRaises(TestException): + thread.ReraiseIfException() + + +class TestReraiserThreadGroup(unittest.TestCase): + """Tests for reraiser_thread.ReraiserThreadGroup.""" + + def testInit(self): + ran = [False] * 5 + + def f(i): + ran[i] = True + + group = reraiser_thread.ReraiserThreadGroup( + [reraiser_thread.ReraiserThread(f, args=[i]) for i in range(5)]) + group.StartAll() + group.JoinAll() + for v in ran: + self.assertTrue(v) + + def testAdd(self): + ran = [False] * 5 + + def f(i): + ran[i] = True + + group = reraiser_thread.ReraiserThreadGroup() + for i in xrange(5): + group.Add(reraiser_thread.ReraiserThread(f, args=[i])) + group.StartAll() + group.JoinAll() + for v in ran: + self.assertTrue(v) + + def testJoinRaise(self): + def f(): + raise TestException + group = reraiser_thread.ReraiserThreadGroup( + [reraiser_thread.ReraiserThread(f) for _ in xrange(5)]) + group.StartAll() + with self.assertRaises(TestException): + group.JoinAll() + + def testJoinTimeout(self): + def f(): + pass + event = threading.Event() + + def g(): + event.wait() + group = reraiser_thread.ReraiserThreadGroup( + [reraiser_thread.ReraiserThread(g), + reraiser_thread.ReraiserThread(f)]) + group.StartAll() + with self.assertRaises(reraiser_thread.TimeoutError): + group.JoinAll(watchdog_timer.WatchdogTimer(0.01)) + event.set() + + +class TestRunAsync(unittest.TestCase): + """Tests for reraiser_thread.RunAsync.""" + + def testNoArgs(self): + results = reraiser_thread.RunAsync([]) + self.assertEqual([], results) + + def testOneArg(self): + results = reraiser_thread.RunAsync([lambda: 1]) + self.assertEqual([1], results) + + def testTwoArgs(self): + a, b = reraiser_thread.RunAsync((lambda: 1, lambda: 2)) + self.assertEqual(1, a) + self.assertEqual(2, b) + +if __name__ == '__main__': + unittest.main() diff --git a/catapult/devil/devil/utils/reset_usb.py b/catapult/devil/devil/utils/reset_usb.py new file mode 100755 index 00000000..3f3b30ac --- /dev/null +++ b/catapult/devil/devil/utils/reset_usb.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import fcntl +import logging +import re +import sys + +from devil.android import device_errors +from devil.utils import lsusb +from devil.utils import run_tests_helper + +_INDENTATION_RE = re.compile(r'^( *)') +_LSUSB_BUS_DEVICE_RE = re.compile(r'^Bus (\d{3}) Device (\d{3}):') +_LSUSB_ENTRY_RE = re.compile(r'^ *([^ ]+) +([^ ]+) *([^ ].*)?$') +_LSUSB_GROUP_RE = re.compile(r'^ *([^ ]+.*):$') + +_USBDEVFS_RESET = ord('U') << 8 | 20 + + +def reset_usb(bus, device): + """Reset the USB device with the given bus and device.""" + usb_file_path = '/dev/bus/usb/%03d/%03d' % (bus, device) + with open(usb_file_path, 'w') as usb_file: + logging.debug('fcntl.ioctl(%s, %d)', usb_file_path, _USBDEVFS_RESET) + fcntl.ioctl(usb_file, _USBDEVFS_RESET) + + +def reset_android_usb(serial): + """Reset the USB device for the given Android device.""" + lsusb_info = lsusb.lsusb() + + bus = None + device = None + for device_info in lsusb_info: + device_serial = lsusb.get_lsusb_serial(device_info) + if device_serial == serial: + bus = int(device_info.get('bus')) + device = int(device_info.get('device')) + + if bus and device: + reset_usb(bus, device) + else: + raise device_errors.DeviceUnreachableError( + 'Unable to determine bus or device for device %s' % serial) + + +def reset_all_android_devices(): + """Reset all USB devices that look like an Android device.""" + _reset_all_matching(lambda i: bool(lsusb.get_lsusb_serial(i))) + + +def _reset_all_matching(condition): + lsusb_info = lsusb.lsusb() + for device_info in lsusb_info: + if int(device_info.get('device')) != 1 and condition(device_info): + bus = int(device_info.get('bus')) + device = int(device_info.get('device')) + try: + reset_usb(bus, device) + serial = lsusb.get_lsusb_serial(device_info) + if serial: + logging.info('Reset USB device (bus: %03d, device: %03d, serial: %s)', + bus, device, serial) + else: + logging.info('Reset USB device (bus: %03d, device: %03d)', + bus, device) + except IOError: + logging.error( + 'Failed to reset USB device (bus: %03d, device: %03d)', + bus, device) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--verbose', action='count') + parser.add_argument('-s', '--serial') + parser.add_argument('--bus', type=int) + parser.add_argument('--device', type=int) + args = parser.parse_args() + + run_tests_helper.SetLogLevel(args.verbose) + + if args.serial: + reset_android_usb(args.serial) + elif args.bus and args.device: + reset_usb(args.bus, args.device) + else: + parser.error('Unable to determine target. ' + 'Specify --serial or BOTH --bus and --device.') + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/catapult/devil/devil/utils/run_tests_helper.py b/catapult/devil/devil/utils/run_tests_helper.py new file mode 100644 index 00000000..7df2da65 --- /dev/null +++ b/catapult/devil/devil/utils/run_tests_helper.py @@ -0,0 +1,44 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Helper functions common to native, java and host-driven test runners.""" + +import logging +import sys +import time + + +class CustomFormatter(logging.Formatter): + """Custom log formatter.""" + + # override + def __init__(self, fmt='%(threadName)-4s %(message)s'): + # Can't use super() because in older Python versions logging.Formatter does + # not inherit from object. + logging.Formatter.__init__(self, fmt=fmt) + self._creation_time = time.time() + + # override + def format(self, record): + # Can't use super() because in older Python versions logging.Formatter does + # not inherit from object. + msg = logging.Formatter.format(self, record) + if 'MainThread' in msg[:19]: + msg = msg.replace('MainThread', 'Main', 1) + timediff = time.time() - self._creation_time + return '%s %8.3fs %s' % (record.levelname[0], timediff, msg) + + +def SetLogLevel(verbose_count): + """Sets log level as |verbose_count|.""" + log_level = logging.WARNING # Default. + if verbose_count == 1: + log_level = logging.INFO + elif verbose_count >= 2: + log_level = logging.DEBUG + logger = logging.getLogger() + logger.setLevel(log_level) + custom_handler = logging.StreamHandler(sys.stdout) + custom_handler.setFormatter(CustomFormatter()) + logging.getLogger().addHandler(custom_handler) diff --git a/catapult/devil/devil/utils/timeout_retry.py b/catapult/devil/devil/utils/timeout_retry.py new file mode 100644 index 00000000..95e90ee5 --- /dev/null +++ b/catapult/devil/devil/utils/timeout_retry.py @@ -0,0 +1,181 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A utility to run functions with timeouts and retries.""" +# pylint: disable=W0702 + +import logging +import threading +import time +import traceback + +from devil.utils import reraiser_thread +from devil.utils import watchdog_timer + + +class TimeoutRetryThreadGroup(reraiser_thread.ReraiserThreadGroup): + + def __init__(self, timeout, threads=None): + super(TimeoutRetryThreadGroup, self).__init__(threads) + self._watcher = watchdog_timer.WatchdogTimer(timeout) + + def GetWatcher(self): + """Returns the watchdog keeping track of this thread's time.""" + return self._watcher + + def GetElapsedTime(self): + return self._watcher.GetElapsed() + + def GetRemainingTime(self, required=0, msg=None): + """Get the remaining time before the thread times out. + + Useful to send as the |timeout| parameter of async IO operations. + + Args: + required: minimum amount of time that will be required to complete, e.g., + some sleep or IO operation. + msg: error message to show if timing out. + + Returns: + The number of seconds remaining before the thread times out, or None + if the thread never times out. + + Raises: + reraiser_thread.TimeoutError if the remaining time is less than the + required time. + """ + remaining = self._watcher.GetRemaining() + if remaining is not None and remaining < required: + if msg is None: + msg = 'Timeout expired' + if remaining > 0: + msg += (', wait of %.1f secs required but only %.1f secs left' + % (required, remaining)) + raise reraiser_thread.TimeoutError(msg) + return remaining + + +def CurrentTimeoutThreadGroup(): + """Returns the thread group that owns or is blocked on the active thread. + + Returns: + Returns None if no TimeoutRetryThreadGroup is tracking the current thread. + """ + thread_group = reraiser_thread.CurrentThreadGroup() + while thread_group: + if isinstance(thread_group, TimeoutRetryThreadGroup): + return thread_group + thread_group = thread_group.blocked_parent_thread_group + return None + + +def WaitFor(condition, wait_period=5, max_tries=None): + """Wait for a condition to become true. + + Repeatedly call the function condition(), with no arguments, until it returns + a true value. + + If called within a TimeoutRetryThreadGroup, it cooperates nicely with it. + + Args: + condition: function with the condition to check + wait_period: number of seconds to wait before retrying to check the + condition + max_tries: maximum number of checks to make, the default tries forever + or until the TimeoutRetryThreadGroup expires. + + Returns: + The true value returned by the condition, or None if the condition was + not met after max_tries. + + Raises: + reraiser_thread.TimeoutError: if the current thread is a + TimeoutRetryThreadGroup and the timeout expires. + """ + condition_name = condition.__name__ + timeout_thread_group = CurrentTimeoutThreadGroup() + while max_tries is None or max_tries > 0: + result = condition() + if max_tries is not None: + max_tries -= 1 + msg = ['condition', repr(condition_name), 'met' if result else 'not met'] + if timeout_thread_group: + # pylint: disable=no-member + msg.append('(%.1fs)' % timeout_thread_group.GetElapsedTime()) + logging.info(' '.join(msg)) + if result: + return result + if timeout_thread_group: + # pylint: disable=no-member + timeout_thread_group.GetRemainingTime(wait_period, + msg='Timed out waiting for %r' % condition_name) + time.sleep(wait_period) + return None + + +def _LogLastException(thread_name, attempt, max_attempts, log_func): + log_func('*' * 80) + log_func('Exception on thread %s (attempt %d of %d)', thread_name, + attempt, max_attempts) + log_func('*' * 80) + fmt_exc = ''.join(traceback.format_exc()) + for line in fmt_exc.splitlines(): + log_func(line.rstrip()) + log_func('*' * 80) + + +def AlwaysRetry(_exception): + return True + + +def Run(func, timeout, retries, args=None, kwargs=None, desc=None, + error_log_func=logging.critical, retry_if_func=AlwaysRetry): + """Runs the passed function in a separate thread with timeouts and retries. + + Args: + func: the function to be wrapped. + timeout: the timeout in seconds for each try. + retries: the number of retries. + args: list of positional args to pass to |func|. + kwargs: dictionary of keyword args to pass to |func|. + desc: An optional description of |func| used in logging. If omitted, + |func.__name__| will be used. + error_log_func: Logging function when logging errors. + retry_if_func: Unary callable that takes an exception and returns + whether |func| should be retried. Defaults to always retrying. + + Returns: + The return value of func(*args, **kwargs). + """ + if not args: + args = [] + if not kwargs: + kwargs = {} + + num_try = 1 + while True: + thread_name = 'TimeoutThread-%d-for-%s' % (num_try, + threading.current_thread().name) + child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), + name=thread_name) + try: + thread_group = TimeoutRetryThreadGroup(timeout, threads=[child_thread]) + thread_group.StartAll(will_block=True) + while True: + thread_group.JoinAll(watcher=thread_group.GetWatcher(), timeout=60, + error_log_func=error_log_func) + if thread_group.IsAlive(): + logging.info('Still working on %s', desc if desc else func.__name__) + else: + return thread_group.GetAllReturnValues()[0] + except reraiser_thread.TimeoutError as e: + # Timeouts already get their stacks logged. + if num_try > retries or not retry_if_func(e): + raise + # Do not catch KeyboardInterrupt. + except Exception as e: # pylint: disable=broad-except + if num_try > retries or not retry_if_func(e): + raise + _LogLastException(thread_name, num_try, retries + 1, error_log_func) + num_try += 1 diff --git a/catapult/devil/devil/utils/timeout_retry_unittest.py b/catapult/devil/devil/utils/timeout_retry_unittest.py new file mode 100755 index 00000000..84982889 --- /dev/null +++ b/catapult/devil/devil/utils/timeout_retry_unittest.py @@ -0,0 +1,79 @@ +#!/usr/bin/python +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unittests for timeout_and_retry.py.""" + +import logging +import time +import unittest + +from devil.utils import reraiser_thread +from devil.utils import timeout_retry + + +_DEFAULT_TIMEOUT = .1 + + +class TestException(Exception): + pass + + +def _CountTries(tries): + tries[0] += 1 + raise TestException + + +class TestRun(unittest.TestCase): + """Tests for timeout_retry.Run.""" + + def testRun(self): + self.assertTrue(timeout_retry.Run( + lambda x: x, 30, 3, [True], {})) + + def testTimeout(self): + tries = [0] + + def _sleep(): + tries[0] += 1 + time.sleep(1) + + self.assertRaises( + reraiser_thread.TimeoutError, timeout_retry.Run, _sleep, .0001, 1, + error_log_func=logging.debug) + self.assertEqual(tries[0], 2) + + def testRetries(self): + tries = [0] + self.assertRaises( + TestException, timeout_retry.Run, lambda: _CountTries(tries), + _DEFAULT_TIMEOUT, 3, error_log_func=logging.debug) + self.assertEqual(tries[0], 4) + + def testNoRetries(self): + tries = [0] + self.assertRaises( + TestException, timeout_retry.Run, lambda: _CountTries(tries), + _DEFAULT_TIMEOUT, 0, error_log_func=logging.debug) + self.assertEqual(tries[0], 1) + + def testReturnValue(self): + self.assertTrue(timeout_retry.Run(lambda: True, _DEFAULT_TIMEOUT, 3)) + + def testCurrentTimeoutThreadGroup(self): + def InnerFunc(): + current_thread_group = timeout_retry.CurrentTimeoutThreadGroup() + self.assertIsNotNone(current_thread_group) + + def InnerInnerFunc(): + self.assertEqual(current_thread_group, + timeout_retry.CurrentTimeoutThreadGroup()) + return True + return reraiser_thread.RunAsync((InnerInnerFunc,))[0] + + self.assertTrue(timeout_retry.Run(InnerFunc, _DEFAULT_TIMEOUT, 3)) + + +if __name__ == '__main__': + unittest.main() diff --git a/catapult/devil/devil/utils/watchdog_timer.py b/catapult/devil/devil/utils/watchdog_timer.py new file mode 100644 index 00000000..2f4c4645 --- /dev/null +++ b/catapult/devil/devil/utils/watchdog_timer.py @@ -0,0 +1,47 @@ +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""WatchdogTimer timeout objects.""" + +import time + + +class WatchdogTimer(object): + """A resetable timeout-based watchdog. + + This object is threadsafe. + """ + + def __init__(self, timeout): + """Initializes the watchdog. + + Args: + timeout: The timeout in seconds. If timeout is None it will never timeout. + """ + self._start_time = time.time() + self._timeout = timeout + + def Reset(self): + """Resets the timeout countdown.""" + self._start_time = time.time() + + def GetElapsed(self): + """Returns the elapsed time of the watchdog.""" + return time.time() - self._start_time + + def GetRemaining(self): + """Returns the remaining time of the watchdog.""" + if self._timeout: + return self._timeout - self.GetElapsed() + else: + return None + + def IsTimedOut(self): + """Whether the watchdog has timed out. + + Returns: + True if the watchdog has timed out, False otherwise. + """ + remaining = self.GetRemaining() + return remaining is not None and remaining < 0 diff --git a/catapult/devil/devil/utils/zip_utils.py b/catapult/devil/devil/utils/zip_utils.py new file mode 100644 index 00000000..d799463f --- /dev/null +++ b/catapult/devil/devil/utils/zip_utils.py @@ -0,0 +1,31 @@ +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import logging +import os +import zipfile + + +def WriteToZipFile(zip_file, path, arc_path): + """Recursively write |path| to |zip_file| as |arc_path|. + + zip_file: An open instance of zipfile.ZipFile. + path: An absolute path to the file or directory to be zipped. + arc_path: A relative path within the zip file to which the file or directory + located at |path| should be written. + """ + if os.path.isdir(path): + for dir_path, _, file_names in os.walk(path): + dir_arc_path = os.path.join(arc_path, os.path.relpath(dir_path, path)) + logging.debug('dir: %s -> %s', dir_path, dir_arc_path) + zip_file.write(dir_path, dir_arc_path, zipfile.ZIP_STORED) + for f in file_names: + file_path = os.path.join(dir_path, f) + file_arc_path = os.path.join(dir_arc_path, f) + logging.debug('file: %s -> %s', file_path, file_arc_path) + zip_file.write(file_path, file_arc_path, zipfile.ZIP_DEFLATED) + else: + logging.debug('file: %s -> %s', path, arc_path) + zip_file.write(path, arc_path, zipfile.ZIP_DEFLATED) + diff --git a/catapult/devil/pylintrc b/catapult/devil/pylintrc new file mode 100644 index 00000000..7e024a25 --- /dev/null +++ b/catapult/devil/pylintrc @@ -0,0 +1,68 @@ +[MESSAGES CONTROL] + +# Disable the message, report, category or checker with the given id(s). +# TODO: Shrink this list to as small as possible. +disable= + design, + similarities, + + bad-continuation, + fixme, + import-error, + invalid-name, + locally-disabled, + locally-enabled, + missing-docstring, + star-args, + + +[REPORTS] + +# Don't write out full reports, just messages. +reports=no + + +[BASIC] + +# Regular expression which should only match correct function names. +function-rgx=^(?:(?P<exempt>setUp|tearDown|setUpModule|tearDownModule)|(?P<camel_case>_?[A-Z][a-zA-Z0-9]*))$ + +# Regular expression which should only match correct method names. +method-rgx=^(?:(?P<exempt>_[a-z0-9_]+__|get|post|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass)|(?P<camel_case>(_{0,2}|test|assert)[A-Z][a-zA-Z0-9_]*))$ + +# Regular expression which should only match correct argument names. +argument-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression which should only match correct variable names. +variable-rgx=^[a-z][a-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma. +good-names=main,_ + +# List of builtins function names that should not be used, separated by a comma. +bad-functions=apply,input,reduce + + +[VARIABLES] + +# Tells wether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching names used for dummy variables (i.e. not used). +dummy-variables-rgx=^_.*$|dummy + + +[TYPECHECK] + +# Tells wether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + + +[FORMAT] + +# Maximum number of lines in a module. +max-module-lines=10000 + +# We use two spaces for indents, instead of the usual four spaces or tab. +indent-string=' ' |