diff options
author | Chris Craik <ccraik@google.com> | 2016-08-19 14:42:29 -0700 |
---|---|---|
committer | Chris Craik <ccraik@google.com> | 2016-08-22 12:30:46 -0700 |
commit | a23c9e9f6fc22fe5611def685e1984062b13b560 (patch) | |
tree | c91891dae0ff370a6d8952977fefb8197ed53f18 /catapult/devil | |
parent | cf3bd4e119057b93aabc136f2a2f3199aa24f1ad (diff) | |
download | chromium-trace-a23c9e9f6fc22fe5611def685e1984062b13b560.tar.gz |
Update to latest catapult (1ff7619f)
bug:21925298
bug:29643805
bug:30299278
bug:30397774
bug:30953297
Change-Id: I4d73c3c5454541a50703253ebeb63e5b2ea01fec
Diffstat (limited to 'catapult/devil')
45 files changed, 2984 insertions, 600 deletions
diff --git a/catapult/devil/bin/run_py_devicetests b/catapult/devil/bin/run_py_devicetests new file mode 100755 index 00000000..c23839f3 --- /dev/null +++ b/catapult/devil/bin/run_py_devicetests @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import sys + +_CATAPULT_PATH = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', '..')) +_DEVIL_PATH = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..')) + +sys.path.append(_CATAPULT_PATH) +from catapult_build import run_with_typ + + +def main(): + sys.argv.extend(['--suffixes', '*_devicetest.py', '-j', '1']) + return run_with_typ.Run(top_level_dir=_DEVIL_PATH) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/catapult/devil/devil/android/app_ui.py b/catapult/devil/devil/android/app_ui.py index d5025f40..2b04e8b8 100644 --- a/catapult/devil/devil/android/app_ui.py +++ b/catapult/devil/devil/android/app_ui.py @@ -4,6 +4,7 @@ """Provides functionality to interact with UI elements of an Android app.""" +import collections import re from xml.etree import ElementTree as element_tree @@ -75,6 +76,27 @@ class _UiNode(object): x, y = (str(int(v)) for v in point) self._device.RunShellCommand(['input', 'tap', x, y], check_return=True) + def Dump(self): + """Get a brief summary of the child nodes that can be found on this node. + + Returns: + A list of lines that can be logged or otherwise printed. + """ + summary = collections.defaultdict(set) + for node in self._xml_node.iter(): + package = node.get('package') or '(no package)' + label = node.get('resource-id') or '(no id)' + text = node.get('text') + if text: + label = '%s[%r]' % (label, text) + summary[package].add(label) + lines = [] + for package, labels in sorted(summary.iteritems()): + lines.append('- %s:' % package) + for label in sorted(labels): + lines.append(' - %s' % label) + return lines + def __getitem__(self, key): """Retrieve a child of this node by its index. @@ -181,6 +203,14 @@ class AppUi(object): self._device.ReadFile(dtemp.name, force_pull=True)) return _UiNode(self._device, xml_node, package=self._package) + def ScreenDump(self): + """Get a brief summary of the nodes that can be found on the screen. + + Returns: + A list of lines that can be logged or otherwise printed. + """ + return self._GetRootUiNode().Dump() + def GetUiNode(self, **kwargs): """Get the first node found matching a specified criteria. diff --git a/catapult/devil/devil/android/battery_utils.py b/catapult/devil/devil/android/battery_utils.py index 4c8f5431..7a735f35 100644 --- a/catapult/devil/devil/android/battery_utils.py +++ b/catapult/devil/devil/android/battery_utils.py @@ -95,6 +95,19 @@ _DEVICE_PROFILES = [ 'current': '/sys/class/power_supply/ds2784-fuelgauge/current_now', }, + { + 'name': 'Nexus 5X', + 'witness_file': None, + 'enable_command': ( + 'echo 1 > /sys/class/power_supply/battery/charging_enabled && ' + 'dumpsys battery reset'), + 'disable_command': ( + 'echo 0 > /sys/class/power_supply/battery/charging_enabled && ' + 'dumpsys battery set ac 0 && dumpsys battery set usb 0'), + 'charge_counter': None, + 'voltage': None, + 'current': None, + }, ] # The list of useful dumpsys columns. @@ -253,7 +266,7 @@ class BatteryUtils(object): if entry[_DUMP_VERSION_INDEX] not in ['8', '9']: # Wrong dumpsys version. raise device_errors.DeviceVersionError( - 'Dumpsys version must be 8 or 9. %s found.' + 'Dumpsys version must be 8 or 9. "%s" found.' % entry[_DUMP_VERSION_INDEX]) if _ROW_TYPE_INDEX < len(entry) and entry[_ROW_TYPE_INDEX] == 'uid': current_package = entry[_PACKAGE_NAME_INDEX] @@ -608,22 +621,27 @@ class BatteryUtils(object): ['dumpsys', 'battery', 'set', 'usb', '1'], check_return=True) self._device.RunShellCommand( ['dumpsys', 'battery', 'set', 'ac', '1'], check_return=True) - self._device.RunShellCommand( - ['dumpsys', 'batterystats', '--reset'], check_return=True) - battery_data = self._device.RunShellCommand( - ['dumpsys', 'batterystats', '--charged', '-c'], - check_return=True, large_output=True) - for line in battery_data: - l = line.split(',') - if (len(l) > _PWI_POWER_CONSUMPTION_INDEX and l[_ROW_TYPE_INDEX] == 'pwi' - and l[_PWI_POWER_CONSUMPTION_INDEX] != 0): - self._device.RunShellCommand( - ['dumpsys', 'battery', 'reset'], check_return=True) - raise device_errors.CommandFailedError( - 'Non-zero pmi value found after reset.') - self._device.RunShellCommand( - ['dumpsys', 'battery', 'reset'], check_return=True) - return True + + def test_if_clear(): + self._device.RunShellCommand( + ['dumpsys', 'batterystats', '--reset'], check_return=True) + battery_data = self._device.RunShellCommand( + ['dumpsys', 'batterystats', '--charged', '-c'], + check_return=True, large_output=True) + for line in battery_data: + l = line.split(',') + if (len(l) > _PWI_POWER_CONSUMPTION_INDEX + and l[_ROW_TYPE_INDEX] == 'pwi' + and float(l[_PWI_POWER_CONSUMPTION_INDEX]) != 0.0): + return False + return True + + try: + timeout_retry.WaitFor(test_if_clear, wait_period=1) + return True + finally: + self._device.RunShellCommand( + ['dumpsys', 'battery', 'reset'], check_return=True) def _DiscoverDeviceProfile(self): """Checks and caches device information. diff --git a/catapult/devil/devil/android/battery_utils_test.py b/catapult/devil/devil/android/battery_utils_test.py index 79939217..9fbd1276 100755 --- a/catapult/devil/devil/android/battery_utils_test.py +++ b/catapult/devil/devil/android/battery_utils_test.py @@ -650,6 +650,7 @@ class BatteryUtilsClearPowerData(BatteryUtilsTest): ['dumpsys', 'battery', 'reset'], check_return=True), [])): self.assertTrue(self.battery._ClearPowerData()) + @mock.patch('time.sleep', mock.Mock()) def testClearPowerData_notClearedL(self): with self.patch_call(self.call.device.build_version_sdk, return_value=22): @@ -666,9 +667,26 @@ class BatteryUtilsClearPowerData(BatteryUtilsTest): check_return=True, large_output=True), ['9,1000,l,pwi,uid,0.0327']), (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '--reset'], check_return=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '--charged', '-c'], + check_return=True, large_output=True), + ['9,1000,l,pwi,uid,0.0327']), + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '--reset'], check_return=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '--charged', '-c'], + check_return=True, large_output=True), + ['9,1000,l,pwi,uid,0.0327']), + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '--reset'], check_return=True), []), + (self.call.device.RunShellCommand( + ['dumpsys', 'batterystats', '--charged', '-c'], + check_return=True, large_output=True), + ['9,1000,l,pwi,uid,0.0']), + (self.call.device.RunShellCommand( ['dumpsys', 'battery', 'reset'], check_return=True), [])): - with self.assertRaises(device_errors.CommandFailedError): - self.battery._ClearPowerData() + self.battery._ClearPowerData() if __name__ == '__main__': diff --git a/catapult/devil/devil/android/constants/chrome.py b/catapult/devil/devil/android/constants/chrome.py index 5190ff93..006764f2 100644 --- a/catapult/devil/devil/android/constants/chrome.py +++ b/catapult/devil/devil/android/constants/chrome.py @@ -6,55 +6,52 @@ import collections PackageInfo = collections.namedtuple( 'PackageInfo', - ['package', 'activity', 'cmdline_file', 'devtools_socket', 'test_package']) + ['package', 'activity', 'cmdline_file', 'devtools_socket']) PACKAGE_INFO = { 'chrome_document': PackageInfo( 'com.google.android.apps.chrome.document', 'com.google.android.apps.chrome.document.ChromeLauncherActivity', '/data/local/chrome-command-line', - 'chrome_devtools_remote', - None), + 'chrome_devtools_remote'), 'chrome': PackageInfo( 'com.google.android.apps.chrome', 'com.google.android.apps.chrome.Main', '/data/local/chrome-command-line', - 'chrome_devtools_remote', - 'com.google.android.apps.chrome.tests'), + 'chrome_devtools_remote'), 'chrome_beta': PackageInfo( 'com.chrome.beta', 'com.google.android.apps.chrome.Main', '/data/local/chrome-command-line', - 'chrome_devtools_remote', - None), + 'chrome_devtools_remote'), 'chrome_stable': PackageInfo( 'com.android.chrome', 'com.google.android.apps.chrome.Main', '/data/local/chrome-command-line', - 'chrome_devtools_remote', - None), + 'chrome_devtools_remote'), 'chrome_dev': PackageInfo( 'com.chrome.dev', 'com.google.android.apps.chrome.Main', '/data/local/chrome-command-line', - 'chrome_devtools_remote', - None), + 'chrome_devtools_remote'), 'chrome_canary': PackageInfo( 'com.chrome.canary', 'com.google.android.apps.chrome.Main', '/data/local/chrome-command-line', - 'chrome_devtools_remote', - None), + 'chrome_devtools_remote'), 'chrome_work': PackageInfo( 'com.chrome.work', 'com.google.android.apps.chrome.Main', '/data/local/chrome-command-line', - 'chrome_devtools_remote', - None), + 'chrome_devtools_remote'), 'chromium': PackageInfo( 'org.chromium.chrome', 'com.google.android.apps.chrome.Main', '/data/local/chrome-command-line', - 'chrome_devtools_remote', - 'org.chromium.chrome.tests'), + 'chrome_devtools_remote'), + 'content_shell': PackageInfo( + 'org.chromium.content_shell_apk', + '.ContentShellActivity', + '/data/local/tmp/content-shell-command-line', + 'content_shell_devtools_remote'), } diff --git a/catapult/devil/devil/android/device_blacklist.py b/catapult/devil/devil/android/device_blacklist.py index 94f9cbec..e8055a77 100644 --- a/catapult/devil/devil/android/device_blacklist.py +++ b/catapult/devil/devil/android/device_blacklist.py @@ -22,15 +22,22 @@ class Blacklist(object): A dict containing bad devices. """ with self._blacklist_lock: + blacklist = dict() if not os.path.exists(self._path): - return dict() + return blacklist + + try: + with open(self._path, 'r') as f: + blacklist = json.load(f) + except (IOError, ValueError) as e: + logging.warning('Unable to read blacklist: %s', str(e)) + os.remove(self._path) - with open(self._path, 'r') as f: - blacklist = json.load(f) if not isinstance(blacklist, dict): logging.warning('Ignoring %s: %s (a dict was expected instead)', self._path, blacklist) blacklist = dict() + return blacklist def Write(self, blacklist): diff --git a/catapult/devil/devil/android/device_blacklist_test.py b/catapult/devil/devil/android/device_blacklist_test.py new file mode 100644 index 00000000..bc44da55 --- /dev/null +++ b/catapult/devil/devil/android/device_blacklist_test.py @@ -0,0 +1,38 @@ +#! /usr/bin/env python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import tempfile +import unittest + +from devil.android import device_blacklist + + +class DeviceBlacklistTest(unittest.TestCase): + + def testBlacklistFileDoesNotExist(self): + with tempfile.NamedTemporaryFile() as blacklist_file: + # Allow the temporary file to be deleted. + pass + + test_blacklist = device_blacklist.Blacklist(blacklist_file.name) + self.assertEquals({}, test_blacklist.Read()) + + def testBlacklistFileIsEmpty(self): + try: + with tempfile.NamedTemporaryFile(delete=False) as blacklist_file: + # Allow the temporary file to be closed. + pass + + test_blacklist = device_blacklist.Blacklist(blacklist_file.name) + self.assertEquals({}, test_blacklist.Read()) + + finally: + if os.path.exists(blacklist_file.name): + os.remove(blacklist_file.name) + + +if __name__ == '__main__': + unittest.main() diff --git a/catapult/devil/devil/android/device_errors.py b/catapult/devil/devil/android/device_errors.py index b1b8890f..67faa488 100644 --- a/catapult/devil/devil/android/device_errors.py +++ b/catapult/devil/devil/android/device_errors.py @@ -8,6 +8,7 @@ Exception classes raised by AdbWrapper and DeviceUtils. from devil import base_error from devil.utils import cmd_helper +from devil.utils import parallelizer class CommandFailedError(base_error.BaseError): @@ -109,6 +110,21 @@ class NoDevicesError(base_error.BaseError): 'No devices attached.', is_infra_error=True) +class MultipleDevicesError(base_error.BaseError): + """Exception for having multiple attached devices without selecting one.""" + + def __init__(self, devices): + parallel_devices = parallelizer.Parallelizer(devices) + descriptions = parallel_devices.pMap( + lambda d: d.build_description).pGet(None) + msg = ('More than one device available. Use -d/--device to select a device ' + 'by serial.\n\nAvailable devices:\n') + for d, desc in zip(devices, descriptions): + msg += ' %s (%s)\n' % (d, desc) + + super(MultipleDevicesError, self).__init__(msg, is_infra_error=True) + + class NoAdbError(base_error.BaseError): """Exception for being unable to find ADB.""" diff --git a/catapult/devil/devil/android/device_list.py b/catapult/devil/devil/android/device_list.py index 0eb6acba..36a03b16 100644 --- a/catapult/devil/devil/android/device_list.py +++ b/catapult/devil/devil/android/device_list.py @@ -4,11 +4,10 @@ """A module to keep track of devices across builds.""" +import json +import logging import os -LAST_DEVICES_FILENAME = '.last_devices' -LAST_MISSING_DEVICES_FILENAME = '.last_missing' - def GetPersistentDeviceList(file_name): """Returns a list of devices. @@ -18,13 +17,34 @@ def GetPersistentDeviceList(file_name): Returns: List of device serial numbers that were on the bot. """ + if not os.path.isfile(file_name): + logging.warning('Device file %s doesn\'t exist.', file_name) + return [] + + try: + with open(file_name) as f: + devices = json.load(f) + if not isinstance(devices, list) or not all(isinstance(d, basestring) + for d in devices): + logging.warning('Unrecognized device file format: %s', devices) + return [] + return [d for d in devices if d != '(error)'] + except ValueError: + logging.exception( + 'Error reading device file %s. Falling back to old format.', file_name) + + # TODO(bpastene) Remove support for old unstructured file format. with open(file_name) as f: - return f.read().splitlines() + return [d for d in f.read().splitlines() if d != '(error)'] def WritePersistentDeviceList(file_name, device_list): path = os.path.dirname(file_name) + assert isinstance(device_list, list) + # If there is a problem with ADB "(error)" can be added to the device list. + # These should be removed before saving. + device_list = [d for d in device_list if d != '(error)'] if not os.path.exists(path): os.makedirs(path) with open(file_name, 'w') as f: - f.write('\n'.join(set(device_list))) + json.dump(device_list, f) diff --git a/catapult/devil/devil/android/device_utils.py b/catapult/devil/devil/android/device_utils.py index 5cea40ba..93aaaf92 100644 --- a/catapult/devil/devil/android/device_utils.py +++ b/catapult/devil/devil/android/device_utils.py @@ -8,6 +8,7 @@ Eventually, this will be based on adb_wrapper. """ # pylint: disable=unused-argument +import calendar import collections import itertools import json @@ -17,8 +18,11 @@ import os import posixpath import re import shutil +import stat import tempfile import time +import threading +import uuid import zipfile from devil import base_error @@ -32,6 +36,7 @@ from devil.android import device_temp_file from devil.android import install_commands from devil.android import logcat_monitor from devil.android import md5sum +from devil.android.constants import chrome from devil.android.sdk import adb_wrapper from devil.android.sdk import gce_adb_wrapper from devil.android.sdk import intent @@ -89,16 +94,15 @@ _PERMISSIONS_BLACKLIST = [ 'com.android.browser.permission.WRITE_HISTORY_BOOKMARKS', 'com.android.launcher.permission.INSTALL_SHORTCUT', 'com.chrome.permission.DEVICE_EXTRAS', - 'com.google.android.apps.chrome.permission.C2D_MESSAGE', - 'com.google.android.apps.chrome.permission.READ_WRITE_BOOKMARK_FOLDERS', - 'com.google.android.apps.chrome.TOS_ACKED', 'com.google.android.c2dm.permission.RECEIVE', 'com.google.android.providers.gsf.permission.READ_GSERVICES', 'com.sec.enterprise.knox.MDM_CONTENT_PROVIDER', - 'org.chromium.chrome.permission.C2D_MESSAGE', - 'org.chromium.chrome.permission.READ_WRITE_BOOKMARK_FOLDERS', - 'org.chromium.chrome.TOS_ACKED', ] +for package_info in chrome.PACKAGE_INFO.itervalues(): + _PERMISSIONS_BLACKLIST.extend([ + '%s.permission.C2D_MESSAGE' % package_info.package, + '%s.permission.READ_WRITE_BOOKMARK_FOLDERS' % package_info.package, + '%s.TOS_ACKED' % package_info.package]) _CURRENT_FOCUS_CRASH_RE = re.compile( r'\s*mCurrentFocus.*Application (Error|Not Responding): (\S+)}') @@ -106,6 +110,40 @@ _CURRENT_FOCUS_CRASH_RE = re.compile( _GETPROP_RE = re.compile(r'\[(.*?)\]: \[(.*?)\]') _IPV4_ADDRESS_RE = re.compile(r'([0-9]{1,3}\.){3}[0-9]{1,3}\:[0-9]{4,5}') +# Regex to parse the long (-l) output of 'ls' command, c.f. +# https://github.com/landley/toybox/blob/master/toys/posix/ls.c#L446 +_LONG_LS_OUTPUT_RE = re.compile( + r'(?P<st_mode>[\w-]{10})\s+' # File permissions + r'(?:(?P<st_nlink>\d+)\s+)?' # Number of links (optional) + r'(?P<st_owner>\w+)\s+' # Name of owner + r'(?P<st_group>\w+)\s+' # Group of owner + r'(?:' # Either ... + r'(?P<st_rdev_major>\d+),\s+' # Device major, and + r'(?P<st_rdev_minor>\d+)\s+' # Device minor + r'|' # .. or + r'(?P<st_size>\d+)\s+' # Size in bytes + r')?' # .. or nothing + r'(?P<st_mtime>\d{4}-\d\d-\d\d \d\d:\d\d)\s+' # Modification date/time + r'(?P<filename>.+?)' # File name + r'(?: -> (?P<symbolic_link_to>.+))?' # Symbolic link (optional) + r'$' # End of string +) +_LS_DATE_FORMAT = '%Y-%m-%d %H:%M' +_FILE_MODE_RE = re.compile(r'[dbclps-](?:[r-][w-][xSs-]){2}[r-][w-][xTt-]$') +_FILE_MODE_KIND = { + 'd': stat.S_IFDIR, 'b': stat.S_IFBLK, 'c': stat.S_IFCHR, + 'l': stat.S_IFLNK, 'p': stat.S_IFIFO, 's': stat.S_IFSOCK, + '-': stat.S_IFREG} +_FILE_MODE_PERMS = [ + stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR, + stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, + stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH, +] +_FILE_MODE_SPECIAL = [ + ('s', stat.S_ISUID), + ('s', stat.S_ISGID), + ('t', stat.S_ISVTX), +] @decorators.WithExplicitTimeoutAndRetries( _DEFAULT_TIMEOUT, _DEFAULT_RETRIES) @@ -152,6 +190,24 @@ def RestartServer(): raise device_errors.CommandFailedError('Failed to start adb server') +def _ParseModeString(mode_str): + """Parse a mode string, e.g. 'drwxrwxrwx', into a st_mode value. + + Effectively the reverse of |mode_to_string| in, e.g.: + https://github.com/landley/toybox/blob/master/lib/lib.c#L896 + """ + if not _FILE_MODE_RE.match(mode_str): + raise ValueError('Unexpected file mode %r', mode_str) + mode = _FILE_MODE_KIND[mode_str[0]] + for c, flag in zip(mode_str[1:], _FILE_MODE_PERMS): + if c != '-' and c.islower(): + mode |= flag + for c, (t, flag) in zip(mode_str[3::3], _FILE_MODE_SPECIAL): + if c.lower() == t: + mode |= flag + return mode + + def _GetTimeStamp(): """Return a basic ISO 8601 time stamp with the current local time.""" return time.strftime('%Y%m%dT%H%M%S', time.localtime()) @@ -178,6 +234,18 @@ def _CreateAdbWrapper(device): return adb_wrapper.AdbWrapper(device) +def _FormatPartialOutputError(output): + lines = output.splitlines() if isinstance(output, basestring) else output + message = ['Partial output found:'] + if len(lines) > 11: + message.extend('- %s' % line for line in lines[:5]) + message.extend('<snip>') + message.extend('- %s' % line for line in lines[-5:]) + else: + message.extend('- %s' % line for line in lines) + return '\n'.join(message) + + class DeviceUtils(object): _MAX_ADB_COMMAND_LENGTH = 512 @@ -219,6 +287,7 @@ class DeviceUtils(object): self._enable_device_files_cache = enable_device_files_cache self._cache = {} self._client_caches = {} + self._cache_lock = threading.RLock() assert hasattr(self, decorators.DEFAULT_TIMEOUT_ATTR) assert hasattr(self, decorators.DEFAULT_RETRIES_ATTR) @@ -287,7 +356,7 @@ class DeviceUtils(object): DeviceUnreachableError on missing device. """ try: - self.RunShellCommand('ls /root', check_return=True) + self.RunShellCommand(['ls', '/root'], check_return=True) return True except device_errors.AdbCommandFailedError: return False @@ -379,17 +448,11 @@ class DeviceUtils(object): CommandTimeoutError on timeout. DeviceUnreachableError on missing device. """ - if 'external_storage' in self._cache: - return self._cache['external_storage'] - - value = self.RunShellCommand('echo $EXTERNAL_STORAGE', - single_line=True, - check_return=True) - if not value: + self._EnsureCacheInitialized() + if not self._cache['external_storage']: raise device_errors.CommandFailedError('$EXTERNAL_STORAGE is not set', str(self)) - self._cache['external_storage'] = value - return value + return self._cache['external_storage'] @decorators.WithTimeoutAndRetriesFromInstance() def GetApplicationPaths(self, package, timeout=None, retries=None): @@ -509,7 +572,10 @@ class DeviceUtils(object): return False def boot_completed(): - return self.GetProp('sys.boot_completed', cache=False) == '1' + try: + return self.GetProp('sys.boot_completed', cache=False) == '1' + except device_errors.CommandFailedError: + return False def wifi_enabled(): return 'Wi-Fi is enabled' in self.RunShellCommand(['dumpsys', 'wifi'], @@ -716,7 +782,7 @@ class DeviceUtils(object): @decorators.WithTimeoutAndRetriesFromInstance() def RunShellCommand(self, cmd, check_return=False, cwd=None, env=None, as_root=False, single_line=False, large_output=False, - timeout=None, retries=None): + raw_output=False, timeout=None, retries=None): """Run an ADB shell command. The command to run |cmd| should be a sequence of program arguments or else @@ -751,6 +817,8 @@ class DeviceUtils(object): expected. large_output: Uses a work-around for large shell command output. Without this large output will be truncated. + raw_output: Whether to only return the raw output + (no splitting into lines). timeout: timeout in seconds retries: number of retries @@ -808,7 +876,7 @@ class DeviceUtils(object): return handle_large_command(cmd) except device_errors.AdbCommandFailedError as exc: if exc.status is None: - logging.exception('No output found for %s', cmd) + logging.error(_FormatPartialOutputError(exc.output)) logging.warning('Attempting to run in large_output mode.') logging.warning('Use RunShellCommand(..., large_output=True) for ' 'shell commands that expect a lot of output.') @@ -827,8 +895,12 @@ class DeviceUtils(object): # "su -c sh -c" allows using shell features in |cmd| cmd = self._Su('sh -c %s' % cmd_helper.SingleQuote(cmd)) - output = handle_large_output(cmd, large_output).splitlines() + output = handle_large_output(cmd, large_output) + if raw_output: + return output + + output = output.splitlines() if single_line: if not output: return '' @@ -1130,13 +1202,14 @@ class DeviceUtils(object): all_changed_files += changed_files all_stale_files += stale_files cache_commit_funcs.append(cache_commit_func) - if (os.path.isdir(h) and changed_files and not up_to_date_files - and not stale_files): - missing_dirs.append(d) + if changed_files and not up_to_date_files and not stale_files: + if os.path.isdir(h): + missing_dirs.append(d) + else: + missing_dirs.append(posixpath.dirname(d)) if delete_device_stale and all_stale_files: - self.RunShellCommand(['rm', '-f'] + all_stale_files, - check_return=True) + self.RunShellCommand(['rm', '-f'] + all_stale_files, check_return=True) if all_changed_files: if missing_dirs: @@ -1154,11 +1227,12 @@ class DeviceUtils(object): track_stale: whether to bother looking for stale files (slower) Returns: - a three-element tuple + a four-element tuple 1st element: a list of (host_files_path, device_files_path) tuples to push 2nd element: a list of host_files_path that are up-to-date 3rd element: a list of stale files under device_path, or [] when track_stale == False + 4th element: a cache commit function. """ try: # Length calculations below assume no trailing /. @@ -1268,7 +1342,10 @@ class DeviceUtils(object): len(host_device_tuples), dir_file_count, dir_size, False) zip_duration = self._ApproximateDuration(1, 1, size, True) - if dir_push_duration < push_duration and dir_push_duration < zip_duration: + if (dir_push_duration < push_duration and dir_push_duration < zip_duration + # TODO(jbudorick): Resume directory pushing once clients have switched + # to 1.0.36-compatible syntax. + and False): self._PushChangedFilesIndividually(host_device_tuples) elif push_duration < zip_duration: self._PushChangedFilesIndividually(files) @@ -1435,10 +1512,6 @@ class DeviceUtils(object): if os.path.exists(d): shutil.rmtree(d) - _LS_RE = re.compile( - r'(?P<perms>\S+) (?:(?P<inodes>\d+) +)?(?P<owner>\S+) +(?P<group>\S+) +' - r'(?:(?P<size>\d+) +)?(?P<date>\S+) +(?P<time>\S+) +(?P<name>.+)$') - @decorators.WithTimeoutAndRetriesFromInstance() def ReadFile(self, device_path, as_root=False, force_pull=False, timeout=None, retries=None): @@ -1466,17 +1539,7 @@ class DeviceUtils(object): DeviceUnreachableError on missing device. """ def get_size(path): - # TODO(jbudorick): Implement a generic version of Stat() that handles - # as_root=True, then switch this implementation to use that. - ls_out = self.RunShellCommand(['ls', '-l', device_path], as_root=as_root, - check_return=True) - file_name = posixpath.basename(device_path) - for line in ls_out: - m = self._LS_RE.match(line) - if m and file_name == posixpath.basename(m.group('name')): - return int(m.group('size')) - logging.warning('Could not determine size of %s.', device_path) - return None + return self.FileSize(path, as_root=as_root) if (not force_pull and 0 < get_size(device_path) <= self._MAX_ADB_OUTPUT_LENGTH): @@ -1540,19 +1603,43 @@ class DeviceUtils(object): # If root is not needed, we can push directly to the desired location. self._WriteFileWithPush(device_path, contents) - @decorators.WithTimeoutAndRetriesFromInstance() - def Ls(self, device_path, timeout=None, retries=None): - """Lists the contents of a directory on the device. + def _ParseLongLsOutput(self, device_path, as_root=False, **kwargs): + """Run and scrape the output of 'ls -a -l' on a device directory.""" + device_path = posixpath.join(device_path, '') # Force trailing '/'. + output = self.RunShellCommand( + ['ls', '-a', '-l', device_path], as_root=as_root, + check_return=True, env={'TZ': 'utc'}, **kwargs) + if output and output[0].startswith('total '): + output.pop(0) # pylint: disable=maybe-no-member + + entries = [] + for line in output: + m = _LONG_LS_OUTPUT_RE.match(line) + if m: + if m.group('filename') not in ['.', '..']: + entries.append(m.groupdict()) + else: + logging.info('Skipping: %s', line) + + return entries + + def ListDirectory(self, device_path, as_root=False, **kwargs): + """List all files on a device directory. + + Mirroring os.listdir (and most client expectations) the resulting list + does not include the special entries '.' and '..' even if they are present + in the directory. Args: device_path: A string containing the path of the directory on the device to list. + as_root: A boolean indicating whether the to use root privileges to list + the directory contents. timeout: timeout in seconds retries: number of retries Returns: - A list of pairs (filename, stat) for each file found in the directory, - where the stat object has the properties: st_mode, st_size, and st_time. + A list of filenames for all entries contained in the directory. Raises: AdbCommandFailedError if |device_path| does not specify a valid and @@ -1560,33 +1647,120 @@ class DeviceUtils(object): CommandTimeoutError on timeout. DeviceUnreachableError on missing device. """ - return self.adb.Ls(device_path) + entries = self._ParseLongLsOutput(device_path, as_root=as_root, **kwargs) + return [d['filename'] for d in entries] - @decorators.WithTimeoutAndRetriesFromInstance() - def Stat(self, device_path, timeout=None, retries=None): + def StatDirectory(self, device_path, as_root=False, **kwargs): + """List file and stat info for all entries on a device directory. + + Implementation notes: this is currently implemented by parsing the output + of 'ls -a -l' on the device. Whether possible and convenient, we attempt to + make parsing strict and return values mirroring those of the standard |os| + and |stat| Python modules. + + Mirroring os.listdir (and most client expectations) the resulting list + does not include the special entries '.' and '..' even if they are present + in the directory. + + Args: + device_path: A string containing the path of the directory on the device + to list. + as_root: A boolean indicating whether the to use root privileges to list + the directory contents. + timeout: timeout in seconds + retries: number of retries + + Returns: + A list of dictionaries, each containing the following keys: + filename: A string with the file name. + st_mode: File permissions, use the stat module to interpret these. + st_nlink: Number of hard links (may be missing). + st_owner: A string with the user name of the owner. + st_group: A string with the group name of the owner. + st_rdev_pair: Device type as (major, minior) (only if inode device). + st_size: Size of file, in bytes (may be missing for non-regular files). + st_mtime: Time of most recent modification, in seconds since epoch + (although resolution is in minutes). + symbolic_link_to: If entry is a symbolic link, path where it points to; + missing otherwise. + + Raises: + AdbCommandFailedError if |device_path| does not specify a valid and + accessible directory in the device. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + entries = self._ParseLongLsOutput(device_path, as_root=as_root, **kwargs) + for d in entries: + for key, value in d.items(): + if value is None: + del d[key] # Remove missing fields. + d['st_mode'] = _ParseModeString(d['st_mode']) + d['st_mtime'] = calendar.timegm( + time.strptime(d['st_mtime'], _LS_DATE_FORMAT)) + for key in ['st_nlink', 'st_size', 'st_rdev_major', 'st_rdev_minor']: + if key in d: + d[key] = int(d[key]) + if 'st_rdev_major' in d and 'st_rdev_minor' in d: + d['st_rdev_pair'] = (d.pop('st_rdev_major'), d.pop('st_rdev_minor')) + return entries + + def StatPath(self, device_path, as_root=False, **kwargs): """Get the stat attributes of a file or directory on the device. Args: - device_path: A string containing the path of from which to get attributes - on the device. + device_path: A string containing the path of a file or directory from + which to get attributes. + as_root: A boolean indicating whether the to use root privileges to + access the file information. timeout: timeout in seconds retries: number of retries Returns: - A stat object with the properties: st_mode, st_size, and st_time + A dictionary with the stat info collected; see StatDirectory for details. Raises: CommandFailedError if device_path cannot be found on the device. CommandTimeoutError on timeout. DeviceUnreachableError on missing device. """ - dirname, target = device_path.rsplit('/', 1) - for filename, stat in self.adb.Ls(dirname): - if filename == target: - return stat + dirname, filename = posixpath.split(posixpath.normpath(device_path)) + for entry in self.StatDirectory(dirname, as_root=as_root, **kwargs): + if entry['filename'] == filename: + return entry raise device_errors.CommandFailedError( 'Cannot find file or directory: %r' % device_path, str(self)) + def FileSize(self, device_path, as_root=False, **kwargs): + """Get the size of a file on the device. + + Note: This is implemented by parsing the output of the 'ls' command on + the device. On some Android versions, when passing a directory or special + file, the size is *not* reported and this function will throw an exception. + + Args: + device_path: A string containing the path of a file on the device. + as_root: A boolean indicating whether the to use root privileges to + access the file information. + timeout: timeout in seconds + retries: number of retries + + Returns: + The size of the file in bytes. + + Raises: + CommandFailedError if device_path cannot be found on the device, or + its size cannot be determited for some reason. + CommandTimeoutError on timeout. + DeviceUnreachableError on missing device. + """ + entry = self.StatPath(device_path, as_root=as_root, **kwargs) + try: + return entry['st_size'] + except KeyError: + raise device_errors.CommandFailedError( + 'Could not determine the size of: %s' % device_path, str(self)) + @decorators.WithTimeoutAndRetriesFromInstance() def SetJavaAsserts(self, enabled, timeout=None, retries=None): """Enables or disables Java asserts. @@ -1749,8 +1923,37 @@ class DeviceUtils(object): """Returns the product board name of the device (e.g. 'shamu').""" return self.GetProp('ro.product.board', cache=True) - def GetProp(self, property_name, cache=False, timeout=DEFAULT, - retries=DEFAULT): + def _EnsureCacheInitialized(self): + """Populates cache token, runs getprop and fetches $EXTERNAL_STORAGE.""" + if self._cache['token']: + return + with self._cache_lock: + if self._cache['token']: + return + # Change the token every time to ensure that it will match only the + # previously dumped cache. + token = str(uuid.uuid1()) + cmd = ( + 'c=/data/local/tmp/cache_token;' + 'echo $EXTERNAL_STORAGE;' + 'cat $c 2>/dev/null||echo;' + 'echo "%s">$c &&' % token + + 'getprop' + ) + output = self.RunShellCommand(cmd, check_return=True, large_output=True) + # Error-checking for this existing is done in GetExternalStoragePath(). + self._cache['external_storage'] = output[0] + self._cache['prev_token'] = output[1] + output = output[2:] + + prop_cache = self._cache['getprop'] + prop_cache.clear() + for key, value in _GETPROP_RE.findall(''.join(output)): + prop_cache[key] = value + self._cache['token'] = token + + @decorators.WithTimeoutAndRetriesFromInstance() + def GetProp(self, property_name, cache=False, timeout=None, retries=None): """Gets a property from the device. Args: @@ -1769,29 +1972,19 @@ class DeviceUtils(object): assert isinstance(property_name, basestring), ( "property_name is not a string: %r" % property_name) - prop_cache = self._cache['getprop'] if cache: - if property_name not in prop_cache: - # It takes ~120ms to query a single property, and ~130ms to query all - # properties. So, when caching we always query all properties. - output = self.RunShellCommand( - ['getprop'], check_return=True, large_output=True, - timeout=self._default_timeout if timeout is DEFAULT else timeout, - retries=self._default_retries if retries is DEFAULT else retries) - prop_cache.clear() - for key, value in _GETPROP_RE.findall(''.join(output)): - prop_cache[key] = value - if property_name not in prop_cache: - prop_cache[property_name] = '' + # It takes ~120ms to query a single property, and ~130ms to query all + # properties. So, when caching we always query all properties. + self._EnsureCacheInitialized() else: # timeout and retries are handled down at run shell, because we don't # want to apply them in the other branch when reading from the cache value = self.RunShellCommand( ['getprop', property_name], single_line=True, check_return=True, - timeout=self._default_timeout if timeout is DEFAULT else timeout, - retries=self._default_retries if retries is DEFAULT else retries) - prop_cache[property_name] = value - return prop_cache[property_name] + timeout=timeout, retries=retries) + self._cache['getprop'][property_name] = value + # Non-existent properties are treated as empty strings by getprop. + return self._cache['getprop'].get(property_name, '') @decorators.WithTimeoutAndRetriesFromInstance() def SetProp(self, property_name, value, check=False, timeout=None, @@ -2031,11 +2224,36 @@ class DeviceUtils(object): 'getprop': {}, # Map of device_path -> [ignore_other_files, map of path->checksum] 'device_path_checksums': {}, + # Location of sdcard ($EXTERNAL_STORAGE). + 'external_storage': None, + # Token used to detect when LoadCacheData is stale. + 'token': None, + 'prev_token': None, } - def LoadCacheData(self, data): - """Initializes the cache from data created using DumpCacheData.""" + @decorators.WithTimeoutAndRetriesFromInstance() + def LoadCacheData(self, data, timeout=None, retries=None): + """Initializes the cache from data created using DumpCacheData. + + The cache is used only if its token matches the one found on the device. + This prevents a stale cache from being used (which can happen when sharing + devices). + + Args: + data: A previously serialized cache (string). + timeout: timeout in seconds + retries: number of retries + + Returns: + Whether the cache was loaded. + """ obj = json.loads(data) + self._EnsureCacheInitialized() + given_token = obj.get('token') + if not given_token or self._cache['prev_token'] != given_token: + logging.warning('Stale cache detected. Not using it.') + return False + self._cache['package_apk_paths'] = obj.get('package_apk_paths', {}) # When using a cache across script invokations, verify that apps have # not been uninstalled. @@ -2048,10 +2266,22 @@ class DeviceUtils(object): self._cache['package_apk_checksums'] = package_apk_checksums device_path_checksums = obj.get('device_path_checksums', {}) self._cache['device_path_checksums'] = device_path_checksums + return True - def DumpCacheData(self): - """Dumps the current cache state to a string.""" + @decorators.WithTimeoutAndRetriesFromInstance() + def DumpCacheData(self, timeout=None, retries=None): + """Dumps the current cache state to a string. + + Args: + timeout: timeout in seconds + retries: number of retries + + Returns: + A serialized cache as a string. + """ + self._EnsureCacheInitialized() obj = {} + obj['token'] = self._cache['token'] obj['package_apk_paths'] = self._cache['package_apk_paths'] obj['package_apk_checksums'] = self._cache['package_apk_checksums'] # JSON can't handle sets. @@ -2073,13 +2303,7 @@ class DeviceUtils(object): Returns: A Parallelizer operating over |devices|. - - Raises: - device_errors.NoDevicesError: If no devices are passed. """ - if not devices: - raise device_errors.NoDevicesError() - devices = [d if isinstance(d, cls) else cls(d) for d in devices] if async: return parallelizer.Parallelizer(devices) @@ -2087,20 +2311,76 @@ class DeviceUtils(object): return parallelizer.SyncParallelizer(devices) @classmethod - def HealthyDevices(cls, blacklist=None, **kwargs): + def HealthyDevices(cls, blacklist=None, device_arg='default', **kwargs): + """Returns a list of DeviceUtils instances. + + Returns a list of DeviceUtils instances that are attached, not blacklisted, + and optionally filtered by --device flags or ANDROID_SERIAL environment + variable. + + Args: + blacklist: A DeviceBlacklist instance (optional). Device serials in this + blacklist will never be returned, but a warning will be logged if they + otherwise would have been. + device_arg: The value of the --device flag. This can be: + 'default' -> Same as [], but returns an empty list rather than raise a + NoDevicesError. + [] -> Returns all devices, unless $ANDROID_SERIAL is set. + None -> Use $ANDROID_SERIAL if set, otherwise looks for a single + attached device. Raises an exception if multiple devices are + attached. + 'serial' -> Returns an instance for the given serial, if not + blacklisted. + ['A', 'B', ...] -> Returns instances for the subset that is not + blacklisted. + A device serial, or a list of device serials (optional). + + Returns: + A list of one or more DeviceUtils instances. + + Raises: + NoDevicesError: Raised when no non-blacklisted devices exist and + device_arg is passed. + MultipleDevicesError: Raise when multiple devices exist, but |device_arg| + is None. + """ + allow_no_devices = False + if device_arg == 'default': + allow_no_devices = True + device_arg = () + + select_multiple = True + if not (isinstance(device_arg, tuple) or isinstance(device_arg, list)): + select_multiple = False + if device_arg: + device_arg = (device_arg,) + blacklisted_devices = blacklist.Read() if blacklist else [] - def blacklisted(adb): - if adb.GetDeviceSerial() in blacklisted_devices: - logging.warning('Device %s is blacklisted.', adb.GetDeviceSerial()) + # adb looks for ANDROID_SERIAL, so support it as well. + android_serial = os.environ.get('ANDROID_SERIAL') + if not device_arg and android_serial: + device_arg = (android_serial,) + + def blacklisted(serial): + if serial in blacklisted_devices: + logging.warning('Device %s is blacklisted.', serial) return True return False - devices = [] - for adb in adb_wrapper.AdbWrapper.Devices(): - if not blacklisted(adb): - devices.append(cls(_CreateAdbWrapper(adb), **kwargs)) - return devices + if device_arg: + devices = [cls(x, **kwargs) for x in device_arg if not blacklisted(x)] + else: + devices = [] + for adb in adb_wrapper.AdbWrapper.Devices(): + if not blacklisted(adb.GetDeviceSerial()): + devices.append(cls(_CreateAdbWrapper(adb), **kwargs)) + + if len(devices) == 0 and not allow_no_devices: + raise device_errors.NoDevicesError() + if len(devices) > 1 and not select_multiple: + raise device_errors.MultipleDevicesError(devices) + return sorted(devices) @decorators.WithTimeoutAndRetriesFromInstance() def RestartAdbd(self, timeout=None, retries=None): diff --git a/catapult/devil/devil/android/device_utils_devicetest.py b/catapult/devil/devil/android/device_utils_devicetest.py index 9a503738..33c1fb93 100755 --- a/catapult/devil/devil/android/device_utils_devicetest.py +++ b/catapult/devil/devil/android/device_utils_devicetest.py @@ -9,9 +9,14 @@ The test will invoke real devices """ import os +import sys import tempfile import unittest +if __name__ == '__main__': + sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', ))) + from devil.android import device_utils from devil.android.sdk import adb_wrapper from devil.utils import cmd_helper diff --git a/catapult/devil/devil/android/device_utils_test.py b/catapult/devil/devil/android/device_utils_test.py index 38849eca..18fda546 100755 --- a/catapult/devil/devil/android/device_utils_test.py +++ b/catapult/devil/devil/android/device_utils_test.py @@ -10,7 +10,10 @@ Unit tests for the contents of device_utils.py (mostly DeviceUtils). # pylint: disable=protected-access # pylint: disable=unused-argument +import json import logging +import os +import stat import unittest from devil import devil_env @@ -27,6 +30,17 @@ with devil_env.SysPath(devil_env.PYMOCK_PATH): import mock # pylint: disable=import-error +class AnyStringWith(object): + def __init__(self, value): + self._value = value + + def __eq__(self, other): + return self._value in other + + def __repr__(self): + return '<AnyStringWith: %s>' % self._value + + class _MockApkHelper(object): def __init__(self, path, package_name, perms=None): @@ -41,6 +55,10 @@ class _MockApkHelper(object): return self.perms +class _MockMultipleDevicesError(Exception): + pass + + class DeviceUtilsInitTest(unittest.TestCase): def testInitWithStr(self): @@ -172,6 +190,12 @@ class DeviceUtilsTest(mock_calls.TestCase): return mock.Mock(side_effect=device_errors.CommandTimeoutError( msg, str(self.device))) + def EnsureCacheInitialized(self, props=None, sdcard='/sdcard'): + props = props or [] + ret = [sdcard, 'TOKEN'] + props + return (self.call.device.RunShellCommand( + AnyStringWith('getprop'), check_return=True, large_output=True), ret) + class DeviceUtilsEqTest(DeviceUtilsTest): @@ -305,13 +329,14 @@ class DeviceUtilsIsUserBuildTest(DeviceUtilsTest): class DeviceUtilsGetExternalStoragePathTest(DeviceUtilsTest): def testGetExternalStoragePath_succeeds(self): - with self.assertCall( - self.call.adb.Shell('echo $EXTERNAL_STORAGE'), '/fake/storage/path\n'): + with self.assertCalls( + self.EnsureCacheInitialized(sdcard='/fake/storage/path')): self.assertEquals('/fake/storage/path', self.device.GetExternalStoragePath()) def testGetExternalStoragePath_fails(self): - with self.assertCall(self.call.adb.Shell('echo $EXTERNAL_STORAGE'), '\n'): + with self.assertCalls( + self.EnsureCacheInitialized(sdcard='')): with self.assertRaises(device_errors.CommandFailedError): self.device.GetExternalStoragePath() @@ -451,6 +476,23 @@ class DeviceUtilsWaitUntilFullyBootedTest(DeviceUtilsTest): (self.call.device.GetProp('sys.boot_completed', cache=False), '1')): self.device.WaitUntilFullyBooted(wifi=False) + def testWaitUntilFullyBooted_deviceBrieflyOffline(self): + with self.assertCalls( + self.call.adb.WaitForDevice(), + # sd_card_ready + (self.call.device.GetExternalStoragePath(), '/fake/storage/path'), + (self.call.adb.Shell('test -d /fake/storage/path'), ''), + # pm_ready + (self.call.device._GetApplicationPathsInternal('android', + skip_cache=True), + ['package:/some/fake/path']), + # boot_completed + (self.call.device.GetProp('sys.boot_completed', cache=False), + self.AdbCommandError()), + # boot_completed + (self.call.device.GetProp('sys.boot_completed', cache=False), '1')): + self.device.WaitUntilFullyBooted(wifi=False) + def testWaitUntilFullyBooted_sdCardReadyFails_noPath(self): with self.assertCalls( self.call.adb.WaitForDevice(), @@ -835,6 +877,12 @@ class DeviceUtilsRunShellCommandTest(DeviceUtilsTest): self.assertEquals(['file1', 'file2', 'file3'], self.device.RunShellCommand(cmd)) + def testRunShellCommand_manyLinesRawOutput(self): + cmd = 'ls /some/path' + with self.assertCall(self.call.adb.Shell(cmd), '\rfile1\nfile2\r\nfile3\n'): + self.assertEquals('\rfile1\nfile2\r\nfile3\n', + self.device.RunShellCommand(cmd, raw_output=True)) + def testRunShellCommand_singleLine_success(self): cmd = 'echo $VALUE' with self.assertCall(self.call.adb.Shell(cmd), 'some value\n'): @@ -1198,12 +1246,15 @@ class DeviceUtilsStartActivityTest(DeviceUtilsTest): test_intent = intent.Intent(action='android.intent.action.VIEW', package='test.package', activity='.Main', - flags='0x10000000') + flags=[ + intent.FLAG_ACTIVITY_NEW_TASK, + intent.FLAG_ACTIVITY_RESET_TASK_IF_NEEDED + ]) with self.assertCall( self.call.adb.Shell('am start ' '-a android.intent.action.VIEW ' '-n test.package/.Main ' - '-f 0x10000000'), + '-f 0x10200000'), 'Starting: Intent { act=android.intent.action.VIEW }'): self.device.StartActivity(test_intent) @@ -1599,10 +1650,7 @@ class DeviceUtilsReadFileTest(DeviceUtilsTest): def testReadFile_exists(self): with self.assertCalls( - (self.call.device.RunShellCommand( - ['ls', '-l', '/read/this/test/file'], - as_root=False, check_return=True), - ['-rw-rw---- root foo 256 1970-01-01 00:00 file']), + (self.call.device.FileSize('/read/this/test/file', as_root=False), 256), (self.call.device.RunShellCommand( ['cat', '/read/this/test/file'], as_root=False, check_return=True), @@ -1613,10 +1661,7 @@ class DeviceUtilsReadFileTest(DeviceUtilsTest): def testReadFile_exists2(self): # Same as testReadFile_exists, but uses Android N ls output. with self.assertCalls( - (self.call.device.RunShellCommand( - ['ls', '-l', '/read/this/test/file'], - as_root=False, check_return=True), - ['-rw-rw-rw- 1 root root 256 2016-03-15 03:27 /read/this/test/file']), + (self.call.device.FileSize('/read/this/test/file', as_root=False), 256), (self.call.device.RunShellCommand( ['cat', '/read/this/test/file'], as_root=False, check_return=True), @@ -1626,19 +1671,15 @@ class DeviceUtilsReadFileTest(DeviceUtilsTest): def testReadFile_doesNotExist(self): with self.assertCall( - self.call.device.RunShellCommand( - ['ls', '-l', '/this/file/does.not.exist'], - as_root=False, check_return=True), + self.call.device.FileSize('/this/file/does.not.exist', as_root=False), self.CommandError('File does not exist')): with self.assertRaises(device_errors.CommandFailedError): self.device.ReadFile('/this/file/does.not.exist') def testReadFile_zeroSize(self): with self.assertCalls( - (self.call.device.RunShellCommand( - ['ls', '-l', '/this/file/has/zero/size'], - as_root=False, check_return=True), - ['-r--r--r-- root foo 0 1970-01-01 00:00 zero_size_file']), + (self.call.device.FileSize('/this/file/has/zero/size', as_root=False), + 0), (self.call.device._ReadFileWithPull('/this/file/has/zero/size'), 'but it has contents\n')): self.assertEqual('but it has contents\n', @@ -1646,10 +1687,8 @@ class DeviceUtilsReadFileTest(DeviceUtilsTest): def testReadFile_withSU(self): with self.assertCalls( - (self.call.device.RunShellCommand( - ['ls', '-l', '/this/file/can.be.read.with.su'], - as_root=True, check_return=True), - ['-rw------- root root 256 1970-01-01 00:00 can.be.read.with.su']), + (self.call.device.FileSize( + '/this/file/can.be.read.with.su', as_root=True), 256), (self.call.device.RunShellCommand( ['cat', '/this/file/can.be.read.with.su'], as_root=True, check_return=True), @@ -1662,10 +1701,8 @@ class DeviceUtilsReadFileTest(DeviceUtilsTest): def testReadFile_withPull(self): contents = 'a' * 123456 with self.assertCalls( - (self.call.device.RunShellCommand( - ['ls', '-l', '/read/this/big/test/file'], - as_root=False, check_return=True), - ['-rw-rw---- root foo 123456 1970-01-01 00:00 file']), + (self.call.device.FileSize('/read/this/big/test/file', as_root=False), + 123456), (self.call.device._ReadFileWithPull('/read/this/big/test/file'), contents)): self.assertEqual( @@ -1674,10 +1711,8 @@ class DeviceUtilsReadFileTest(DeviceUtilsTest): def testReadFile_withPullAndSU(self): contents = 'b' * 123456 with self.assertCalls( - (self.call.device.RunShellCommand( - ['ls', '-l', '/this/big/file/can.be.read.with.su'], - as_root=True, check_return=True), - ['-rw------- root root 123456 1970-01-01 00:00 can.be.read.with.su']), + (self.call.device.FileSize( + '/this/big/file/can.be.read.with.su', as_root=True), 123456), (self.call.device.NeedsSU(), True), (mock.call.devil.android.device_temp_file.DeviceTempFile(self.adb), MockTempFile('/sdcard/tmp/on.device')), @@ -1768,52 +1803,189 @@ class DeviceUtilsWriteFileTest(DeviceUtilsTest): self.device.WriteFile('/test/file', 'contents', as_root=True) -class DeviceUtilsLsTest(DeviceUtilsTest): +class DeviceUtilsStatDirectoryTest(DeviceUtilsTest): + # Note: Also tests ListDirectory in testStatDirectory_fileList. + + EXAMPLE_LS_OUTPUT = [ + 'total 12345', + 'drwxr-xr-x 19 root root 0 1970-04-06 18:03 .', + 'drwxr-xr-x 19 root root 0 1970-04-06 18:03 ..', + 'drwxr-xr-x 6 root root 1970-01-01 00:00 some_dir', + '-rw-r--r-- 1 root root 723 1971-01-01 07:04 some_file', + '-rw-r----- 1 root root 327 2009-02-13 23:30 My Music File', + # Older Android versions do not print st_nlink + 'lrwxrwxrwx root root 1970-01-01 00:00 lnk -> /some/path', + 'srwxrwx--- system system 2016-05-31 17:25 a_socket1', + 'drwxrwxrwt system misc 1970-11-23 02:25 tmp', + 'drwxr-s--- system shell 1970-11-23 02:24 my_cmd', + 'cr--r----- root system 10, 183 1971-01-01 07:04 random', + 'brw------- root root 7, 0 1971-01-01 07:04 block_dev', + '-rwS------ root shell 157404 2015-04-13 15:44 silly', + ] + + FILENAMES = [ + 'some_dir', 'some_file', 'My Music File', 'lnk', 'a_socket1', + 'tmp', 'my_cmd', 'random', 'block_dev', 'silly'] + + def getStatEntries(self, path_given='/', path_listed='/'): + with self.assertCall( + self.call.device.RunShellCommand( + ['ls', '-a', '-l', path_listed], + check_return=True, as_root=False, env={'TZ': 'utc'}), + self.EXAMPLE_LS_OUTPUT): + entries = self.device.StatDirectory(path_given) + return {f['filename']: f for f in entries} - def testLs_directory(self): - result = [('.', adb_wrapper.DeviceStat(16889, 4096, 1417436123)), - ('..', adb_wrapper.DeviceStat(16873, 4096, 12382237)), - ('testfile.txt', adb_wrapper.DeviceStat(33206, 3, 1417436122))] - with self.assertCalls( - (self.call.adb.Ls('/data/local/tmp'), result)): - self.assertEquals(result, - self.device.Ls('/data/local/tmp')) + def getListEntries(self): + with self.assertCall( + self.call.device.RunShellCommand( + ['ls', '-a', '-l', '/'], + check_return=True, as_root=False, env={'TZ': 'utc'}), + self.EXAMPLE_LS_OUTPUT): + return self.device.ListDirectory('/') + + def testStatDirectory_forceTrailingSlash(self): + self.getStatEntries(path_given='/foo/bar/', path_listed='/foo/bar/') + self.getStatEntries(path_given='/foo/bar', path_listed='/foo/bar/') + + def testStatDirectory_fileList(self): + self.assertItemsEqual(self.getStatEntries().keys(), self.FILENAMES) + self.assertItemsEqual(self.getListEntries(), self.FILENAMES) + + def testStatDirectory_fileModes(self): + expected_modes = ( + ('some_dir', stat.S_ISDIR), + ('some_file', stat.S_ISREG), + ('lnk', stat.S_ISLNK), + ('a_socket1', stat.S_ISSOCK), + ('block_dev', stat.S_ISBLK), + ('random', stat.S_ISCHR), + ) + entries = self.getStatEntries() + for filename, check in expected_modes: + self.assertTrue(check(entries[filename]['st_mode'])) + + def testStatDirectory_filePermissions(self): + should_have = ( + ('some_file', stat.S_IWUSR), # Owner can write. + ('tmp', stat.S_IXOTH), # Others can execute. + ('tmp', stat.S_ISVTX), # Has sticky bit. + ('my_cmd', stat.S_ISGID), # Has set-group-ID bit. + ('silly', stat.S_ISUID), # Has set UID bit. + ) + should_not_have = ( + ('some_file', stat.S_IWOTH), # Others can't write. + ('block_dev', stat.S_IRGRP), # Group can't read. + ('silly', stat.S_IXUSR), # Owner can't execute. + ) + entries = self.getStatEntries() + for filename, bit in should_have: + self.assertTrue(entries[filename]['st_mode'] & bit) + for filename, bit in should_not_have: + self.assertFalse(entries[filename]['st_mode'] & bit) + + def testStatDirectory_numHardLinks(self): + entries = self.getStatEntries() + self.assertEqual(entries['some_dir']['st_nlink'], 6) + self.assertEqual(entries['some_file']['st_nlink'], 1) + self.assertFalse('st_nlink' in entries['tmp']) + + def testStatDirectory_fileOwners(self): + entries = self.getStatEntries() + self.assertEqual(entries['some_dir']['st_owner'], 'root') + self.assertEqual(entries['my_cmd']['st_owner'], 'system') + self.assertEqual(entries['my_cmd']['st_group'], 'shell') + self.assertEqual(entries['tmp']['st_group'], 'misc') + + def testStatDirectory_fileSize(self): + entries = self.getStatEntries() + self.assertEqual(entries['some_file']['st_size'], 723) + self.assertEqual(entries['My Music File']['st_size'], 327) + # Sizes are sometimes not reported for non-regular files, don't try to + # guess the size in those cases. + self.assertFalse('st_size' in entries['some_dir']) + + def testStatDirectory_fileDateTime(self): + entries = self.getStatEntries() + self.assertEqual(entries['some_dir']['st_mtime'], 0) # Epoch! + self.assertEqual(entries['My Music File']['st_mtime'], 1234567800) + + def testStatDirectory_deviceType(self): + entries = self.getStatEntries() + self.assertEqual(entries['random']['st_rdev_pair'], (10, 183)) + self.assertEqual(entries['block_dev']['st_rdev_pair'], (7, 0)) + + def testStatDirectory_symbolicLinks(self): + entries = self.getStatEntries() + self.assertEqual(entries['lnk']['symbolic_link_to'], '/some/path') + for d in entries.itervalues(): + self.assertEqual('symbolic_link_to' in d, stat.S_ISLNK(d['st_mode'])) + + +class DeviceUtilsStatPathTest(DeviceUtilsTest): + + EXAMPLE_DIRECTORY = [ + {'filename': 'foo.txt', 'st_size': 123, 'st_time': 456}, + {'filename': 'some_dir', 'st_time': 0} + ] + INDEX = {e['filename']: e for e in EXAMPLE_DIRECTORY} + + def testStatPath_file(self): + with self.assertCall( + self.call.device.StatDirectory('/data/local/tmp', as_root=False), + self.EXAMPLE_DIRECTORY): + self.assertEquals(self.INDEX['foo.txt'], + self.device.StatPath('/data/local/tmp/foo.txt')) - def testLs_nothing(self): - with self.assertCalls( - (self.call.adb.Ls('/data/local/tmp/testfile.txt'), [])): - self.assertEquals([], - self.device.Ls('/data/local/tmp/testfile.txt')) + def testStatPath_directory(self): + with self.assertCall( + self.call.device.StatDirectory('/data/local/tmp', as_root=False), + self.EXAMPLE_DIRECTORY): + self.assertEquals(self.INDEX['some_dir'], + self.device.StatPath('/data/local/tmp/some_dir')) + def testStatPath_directoryWithTrailingSlash(self): + with self.assertCall( + self.call.device.StatDirectory('/data/local/tmp', as_root=False), + self.EXAMPLE_DIRECTORY): + self.assertEquals(self.INDEX['some_dir'], + self.device.StatPath('/data/local/tmp/some_dir/')) -class DeviceUtilsStatTest(DeviceUtilsTest): + def testStatPath_doesNotExist(self): + with self.assertCall( + self.call.device.StatDirectory('/data/local/tmp', as_root=False), + self.EXAMPLE_DIRECTORY): + with self.assertRaises(device_errors.CommandFailedError): + self.device.StatPath('/data/local/tmp/does.not.exist.txt') - def testStat_file(self): - result = [('.', adb_wrapper.DeviceStat(16889, 4096, 1417436123)), - ('..', adb_wrapper.DeviceStat(16873, 4096, 12382237)), - ('testfile.txt', adb_wrapper.DeviceStat(33206, 3, 1417436122))] - with self.assertCalls( - (self.call.adb.Ls('/data/local/tmp'), result)): - self.assertEquals(adb_wrapper.DeviceStat(33206, 3, 1417436122), - self.device.Stat('/data/local/tmp/testfile.txt')) - def testStat_directory(self): - result = [('.', adb_wrapper.DeviceStat(16873, 4096, 12382237)), - ('..', adb_wrapper.DeviceStat(16873, 4096, 12382237)), - ('tmp', adb_wrapper.DeviceStat(16889, 4096, 1417436123))] - with self.assertCalls( - (self.call.adb.Ls('/data/local'), result)): - self.assertEquals(adb_wrapper.DeviceStat(16889, 4096, 1417436123), - self.device.Stat('/data/local/tmp')) +class DeviceUtilsFileSizeTest(DeviceUtilsTest): - def testStat_doesNotExist(self): - result = [('.', adb_wrapper.DeviceStat(16889, 4096, 1417436123)), - ('..', adb_wrapper.DeviceStat(16873, 4096, 12382237)), - ('testfile.txt', adb_wrapper.DeviceStat(33206, 3, 1417436122))] - with self.assertCalls( - (self.call.adb.Ls('/data/local/tmp'), result)): + EXAMPLE_DIRECTORY = [ + {'filename': 'foo.txt', 'st_size': 123, 'st_mtime': 456}, + {'filename': 'some_dir', 'st_mtime': 0} + ] + + def testFileSize_file(self): + with self.assertCall( + self.call.device.StatDirectory('/data/local/tmp', as_root=False), + self.EXAMPLE_DIRECTORY): + self.assertEquals(123, + self.device.FileSize('/data/local/tmp/foo.txt')) + + def testFileSize_doesNotExist(self): + with self.assertCall( + self.call.device.StatDirectory('/data/local/tmp', as_root=False), + self.EXAMPLE_DIRECTORY): with self.assertRaises(device_errors.CommandFailedError): - self.device.Stat('/data/local/tmp/does.not.exist.txt') + self.device.FileSize('/data/local/tmp/does.not.exist.txt') + + def testFileSize_directoryWithNoSize(self): + with self.assertCall( + self.call.device.StatDirectory('/data/local/tmp', as_root=False), + self.EXAMPLE_DIRECTORY): + with self.assertRaises(device_errors.CommandFailedError): + self.device.FileSize('/data/local/tmp/some_dir') class DeviceUtilsSetJavaAssertsTest(DeviceUtilsTest): @@ -1866,6 +2038,34 @@ class DeviceUtilsSetJavaAssertsTest(DeviceUtilsTest): self.assertFalse(self.device.SetJavaAsserts(True)) +class DeviceUtilsEnsureCacheInitializedTest(DeviceUtilsTest): + + def testEnsureCacheInitialized_noCache_success(self): + self.assertIsNone(self.device._cache['token']) + with self.assertCall( + self.call.device.RunShellCommand( + AnyStringWith('getprop'), check_return=True, large_output=True), + ['/sdcard', 'TOKEN']): + self.device._EnsureCacheInitialized() + self.assertIsNotNone(self.device._cache['token']) + + def testEnsureCacheInitialized_noCache_failure(self): + self.assertIsNone(self.device._cache['token']) + with self.assertCall( + self.call.device.RunShellCommand( + AnyStringWith('getprop'), check_return=True, large_output=True), + self.TimeoutError()): + with self.assertRaises(device_errors.CommandTimeoutError): + self.device._EnsureCacheInitialized() + self.assertIsNone(self.device._cache['token']) + + def testEnsureCacheInitialized_cache(self): + self.device._cache['token'] = 'TOKEN' + with self.assertCalls(): + self.device._EnsureCacheInitialized() + self.assertIsNotNone(self.device._cache['token']) + + class DeviceUtilsGetPropTest(DeviceUtilsTest): def testGetProp_exists(self): @@ -1889,31 +2089,13 @@ class DeviceUtilsGetPropTest(DeviceUtilsTest): self.assertEqual('', self.device.GetProp('property.does.not.exist')) def testGetProp_cachedRoProp(self): - with self.assertCall( - self.call.device.RunShellCommand( - ['getprop'], check_return=True, large_output=True, - timeout=self.device._default_timeout, - retries=self.device._default_retries), - ['[ro.build.type]: [userdebug]']): + with self.assertCalls( + self.EnsureCacheInitialized(props=['[ro.build.type]: [userdebug]'])): self.assertEqual('userdebug', self.device.GetProp('ro.build.type', cache=True)) self.assertEqual('userdebug', self.device.GetProp('ro.build.type', cache=True)) - def testGetProp_retryAndCache(self): - with self.assertCalls( - (self.call.device.RunShellCommand( - ['getprop'], check_return=True, large_output=True, - timeout=self.device._default_timeout, - retries=3), - ['[ro.build.type]: [userdebug]'])): - self.assertEqual('userdebug', - self.device.GetProp('ro.build.type', - cache=True, retries=3)) - self.assertEqual('userdebug', - self.device.GetProp('ro.build.type', - cache=True, retries=3)) - class DeviceUtilsSetPropTest(DeviceUtilsTest): @@ -2139,7 +2321,7 @@ class DeviceUtilsClientCache(DeviceUtilsTest): class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): - def testHealthyDevices_emptyBlacklist(self): + def testHealthyDevices_emptyBlacklist_defaultDeviceArg(self): test_serials = ['0123456789abcdef', 'fedcba9876543210'] with self.assertCalls( (mock.call.devil.android.sdk.adb_wrapper.AdbWrapper.Devices(), @@ -2150,7 +2332,7 @@ class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): self.assertTrue(isinstance(device, device_utils.DeviceUtils)) self.assertEquals(serial, device.adb.GetDeviceSerial()) - def testHealthyDevices_blacklist(self): + def testHealthyDevices_blacklist_defaultDeviceArg(self): test_serials = ['0123456789abcdef', 'fedcba9876543210'] with self.assertCalls( (mock.call.devil.android.sdk.adb_wrapper.AdbWrapper.Devices(), @@ -2162,6 +2344,81 @@ class DeviceUtilsHealthyDevicesTest(mock_calls.TestCase): self.assertTrue(isinstance(devices[0], device_utils.DeviceUtils)) self.assertEquals('0123456789abcdef', devices[0].adb.GetDeviceSerial()) + def testHealthyDevices_noneDeviceArg_multiple_attached(self): + test_serials = ['0123456789abcdef', 'fedcba9876543210'] + with self.assertCalls( + (mock.call.devil.android.sdk.adb_wrapper.AdbWrapper.Devices(), + [_AdbWrapperMock(s) for s in test_serials]), + (mock.call.devil.android.device_errors.MultipleDevicesError(mock.ANY), + _MockMultipleDevicesError())): + with self.assertRaises(_MockMultipleDevicesError): + device_utils.DeviceUtils.HealthyDevices(device_arg=None) + + def testHealthyDevices_noneDeviceArg_one_attached(self): + test_serials = ['0123456789abcdef'] + with self.assertCalls( + (mock.call.devil.android.sdk.adb_wrapper.AdbWrapper.Devices(), + [_AdbWrapperMock(s) for s in test_serials])): + devices = device_utils.DeviceUtils.HealthyDevices(device_arg=None) + self.assertEquals(1, len(devices)) + + def testHealthyDevices_noneDeviceArg_no_attached(self): + test_serials = [] + with self.assertCalls( + (mock.call.devil.android.sdk.adb_wrapper.AdbWrapper.Devices(), + [_AdbWrapperMock(s) for s in test_serials])): + with self.assertRaises(device_errors.NoDevicesError): + device_utils.DeviceUtils.HealthyDevices(device_arg=None) + + def testHealthyDevices_noneDeviceArg_multiple_attached_ANDROID_SERIAL(self): + try: + os.environ['ANDROID_SERIAL'] = '0123456789abcdef' + with self.assertCalls(): # Should skip adb devices when device is known. + device_utils.DeviceUtils.HealthyDevices(device_arg=None) + finally: + del os.environ['ANDROID_SERIAL'] + + def testHealthyDevices_stringDeviceArg(self): + with self.assertCalls(): # Should skip adb devices when device is known. + devices = device_utils.DeviceUtils.HealthyDevices( + device_arg='0123456789abcdef') + self.assertEquals(1, len(devices)) + + def testHealthyDevices_EmptyListDeviceArg_multiple_attached(self): + test_serials = ['0123456789abcdef', 'fedcba9876543210'] + with self.assertCalls( + (mock.call.devil.android.sdk.adb_wrapper.AdbWrapper.Devices(), + [_AdbWrapperMock(s) for s in test_serials])): + devices = device_utils.DeviceUtils.HealthyDevices(device_arg=()) + self.assertEquals(2, len(devices)) + + def testHealthyDevices_EmptyListDeviceArg_ANDROID_SERIAL(self): + try: + os.environ['ANDROID_SERIAL'] = '0123456789abcdef' + with self.assertCalls(): # Should skip adb devices when device is known. + devices = device_utils.DeviceUtils.HealthyDevices(device_arg=()) + finally: + del os.environ['ANDROID_SERIAL'] + self.assertEquals(1, len(devices)) + + def testHealthyDevices_EmptyListDeviceArg_no_attached(self): + test_serials = [] + with self.assertCalls( + (mock.call.devil.android.sdk.adb_wrapper.AdbWrapper.Devices(), + [_AdbWrapperMock(s) for s in test_serials])): + with self.assertRaises(device_errors.NoDevicesError): + device_utils.DeviceUtils.HealthyDevices(device_arg=[]) + + def testHealthyDevices_ListDeviceArg(self): + device_arg = ['0123456789abcdef', 'fedcba9876543210'] + try: + os.environ['ANDROID_SERIAL'] = 'should-not-apply' + with self.assertCalls(): # Should skip adb devices when device is known. + devices = device_utils.DeviceUtils.HealthyDevices(device_arg=device_arg) + finally: + del os.environ['ANDROID_SERIAL'] + self.assertEquals(2, len(devices)) + class DeviceUtilsRestartAdbdTest(DeviceUtilsTest): @@ -2307,6 +2564,31 @@ class DeviecUtilsSetScreen(DeviceUtilsTest): (self.call.device.IsScreenOn(), False)): self.device.SetScreen(False) +class DeviecUtilsLoadCacheData(DeviceUtilsTest): + + def testTokenMissing(self): + with self.assertCalls( + self.EnsureCacheInitialized()): + self.assertFalse(self.device.LoadCacheData('{}')) + + def testTokenStale(self): + with self.assertCalls( + self.EnsureCacheInitialized()): + self.assertFalse(self.device.LoadCacheData('{"token":"foo"}')) + + def testTokenMatches(self): + with self.assertCalls( + self.EnsureCacheInitialized()): + self.assertTrue(self.device.LoadCacheData('{"token":"TOKEN"}')) + + def testDumpThenLoad(self): + with self.assertCalls( + self.EnsureCacheInitialized()): + data = json.loads(self.device.DumpCacheData()) + data['token'] = 'TOKEN' + self.assertTrue(self.device.LoadCacheData(json.dumps(data))) + + if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) unittest.main(verbosity=2) diff --git a/catapult/devil/devil/android/fastboot_utils.py b/catapult/devil/devil/android/fastboot_utils.py index f1287d1a..c5e8a498 100644 --- a/catapult/devil/devil/android/fastboot_utils.py +++ b/catapult/devil/devil/android/fastboot_utils.py @@ -5,6 +5,7 @@ """Provides a variety of device interactions based on fastboot.""" # pylint: disable=unused-argument +import collections import contextlib import fnmatch import logging @@ -19,31 +20,57 @@ from devil.utils import timeout_retry _DEFAULT_TIMEOUT = 30 _DEFAULT_RETRIES = 3 _FASTBOOT_REBOOT_TIMEOUT = 10 * _DEFAULT_TIMEOUT -ALL_PARTITIONS = [ - 'bootloader', - 'radio', - 'boot', - 'recovery', - 'system', - 'userdata', - 'cache', -] +_KNOWN_PARTITIONS = collections.OrderedDict([ + ('bootloader', {'image': 'bootloader*.img', 'restart': True}), + ('radio', {'image': 'radio*.img', 'restart': True}), + ('boot', {'image': 'boot.img'}), + ('recovery', {'image': 'recovery.img'}), + ('system', {'image': 'system.img'}), + ('userdata', {'image': 'userdata.img', 'wipe_only': True}), + ('cache', {'image': 'cache.img', 'wipe_only': True}), + ('vendor', {'image': 'vendor*.img', 'optional': True}), + ]) +ALL_PARTITIONS = _KNOWN_PARTITIONS.keys() + + +def _FindAndVerifyPartitionsAndImages(partitions, directory): + """Validate partitions and images. + + Validate all partition names and partition directories. Cannot stop mid + flash so its important to validate everything first. + + Args: + Partitions: partitions to be tested. + directory: directory containing the images. + + Returns: + Dictionary with exact partition, image name mapping. + """ + + files = os.listdir(directory) + return_dict = collections.OrderedDict() + + def find_file(pattern): + for filename in files: + if fnmatch.fnmatch(filename, pattern): + return os.path.join(directory, filename) + return None + for partition in partitions: + partition_info = _KNOWN_PARTITIONS[partition] + image_file = find_file(partition_info['image']) + if image_file: + return_dict[partition] = image_file + elif not partition_info.get('optional'): + raise device_errors.FastbootCommandFailedError( + 'Failed to flash device. Could not find image for %s.', + partition_info['image']) + return return_dict class FastbootUtils(object): _FASTBOOT_WAIT_TIME = 1 - _RESTART_WHEN_FLASHING = ['bootloader', 'radio'] _BOARD_VERIFICATION_FILE = 'android-info.txt' - _FLASH_IMAGE_FILES = { - 'bootloader': 'bootloader*.img', - 'radio': 'radio*.img', - 'boot': 'boot.img', - 'recovery': 'recovery.img', - 'system': 'system.img', - 'userdata': 'userdata.img', - 'cache': 'cache.img', - } def __init__(self, device, fastbooter=None, default_timeout=_DEFAULT_TIMEOUT, default_retries=_DEFAULT_RETRIES): @@ -96,7 +123,8 @@ class FastbootUtils(object): @decorators.WithTimeoutAndRetriesFromInstance( min_default_timeout=_FASTBOOT_REBOOT_TIMEOUT) - def Reboot(self, bootloader=False, timeout=None, retries=None): + def Reboot( + self, bootloader=False, wait_for_reboot=True, timeout=None, retries=None): """Reboots out of fastboot mode. It reboots the phone either back into fastboot, or to a regular boot. It @@ -110,7 +138,8 @@ class FastbootUtils(object): self.WaitForFastbootMode() else: self.fastboot.Reboot() - self._device.WaitUntilFullyBooted(timeout=_FASTBOOT_REBOOT_TIMEOUT) + if wait_for_reboot: + self._device.WaitUntilFullyBooted(timeout=_FASTBOOT_REBOOT_TIMEOUT) def _VerifyBoard(self, directory): """Validate as best as possible that the android build matches the device. @@ -147,31 +176,6 @@ class FastbootUtils(object): return False - def _FindAndVerifyPartitionsAndImages(self, partitions, directory): - """Validate partitions and images. - - Validate all partition names and partition directories. Cannot stop mid - flash so its important to validate everything first. - - Args: - Partitions: partitions to be tested. - directory: directory containing the images. - - Returns: - Dictionary with exact partition, image name mapping. - """ - files = os.listdir(directory) - - def find_file(pattern): - for filename in files: - if fnmatch.fnmatch(filename, pattern): - return os.path.join(directory, filename) - raise device_errors.FastbootCommandFailedError( - 'Failed to flash device. Counld not find image for %s.', pattern) - - return {name: find_file(self._FLASH_IMAGE_FILES[name]) - for name in partitions} - def _FlashPartitions(self, partitions, directory, wipe=False, force=False): """Flashes all given partiitons with all given images. @@ -197,21 +201,21 @@ class FastbootUtils(object): 'device type. Run again with force=True to force flashing with an ' 'unverified board.') - flash_image_files = self._FindAndVerifyPartitionsAndImages(partitions, - directory) + flash_image_files = _FindAndVerifyPartitionsAndImages(partitions, directory) + partitions = flash_image_files.keys() for partition in partitions: - if partition in ['cache', 'userdata'] and not wipe: + if _KNOWN_PARTITIONS[partition].get('wipe_only') and not wipe: logging.info( 'Not flashing in wipe mode. Skipping partition %s.', partition) else: logging.info( 'Flashing %s with %s', partition, flash_image_files[partition]) self.fastboot.Flash(partition, flash_image_files[partition]) - if partition in self._RESTART_WHEN_FLASHING: + if _KNOWN_PARTITIONS[partition].get('restart', False): self.Reboot(bootloader=True) @contextlib.contextmanager - def FastbootMode(self, timeout=None, retries=None): + def FastbootMode(self, wait_for_reboot=True, timeout=None, retries=None): """Context manager that enables fastboot mode, and reboots after. Example usage: @@ -225,7 +229,7 @@ class FastbootUtils(object): yield self finally: self.fastboot.SetOemOffModeCharge(True) - self.Reboot() + self.Reboot(wait_for_reboot=wait_for_reboot) def FlashDevice(self, directory, partitions=None, wipe=False): """Flash device with build in |directory|. @@ -242,5 +246,9 @@ class FastbootUtils(object): """ if partitions is None: partitions = ALL_PARTITIONS - with self.FastbootMode(): + # If a device is wiped, then it will no longer have adb keys so it cannot be + # communicated with to verify that it is rebooted. It is up to the user of + # this script to ensure that the adb keys are set on the device after using + # this to wipe a device. + with self.FastbootMode(wait_for_reboot=not wipe): self._FlashPartitions(partitions, directory, wipe=wipe) diff --git a/catapult/devil/devil/android/fastboot_utils_test.py b/catapult/devil/devil/android/fastboot_utils_test.py index 8e6fc88b..05629746 100755 --- a/catapult/devil/devil/android/fastboot_utils_test.py +++ b/catapult/devil/devil/android/fastboot_utils_test.py @@ -9,6 +9,7 @@ Unit tests for the contents of fastboot_utils.py # pylint: disable=protected-access,unused-argument +import collections import io import logging import unittest @@ -25,14 +26,17 @@ with devil_env.SysPath(devil_env.PYMOCK_PATH): _BOARD = 'board_type' _SERIAL = '0123456789abcdef' -_PARTITIONS = ['cache', 'userdata', 'system', 'bootloader', 'radio'] -_IMAGES = { - 'cache': 'cache.img', - 'userdata': 'userdata.img', - 'system': 'system.img', - 'bootloader': 'bootloader.img', - 'radio': 'radio.img', -} +_PARTITIONS = [ + 'bootloader', 'radio', 'boot', 'recovery', 'system', 'userdata', 'cache'] +_IMAGES = collections.OrderedDict([ + ('bootloader', 'bootloader.img'), + ('radio', 'radio.img'), + ('boot', 'boot.img'), + ('recovery', 'recovery.img'), + ('system', 'system.img'), + ('userdata', 'userdata.img'), + ('cache', 'cache.img') +]) _VALID_FILES = [_BOARD + '.zip', 'android-info.txt'] _INVALID_FILES = ['test.zip', 'android-info.txt'] @@ -92,6 +96,11 @@ class FastbootUtilsInitTest(FastbootUtilsTest): with self.assertRaises(AttributeError): fastboot_utils.FastbootUtils('') + def testPartitionOrdering(self): + parts = ['bootloader', 'radio', 'boot', 'recovery', 'system', 'userdata', + 'cache', 'vendor'] + self.assertListEqual(fastboot_utils.ALL_PARTITIONS, parts) + class FastbootUtilsWaitForFastbootMode(FastbootUtilsTest): @@ -131,47 +140,60 @@ class FastbootUtilsFlashPartitions(FastbootUtilsTest): def testFlashPartitions_wipe(self): with self.assertCalls( (self.call.fastboot._VerifyBoard('test'), True), - (self.call.fastboot._FindAndVerifyPartitionsAndImages( - _PARTITIONS, 'test'), _IMAGES), - (self.call.fastboot.fastboot.Flash('cache', 'cache.img')), - (self.call.fastboot.fastboot.Flash('userdata', 'userdata.img')), - (self.call.fastboot.fastboot.Flash('system', 'system.img')), + (mock.call.devil.android.fastboot_utils. + _FindAndVerifyPartitionsAndImages(_PARTITIONS, 'test'), _IMAGES), (self.call.fastboot.fastboot.Flash('bootloader', 'bootloader.img')), (self.call.fastboot.Reboot(bootloader=True)), (self.call.fastboot.fastboot.Flash('radio', 'radio.img')), - (self.call.fastboot.Reboot(bootloader=True))): + (self.call.fastboot.Reboot(bootloader=True)), + (self.call.fastboot.fastboot.Flash('boot', 'boot.img')), + (self.call.fastboot.fastboot.Flash('recovery', 'recovery.img')), + (self.call.fastboot.fastboot.Flash('system', 'system.img')), + (self.call.fastboot.fastboot.Flash('userdata', 'userdata.img')), + (self.call.fastboot.fastboot.Flash('cache', 'cache.img'))): self.fastboot._FlashPartitions(_PARTITIONS, 'test', wipe=True) def testFlashPartitions_noWipe(self): with self.assertCalls( (self.call.fastboot._VerifyBoard('test'), True), - (self.call.fastboot._FindAndVerifyPartitionsAndImages( - _PARTITIONS, 'test'), _IMAGES), - (self.call.fastboot.fastboot.Flash('system', 'system.img')), + (mock.call.devil.android.fastboot_utils. + _FindAndVerifyPartitionsAndImages(_PARTITIONS, 'test'), _IMAGES), (self.call.fastboot.fastboot.Flash('bootloader', 'bootloader.img')), (self.call.fastboot.Reboot(bootloader=True)), (self.call.fastboot.fastboot.Flash('radio', 'radio.img')), - (self.call.fastboot.Reboot(bootloader=True))): + (self.call.fastboot.Reboot(bootloader=True)), + (self.call.fastboot.fastboot.Flash('boot', 'boot.img')), + (self.call.fastboot.fastboot.Flash('recovery', 'recovery.img')), + (self.call.fastboot.fastboot.Flash('system', 'system.img'))): self.fastboot._FlashPartitions(_PARTITIONS, 'test') class FastbootUtilsFastbootMode(FastbootUtilsTest): - def testFastbootMode_good(self): + def testFastbootMode_goodWait(self): with self.assertCalls( self.call.fastboot.EnableFastbootMode(), self.call.fastboot.fastboot.SetOemOffModeCharge(False), self.call.fastboot.fastboot.SetOemOffModeCharge(True), - self.call.fastboot.Reboot()): + self.call.fastboot.Reboot(wait_for_reboot=True)): with self.fastboot.FastbootMode() as fbm: self.assertEqual(self.fastboot, fbm) + def testFastbootMode_goodNoWait(self): + with self.assertCalls( + self.call.fastboot.EnableFastbootMode(), + self.call.fastboot.fastboot.SetOemOffModeCharge(False), + self.call.fastboot.fastboot.SetOemOffModeCharge(True), + self.call.fastboot.Reboot(wait_for_reboot=False)): + with self.fastboot.FastbootMode(wait_for_reboot=False) as fbm: + self.assertEqual(self.fastboot, fbm) + def testFastbootMode_exception(self): with self.assertCalls( self.call.fastboot.EnableFastbootMode(), self.call.fastboot.fastboot.SetOemOffModeCharge(False), self.call.fastboot.fastboot.SetOemOffModeCharge(True), - self.call.fastboot.Reboot()): + self.call.fastboot.Reboot(wait_for_reboot=True)): with self.assertRaises(NotImplementedError): with self.fastboot.FastbootMode() as fbm: self.assertEqual(self.fastboot, fbm) @@ -236,9 +258,10 @@ class FastbootUtilsVerifyBoard(FastbootUtilsTest): class FastbootUtilsFindAndVerifyPartitionsAndImages(FastbootUtilsTest): - def testFindAndVerifyPartitionsAndImages_valid(self): + def testFindAndVerifyPartitionsAndImages_validNoVendor(self): PARTITIONS = [ - 'bootloader', 'radio', 'boot', 'recovery', 'system', 'userdata', 'cache' + 'bootloader', 'radio', 'boot', 'recovery', 'system', 'userdata', + 'cache', 'vendor' ] files = [ 'bootloader-test-.img', @@ -249,7 +272,42 @@ class FastbootUtilsFindAndVerifyPartitionsAndImages(FastbootUtilsTest): 'userdata.img', 'cache.img' ] - return_check = { + img_check = collections.OrderedDict([ + ('bootloader', 'test/bootloader-test-.img'), + ('radio', 'test/radio123.img'), + ('boot', 'test/boot.img'), + ('recovery', 'test/recovery.img'), + ('system', 'test/system.img'), + ('userdata', 'test/userdata.img'), + ('cache', 'test/cache.img'), + ]) + parts_check = [ + 'bootloader', 'radio', 'boot', 'recovery', 'system', 'userdata', + 'cache' + ] + with mock.patch('os.listdir', return_value=files): + imgs = fastboot_utils._FindAndVerifyPartitionsAndImages( + PARTITIONS, 'test') + parts = imgs.keys() + self.assertDictEqual(imgs, img_check) + self.assertListEqual(parts, parts_check) + + def testFindAndVerifyPartitionsAndImages_validVendor(self): + PARTITIONS = [ + 'bootloader', 'radio', 'boot', 'recovery', 'system', 'userdata', + 'cache', 'vendor' + ] + files = [ + 'bootloader-test-.img', + 'radio123.img', + 'boot.img', + 'recovery.img', + 'system.img', + 'userdata.img', + 'cache.img', + 'vendor.img' + ] + img_check = { 'bootloader': 'test/bootloader-test-.img', 'radio': 'test/radio123.img', 'boot': 'test/boot.img', @@ -257,22 +315,59 @@ class FastbootUtilsFindAndVerifyPartitionsAndImages(FastbootUtilsTest): 'system': 'test/system.img', 'userdata': 'test/userdata.img', 'cache': 'test/cache.img', + 'vendor': 'test/vendor.img', } + parts_check = [ + 'bootloader', 'radio', 'boot', 'recovery', 'system', 'userdata', + 'cache', 'vendor' + ] with mock.patch('os.listdir', return_value=files): - return_value = self.fastboot._FindAndVerifyPartitionsAndImages( + imgs = fastboot_utils._FindAndVerifyPartitionsAndImages( PARTITIONS, 'test') - self.assertDictEqual(return_value, return_check) + parts = imgs.keys() + self.assertDictEqual(imgs, img_check) + self.assertListEqual(parts, parts_check) def testFindAndVerifyPartitionsAndImages_badPartition(self): with mock.patch('os.listdir', return_value=['test']): with self.assertRaises(KeyError): - self.fastboot._FindAndVerifyPartitionsAndImages(['test'], 'test') + fastboot_utils._FindAndVerifyPartitionsAndImages(['test'], 'test') def testFindAndVerifyPartitionsAndImages_noFile(self): with mock.patch('os.listdir', return_value=['test']): with self.assertRaises(device_errors.FastbootCommandFailedError): - self.fastboot._FindAndVerifyPartitionsAndImages(['cache'], 'test') + fastboot_utils._FindAndVerifyPartitionsAndImages(['cache'], 'test') + + +class FastbootUtilsFlashDevice(FastbootUtilsTest): + + def testFlashDevice_wipe(self): + with self.assertCalls( + self.call.fastboot.EnableFastbootMode(), + self.call.fastboot.fastboot.SetOemOffModeCharge(False), + self.call.fastboot._FlashPartitions(mock.ANY, 'test', wipe=True), + self.call.fastboot.fastboot.SetOemOffModeCharge(True), + self.call.fastboot.Reboot(wait_for_reboot=False)): + self.fastboot.FlashDevice('test', wipe=True) + + def testFlashDevice_noWipe(self): + with self.assertCalls( + self.call.fastboot.EnableFastbootMode(), + self.call.fastboot.fastboot.SetOemOffModeCharge(False), + self.call.fastboot._FlashPartitions(mock.ANY, 'test', wipe=False), + self.call.fastboot.fastboot.SetOemOffModeCharge(True), + self.call.fastboot.Reboot(wait_for_reboot=True)): + self.fastboot.FlashDevice('test', wipe=False) + + def testFlashDevice_partitions(self): + with self.assertCalls( + self.call.fastboot.EnableFastbootMode(), + self.call.fastboot.fastboot.SetOemOffModeCharge(False), + self.call.fastboot._FlashPartitions(['boot'], 'test', wipe=False), + self.call.fastboot.fastboot.SetOemOffModeCharge(True), + self.call.fastboot.Reboot(wait_for_reboot=True)): + self.fastboot.FlashDevice('test', partitions=['boot'], wipe=False) if __name__ == '__main__': diff --git a/catapult/devil/devil/android/forwarder.py b/catapult/devil/devil/android/forwarder.py index 21f52236..99e8343b 100644 --- a/catapult/devil/devil/android/forwarder.py +++ b/catapult/devil/devil/android/forwarder.py @@ -11,6 +11,7 @@ import psutil from devil import base_error from devil import devil_env +from devil.android import device_errors from devil.android.constants import file_system from devil.android.valgrind_tools import base_tool from devil.utils import cmd_helper @@ -288,7 +289,11 @@ class Forwarder(object): device_serial = str(device) if device_serial in self._initialized_devices: return - Forwarder._KillDeviceLocked(device, tool) + try: + Forwarder._KillDeviceLocked(device, tool) + except device_errors.CommandFailedError: + logging.warning('Failed to kill device forwarder. Rebooting.') + device.Reboot() forwarder_device_path_on_host = devil_env.config.FetchPath( 'forwarder_device', device=device) forwarder_device_path_on_device = ( diff --git a/catapult/devil/devil/android/logcat_monitor.py b/catapult/devil/devil/android/logcat_monitor.py index 9ec94125..bf30d409 100644 --- a/catapult/devil/devil/android/logcat_monitor.py +++ b/catapult/devil/devil/android/logcat_monitor.py @@ -236,6 +236,10 @@ class LogcatMonitor(object): 'Need to call |Close| on the logcat monitor when done!') self._record_file.close() + @property + def adb(self): + return self._adb + class LogcatMonitorCommandError(device_errors.CommandFailedError): """Exception for errors with logcat monitor commands.""" diff --git a/catapult/devil/devil/android/perf/perf_control.py b/catapult/devil/devil/android/perf/perf_control.py index af1d52c3..383b4fb2 100644 --- a/catapult/devil/devil/android/perf/perf_control.py +++ b/catapult/devil/devil/android/perf/perf_control.py @@ -4,6 +4,7 @@ import atexit import logging +import re from devil.android import device_errors @@ -12,12 +13,14 @@ class PerfControl(object): """Provides methods for setting the performance mode of a device.""" _CPU_PATH = '/sys/devices/system/cpu' _KERNEL_MAX = '/sys/devices/system/cpu/kernel_max' + _CPU_FILE_PATTERN = re.compile(r'^cpu\d+$') def __init__(self, device): self._device = device - # this will raise an AdbCommandFailedError if no CPU files are found - self._cpu_files = self._device.RunShellCommand( - 'ls -d cpu[0-9]*', cwd=self._CPU_PATH, check_return=True, as_root=True) + self._cpu_files = [ + filename + for filename in self._device.ListDirectory(self._CPU_PATH, as_root=True) + if self._CPU_FILE_PATTERN.match(filename)] assert self._cpu_files, 'Failed to detect CPUs.' self._cpu_file_list = ' '.join(self._cpu_files) logging.info('CPUs found: %s', self._cpu_file_list) @@ -146,8 +149,8 @@ class PerfControl(object): """ if self._have_mpdecision: - script = 'stop mpdecision' if force_online else 'start mpdecision' - self._device.RunShellCommand(script, check_return=True, as_root=True) + cmd = ['stop', 'mpdecision'] if force_online else ['start', 'mpdecision'] + self._device.RunShellCommand(cmd, check_return=True, as_root=True) if not self._have_mpdecision and not self._AllCpusAreOnline(): logging.warning('Unexpected cpu hot plugging detected.') diff --git a/catapult/devil/devil/android/perf/thermal_throttle.py b/catapult/devil/devil/android/perf/thermal_throttle.py index 9aad4bb3..362a9d44 100644 --- a/catapult/devil/devil/android/perf/thermal_throttle.py +++ b/catapult/devil/devil/android/perf/thermal_throttle.py @@ -95,7 +95,8 @@ class ThermalThrottle(object): return False has_been_throttled = False serial_number = str(self._device) - log = self._device.RunShellCommand('dmesg -c') + log = self._device.RunShellCommand( + ['dmesg', '-c'], large_output=True, check_return=True) degree_symbol = unichr(0x00B0) for line in log: if self._detector.BecameThrottled(line): @@ -129,4 +130,3 @@ class ThermalThrottle(object): serial_number, btemp, degree_symbol) return has_been_throttled - diff --git a/catapult/devil/devil/android/ports.py b/catapult/devil/devil/android/ports.py index 4783082c..6384d74b 100644 --- a/catapult/devil/devil/android/ports.py +++ b/catapult/devil/devil/android/ports.py @@ -34,8 +34,6 @@ def ResetTestServerPortAllocation(): try: with open(_TEST_SERVER_PORT_FILE, 'w') as fp: fp.write('%d' % _TEST_SERVER_PORT_FIRST) - if os.path.exists(_TEST_SERVER_PORT_LOCKFILE): - os.unlink(_TEST_SERVER_PORT_LOCKFILE) return True except Exception: # pylint: disable=broad-except logging.exception('Error while resetting port allocation') @@ -70,7 +68,7 @@ def AllocateTestServerPort(): fp.seek(0, os.SEEK_SET) fp.write('%d' % (port + 1)) except Exception: # pylint: disable=broad-except - logging.exception('ERror while allocating port') + logging.exception('Error while allocating port') finally: if fp_lock: fcntl.flock(fp_lock, fcntl.LOCK_UN) diff --git a/catapult/devil/devil/android/sdk/adb_compatibility_devicetest.py b/catapult/devil/devil/android/sdk/adb_compatibility_devicetest.py index de08e21a..6f670168 100755..100644 --- a/catapult/devil/devil/android/sdk/adb_compatibility_devicetest.py +++ b/catapult/devil/devil/android/sdk/adb_compatibility_devicetest.py @@ -3,24 +3,27 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import contextlib import os +import posixpath +import random import signal import sys import unittest +_CATAPULT_BASE_DIR = os.path.abspath(os.path.join( + os.path.dirname(__file__), '..', '..', '..', '..')) + +sys.path.append(os.path.join(_CATAPULT_BASE_DIR, 'devil')) from devil import devil_env +from devil.android import device_errors from devil.android.sdk import adb_wrapper from devil.utils import cmd_helper from devil.utils import timeout_retry -_PYMOCK_PATH = os.path.abspath(os.path.join( - os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, - 'third_party', 'mock')) -with devil_env.SysPath(_PYMOCK_PATH): - import mock # pylint: disable=import-error - -_ADB_PATH = os.environ.get('ADB_PATH', 'adb') +_TEST_DATA_DIR = os.path.abspath(os.path.join( + os.path.dirname(__file__), 'test', 'data')) def _hostAdbPids(): @@ -34,11 +37,26 @@ def _hostAdbPids(): if name == 'adb'] -@mock.patch('devil.android.sdk.adb_wrapper.AdbWrapper.GetAdbPath', - return_value=_ADB_PATH) class AdbCompatibilityTest(unittest.TestCase): - def testStartServer(self, *_args): + @classmethod + def setUpClass(cls): + custom_adb_path = os.environ.get('ADB_PATH') + custom_deps = { + 'config_type': 'BaseConfig', + 'dependencies': {}, + } + if custom_adb_path: + custom_deps['dependencies']['adb'] = { + 'file_info': { + devil_env.GetPlatform(): { + 'local_paths': [custom_adb_path], + }, + }, + } + devil_env.config.Initialize(configs=[custom_deps]) + + def testStartServer(self): # Manually kill off any instances of adb. adb_pids = _hostAdbPids() for p in adb_pids: @@ -50,7 +68,7 @@ class AdbCompatibilityTest(unittest.TestCase): # start the adb server start_server_status, _ = cmd_helper.GetCmdStatusAndOutput( - [_ADB_PATH, 'start-server']) + [adb_wrapper.AdbWrapper.GetAdbPath(), 'start-server']) # verify that the server is now online self.assertEquals(0, start_server_status) @@ -58,7 +76,7 @@ class AdbCompatibilityTest(unittest.TestCase): timeout_retry.WaitFor( lambda: bool(_hostAdbPids()), wait_period=0.1, max_tries=10)) - def testKillServer(self, *_args): + def testKillServer(self): adb_pids = _hostAdbPids() if not adb_pids: adb_wrapper.AdbWrapper.StartServer() @@ -67,16 +85,112 @@ class AdbCompatibilityTest(unittest.TestCase): self.assertEqual(1, len(adb_pids)) kill_server_status, _ = cmd_helper.GetCmdStatusAndOutput( - [_ADB_PATH, 'kill-server']) + [adb_wrapper.AdbWrapper.GetAdbPath(), 'kill-server']) self.assertEqual(0, kill_server_status) adb_pids = _hostAdbPids() self.assertEqual(0, len(adb_pids)) + def testDevices(self): + devices = adb_wrapper.AdbWrapper.Devices() + self.assertNotEqual(0, len(devices), 'No devices found.') + + def getTestInstance(self): + """Creates a real AdbWrapper instance for testing.""" + devices = adb_wrapper.AdbWrapper.Devices() + if not devices: + self.skipTest('No test device available.') + return adb_wrapper.AdbWrapper(devices[0]) + + def testShell(self): + under_test = self.getTestInstance() + shell_ls_result = under_test.Shell('ls') + self.assertIsInstance(shell_ls_result, str) + self.assertTrue(bool(shell_ls_result)) + + def testShell_failed(self): + under_test = self.getTestInstance() + with self.assertRaises(device_errors.AdbShellCommandFailedError): + under_test.Shell('ls /foo/bar/baz') + + def testShell_externalStorageDefined(self): + under_test = self.getTestInstance() + external_storage = under_test.Shell('echo $EXTERNAL_STORAGE') + self.assertIsInstance(external_storage, str) + self.assertTrue(posixpath.isabs(external_storage)) + + @contextlib.contextmanager + def getTestPushDestination(self, under_test): + """Creates a temporary directory suitable for pushing to.""" + external_storage = under_test.Shell('echo $EXTERNAL_STORAGE').strip() + if not external_storage: + self.skipTest('External storage not available.') + while True: + random_hex = hex(random.randint(0, 2 ** 52))[2:] + name = 'tmp_push_test%s' % random_hex + path = posixpath.join(external_storage, name) + try: + under_test.Shell('ls %s' % path) + except device_errors.AdbShellCommandFailedError: + break + under_test.Shell('mkdir %s' % path) + try: + yield path + finally: + under_test.Shell('rm -rf %s' % path) + + def testPush_fileToFile(self): + under_test = self.getTestInstance() + with self.getTestPushDestination(under_test) as push_target_directory: + src = os.path.join(_TEST_DATA_DIR, 'push_file.txt') + dest = posixpath.join(push_target_directory, 'push_file.txt') + with self.assertRaises(device_errors.AdbShellCommandFailedError): + under_test.Shell('ls %s' % dest) + under_test.Push(src, dest) + self.assertEquals(dest, under_test.Shell('ls %s' % dest).strip()) + + def testPush_fileToDirectory(self): + under_test = self.getTestInstance() + with self.getTestPushDestination(under_test) as push_target_directory: + src = os.path.join(_TEST_DATA_DIR, 'push_file.txt') + dest = push_target_directory + resulting_file = posixpath.join(dest, 'push_file.txt') + with self.assertRaises(device_errors.AdbShellCommandFailedError): + under_test.Shell('ls %s' % resulting_file) + under_test.Push(src, dest) + self.assertEquals( + resulting_file, + under_test.Shell('ls %s' % resulting_file).strip()) + + def testPush_directoryToDirectory(self): + under_test = self.getTestInstance() + with self.getTestPushDestination(under_test) as push_target_directory: + src = os.path.join(_TEST_DATA_DIR, 'push_directory') + dest = posixpath.join(push_target_directory, 'push_directory') + with self.assertRaises(device_errors.AdbShellCommandFailedError): + under_test.Shell('ls %s' % dest) + under_test.Push(src, dest) + self.assertEquals( + sorted(os.listdir(src)), + sorted(under_test.Shell('ls %s' % dest).strip().split())) + + def testPush_directoryToExistingDirectory(self): + under_test = self.getTestInstance() + with self.getTestPushDestination(under_test) as push_target_directory: + src = os.path.join(_TEST_DATA_DIR, 'push_directory') + dest = push_target_directory + resulting_directory = posixpath.join(dest, 'push_directory') + with self.assertRaises(device_errors.AdbShellCommandFailedError): + under_test.Shell('ls %s' % resulting_directory) + under_test.Shell('mkdir %s' % resulting_directory) + under_test.Push(src, dest) + self.assertEquals( + sorted(os.listdir(src)), + sorted(under_test.Shell('ls %s' % resulting_directory).split())) + # TODO(jbudorick): Implement tests for the following: # taskset -c # devices [-l] - # push # pull # shell # ls @@ -99,17 +213,19 @@ class AdbCompatibilityTest(unittest.TestCase): @classmethod def tearDownClass(cls): - version_status, version_output = cmd_helper.GetCmdStatusAndOutput( - [_ADB_PATH, 'version']) - if version_status != 0: - version = ['(unable to determine version)'] - else: - version = version_output.splitlines() - print - print 'tested %s' % _ADB_PATH - for l in version: - print ' %s' % l + print + print 'tested %s' % adb_wrapper.AdbWrapper.GetAdbPath() + print ' %s' % adb_wrapper.AdbWrapper.Version() + print 'connected devices:' + try: + for d in adb_wrapper.AdbWrapper.Devices(): + print ' %s' % d + except device_errors.AdbCommandFailedError: + print ' <failed to list devices>' + raise + finally: + print if __name__ == '__main__': diff --git a/catapult/devil/devil/android/sdk/adb_wrapper.py b/catapult/devil/devil/android/sdk/adb_wrapper.py index a65ab7cb..57e6396a 100644 --- a/catapult/devil/devil/android/sdk/adb_wrapper.py +++ b/catapult/devil/devil/android/sdk/adb_wrapper.py @@ -9,10 +9,13 @@ should be delegated to a higher level (ex. DeviceUtils). """ import collections +import distutils.version import errno import logging import os +import posixpath import re +import subprocess from devil import devil_env from devil.android import decorators @@ -25,12 +28,14 @@ with devil_env.SysPath(devil_env.DEPENDENCY_MANAGER_PATH): import dependency_manager # pylint: disable=import-error -_DEFAULT_TIMEOUT = 30 -_DEFAULT_RETRIES = 2 +DEFAULT_TIMEOUT = 30 +DEFAULT_RETRIES = 2 +_ADB_VERSION_RE = re.compile(r'Android Debug Bridge version (\d+\.\d+\.\d+)') _EMULATOR_RE = re.compile(r'^emulator-[0-9]+$') - _READY_STATE = 'device' +_VERITY_DISABLE_RE = re.compile('Verity (already)? disabled') +_VERITY_ENABLE_RE = re.compile('Verity (already)? enabled') def VerifyLocalFileExists(path): @@ -64,6 +69,16 @@ def _FindAdb(): raise device_errors.NoAdbError() +def _GetVersion(): + # pylint: disable=protected-access + raw_version = AdbWrapper._RunAdbCmd(['version'], timeout=2, retries=0) + for l in raw_version.splitlines(): + m = _ADB_VERSION_RE.search(l) + if m: + return m.group(1) + return None + + def _ShouldRetryAdbCmd(exc): return not isinstance(exc, device_errors.NoAdbError) @@ -72,10 +87,29 @@ DeviceStat = collections.namedtuple('DeviceStat', ['st_mode', 'st_size', 'st_time']) +def _IsExtraneousLine(line, send_cmd): + """Determine if a line read from stdout in persistent shell is extraneous. + + The results output to stdout by the persistent shell process + (in PersistentShell below) often include "extraneous" lines that are + not part of the output of the shell command. These "extraneous" lines + do not always appear and are of two forms: shell prompt lines and lines + that just duplicate what the input command was. This function + detects these extraneous lines. Since all these lines have the + original command in them, that is what it detects ror. + + Args: + line: Output line to check. + send_cmd: Command that was sent to adb persistent shell. + """ + return send_cmd.rstrip() in line + + class AdbWrapper(object): """A wrapper around a local Android Debug Bridge executable.""" _adb_path = lazy.WeakConstant(_FindAdb) + _adb_version = lazy.WeakConstant(_GetVersion) def __init__(self, device_serial): """Initializes the AdbWrapper. @@ -87,11 +121,107 @@ class AdbWrapper(object): raise ValueError('A device serial must be specified') self._device_serial = str(device_serial) + class PersistentShell(object): + '''Class to use persistent shell for ADB. + + This class allows a persistent ADB shell to be created, where multiple + commands can be passed into it. This avoids the overhead of starting + up a new ADB shell for each command. + + Example of use: + with pshell as PersistentShell('123456789'): + pshell.RunCommand('which ls') + pshell.RunCommandAndClose('echo TEST') + ''' + def __init__(self, serial): + """Initialization function: + + Args: + serial: Serial number of device. + """ + self._cmd = [AdbWrapper.GetAdbPath(), '-s', serial, 'shell'] + self._process = None + + def __enter__(self): + self.Start() + self.WaitForReady() + return self + + def __exit__(self, exc_type, exc_value, tb): + self.Stop() + + def Start(self): + """Start the shell.""" + if self._process is not None: + raise RuntimeError('Persistent shell already running.') + self._process = subprocess.Popen(self._cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + shell=False) + + def WaitForReady(self): + """Wait for the shell to be ready after starting. + + Sends an echo command, then waits until it gets a response. + """ + self._process.stdin.write('echo\n') + output_line = self._process.stdout.readline() + while output_line.rstrip() != '': + output_line = self._process.stdout.readline() + + def RunCommand(self, command, close=False): + """Runs an ADB command and returns the output. + + Note that there can be approximately 40 ms of additional latency + between sending the command and receiving the results if close=False + due to the use of Nagle's algorithm in the TCP socket between the + adb server and client. To avoid this extra latency, set close=True. + + Args: + command: Command to send. + Returns: + The command output, given as a list of lines, and the exit code + """ + + if close: + def run_cmd(cmd): + send_cmd = '( %s ); echo $?; exit;\n' % cmd.rstrip() + (output, _) = self._process.communicate(send_cmd) + self._process = None + for x in output.splitlines(): + yield x + + else: + def run_cmd(cmd): + send_cmd = '( %s ); echo DONE:$?;\n' % cmd.rstrip() + self._process.stdin.write(send_cmd) + while True: + output_line = self._process.stdout.readline().rstrip() + if output_line[:5] == 'DONE:': + yield output_line[5:] + break + yield output_line + + result = [line for line in run_cmd(command) + if not _IsExtraneousLine(line, command)] + + return (result[:-1], int(result[-1])) + + def Stop(self): + """Stops the ADB process if it is still running.""" + if self._process is not None: + self._process.stdin.write('exit\n') + self._process = None + @classmethod def GetAdbPath(cls): return cls._adb_path.read() @classmethod + def Version(cls): + return cls._adb_version.read() + + @classmethod def _BuildAdbCmd(cls, args, device_serial, cpu_affinity=None): if cpu_affinity is not None: cmd = ['taskset', '-c', str(cpu_affinity)] @@ -192,24 +322,24 @@ class AdbWrapper(object): # pylint: enable=unused-argument @classmethod - def KillServer(cls, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + def KillServer(cls, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): cls._RunAdbCmd(['kill-server'], timeout=timeout, retries=retries) @classmethod - def StartServer(cls, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + def StartServer(cls, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): # CPU affinity is used to reduce adb instability http://crbug.com/268450 cls._RunAdbCmd(['start-server'], timeout=timeout, retries=retries, cpu_affinity=0) @classmethod - def GetDevices(cls, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + def GetDevices(cls, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): """DEPRECATED. Refer to Devices(...) below.""" # TODO(jbudorick): Remove this function once no more clients are using it. return cls.Devices(timeout=timeout, retries=retries) @classmethod def Devices(cls, desired_state=_READY_STATE, long_list=False, - timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): """Get the list of active attached devices. Args: @@ -238,8 +368,8 @@ class AdbWrapper(object): ] @classmethod - def _RawDevices(cls, long_list=False, timeout=_DEFAULT_TIMEOUT, - retries=_DEFAULT_RETRIES): + def _RawDevices(cls, long_list=False, timeout=DEFAULT_TIMEOUT, + retries=DEFAULT_RETRIES): cmd = ['devices'] if long_list: cmd.append('-l') @@ -254,7 +384,7 @@ class AdbWrapper(object): """ return self._device_serial - def Push(self, local, remote, timeout=60 * 5, retries=_DEFAULT_RETRIES): + def Push(self, local, remote, timeout=60 * 5, retries=DEFAULT_RETRIES): """Pushes a file from the host to the device. Args: @@ -264,9 +394,47 @@ class AdbWrapper(object): retries: (optional) Number of retries to attempt. """ VerifyLocalFileExists(local) + + if (distutils.version.LooseVersion(self.Version()) < + distutils.version.LooseVersion('1.0.36')): + + # Different versions of adb handle pushing a directory to an existing + # directory differently. + + # In the version packaged with the M SDK, 1.0.32, the following push: + # foo/bar -> /sdcard/foo/bar + # where bar is an existing directory both on the host and the device + # results in the contents of bar/ on the host being pushed to bar/ on + # the device, i.e. + # foo/bar/A -> /sdcard/foo/bar/A + # foo/bar/B -> /sdcard/foo/bar/B + # ... etc. + + # In the version packaged with the N SDK, 1.0.36, the same push under + # the same conditions results in a second bar/ directory being created + # underneath the first bar/ directory on the device, i.e. + # foo/bar/A -> /sdcard/foo/bar/bar/A + # foo/bar/B -> /sdcard/foo/bar/bar/B + # ... etc. + + # In order to provide a consistent interface to clients, we check whether + # the target is an existing directory on the device and, if so, modifies + # the target passed to adb to emulate the behavior on 1.0.36 and above. + + # Note that this behavior may have started before 1.0.36; that's simply + # the earliest version we've confirmed thus far. + + try: + self.Shell('test -d %s' % remote, timeout=timeout, retries=retries) + remote = posixpath.join(remote, posixpath.basename(local)) + except device_errors.AdbShellCommandFailedError: + # The target directory doesn't exist on the device, so we can use it + # without modification. + pass + self._RunDeviceAdbCmd(['push', local, remote], timeout, retries) - def Pull(self, remote, local, timeout=60 * 5, retries=_DEFAULT_RETRIES): + def Pull(self, remote, local, timeout=60 * 5, retries=DEFAULT_RETRIES): """Pulls a file from the device to the host. Args: @@ -283,8 +451,8 @@ class AdbWrapper(object): raise device_errors.AdbCommandFailedError( cmd, 'File not found on host: %s' % local, device_serial=str(self)) - def Shell(self, command, expect_status=0, timeout=_DEFAULT_TIMEOUT, - retries=_DEFAULT_RETRIES): + def Shell(self, command, expect_status=0, timeout=DEFAULT_TIMEOUT, + retries=DEFAULT_RETRIES): """Runs a shell command on the device. Args: @@ -338,7 +506,7 @@ class AdbWrapper(object): return cmd_helper.IterCmdOutputLines( self._BuildAdbCmd(args, self._device_serial), timeout=timeout) - def Ls(self, path, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + def Ls(self, path, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): """List the contents of a directory on the device. Args: @@ -377,7 +545,7 @@ class AdbWrapper(object): def Logcat(self, clear=False, dump=False, filter_specs=None, logcat_format=None, ring_buffer=None, timeout=None, - retries=_DEFAULT_RETRIES): + retries=DEFAULT_RETRIES): """Get an iterable over the logcat output. Args: @@ -391,7 +559,7 @@ class AdbWrapper(object): Options include "main", "system", "radio", "events", "crash" or "all". The default is equivalent to ["main", "system", "crash"]. timeout: (optional) If set, timeout per try in seconds. If clear or dump - is set, defaults to _DEFAULT_TIMEOUT. + is set, defaults to DEFAULT_TIMEOUT. retries: (optional) If clear or dump is set, the number of retries to attempt. Otherwise, does nothing. @@ -417,11 +585,11 @@ class AdbWrapper(object): if use_iter: return self._IterRunDeviceAdbCmd(cmd, timeout) else: - timeout = timeout if timeout is not None else _DEFAULT_TIMEOUT + timeout = timeout if timeout is not None else DEFAULT_TIMEOUT return self._RunDeviceAdbCmd(cmd, timeout, retries).splitlines() - def Forward(self, local, remote, timeout=_DEFAULT_TIMEOUT, - retries=_DEFAULT_RETRIES): + def Forward(self, local, remote, allow_rebind=False, + timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): """Forward socket connections from the local socket to the remote socket. Sockets are specified by one of: @@ -435,14 +603,20 @@ class AdbWrapper(object): Args: local: The host socket. remote: The device socket. + allow_rebind: A boolean indicating whether adb may rebind a local socket; + otherwise, the default, an exception is raised if the local socket is + already being forwarded. timeout: (optional) Timeout per try in seconds. retries: (optional) Number of retries to attempt. """ - self._RunDeviceAdbCmd(['forward', str(local), str(remote)], timeout, - retries) + cmd = ['forward'] + if not allow_rebind: + cmd.append('--no-rebind') + cmd.extend([str(local), str(remote)]) + self._RunDeviceAdbCmd(cmd, timeout, retries) - def ForwardRemove(self, local, timeout=_DEFAULT_TIMEOUT, - retries=_DEFAULT_RETRIES): + def ForwardRemove(self, local, timeout=DEFAULT_TIMEOUT, + retries=DEFAULT_RETRIES): """Remove a forward socket connection. Args: @@ -453,7 +627,7 @@ class AdbWrapper(object): self._RunDeviceAdbCmd(['forward', '--remove', str(local)], timeout, retries) - def ForwardList(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + def ForwardList(self, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): """List all currently forwarded socket connections. Args: @@ -462,7 +636,7 @@ class AdbWrapper(object): """ return self._RunDeviceAdbCmd(['forward', '--list'], timeout, retries) - def JDWP(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + def JDWP(self, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): """List of PIDs of processes hosting a JDWP transport. Args: @@ -477,7 +651,7 @@ class AdbWrapper(object): def Install(self, apk_path, forward_lock=False, allow_downgrade=False, reinstall=False, sd_card=False, timeout=60 * 2, - retries=_DEFAULT_RETRIES): + retries=DEFAULT_RETRIES): """Install an apk on the device. Args: @@ -507,7 +681,7 @@ class AdbWrapper(object): def InstallMultiple(self, apk_paths, forward_lock=False, reinstall=False, sd_card=False, allow_downgrade=False, partial=False, - timeout=60 * 2, retries=_DEFAULT_RETRIES): + timeout=60 * 2, retries=DEFAULT_RETRIES): """Install an apk with splits on the device. Args: @@ -539,8 +713,8 @@ class AdbWrapper(object): raise device_errors.AdbCommandFailedError( cmd, output, device_serial=self._device_serial) - def Uninstall(self, package, keep_data=False, timeout=_DEFAULT_TIMEOUT, - retries=_DEFAULT_RETRIES): + def Uninstall(self, package, keep_data=False, timeout=DEFAULT_TIMEOUT, + retries=DEFAULT_RETRIES): """Remove the app |package| from the device. Args: @@ -559,8 +733,8 @@ class AdbWrapper(object): cmd, output, device_serial=self._device_serial) def Backup(self, path, packages=None, apk=False, shared=False, - nosystem=True, include_all=False, timeout=_DEFAULT_TIMEOUT, - retries=_DEFAULT_RETRIES): + nosystem=True, include_all=False, timeout=DEFAULT_TIMEOUT, + retries=DEFAULT_RETRIES): """Write an archive of the device's data to |path|. Args: @@ -591,7 +765,7 @@ class AdbWrapper(object): VerifyLocalFileExists(path) return ret - def Restore(self, path, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + def Restore(self, path, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): """Restore device contents from the backup archive. Args: @@ -602,7 +776,7 @@ class AdbWrapper(object): VerifyLocalFileExists(path) self._RunDeviceAdbCmd(['restore'] + [path], timeout, retries) - def WaitForDevice(self, timeout=60 * 5, retries=_DEFAULT_RETRIES): + def WaitForDevice(self, timeout=60 * 5, retries=DEFAULT_RETRIES): """Block until the device is online. Args: @@ -611,7 +785,7 @@ class AdbWrapper(object): """ self._RunDeviceAdbCmd(['wait-for-device'], timeout, retries) - def GetState(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + def GetState(self, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): """Get device state. Args: @@ -631,7 +805,7 @@ class AdbWrapper(object): return line[1] return 'offline' - def GetDevPath(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + def GetDevPath(self, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): """Gets the device path. Args: @@ -643,12 +817,12 @@ class AdbWrapper(object): """ return self._RunDeviceAdbCmd(['get-devpath'], timeout, retries) - def Remount(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + def Remount(self, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): """Remounts the /system partition on the device read-write.""" self._RunDeviceAdbCmd(['remount'], timeout, retries) def Reboot(self, to_bootloader=False, timeout=60 * 5, - retries=_DEFAULT_RETRIES): + retries=DEFAULT_RETRIES): """Reboots the device. Args: @@ -662,7 +836,7 @@ class AdbWrapper(object): cmd = ['reboot'] self._RunDeviceAdbCmd(cmd, timeout, retries) - def Root(self, timeout=_DEFAULT_TIMEOUT, retries=_DEFAULT_RETRIES): + def Root(self, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): """Restarts the adbd daemon with root permissions, if possible. Args: @@ -674,8 +848,8 @@ class AdbWrapper(object): raise device_errors.AdbCommandFailedError( ['root'], output, device_serial=self._device_serial) - def Emu(self, cmd, timeout=_DEFAULT_TIMEOUT, - retries=_DEFAULT_RETRIES): + def Emu(self, cmd, timeout=DEFAULT_TIMEOUT, + retries=DEFAULT_RETRIES): """Runs an emulator console command. See http://developer.android.com/tools/devices/emulator.html#console @@ -692,6 +866,20 @@ class AdbWrapper(object): cmd = [cmd] return self._RunDeviceAdbCmd(['emu'] + cmd, timeout, retries) + def DisableVerity(self, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): + """Disable Marshmallow's Verity security feature""" + output = self._RunDeviceAdbCmd(['disable-verity'], timeout, retries) + if output and _VERITY_DISABLE_RE.search(output): + raise device_errors.AdbCommandFailedError( + ['disable-verity'], output, device_serial=self._device_serial) + + def EnableVerity(self, timeout=DEFAULT_TIMEOUT, retries=DEFAULT_RETRIES): + """Enable Marshmallow's Verity security feature""" + output = self._RunDeviceAdbCmd(['enable-verity'], timeout, retries) + if output and _VERITY_ENABLE_RE.search(output): + raise device_errors.AdbCommandFailedError( + ['enable-verity'], output, device_serial=self._device_serial) + @property def is_emulator(self): return _EMULATOR_RE.match(self._device_serial) diff --git a/catapult/devil/devil/android/sdk/adb_wrapper_devicetest.py b/catapult/devil/devil/android/sdk/adb_wrapper_devicetest.py index 59755c00..9a38c6cd 100644..100755 --- a/catapult/devil/devil/android/sdk/adb_wrapper_devicetest.py +++ b/catapult/devil/devil/android/sdk/adb_wrapper_devicetest.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -44,6 +46,20 @@ class TestAdbWrapper(unittest.TestCase): with self.assertRaises(device_errors.AdbCommandFailedError): self._adb.Shell('echo test', expect_status=1) + @unittest.skip("https://github.com/catapult-project/catapult/issues/2574") + def testPersistentShell(self): + # We need to access the device serial number here in order + # to create the persistent shell. + serial = self._adb.GetDeviceSerial() # pylint: disable=protected-access + with self._adb.PersistentShell(serial) as pshell: + (res1, code1) = pshell.RunCommand('echo TEST') + (res2, code2) = pshell.RunCommand('echo TEST2') + self.assertEqual(len(res1), 1) + self.assertEqual(res1[0], 'TEST') + self.assertEqual(res2[-1], 'TEST2') + self.assertEqual(code1, 0) + self.assertEqual(code2, 0) + def testPushLsPull(self): path = self._MakeTempFile('foo') device_path = '/data/local/tmp/testfile.txt' diff --git a/catapult/devil/devil/android/sdk/fastboot.py b/catapult/devil/devil/android/sdk/fastboot.py index d9fa653b..d7f9f624 100644 --- a/catapult/devil/devil/android/sdk/fastboot.py +++ b/catapult/devil/devil/android/sdk/fastboot.py @@ -9,8 +9,6 @@ should be delegated to a higher level (ex. FastbootUtils). """ # pylint: disable=unused-argument -import os - from devil import devil_env from devil.android import decorators from devil.android import device_errors @@ -24,8 +22,8 @@ _FLASH_TIMEOUT = _DEFAULT_TIMEOUT * 10 class Fastboot(object): - _fastboot_path = lazy.WeakConstant(lambda: os.path.join( - devil_env.config.LocalPath('android_sdk'), 'platform-tools', 'adb')) + _fastboot_path = lazy.WeakConstant( + lambda: devil_env.config.FetchPath('fastboot')) def __init__(self, device_serial, default_timeout=_DEFAULT_TIMEOUT, default_retries=_DEFAULT_RETRIES): @@ -40,8 +38,7 @@ class Fastboot(object): self._default_timeout = default_timeout self._default_retries = default_retries - @decorators.WithTimeoutAndRetriesFromInstance() - def _RunFastbootCommand(self, cmd, timeout=None, retries=None): + def _RunFastbootCommand(self, cmd): """Run a command line command using the fastboot android tool. Args: diff --git a/catapult/devil/devil/android/sdk/gce_adb_wrapper.py b/catapult/devil/devil/android/sdk/gce_adb_wrapper.py index 5ee7959c..a85d2bd2 100644 --- a/catapult/devil/devil/android/sdk/gce_adb_wrapper.py +++ b/catapult/devil/devil/android/sdk/gce_adb_wrapper.py @@ -19,16 +19,27 @@ from devil.android.sdk import adb_wrapper from devil.utils import cmd_helper -# SSH key file for accessing the instances. The keys are created at -# startup and removed & revoked at teardown. -_SSH_KEY_FILE = '/tmp/ssh_android_gce_instance' - - class GceAdbWrapper(adb_wrapper.AdbWrapper): def __init__(self, device_serial): super(GceAdbWrapper, self).__init__(device_serial) - self._instance_ip = self.Shell('getprop net.gce.ip_address').strip() + self._Connect() + self.Root() + self._instance_ip = self.Shell('getprop net.gce.ip').strip() + + def _Connect(self, timeout=adb_wrapper.DEFAULT_TIMEOUT, + retries=adb_wrapper.DEFAULT_RETRIES): + """Connects ADB to the android gce instance.""" + cmd = ['connect', self._device_serial] + output = self._RunAdbCmd(cmd, timeout=timeout, retries=retries) + if 'unable to connect' in output: + raise device_errors.AdbCommandFailedError(cmd, output) + self.WaitForDevice() + + # override + def Root(self, **kwargs): + super(GceAdbWrapper, self).Root() + self._Connect() # override def Push(self, local, remote, **kwargs): @@ -38,7 +49,6 @@ class GceAdbWrapper(adb_wrapper.AdbWrapper): local: Path on the host filesystem. remote: Path on the instance filesystem. """ - adb_wrapper.VerifyLocalFileExists(_SSH_KEY_FILE) adb_wrapper.VerifyLocalFileExists(local) if os.path.isdir(local): self.Shell('mkdir -p %s' % cmd_helper.SingleQuote(remote)) @@ -66,7 +76,6 @@ class GceAdbWrapper(adb_wrapper.AdbWrapper): cmd = [ 'scp', '-r', - '-i', _SSH_KEY_FILE, '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', local, @@ -86,12 +95,10 @@ class GceAdbWrapper(adb_wrapper.AdbWrapper): remote: Path on the instance filesystem. local: Path on the host filesystem. """ - adb_wrapper.VerifyLocalFileExists(_SSH_KEY_FILE) cmd = [ 'scp', '-p', '-r', - '-i', _SSH_KEY_FILE, '-o', 'UserKnownHostsFile=/dev/null', '-o', 'StrictHostKeyChecking=no', 'root@%s:%s' % (self._instance_ip, remote), @@ -122,7 +129,6 @@ class GceAdbWrapper(adb_wrapper.AdbWrapper): reinstall: (optional) If set reinstalls the app, keeping its data. sd_card: (optional) If set installs on the SD card. """ - adb_wrapper.VerifyLocalFileExists(_SSH_KEY_FILE) adb_wrapper.VerifyLocalFileExists(apk_path) cmd = ['install'] if forward_lock: diff --git a/catapult/devil/devil/android/sdk/intent.py b/catapult/devil/devil/android/sdk/intent.py index e612f76b..cdefb463 100644 --- a/catapult/devil/devil/android/sdk/intent.py +++ b/catapult/devil/devil/android/sdk/intent.py @@ -8,6 +8,21 @@ This is generally intended to be used with functions that calls Android's Am command. """ +# Some common flag constants that can be used to construct intents. +# Full list: http://developer.android.com/reference/android/content/Intent.html +FLAG_ACTIVITY_CLEAR_TASK = 0x00008000 +FLAG_ACTIVITY_CLEAR_TOP = 0x04000000 +FLAG_ACTIVITY_NEW_TASK = 0x10000000 +FLAG_ACTIVITY_REORDER_TO_FRONT = 0x00020000 +FLAG_ACTIVITY_RESET_TASK_IF_NEEDED = 0x00200000 + + +def _bitwise_or(flags): + result = 0 + for flag in flags: + result |= flag + return result + class Intent(object): @@ -25,7 +40,7 @@ class Intent(object): data: A string containing a data URI. extras: A dict containing extra parameters to be passed along with the intent. - flags: A string containing flags to pass. + flags: A sequence of flag constants to be passed with the intent. package: A string that, with activity, can be used to specify the component. """ @@ -38,7 +53,7 @@ class Intent(object): self._component = component self._data = data self._extras = extras - self._flags = flags + self._flags = '0x%0.8x' % _bitwise_or(flags) if flags else None self._package = package if self._component and '/' in component: diff --git a/catapult/devil/devil/android/sdk/test/data/push_directory/push_directory_contents.txt b/catapult/devil/devil/android/sdk/test/data/push_directory/push_directory_contents.txt new file mode 100644 index 00000000..573df2e9 --- /dev/null +++ b/catapult/devil/devil/android/sdk/test/data/push_directory/push_directory_contents.txt @@ -0,0 +1 @@ +Goodnight, moon. diff --git a/catapult/devil/devil/android/sdk/test/data/push_file.txt b/catapult/devil/devil/android/sdk/test/data/push_file.txt new file mode 100644 index 00000000..af5626b4 --- /dev/null +++ b/catapult/devil/devil/android/sdk/test/data/push_file.txt @@ -0,0 +1 @@ +Hello, world! diff --git a/catapult/devil/devil/android/tools/adb_run_shell_cmd.py b/catapult/devil/devil/android/tools/adb_run_shell_cmd.py index f995d272..a826ab12 100755 --- a/catapult/devil/devil/android/tools/adb_run_shell_cmd.py +++ b/catapult/devil/devil/android/tools/adb_run_shell_cmd.py @@ -5,11 +5,9 @@ import argparse import json -import logging import sys from devil.android import device_blacklist -from devil.android import device_errors from devil.android import device_utils from devil.utils import run_tests_helper @@ -19,6 +17,7 @@ def main(): 'Run an adb shell command on selected devices') parser.add_argument('cmd', help='Adb shell command to run.', nargs="+") parser.add_argument('-d', '--device', action='append', dest='devices', + default=[], help='Device to run cmd on. Runs on all devices if not ' 'specified. Set multiple times for multiple devices') parser.add_argument('-v', '--verbose', default=0, action='count', @@ -32,29 +31,15 @@ def main(): args.blacklist_file = device_blacklist.Blacklist( args.blacklist_file) if args.blacklist_file else None - attached_devices = device_utils.DeviceUtils.HealthyDevices( - blacklist=args.blacklist_file) + devices = device_utils.DeviceUtils.HealthyDevices( + blacklist=args.blacklist_file, device_arg=args.devices) - if args.devices: - selected_devices = [] - attached_devices = {str(d): d for d in attached_devices} - for serial in args.devices: - if serial in attached_devices: - selected_devices.append(attached_devices[serial]) - else: - logging.warning('Specified device %s not found.', serial) - else: - selected_devices = attached_devices - - if not selected_devices: - raise device_errors.NoDevicesError - - p_out = (device_utils.DeviceUtils.parallel(selected_devices).RunShellCommand( + p_out = (device_utils.DeviceUtils.parallel(devices).RunShellCommand( args.cmd, large_output=True, as_root=args.as_root, check_return=True) .pGet(None)) data = {} - for device, output in zip(selected_devices, p_out): + for device, output in zip(devices, p_out): for line in output: print '%s: %s' % (device, line) data[str(device)] = output diff --git a/catapult/devil/devil/android/tools/device_recovery.py b/catapult/devil/devil/android/tools/device_recovery.py new file mode 100755 index 00000000..e6456a70 --- /dev/null +++ b/catapult/devil/devil/android/tools/device_recovery.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A script to recover devices in a known bad state.""" + +import argparse +import logging +import os +import psutil +import signal +import sys + +if __name__ == '__main__': + sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..', '..'))) +from devil import devil_env +from devil.android import device_blacklist +from devil.android import device_errors +from devil.android import device_utils +from devil.android.tools import device_status +from devil.utils import lsusb +from devil.utils import reset_usb +from devil.utils import run_tests_helper + + +def KillAllAdb(): + def get_all_adb(): + for p in psutil.process_iter(): + try: + if 'adb' in p.name: + yield p + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]: + for p in get_all_adb(): + try: + logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name, + ' '.join(p.cmdline)) + p.send_signal(sig) + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + for p in get_all_adb(): + try: + logging.error('Unable to kill %d (%s [%s])', p.pid, p.name, + ' '.join(p.cmdline)) + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + + +def RecoverDevice(device, blacklist, should_reboot=lambda device: True): + if device_status.IsBlacklisted(device.adb.GetDeviceSerial(), + blacklist): + logging.debug('%s is blacklisted, skipping recovery.', str(device)) + return + + if should_reboot(device): + try: + device.WaitUntilFullyBooted(retries=0) + except (device_errors.CommandTimeoutError, + device_errors.CommandFailedError): + logging.exception('Failure while waiting for %s. ' + 'Attempting to recover.', str(device)) + try: + try: + device.Reboot(block=False, timeout=5, retries=0) + except device_errors.CommandTimeoutError: + logging.warning('Timed out while attempting to reboot %s normally.' + 'Attempting alternative reboot.', str(device)) + # The device drops offline before we can grab the exit code, so + # we don't check for status. + device.adb.Root() + device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None, + timeout=5, retries=0) + except device_errors.CommandFailedError: + logging.exception('Failed to reboot %s.', str(device)) + if blacklist: + blacklist.Extend([device.adb.GetDeviceSerial()], + reason='reboot_failure') + except device_errors.CommandTimeoutError: + logging.exception('Timed out while rebooting %s.', str(device)) + if blacklist: + blacklist.Extend([device.adb.GetDeviceSerial()], + reason='reboot_timeout') + + try: + device.WaitUntilFullyBooted( + retries=0, timeout=device.REBOOT_DEFAULT_TIMEOUT) + except device_errors.CommandFailedError: + logging.exception('Failure while waiting for %s.', str(device)) + if blacklist: + blacklist.Extend([device.adb.GetDeviceSerial()], + reason='reboot_failure') + except device_errors.CommandTimeoutError: + logging.exception('Timed out while waiting for %s.', str(device)) + if blacklist: + blacklist.Extend([device.adb.GetDeviceSerial()], + reason='reboot_timeout') + + +def RecoverDevices(devices, blacklist): + """Attempts to recover any inoperable devices in the provided list. + + Args: + devices: The list of devices to attempt to recover. + blacklist: The current device blacklist, which will be used then + reset. + """ + + statuses = device_status.DeviceStatus(devices, blacklist) + + should_restart_usb = set( + status['serial'] for status in statuses + if (not status['usb_status'] + or status['adb_status'] in ('offline', 'missing'))) + should_restart_adb = should_restart_usb.union(set( + status['serial'] for status in statuses + if status['adb_status'] == 'unauthorized')) + should_reboot_device = should_restart_adb.union(set( + status['serial'] for status in statuses + if status['blacklisted'])) + + logging.debug('Should restart USB for:') + for d in should_restart_usb: + logging.debug(' %s', d) + logging.debug('Should restart ADB for:') + for d in should_restart_adb: + logging.debug(' %s', d) + logging.debug('Should reboot:') + for d in should_reboot_device: + logging.debug(' %s', d) + + if blacklist: + blacklist.Reset() + + if should_restart_adb: + KillAllAdb() + for serial in should_restart_usb: + try: + reset_usb.reset_android_usb(serial) + except IOError: + logging.exception('Unable to reset USB for %s.', serial) + if blacklist: + blacklist.Extend([serial], reason='USB failure') + except device_errors.DeviceUnreachableError: + logging.exception('Unable to reset USB for %s.', serial) + if blacklist: + blacklist.Extend([serial], reason='offline') + + device_utils.DeviceUtils.parallel(devices).pMap( + RecoverDevice, blacklist, + should_reboot=lambda device: device in should_reboot_device) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--adb-path', + help='Absolute path to the adb binary to use.') + parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') + parser.add_argument('--known-devices-file', action='append', default=[], + dest='known_devices_files', + help='Path to known device lists.') + parser.add_argument('-v', '--verbose', action='count', default=1, + help='Log more information.') + + args = parser.parse_args() + run_tests_helper.SetLogLevel(args.verbose) + + devil_dynamic_config = { + 'config_type': 'BaseConfig', + 'dependencies': {}, + } + + if args.adb_path: + devil_dynamic_config['dependencies'].update({ + 'adb': { + 'file_info': { + devil_env.GetPlatform(): { + 'local_paths': [args.adb_path] + } + } + } + }) + devil_env.config.Initialize(configs=[devil_dynamic_config]) + + blacklist = (device_blacklist.Blacklist(args.blacklist_file) + if args.blacklist_file + else None) + + expected_devices = device_status.GetExpectedDevices(args.known_devices_files) + usb_devices = set(lsusb.get_android_devices()) + devices = [device_utils.DeviceUtils(s) + for s in expected_devices.union(usb_devices)] + + RecoverDevices(devices, blacklist) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/catapult/devil/devil/android/tools/device_status.py b/catapult/devil/devil/android/tools/device_status.py new file mode 100755 index 00000000..7e169634 --- /dev/null +++ b/catapult/devil/devil/android/tools/device_status.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A script to keep track of devices across builds and report state.""" + +import argparse +import json +import logging +import os +import re +import sys + +if __name__ == '__main__': + sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..', '..'))) +from devil import devil_env +from devil.android import battery_utils +from devil.android import device_blacklist +from devil.android import device_errors +from devil.android import device_list +from devil.android import device_utils +from devil.android.sdk import adb_wrapper +from devil.constants import exit_codes +from devil.utils import lsusb +from devil.utils import run_tests_helper + +_RE_DEVICE_ID = re.compile(r'Device ID = (\d+)') + + +def IsBlacklisted(serial, blacklist): + return blacklist and serial in blacklist.Read() + + +def _BatteryStatus(device, blacklist): + battery_info = {} + try: + battery = battery_utils.BatteryUtils(device) + battery_info = battery.GetBatteryInfo(timeout=5) + battery_level = int(battery_info.get('level', 100)) + + if battery_level < 15: + logging.error('Critically low battery level (%d)', battery_level) + battery = battery_utils.BatteryUtils(device) + if not battery.GetCharging(): + battery.SetCharging(True) + if blacklist: + blacklist.Extend([device.adb.GetDeviceSerial()], reason='low_battery') + + except device_errors.CommandFailedError: + logging.exception('Failed to get battery information for %s', + str(device)) + + return battery_info + + +def _IMEISlice(device): + imei_slice = '' + try: + for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'], + check_return=True, timeout=5): + m = _RE_DEVICE_ID.match(l) + if m: + imei_slice = m.group(1)[-6:] + except device_errors.CommandFailedError: + logging.exception('Failed to get IMEI slice for %s', str(device)) + + return imei_slice + + +def DeviceStatus(devices, blacklist): + """Generates status information for the given devices. + + Args: + devices: The devices to generate status for. + blacklist: The current device blacklist. + Returns: + A dict of the following form: + { + '<serial>': { + 'serial': '<serial>', + 'adb_status': str, + 'usb_status': bool, + 'blacklisted': bool, + # only if the device is connected and not blacklisted + 'type': ro.build.product, + 'build': ro.build.id, + 'build_detail': ro.build.fingerprint, + 'battery': { + ... + }, + 'imei_slice': str, + 'wifi_ip': str, + }, + ... + } + """ + adb_devices = { + a[0].GetDeviceSerial(): a + for a in adb_wrapper.AdbWrapper.Devices(desired_state=None, long_list=True) + } + usb_devices = set(lsusb.get_android_devices()) + + def blacklisting_device_status(device): + serial = device.adb.GetDeviceSerial() + adb_status = ( + adb_devices[serial][1] if serial in adb_devices + else 'missing') + usb_status = bool(serial in usb_devices) + + device_status = { + 'serial': serial, + 'adb_status': adb_status, + 'usb_status': usb_status, + } + + if not IsBlacklisted(serial, blacklist): + if adb_status == 'device': + try: + build_product = device.build_product + build_id = device.build_id + build_fingerprint = device.build_fingerprint + build_description = device.build_description + wifi_ip = device.GetProp('dhcp.wlan0.ipaddress') + battery_info = _BatteryStatus(device, blacklist) + imei_slice = _IMEISlice(device) + + if (device.product_name == 'mantaray' and + battery_info.get('AC powered', None) != 'true'): + logging.error('Mantaray device not connected to AC power.') + + device_status.update({ + 'ro.build.product': build_product, + 'ro.build.id': build_id, + 'ro.build.fingerprint': build_fingerprint, + 'ro.build.description': build_description, + 'battery': battery_info, + 'imei_slice': imei_slice, + 'wifi_ip': wifi_ip, + }) + + except device_errors.CommandFailedError: + logging.exception('Failure while getting device status for %s.', + str(device)) + if blacklist: + blacklist.Extend([serial], reason='status_check_failure') + + except device_errors.CommandTimeoutError: + logging.exception('Timeout while getting device status for %s.', + str(device)) + if blacklist: + blacklist.Extend([serial], reason='status_check_timeout') + + elif blacklist: + blacklist.Extend([serial], + reason=adb_status if usb_status else 'offline') + + device_status['blacklisted'] = IsBlacklisted(serial, blacklist) + + return device_status + + parallel_devices = device_utils.DeviceUtils.parallel(devices) + statuses = parallel_devices.pMap(blacklisting_device_status).pGet(None) + return statuses + + +def _LogStatuses(statuses): + # Log the state of all devices. + for status in statuses: + logging.info(status['serial']) + adb_status = status.get('adb_status') + blacklisted = status.get('blacklisted') + logging.info(' USB status: %s', + 'online' if status.get('usb_status') else 'offline') + logging.info(' ADB status: %s', adb_status) + logging.info(' Blacklisted: %s', str(blacklisted)) + if adb_status == 'device' and not blacklisted: + logging.info(' Device type: %s', status.get('ro.build.product')) + logging.info(' OS build: %s', status.get('ro.build.id')) + logging.info(' OS build fingerprint: %s', + status.get('ro.build.fingerprint')) + logging.info(' Battery state:') + for k, v in status.get('battery', {}).iteritems(): + logging.info(' %s: %s', k, v) + logging.info(' IMEI slice: %s', status.get('imei_slice')) + logging.info(' WiFi IP: %s', status.get('wifi_ip')) + + +def _WriteBuildbotFile(file_path, statuses): + buildbot_path, _ = os.path.split(file_path) + if os.path.exists(buildbot_path): + with open(file_path, 'w') as f: + for status in statuses: + try: + if status['adb_status'] == 'device': + f.write('{serial} {adb_status} {build_product} {build_id} ' + '{temperature:.1f}C {level}%\n'.format( + serial=status['serial'], + adb_status=status['adb_status'], + build_product=status['type'], + build_id=status['build'], + temperature=float(status['battery']['temperature']) / 10, + level=status['battery']['level'] + )) + elif status.get('usb_status', False): + f.write('{serial} {adb_status}\n'.format( + serial=status['serial'], + adb_status=status['adb_status'] + )) + else: + f.write('{serial} offline\n'.format( + serial=status['serial'] + )) + except Exception: # pylint: disable=broad-except + pass + + +def GetExpectedDevices(known_devices_files): + expected_devices = set() + try: + for path in known_devices_files: + if os.path.exists(path): + expected_devices.update(device_list.GetPersistentDeviceList(path)) + else: + logging.warning('Could not find known devices file: %s', path) + except IOError: + logging.warning('Problem reading %s, skipping.', path) + + logging.info('Expected devices:') + for device in expected_devices: + logging.info(' %s', device) + return expected_devices + + +def AddArguments(parser): + parser.add_argument('--json-output', + help='Output JSON information into a specified file.') + parser.add_argument('--adb-path', + help='Absolute path to the adb binary to use.') + parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') + parser.add_argument('--known-devices-file', action='append', default=[], + dest='known_devices_files', + help='Path to known device lists.') + parser.add_argument('--buildbot-path', '-b', + default='/home/chrome-bot/.adb_device_info', + help='Absolute path to buildbot file location') + parser.add_argument('-v', '--verbose', action='count', default=1, + help='Log more information.') + parser.add_argument('-w', '--overwrite-known-devices-files', + action='store_true', + help='If set, overwrites known devices files wiht new ' + 'values.') + +def main(): + parser = argparse.ArgumentParser() + AddArguments(parser) + args = parser.parse_args() + + run_tests_helper.SetLogLevel(args.verbose) + + + devil_dynamic_config = { + 'config_type': 'BaseConfig', + 'dependencies': {}, + } + + if args.adb_path: + devil_dynamic_config['dependencies'].update({ + 'adb': { + 'file_info': { + devil_env.GetPlatform(): { + 'local_paths': [args.adb_path] + } + } + } + }) + devil_env.config.Initialize(configs=[devil_dynamic_config]) + + blacklist = (device_blacklist.Blacklist(args.blacklist_file) + if args.blacklist_file + else None) + + expected_devices = GetExpectedDevices(args.known_devices_files) + usb_devices = set(lsusb.get_android_devices()) + devices = [device_utils.DeviceUtils(s) + for s in expected_devices.union(usb_devices)] + + statuses = DeviceStatus(devices, blacklist) + + # Log the state of all devices. + _LogStatuses(statuses) + + # Update the last devices file(s). + if args.overwrite_known_devices_files: + for path in args.known_devices_files: + device_list.WritePersistentDeviceList( + path, [status['serial'] for status in statuses]) + + # Write device info to file for buildbot info display. + _WriteBuildbotFile(args.buildbot_path, statuses) + + # Dump the device statuses to JSON. + if args.json_output: + with open(args.json_output, 'wb') as f: + f.write(json.dumps( + statuses, indent=4, sort_keys=True, separators=(',', ': '))) + + live_devices = [status['serial'] for status in statuses + if (status['adb_status'] == 'device' + and not IsBlacklisted(status['serial'], blacklist))] + + # If all devices failed, or if there are no devices, it's an infra error. + if not live_devices: + logging.error('No available devices.') + return 0 if live_devices else exit_codes.INFRA + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/catapult/devil/devil/devil_dependencies.json b/catapult/devil/devil/devil_dependencies.json index 4c185c0e..d16e792b 100644 --- a/catapult/devil/devil/devil_dependencies.json +++ b/catapult/devil/devil/devil_dependencies.json @@ -6,7 +6,7 @@ "cloud_storage_base_folder": "binary_dependencies", "file_info": { "linux2_x86_64": { - "cloud_storage_hash": "7448de3cb5e834afdedeaad8b40ba63ac53f3dc4", + "cloud_storage_hash": "16ba3180141a2489d7ec99b39fd6e3434a9a373f", "download_path": "../bin/deps/linux2/x86_64/bin/aapt" } } @@ -16,7 +16,7 @@ "cloud_storage_base_folder": "binary_dependencies", "file_info": { "linux2_x86_64": { - "cloud_storage_hash": "0c2043552619c8ec8bb5d986ba75703a598611fc", + "cloud_storage_hash": "8bd43e3930f6eec643d5dc64cab9e5bb4ddf4909", "download_path": "../bin/deps/linux2/x86_64/bin/adb" } } @@ -26,7 +26,7 @@ "cloud_storage_base_folder": "binary_dependencies", "file_info": { "linux2_x86_64": { - "cloud_storage_hash": "52d150a7ccde835f38b4337392152f3013d5f303", + "cloud_storage_hash": "91cdce1e3bd81b2ac1fd380013896d0e2cdb40a0", "download_path": "../bin/deps/linux2/x86_64/lib/libc++.so" } } @@ -36,7 +36,7 @@ "cloud_storage_base_folder": "binary_dependencies", "file_info": { "linux2_x86_64": { - "cloud_storage_hash": "049f482f29bc34e2ed844e2e47b7609f8ffbeb4f", + "cloud_storage_hash": "4e22f641e4757309510e8d9f933f5aa504574ab6", "download_path": "../bin/deps/linux2/x86_64/lib.java/chromium_commands.dex.jar" } } @@ -46,21 +46,31 @@ "cloud_storage_base_folder": "binary_dependencies", "file_info": { "linux2_x86_64": { - "cloud_storage_hash": "38765b5b358c29003e56b1d214606ea13467b6fe", + "cloud_storage_hash": "acfb10f7a868baf9bcf446a2d9f8ed6b5d52c3c6", "download_path": "../bin/deps/linux2/x86_64/bin/dexdump" } } }, + "fastboot": { + "cloud_storage_bucket": "chromium-telemetry", + "cloud_storage_base_folder": "binary_dependencies", + "file_info": { + "linux2_x86_64": { + "cloud_storage_hash": "db9728166f182800eb9d09e9f036d56e105e8235", + "download_path": "../bin/deps/linux2/x86_64/bin/fastboot" + } + } + }, "forwarder_device": { "cloud_storage_bucket": "chromium-telemetry", "cloud_storage_base_folder": "binary_dependencies", "file_info": { "android_armeabi-v7a": { - "cloud_storage_hash": "4858c9e41da72ad8ff24414731feae2137229361", + "cloud_storage_hash": "de54e23327cc04ef7009fe697227617c7eeb1b2e", "download_path": "../bin/deps/android/armeabi-v7a/bin/forwarder_device" }, "android_arm64-v8a": { - "cloud_storage_hash": "8cbd1ac2079ee82ce5f1cf4d3e85fc1e53a8f018", + "cloud_storage_hash": "67b496f6d3ec6371393c2a8f4f403fdb27f5d091", "download_path": "../bin/deps/android/arm64-v8a/bin/forwarder_device" } } @@ -70,7 +80,7 @@ "cloud_storage_base_folder": "binary_dependencies", "file_info": { "linux2_x86_64": { - "cloud_storage_hash": "b3dda9fbdd4a3fb933b64111c11070aa809c7ed4", + "cloud_storage_hash": "02a1854adc15cc9e438f4ecfad2af058bd44fa7e", "download_path": "../bin/deps/linux2/x86_64/forwarder_host" } } @@ -80,15 +90,15 @@ "cloud_storage_base_folder": "binary_dependencies", "file_info": { "android_armeabi-v7a": { - "cloud_storage_hash": "c8894480be71d5e49118483d83ba7a6e0097cba6", + "cloud_storage_hash": "39fd90af0f8828202b687f7128393759181c5e2e", "download_path": "../bin/deps/android/armeabi-v7a/bin/md5sum_device" }, "android_arm64-v8a": { - "cloud_storage_hash": "bbe410e2ffb48367ac4ca0874598d4f85fd16d9d", + "cloud_storage_hash": "4e7d2dedd9c6321fdc152b06869e09a3c5817904", "download_path": "../bin/deps/andorid/arm64-v8a/bin/md5sum_device" }, "android_x86": { - "cloud_storage_hash": "b578a5c2c400ce39761e2558cdf2237567a57257", + "cloud_storage_hash": "d5cf42ab5986a69c31c0177b0df499d6bf708df6", "download_path": "../bin/deps/android/x86/bin/md5sum_device" } } @@ -98,7 +108,7 @@ "cloud_storage_base_folder": "binary_dependencies", "file_info": { "linux2_x86_64": { - "cloud_storage_hash": "49e36c9c4246cfebef26cbd07436c1a8343254aa", + "cloud_storage_hash": "4db5bd5e9bea8880d8bf2caa59d0efb0acc19f74", "download_path": "../bin/deps/linux2/x86_64/bin/md5sum_host" } } @@ -108,7 +118,7 @@ "cloud_storage_base_folder": "binary_dependencies", "file_info": { "linux2_x86_64": { - "cloud_storage_hash": "3327881fa3951a503b9467425ea8e781cdffeb9f", + "cloud_storage_hash": "abb9753a8d3efeea4144e328933931729e01571c", "download_path": "../bin/deps/linux2/x86_64/bin/split-select" } } diff --git a/catapult/devil/devil/utils/__init__.py b/catapult/devil/devil/utils/__init__.py index 50b23dff..ff84988d 100644 --- a/catapult/devil/devil/utils/__init__.py +++ b/catapult/devil/devil/utils/__init__.py @@ -1,3 +1,23 @@ # Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + +import os +import sys + +def _JoinPath(*path_parts): + return os.path.abspath(os.path.join(*path_parts)) + + +def _AddDirToPythonPath(*path_parts): + path = _JoinPath(*path_parts) + if os.path.isdir(path) and path not in sys.path: + # Some call sites that use Telemetry assume that sys.path[0] is the + # directory containing the script, so we add these extra paths to right + # after sys.path[0]. + sys.path.insert(1, path) + +_CATAPULT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), + os.path.pardir, os.path.pardir, os.path.pardir) + +_AddDirToPythonPath(_CATAPULT_DIR, 'common', 'battor') diff --git a/catapult/devil/devil/utils/battor_device_mapping.py b/catapult/devil/devil/utils/battor_device_mapping.py new file mode 100755 index 00000000..5624036c --- /dev/null +++ b/catapult/devil/devil/utils/battor_device_mapping.py @@ -0,0 +1,309 @@ +#!/usr/bin/python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +''' +This script provides tools to map BattOrs to phones. + +Phones are identified by the following string: + +"Phone serial number" - Serial number of the phone. This can be +obtained via 'adb devices' or 'usb-devices', and is not expected +to change for a given phone. + +BattOrs are identified by the following two strings: + +"BattOr serial number" - Serial number of the BattOr. This can be +obtained via 'usb-devices', and is not expected to change for +a given BattOr. + +"BattOr path" - The path of the form '/dev/ttyUSB*' that is used +to communicate with the BattOr (the battor_agent binary takes +this BattOr path as a parameter). The BattOr path is frequently +reassigned by the OS, most often when the device is disconnected +and then reconnected. Thus, the BattOr path cannot be expected +to be stable. + +In a typical application, the user will require the BattOr path +for the BattOr that is plugged into a given phone. For instance, +the user will be running tracing on a particular phone, and will +need to know which BattOr path to use to communicate with the BattOr +to get the corresponding power trace. + +Getting this mapping requires two steps: (1) determining the +mapping between phone serial numbers and BattOr serial numbers, and +(2) getting the BattOr path corresponding to a given BattOr serial +number. + +For step (1), we generate a JSON file giving this mapping. This +JSON file consists of a list of items of the following form: +[{'phone': <phone serial 1>, 'battor': <battor serial 1>}, +{'phone': <phone serial 2>, 'battor': <battor serial 2>}, ...] + +The default way to generate this JSON file is using the function +GenerateSerialMapFile, which generates a mapping based on assuming +that the system has two identical USB hubs connected to it, and +the phone plugged into physical port number 1 on one hub corresponds +to the BattOr plugged into physical port number 1 on the other hub, +and similarly with physical port numbers 2, 3, etc. This generates +the map file based on the structure at the time GenerateSerialMapFile called. +Note that after the map file is generated, port numbers are no longer used; +the user could move around the devices in the ports without affecting +which phone goes with which BattOr. (Thus, if the user wanted to update the +mapping to match the new port connections, the user would have to +re-generate this file.) + +The script update_mapping.py will do this updating from the command line. + +If the user wanted to specify a custom mapping, the user could instead +create the JSON file manually. (In this case, hubs would not be necessary +and the physical ports connected would be irrelevant.) + +Step (2) is conducted through the function GetBattorPathFromPhoneSerial, +which takes a serial number mapping generated via step (1) and a phone +serial number, then gets the corresponding BattOr serial number from the +map and determines its BattOr path (e.g. /dev/ttyUSB0). Since BattOr paths +can change if devices are connected and disconnected (even if connected +or disconnected via the same port) this function should be called to +determine the BattOr path every time before connecting to the BattOr. + +Note that if there is only one BattOr connected to the system, then +GetBattorPathFromPhoneSerial will always return that BattOr and will ignore +the mapping file. Thus, if the user never has more than one BattOr connected +to the system, the user will not need to generate mapping files. +''' + + +import json +import collections + +from battor import battor_error +from devil.utils import find_usb_devices +from devil.utils import usb_hubs + + +def GetBattorList(device_tree_map): + return [x for x in find_usb_devices.GetTTYList() + if IsBattor(x, device_tree_map)] + + +def IsBattor(tty_string, device_tree_map): + (bus, device) = find_usb_devices.GetBusDeviceFromTTY(tty_string) + node = device_tree_map[bus].FindDeviceNumber(device) + return '0403:6001' in node.desc + + +def GetBattorSerialNumbers(device_tree_map): + for x in find_usb_devices.GetTTYList(): + if IsBattor(x, device_tree_map): + (bus, device) = find_usb_devices.GetBusDeviceFromTTY(x) + devnode = device_tree_map[bus].FindDeviceNumber(device) + yield devnode.serial + + +def ReadSerialMapFile(filename): + """Reads JSON file giving phone-to-battor serial number map. + + Parses a JSON file consisting of a list of items of the following form: + [{'phone': <phone serial 1>, 'battor': <battor serial 1>}, + {'phone': <phone serial 2>, 'battor': <battor serial 2>}, ...] + + indicating which phone serial numbers should be matched with + which BattOr serial numbers. Returns dictionary of the form: + + {<phone serial 1>: <BattOr serial 1>, + <phone serial 2>: <BattOr serial 2>} + + Args: + filename: Name of file to read. + """ + result = {} + with open(filename, 'r') as infile: + in_dict = json.load(infile) + for x in in_dict: + result[x['phone']] = x['battor'] + return result + +def WriteSerialMapFile(filename, serial_map): + """Writes a map of phone serial numbers to BattOr serial numbers to file. + + Writes a JSON file consisting of a list of items of the following form: + [{'phone': <phone serial 1>, 'battor': <battor serial 1>}, + {'phone': <phone serial 2>, 'battor': <battor serial 2>}, ...] + + indicating which phone serial numbers should be matched with + which BattOr serial numbers. Mapping is based on the physical port numbers + of the hubs that the BattOrs and phones are connected to. + + Args: + filename: Name of file to write. + serial_map: Serial map {phone: battor} + """ + result = [] + for (phone, battor) in serial_map.iteritems(): + result.append({'phone': phone, 'battor': battor}) + with open(filename, 'w') as outfile: + json.dump(result, outfile) + +def GenerateSerialMap(hub_types=None): + """Generates a map of phone serial numbers to BattOr serial numbers. + + Generates a dict of: + {<phone serial 1>: <battor serial 1>, + <phone serial 2>: <battor serial 2>} + indicating which phone serial numbers should be matched with + which BattOr serial numbers. Mapping is based on the physical port numbers + of the hubs that the BattOrs and phones are connected to. + + Args: + hub_types: List of hub types to check for. If not specified, checks + for all defined hub types. (see usb_hubs.py for details) + """ + if hub_types: + hub_types = [usb_hubs.GetHubType(x) for x in hub_types] + else: + hub_types = usb_hubs.ALL_HUBS + + devtree = find_usb_devices.GetBusNumberToDeviceTreeMap() + + # List of serial numbers in the system that represent BattOrs. + battor_serials = list(GetBattorSerialNumbers(devtree)) + + # If there's only one BattOr in the system, then a serial number ma + # is not necessary. + if len(battor_serials) == 1: + return {} + + # List of dictionaries, one for each hub, that maps the physical + # port number to the serial number of that hub. For instance, in a 2 + # hub system, this could return [{1:'ab', 2:'cd'}, {1:'jkl', 2:'xyz'}] + # where 'ab' and 'cd' are the phone serial numbers and 'jkl' and 'xyz' + # are the BattOr serial numbers. + port_to_serial = find_usb_devices.GetAllPhysicalPortToSerialMaps( + hub_types, device_tree_map=devtree) + + class serials(object): + def __init__(self): + self.phone = None + self.battor = None + + # Map of {physical port number: [phone serial #, BattOr serial #]. This + # map is populated by executing the code below. For instance, in the above + # example, after the code below is executed, port_to_devices would equal + # {1: ['ab', 'jkl'], 2: ['cd', 'xyz']} + port_to_devices = collections.defaultdict(serials) + for hub in port_to_serial: + for (port, serial) in hub.iteritems(): + if serial in battor_serials: + if port_to_devices[port].battor is not None: + raise battor_error.BattorError('Multiple BattOrs on same port number') + else: + port_to_devices[port].battor = serial + else: + if port_to_devices[port].phone is not None: + raise battor_error.BattorError('Multiple phones on same port number') + else: + port_to_devices[port].phone = serial + + # Turn the port_to_devices map into a map of the form + # {phone serial number: BattOr serial number}. + result = {} + for pair in port_to_devices.values(): + if pair.phone is None: + continue + if pair.battor is None: + raise battor_error.BattorError( + 'Phone detected with no corresponding BattOr') + result[pair.phone] = pair.battor + return result + +def GenerateSerialMapFile(filename, hub_types=None): + """Generates a serial map file and writes it.""" + WriteSerialMapFile(filename, GenerateSerialMap(hub_types)) + +def _PhoneToPathMap(serial, serial_map, devtree): + """Maps phone serial number to TTY path, assuming serial map is provided.""" + try: + battor_serial = serial_map[serial] + except KeyError: + raise battor_error.BattorError('Serial number not found in serial map.') + for tree in devtree.values(): + for node in tree.AllNodes(): + if isinstance(node, find_usb_devices.USBDeviceNode): + if node.serial == battor_serial: + bus_device_to_tty = find_usb_devices.GetBusDeviceToTTYMap() + bus_device = (node.bus_num, node.device_num) + try: + return bus_device_to_tty[bus_device] + except KeyError: + raise battor_error.BattorError( + 'Device with given serial number not a BattOr ' + '(does not have TTY path)') + + +def GetBattorPathFromPhoneSerial(serial, serial_map=None, + serial_map_file=None): + """Gets the TTY path (e.g. '/dev/ttyUSB0') to communicate with the BattOr. + + (1) If serial_map is given, it is treated as a dictionary mapping + phone serial numbers to BattOr serial numbers. This function will get the + TTY path for the given BattOr serial number. + + (2) If serial_map_file is given, it is treated as the name of a + phone-to-BattOr mapping file (generated with GenerateSerialMapFile) + and this will be loaded and used as the dict to map port numbers to + BattOr serial numbers. + + You can only give one of serial_map and serial_map_file. + + Args: + serial: Serial number of phone connected on the same physical port that + the BattOr is connected to. + serial_map: Map of phone serial numbers to BattOr serial numbers, given + as a dictionary. + serial_map_file: Map of phone serial numbers to BattOr serial numbers, + given as a file. + hub_types: List of hub types to check for. Used only if serial_map_file + is None. + + Returns: + Device string used to communicate with device. + + Raises: + ValueError: If serial number is not given. + BattorError: If BattOr not found or unexpected USB topology. + """ + # If there's only one BattOr connected to the system, just use that one. + # This allows for use on, e.g., a developer's workstation with no hubs. + devtree = find_usb_devices.GetBusNumberToDeviceTreeMap() + all_battors = GetBattorList(devtree) + if len(all_battors) == 1: + return '/dev/' + all_battors[0] + + if not serial: + raise battor_error.BattorError( + 'Two or more BattOrs connected, no serial provided') + + if serial_map and serial_map_file: + raise ValueError('Cannot specify both serial_map and serial_map_file') + + if serial_map_file: + serial_map = ReadSerialMapFile(serial_map_file) + + tty_string = _PhoneToPathMap(serial, serial_map, devtree) + + if not tty_string: + raise battor_error.BattorError( + 'No device with given serial number detected.') + + if IsBattor(tty_string, devtree): + return '/dev/' + tty_string + else: + raise battor_error.BattorError( + 'Device with given serial number is not a BattOr.') + +if __name__ == '__main__': + # Main function for testing purposes + print GenerateSerialMap() diff --git a/catapult/devil/devil/utils/cmd_helper.py b/catapult/devil/devil/utils/cmd_helper.py index b6237757..c2200522 100644 --- a/catapult/devil/devil/utils/cmd_helper.py +++ b/catapult/devil/devil/utils/cmd_helper.py @@ -172,10 +172,9 @@ def GetCmdStatusAndOutput(args, cwd=None, shell=False): args, cwd=cwd, shell=shell) if stderr: - logging.critical(stderr) - if len(stdout) > 4096: - logging.debug('Truncated output:') - logging.debug(stdout[:4096]) + logging.critical('STDERR: %s', stderr) + logging.debug('STDOUT: %s%s', stdout[:4096].rstrip(), + '<truncated>' if len(stdout) > 4096 else '') return (status, stdout) @@ -273,7 +272,10 @@ def GetCmdStatusAndOutputWithTimeout(args, timeout, cwd=None, shell=False, except TimeoutError: raise TimeoutError(output.getvalue()) - return process.returncode, output.getvalue() + str_output = output.getvalue() + logging.debug('STDOUT+STDERR: %s%s', str_output[:4096].rstrip(), + '<truncated>' if len(str_output) > 4096 else '') + return process.returncode, str_output def IterCmdOutputLines(args, timeout=None, cwd=None, shell=False, diff --git a/catapult/devil/devil/utils/find_usb_devices.py b/catapult/devil/devil/utils/find_usb_devices.py index 4982e46a..0e0f4d56 100755 --- a/catapult/devil/devil/utils/find_usb_devices.py +++ b/catapult/devil/devil/utils/find_usb_devices.py @@ -8,6 +8,7 @@ import sys import argparse from devil.utils import cmd_helper +from devil.utils import usb_hubs from devil.utils import lsusb # Note: In the documentation below, "virtual port" refers to the port number @@ -44,16 +45,6 @@ def GetTTYList(): return [x for x in _GetCommList().splitlines() if 'ttyUSB' in x] -def GetBattorList(device_tree_map): - return [x for x in GetTTYList() if IsBattor(x, device_tree_map)] - - -def IsBattor(tty_string, device_tree_map): - (bus, device) = GetBusDeviceFromTTY(tty_string) - node = device_tree_map[bus].FindDeviceNumber(device) - return 'Future Technology Devices International' in node.desc - - # Class to identify nodes in the USB topology. USB topology is organized as # a tree. class USBNode(object): @@ -251,7 +242,7 @@ _S_LINE_REGEX = re.compile(r'S: SerialNumber=(?P<serial>.*)') _LSUSB_BUS_DEVICE_RE = re.compile(r'^Bus (\d{3}) Device (\d{3}): (.*)') -def GetBusNumberToDeviceTreeMap(fast=False): +def GetBusNumberToDeviceTreeMap(fast=True): """Gets devices currently attached. Args: @@ -292,7 +283,8 @@ def GetBusNumberToDeviceTreeMap(fast=False): # create the new device new_device = USBDeviceNode(bus_num=bus_num, device_num=device_num, - info=info_map[(bus_num, device_num)]) + info=info_map.get((bus_num, device_num), + {'desc': 'NOT AVAILABLE'})) # add device to bus if parent_num != 0: @@ -311,83 +303,12 @@ def GetBusNumberToDeviceTreeMap(fast=False): return tree -class HubType(object): - def __init__(self, id_func, port_mapping): - """Defines a type of hub. - - Args: - id_func: [USBNode -> bool] is a function that can be run on a node - to determine if the node represents this type of hub. - port_mapping: [dict(int:(int|dict))] maps virtual to physical port - numbers. For instance, {3:1, 1:2, 2:3} means that virtual port 3 - corresponds to physical port 1, virtual port 1 corresponds to physical - port 2, and virtual port 2 corresponds to physical port 3. In the - case of hubs with "internal" topology, this is represented by nested - maps. For instance, {1:{1:1,2:2},2:{1:3,2:4}} means, e.g. that the - device plugged into physical port 3 will show up as being connected - to port 1, on a device which is connected to port 2 on the hub. - """ - self._id_func = id_func - # v2p = "virtual to physical" ports - self._v2p_port = port_mapping - - def IsType(self, node): - """Determines if the given Node is a hub of this type. - - Args: - node: [USBNode] Node to check. - """ - return self._id_func(node) - - def GetPhysicalPortToNodeTuples(self, node): - """Gets devices connected to the physical ports on a hub of this type. - - Args: - node: [USBNode] Node representing a hub of this type. - - Yields: - A series of (int, USBNode) tuples giving a physical port - and the USBNode connected to it. - - Raises: - ValueError: If the given node isn't a hub of this type. - """ - if self.IsType(node): - for res in self._GppHelper(node, self._v2p_port): - yield res - else: - raise ValueError('Node must be a hub of this type') - - def _GppHelper(self, node, mapping): - """Helper function for GetPhysicalPortToNodeMap. - - Gets devices connected to physical ports, based on device tree - rooted at the given node and the mapping between virtual and physical - ports. - - Args: - node: [USBNode] Root of tree to search for devices. - mapping: [dict] Mapping between virtual and physical ports. - - Yields: - A series of (int, USBNode) tuples giving a physical port - and the Node connected to it. - """ - for (virtual, physical) in mapping.iteritems(): - if node.HasPort(virtual): - if isinstance(physical, dict): - for res in self._GppHelper(node.PortToDevice(virtual), physical): - yield res - else: - yield (physical, node.PortToDevice(virtual)) - - def GetHubsOnBus(bus, hub_types): """Scans for all hubs on a bus of given hub types. Args: bus: [USBNode] Bus object. - hub_types: [iterable(HubType)] Possible types of hubs. + hub_types: [iterable(usb_hubs.HubType)] Possible types of hubs. Yields: Sequence of tuples representing (hub, type of hub) @@ -402,7 +323,7 @@ def GetPhysicalPortToNodeMap(hub, hub_type): """Gets physical-port:node mapping for a given hub. Args: hub: [USBNode] Hub to get map for. - hub_type: [HubType] Which type of hub it is. + hub_type: [usb_hubs.HubType] Which type of hub it is. Returns: Dict of {physical port: node} @@ -415,7 +336,7 @@ def GetPhysicalPortToBusDeviceMap(hub, hub_type): """Gets physical-port:(bus#, device#) mapping for a given hub. Args: hub: [USBNode] Hub to get map for. - hub_type: [HubType] Which type of hub it is. + hub_type: [usb_hubs.HubType] Which type of hub it is. Returns: Dict of {physical port: (bus number, device number)} @@ -427,9 +348,10 @@ def GetPhysicalPortToBusDeviceMap(hub, hub_type): def GetPhysicalPortToSerialMap(hub, hub_type): """Gets physical-port:serial# mapping for a given hub. + Args: hub: [USBNode] Hub to get map for. - hub_type: [HubType] Which type of hub it is. + hub_type: [usb_hubs.HubType] Which type of hub it is. Returns: Dict of {physical port: serial number)} @@ -444,7 +366,7 @@ def GetPhysicalPortToTTYMap(device, hub_type): """Gets physical-port:tty-string mapping for a given hub. Args: hub: [USBNode] Hub to get map for. - hub_type: [HubType] Which type of hub it is. + hub_type: [usb_hubs.HubType] Which type of hub it is. Returns: Dict of {physical port: tty-string)} @@ -460,7 +382,7 @@ def CollectHubMaps(hub_types, map_func, device_tree_map=None, fast=False): """Runs a function on all hubs in the system and collects their output. Args: - hub_types: [HubType] List of possible hub types. + hub_types: [usb_hubs.HubType] List of possible hub types. map_func: [string] Function to run on each hub. device_tree: Previously constructed device tree map, if any. fast: Whether to construct device tree fast, if not already provided @@ -552,54 +474,36 @@ def GetBusDeviceToTTYMap(): # 4 connects to another 'virtual' hub that itself has the # virtual-to-physical port mapping {1:4, 2:3, 3:2, 4:1}. -PLUGABLE_7PORT_LAYOUT = {1:7, - 2:6, - 3:5, - 4:{1:4, 2:3, 3:2, 4:1}} def TestUSBTopologyScript(): """Test display and hub identification.""" # Identification criteria for Plugable 7-Port Hub - def _is_plugable_7port_hub(node): - """Check if a node is a Plugable 7-Port Hub - (Model USB2-HUB7BC) - The topology of this device is a 4-port hub, - with another 4-port hub connected on port 4. - """ - if not isinstance(node, USBDeviceNode): - return False - if '4-Port HUB' not in node.desc: - return False - if not node.HasPort(4): - return False - return '4-Port HUB' in node.PortToDevice(4).desc - - plugable_7port = HubType(_is_plugable_7port_hub, - PLUGABLE_7PORT_LAYOUT) print '==== USB TOPOLOGY SCRIPT TEST ====' # Display devices print '==== DEVICE DISPLAY ====' - device_trees = GetBusNumberToDeviceTreeMap(fast=True) + device_trees = GetBusNumberToDeviceTreeMap() for device_tree in device_trees.values(): device_tree.Display() print # Display TTY information about devices plugged into hubs. print '==== TTY INFORMATION ====' - for port_map in GetAllPhysicalPortToTTYMaps([plugable_7port], - device_tree_map=device_trees): + for port_map in GetAllPhysicalPortToTTYMaps( + usb_hubs.ALL_HUBS, device_tree_map=device_trees): print port_map print # Display serial number information about devices plugged into hubs. print '==== SERIAL NUMBER INFORMATION ====' - for port_map in GetAllPhysicalPortToSerialMaps([plugable_7port], - device_tree_map=device_trees): + for port_map in GetAllPhysicalPortToSerialMaps( + usb_hubs.ALL_HUBS, device_tree_map=device_trees): print port_map - print '' + + return 0 + def parse_options(argv): """Parses and checks the command-line options. diff --git a/catapult/devil/devil/utils/find_usb_devices_test.py b/catapult/devil/devil/utils/find_usb_devices_test.py index 2e94dcd2..c99e7165 100755 --- a/catapult/devil/devil/utils/find_usb_devices_test.py +++ b/catapult/devil/devil/utils/find_usb_devices_test.py @@ -29,10 +29,15 @@ Bus 002: """ import logging +import os import unittest from devil import devil_env +from devil.utils import battor_device_mapping from devil.utils import find_usb_devices +from devil.utils import lsusb +from devil.utils import usb_hubs + with devil_env.SysPath(devil_env.PYMOCK_PATH): import mock # pylint: disable=import-error @@ -46,15 +51,15 @@ DEVLIST = [(1, 11, 'foo'), (1, 13, 'baz'), (2, 11, 'quux'), (2, 20, 'My Test HUB'), - (2, 21, 'Future Technology Devices International battor_p7_h1_t0'), - (2, 22, 'Future Technology Devices International battor_p5_h1_t1'), + (2, 21, 'ID 0403:6001 battor_p7_h1_t0'), + (2, 22, 'ID 0403:6001 battor_p5_h1_t1'), (2, 23, 'My Test Internal HUB'), - (2, 24, 'Future Technology Devices International battor_p3_h1_t2'), - (2, 25, 'Future Technology Devices International battor_p1_h1_t3'), + (2, 24, 'ID 0403:6001 battor_p3_h1_t2'), + (2, 25, 'ID 0403:6001 battor_p1_h1_t3'), (2, 26, 'Not a Battery Monitor'), (2, 100, 'My Test HUB'), (2, 101, 'My Test Internal HUB'), - (2, 102, 'Future Technology Devices International battor_p1_h1_t4')] + (2, 102, 'ID 0403:6001 battor_p1_h1_t4')] LSUSB_OUTPUT = [ {'bus': b, 'device': d, 'desc': t, 'id': (1000*b)+d} @@ -92,6 +97,23 @@ T: Bus=02 Lev=00 Prnt=100 Port=03 Cnt=00 Dev#=101 Spd=000 MxCh=00 T: Bus=02 Lev=00 Prnt=101 Port=03 Cnt=00 Dev#=102 Spd=000 MxCh=00 ''' +RAW_LSUSB_OUTPUT = ''' +Bus 001 Device 011: FAST foo +Bus 001 Device 012: FAST bar +Bus 001 Device 013: baz +Bus 002 Device 011: quux +Bus 002 Device 020: My Test HUB +Bus 002 Device 021: ID 0403:6001 battor_p7_h1_t0 +Bus 002 Device 022: ID 0403:6001 battor_p5_h1_t1 +Bus 002 Device 023: My Test Internal HUB +Bus 002 Device 024: ID 0403:6001 battor_p3_h1_t2 +Bus 002 Device 025: ID 0403:6001 battor_p1_h1_t3 +Bus 002 Device 026: Not a Battery Monitor +Bus 002 Device 100: My Test HUB +Bus 002 Device 101: My Test Internal HUB +Bus 002 Device 102: ID 0403:6001 battor_p1_h1_t4 +''' + LIST_TTY_OUTPUT = ''' ttyUSB0 Something-else-0 @@ -172,11 +194,11 @@ def isTestHub(node): return False return 'Test Internal HUB' in node.PortToDevice(4).desc -TEST_HUB = find_usb_devices.HubType(isTestHub, - {1:7, - 2:6, - 3:5, - 4:{1:4, 2:3, 3:2, 4:1}}) +TEST_HUB = usb_hubs.HubType(isTestHub, + {1:7, + 2:6, + 3:5, + 4:{1:4, 2:3, 3:2, 4:1}}) class USBScriptTest(unittest.TestCase): def setUp(self): @@ -188,15 +210,17 @@ class USBScriptTest(unittest.TestCase): return_value=USB_DEVICES_OUTPUT) find_usb_devices._GetCommList = mock.Mock( return_value=LIST_TTY_OUTPUT) + lsusb.raw_lsusb = mock.Mock( + return_value=RAW_LSUSB_OUTPUT) def testIsBattor(self): bd = find_usb_devices.GetBusNumberToDeviceTreeMap() - self.assertTrue(find_usb_devices.IsBattor('ttyUSB3', bd)) - self.assertFalse(find_usb_devices.IsBattor('ttyUSB5', bd)) + self.assertTrue(battor_device_mapping.IsBattor('ttyUSB3', bd)) + self.assertFalse(battor_device_mapping.IsBattor('ttyUSB5', bd)) def testGetBattors(self): bd = find_usb_devices.GetBusNumberToDeviceTreeMap() - self.assertEquals(find_usb_devices.GetBattorList(bd), + self.assertEquals(battor_device_mapping.GetBattorList(bd), ['ttyUSB0', 'ttyUSB1', 'ttyUSB2', 'ttyUSB3', 'ttyUSB4']) @@ -229,18 +253,28 @@ class USBScriptTest(unittest.TestCase): 1:'Battor3'}) self.assertEquals(result[1], {}) - def testDeviceDescriptions(self): + def testFastDeviceDescriptions(self): bd = find_usb_devices.GetBusNumberToDeviceTreeMap() dev_foo = bd[1].FindDeviceNumber(11) dev_bar = bd[1].FindDeviceNumber(12) dev_battor_p7_h1_t0 = bd[2].FindDeviceNumber(21) + self.assertEquals(dev_foo.desc, 'FAST foo') + self.assertEquals(dev_bar.desc, 'FAST bar') + self.assertEquals(dev_battor_p7_h1_t0.desc, + 'ID 0403:6001 battor_p7_h1_t0') + + def testDeviceDescriptions(self): + bd = find_usb_devices.GetBusNumberToDeviceTreeMap(fast=False) + dev_foo = bd[1].FindDeviceNumber(11) + dev_bar = bd[1].FindDeviceNumber(12) + dev_battor_p7_h1_t0 = bd[2].FindDeviceNumber(21) self.assertEquals(dev_foo.desc, 'foo') self.assertEquals(dev_bar.desc, 'bar') self.assertEquals(dev_battor_p7_h1_t0.desc, - 'Future Technology Devices International battor_p7_h1_t0') + 'ID 0403:6001 battor_p7_h1_t0') def testDeviceInformation(self): - bd = find_usb_devices.GetBusNumberToDeviceTreeMap() + bd = find_usb_devices.GetBusNumberToDeviceTreeMap(fast=False) dev_foo = bd[1].FindDeviceNumber(11) dev_bar = bd[1].FindDeviceNumber(12) dev_battor_p7_h1_t0 = bd[2].FindDeviceNumber(21) @@ -249,7 +283,7 @@ class USBScriptTest(unittest.TestCase): self.assertEquals(dev_battor_p7_h1_t0.info['id'], 2021) def testSerialNumber(self): - bd = find_usb_devices.GetBusNumberToDeviceTreeMap() + bd = find_usb_devices.GetBusNumberToDeviceTreeMap(fast=False) dev_foo = bd[1].FindDeviceNumber(11) dev_bar = bd[1].FindDeviceNumber(12) dev_battor_p7_h1_t0 = bd[2].FindDeviceNumber(21) @@ -257,6 +291,89 @@ class USBScriptTest(unittest.TestCase): self.assertEquals(dev_bar.serial, 'BarSerial') self.assertEquals(dev_battor_p7_h1_t0.serial, 'Battor0') + def testBattorDictMapping(self): + map_dict = {'Phone1':'Battor1', 'Phone2':'Battor2', 'Phone3':'Battor3'} + a1 = battor_device_mapping.GetBattorPathFromPhoneSerial( + 'Phone1', serial_map=map_dict) + a2 = battor_device_mapping.GetBattorPathFromPhoneSerial( + 'Phone2', serial_map=map_dict) + a3 = battor_device_mapping.GetBattorPathFromPhoneSerial( + 'Phone3', serial_map=map_dict) + self.assertEquals(a1, '/dev/ttyUSB1') + self.assertEquals(a2, '/dev/ttyUSB2') + self.assertEquals(a3, '/dev/ttyUSB3') + + def testBattorDictFromFileMapping(self): + try: + map_dict = {'Phone1':'Battor1', 'Phone2':'Battor2', 'Phone3':'Battor3'} + curr_dir = os.path.dirname(os.path.realpath(__file__)) + filename = os.path.join(curr_dir, 'test', 'data', 'test_write_map.json') + battor_device_mapping.WriteSerialMapFile(filename, map_dict) + a1 = battor_device_mapping.GetBattorPathFromPhoneSerial( + 'Phone1', serial_map_file=filename) + a2 = battor_device_mapping.GetBattorPathFromPhoneSerial( + 'Phone2', serial_map_file=filename) + a3 = battor_device_mapping.GetBattorPathFromPhoneSerial( + 'Phone3', serial_map_file=filename) + finally: + os.remove(filename) + self.assertEquals(a1, '/dev/ttyUSB1') + self.assertEquals(a2, '/dev/ttyUSB2') + self.assertEquals(a3, '/dev/ttyUSB3') + + def testReadSerialMapFile(self): + curr_dir = os.path.dirname(os.path.realpath(__file__)) + map_dict = battor_device_mapping.ReadSerialMapFile( + os.path.join(curr_dir, 'test', 'data', 'test_serial_map.json')) + self.assertEquals(len(map_dict.keys()), 3) + self.assertEquals(map_dict['Phone1'], 'Battor1') + self.assertEquals(map_dict['Phone2'], 'Battor2') + self.assertEquals(map_dict['Phone3'], 'Battor3') + +original_PPTSM = find_usb_devices.GetAllPhysicalPortToSerialMaps +original_PPTTM = find_usb_devices.GetAllPhysicalPortToTTYMaps +original_GBL = battor_device_mapping.GetBattorList +original_GBNDM = find_usb_devices.GetBusNumberToDeviceTreeMap +original_IB = battor_device_mapping.IsBattor +original_GBSM = battor_device_mapping.GetBattorSerialNumbers + +def setup_battor_test(serial, tty, battor, bser=None): + serial_mapper = mock.Mock(return_value=serial) + tty_mapper = mock.Mock(return_value=tty) + battor_lister = mock.Mock(return_value=battor) + devtree = mock.Mock(return_value=None) + is_battor = mock.Mock(side_effect=lambda x, y: x in battor) + battor_serials = mock.Mock(return_value=bser) + find_usb_devices.GetAllPhysicalPortToSerialMaps = serial_mapper + find_usb_devices.GetAllPhysicalPortToTTYMaps = tty_mapper + battor_device_mapping.GetBattorList = battor_lister + find_usb_devices.GetBusNumberToDeviceTreeMap = devtree + battor_device_mapping.IsBattor = is_battor + battor_device_mapping.GetBattorSerialNumbers = battor_serials + +class BattorMappingTest(unittest.TestCase): + def tearDown(self): + find_usb_devices.GetAllPhysicalPortToSerialMaps = original_PPTSM + find_usb_devices.GetAllPhysicalPortToTTYMaps = original_PPTTM + battor_device_mapping.GetBattorList = original_GBL + find_usb_devices.GetBusNumberToDeviceTreeMap = original_GBNDM + battor_device_mapping.IsBattor = original_IB + battor_device_mapping.GetBattorSerialNumbers = original_GBSM + + def test_generate_serial_map(self): + setup_battor_test([{1:'Phn1', 2:'Phn2', 3:'Phn3'}, + {1:'Bat1', 2:'Bat2', 3:'Bat3'}], + [{}, + {1:'ttyUSB0', 2:'ttyUSB1', 3:'ttyUSB2'}], + ['ttyUSB0', 'ttyUSB1', 'ttyUSB2'], + ['Bat1', 'Bat2', 'Bat3']) + result = battor_device_mapping.GenerateSerialMap() + self.assertEqual(len(result), 3) + self.assertEqual(result['Phn1'], 'Bat1') + self.assertEqual(result['Phn2'], 'Bat2') + self.assertEqual(result['Phn3'], 'Bat3') + + if __name__ == "__main__": logging.getLogger().setLevel(logging.DEBUG) unittest.main(verbosity=2) diff --git a/catapult/devil/devil/utils/parallelizer.py b/catapult/devil/devil/utils/parallelizer.py index b8a2824e..35995251 100644 --- a/catapult/devil/devil/utils/parallelizer.py +++ b/catapult/devil/devil/utils/parallelizer.py @@ -65,8 +65,6 @@ class Parallelizer(object): """Allows parallel execution of method calls across a group of objects.""" def __init__(self, objs): - assert (objs is not None and len(objs) > 0), ( - "Passed empty list to 'Parallelizer'") self._orig_objs = objs self._objs = objs @@ -110,8 +108,6 @@ class Parallelizer(object): """ self.pGet(None) - if not self._objs: - raise AttributeError('Nothing to call.') for o in self._objs: if not callable(o): raise AttributeError("'%s' is not callable" % o.__name__) diff --git a/catapult/devil/devil/utils/parallelizer_test.py b/catapult/devil/devil/utils/parallelizer_test.py index 3162a4f5..32ff7ec5 100644 --- a/catapult/devil/devil/utils/parallelizer_test.py +++ b/catapult/devil/devil/utils/parallelizer_test.py @@ -72,13 +72,9 @@ class ParallelizerTestObjectHelper(object): class ParallelizerTest(unittest.TestCase): - def testInitWithNone(self): - with self.assertRaises(AssertionError): - parallelizer.Parallelizer(None) - def testInitEmptyList(self): - with self.assertRaises(AssertionError): - parallelizer.Parallelizer([]) + r = parallelizer.Parallelizer([]).replace('a', 'b').pGet(0.1) + self.assertEquals([], r) def testMethodCall(self): test_data = ['abc_foo', 'def_foo', 'ghi_foo'] diff --git a/catapult/devil/devil/utils/signal_handler.py b/catapult/devil/devil/utils/signal_handler.py new file mode 100644 index 00000000..566bef94 --- /dev/null +++ b/catapult/devil/devil/utils/signal_handler.py @@ -0,0 +1,30 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import contextlib +import signal + + +@contextlib.contextmanager +def AddSignalHandler(signalnum, additional_handler): + """Adds a signal handler for the given signal in the wrapped context. + + This runs the new handler after any existing handler rather than + replacing the existing handler. + + Args: + signum: The signal for which a handler should be added. + additional_handler: The handler to add. + """ + existing_handler = signal.getsignal(signalnum) + + def handler(signum, frame): + if callable(existing_handler): + existing_handler(signum, frame) + additional_handler(signum, frame) + + signal.signal(signalnum, handler) + yield + signal.signal(signalnum, existing_handler) + diff --git a/catapult/devil/devil/utils/test/data/test_serial_map.json b/catapult/devil/devil/utils/test/data/test_serial_map.json new file mode 100644 index 00000000..452df3f2 --- /dev/null +++ b/catapult/devil/devil/utils/test/data/test_serial_map.json @@ -0,0 +1 @@ +[{"phone": "Phone1", "battor": "Battor1"}, {"phone": "Phone2", "battor": "Battor2"}, {"phone": "Phone3", "battor": "Battor3"}] diff --git a/catapult/devil/devil/utils/timeout_retry.py b/catapult/devil/devil/utils/timeout_retry.py index 95e90ee5..e4b5ff8c 100644 --- a/catapult/devil/devil/utils/timeout_retry.py +++ b/catapult/devil/devil/utils/timeout_retry.py @@ -8,7 +8,6 @@ import logging import threading import time -import traceback from devil.utils import reraiser_thread from devil.utils import watchdog_timer @@ -114,17 +113,6 @@ def WaitFor(condition, wait_period=5, max_tries=None): return None -def _LogLastException(thread_name, attempt, max_attempts, log_func): - log_func('*' * 80) - log_func('Exception on thread %s (attempt %d of %d)', thread_name, - attempt, max_attempts) - log_func('*' * 80) - fmt_exc = ''.join(traceback.format_exc()) - for line in fmt_exc.splitlines(): - log_func(line.rstrip()) - log_func('*' * 80) - - def AlwaysRetry(_exception): return True @@ -152,6 +140,8 @@ def Run(func, timeout, retries, args=None, kwargs=None, desc=None, args = [] if not kwargs: kwargs = {} + if not desc: + desc = func.__name__ num_try = 1 while True: @@ -166,7 +156,7 @@ def Run(func, timeout, retries, args=None, kwargs=None, desc=None, thread_group.JoinAll(watcher=thread_group.GetWatcher(), timeout=60, error_log_func=error_log_func) if thread_group.IsAlive(): - logging.info('Still working on %s', desc if desc else func.__name__) + logging.info('Still working on %s', desc) else: return thread_group.GetAllReturnValues()[0] except reraiser_thread.TimeoutError as e: @@ -177,5 +167,7 @@ def Run(func, timeout, retries, args=None, kwargs=None, desc=None, except Exception as e: # pylint: disable=broad-except if num_try > retries or not retry_if_func(e): raise - _LogLastException(thread_name, num_try, retries + 1, error_log_func) + error_log_func( + '(%s) Exception on %s, attempt %d of %d: %r', + thread_name, desc, num_try, retries + 1, e) num_try += 1 diff --git a/catapult/devil/devil/utils/update_mapping.py b/catapult/devil/devil/utils/update_mapping.py new file mode 100755 index 00000000..6666b9b0 --- /dev/null +++ b/catapult/devil/devil/utils/update_mapping.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import sys + +from devil.utils import battor_device_mapping + +def parse_options(): + """Parses and checks the command-line options. + + Returns: + A tuple containing the options structure. + """ + usage = 'Usage: ./update_mapping.py [options]' + desc = ('Example: ./update_mapping.py -o mapping.json.\n' + 'This script generates and stores a file that gives the\n' + 'mapping between phone serial numbers and BattOr serial numbers\n' + 'Mapping is based on which physical ports on the USB hubs the\n' + 'devices are plugged in to. For instance, if there are two hubs,\n' + 'the phone connected to port N on the first hub is mapped to the\n' + 'BattOr connected to port N on the second hub, for each N.') + parser = argparse.ArgumentParser(usage=usage, description=desc) + parser.add_argument('-o', '--output', dest='out_file', + default='mapping.json', type=str, + action='store', help='mapping file name') + parser.add_argument('-u', '--hub', dest='hub_types', + action='append', choices=['plugable_7port', + 'plugable_7port_usb3_part2', + 'plugable_7port_usb3_part3'], + help='USB hub types.') + options = parser.parse_args() + if not options.hub_types: + options.hub_types = ['plugable_7port', 'plugable_7port_usb3_part2', + 'plugable_7port_usb3_part3'] + return options + +def main(): + options = parse_options() + battor_device_mapping.GenerateSerialMapFile(options.out_file, + options.hub_types) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/catapult/devil/devil/utils/usb_hubs.py b/catapult/devil/devil/utils/usb_hubs.py new file mode 100644 index 00000000..1b9566a3 --- /dev/null +++ b/catapult/devil/devil/utils/usb_hubs.py @@ -0,0 +1,165 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +PLUGABLE_7PORT_LAYOUT = {1:7, + 2:6, + 3:5, + 4:{1:4, 2:3, 3:2, 4:1}} + +PLUGABLE_7PORT_USB3_LAYOUT = {1:{1:1, 2:2, 3:3, 4:4}, + 2:5, + 3:6, + 4:7} + +KEEDOX_LAYOUT = {1:1, + 2:2, + 3:3, + 4:{1:4, 2:5, 3:6, 4:7}} + +class HubType(object): + def __init__(self, id_func, port_mapping): + """Defines a type of hub. + + Args: + id_func: [USBNode -> bool] is a function that can be run on a node + to determine if the node represents this type of hub. + port_mapping: [dict(int:(int|dict))] maps virtual to physical port + numbers. For instance, {3:1, 1:2, 2:3} means that virtual port 3 + corresponds to physical port 1, virtual port 1 corresponds to physical + port 2, and virtual port 2 corresponds to physical port 3. In the + case of hubs with "internal" topology, this is represented by nested + maps. For instance, {1:{1:1,2:2},2:{1:3,2:4}} means, e.g. that the + device plugged into physical port 3 will show up as being connected + to port 1, on a device which is connected to port 2 on the hub. + """ + self._id_func = id_func + # v2p = "virtual to physical" ports + self._v2p_port = port_mapping + + def IsType(self, node): + """Determines if the given Node is a hub of this type. + + Args: + node: [USBNode] Node to check. + """ + return self._id_func(node) + + def GetPhysicalPortToNodeTuples(self, node): + """Gets devices connected to the physical ports on a hub of this type. + + Args: + node: [USBNode] Node representing a hub of this type. + + Yields: + A series of (int, USBNode) tuples giving a physical port + and the USBNode connected to it. + + Raises: + ValueError: If the given node isn't a hub of this type. + """ + if self.IsType(node): + for res in self._GppHelper(node, self._v2p_port): + yield res + else: + raise ValueError('Node must be a hub of this type') + + def _GppHelper(self, node, mapping): + """Helper function for GetPhysicalPortToNodeMap. + + Gets devices connected to physical ports, based on device tree + rooted at the given node and the mapping between virtual and physical + ports. + + Args: + node: [USBNode] Root of tree to search for devices. + mapping: [dict] Mapping between virtual and physical ports. + + Yields: + A series of (int, USBNode) tuples giving a physical port + and the Node connected to it. + """ + for (virtual, physical) in mapping.iteritems(): + if node.HasPort(virtual): + if isinstance(physical, dict): + for res in self._GppHelper(node.PortToDevice(virtual), physical): + yield res + else: + yield (physical, node.PortToDevice(virtual)) + +def _is_plugable_7port_hub(node): + """Check if a node is a Plugable 7-Port Hub + (Model USB2-HUB7BC) + The topology of this device is a 4-port hub, + with another 4-port hub connected on port 4. + """ + if '1a40:0101' not in node.desc: + return False + if not node.HasPort(4): + return False + return '1a40:0101' in node.PortToDevice(4).desc + +# Plugable 7-Port USB-3 Hubs show up twice in the USB devices list; they have +# two different "branches", one which has USB2 devices and one which has +# USB3 devices. The "part2" is the "USB-2" branch of the hub, the +# "part3" is the "USB-3" branch of the hub. + +def _is_plugable_7port_usb3_part2_hub(node): + """Check if a node is the "USB2 branch" of + a Plugable 7-Port USB-3 Hub (Model USB3-HUB7BC) + The topology of this device is a 4-port hub, + with another 4-port hub connected on port 1. + """ + if '2109:2811' not in node.desc: + return False + if not node.HasPort(1): + return False + return '2109:2811' in node.PortToDevice(1).desc + +def _is_plugable_7port_usb3_part3_hub(node): + """Check if a node is the "USB3 branch" of + a Plugable 7-Port USB-3 Hub (Model USB3-HUB7BC) + The topology of this device is a 4-port hub, + with another 4-port hub connected on port 1. + """ + if '2109:8110' not in node.desc: + return False + if not node.HasPort(1): + return False + return '2109:8110' in node.PortToDevice(1).desc + +def _is_keedox_hub(node): + """Check if a node is a Keedox hub. + The topology of this device is a 4-port hub, + with another 4-port hub connected on port 4. + """ + if '0bda:5411' not in node.desc: + return False + if not node.HasPort(4): + return False + return '0bda:5411' in node.PortToDevice(4).desc + + +PLUGABLE_7PORT = HubType(_is_plugable_7port_hub, PLUGABLE_7PORT_LAYOUT) +PLUGABLE_7PORT_USB3_PART2 = HubType(_is_plugable_7port_usb3_part2_hub, + PLUGABLE_7PORT_USB3_LAYOUT) +PLUGABLE_7PORT_USB3_PART3 = HubType(_is_plugable_7port_usb3_part3_hub, + PLUGABLE_7PORT_USB3_LAYOUT) +KEEDOX = HubType(_is_keedox_hub, KEEDOX_LAYOUT) + +ALL_HUBS = [PLUGABLE_7PORT, + PLUGABLE_7PORT_USB3_PART2, + PLUGABLE_7PORT_USB3_PART3, + KEEDOX] + +def GetHubType(type_name): + if type_name == 'plugable_7port': + return PLUGABLE_7PORT + if type_name == 'plugable_7port_usb3_part2': + return PLUGABLE_7PORT_USB3_PART2 + if type_name == 'plugable_7port_usb3_part3': + return PLUGABLE_7PORT_USB3_PART3 + if type_name == 'keedox': + return KEEDOX + else: + raise ValueError('Invalid hub type') diff --git a/catapult/devil/docs/device_blacklist.md b/catapult/devil/docs/device_blacklist.md new file mode 100644 index 00000000..c6eed514 --- /dev/null +++ b/catapult/devil/docs/device_blacklist.md @@ -0,0 +1,59 @@ +<!-- Copyright 2016 The Chromium Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. +--> + +# Devil: Device Blacklist + +## What is it? + +The device blacklist is a per-run list of devices detected to be in a known bad +state along with the reason they are suspected of being in a bad state (offline, +not responding, etc). It is stored as a json file. This gets reset every run +during the device recovery step (currently part of `bb_device_status_check`). + +## Bots + +On bots, this is normally found at `//out/bad_devices.json`. If you are having +problems with blacklisted devices locally even though a device is in a good +state, you can safely delete this file. + +# Tools for interacting with device black list. + +You can interact with the device blacklist via [devil.android.device\_blacklist](https://cs.chromium.org/chromium/src/third_party/catapult/devil/devil/android/device_blacklist.py). +This allows for any interaction you would need with a device blacklist: + + - Reading + - Writing + - Extending + - Resetting + +An example usecase of this is: +```python +from devil.android import device_blacklist + +blacklist = device_blacklist.Blacklist(blacklist_path) +blacklisted_devices = blacklist.Read() +for device in blacklisted_devices: + print 'Device %s is blacklisted' % device +blacklist.Reset() +new_blacklist = {'device_id1': {'timestamp': ts, 'reason': reason}} +blacklist.Write(new_blacklist) +blacklist.Extend([device_2, device_3], reason='Reason for blacklisting') +``` + + +## Where it is used. + +The blacklist file path is passed directly to the following scripts in chromium: + + - [test\_runner.py](https://cs.chromium.org/chromium/src/build/android/test_runner.py) + - [provision\_devices.py](https://cs.chromium.org/chromium/src/build/android/provision_devices.py) + - [bb\_device\_status\_check.py](https://cs.chromium.org/chromium/src/build/android/buildbot/bb_device_status_check.py) + +The blacklist is also used in the following scripts: + + - [device\_status.py](https://cs.chromium.org/chromium/src/third_party/catapult/devil/devil/android/tools/device_status.py) + - [device\_recovery.py](https://cs.chromium.org/chromium/src/third_party/catapult/devil/devil/android/tools/device_recovery.py) + + diff --git a/catapult/devil/docs/persistent_device_list.md b/catapult/devil/docs/persistent_device_list.md new file mode 100644 index 00000000..d08d9a9e --- /dev/null +++ b/catapult/devil/docs/persistent_device_list.md @@ -0,0 +1,41 @@ +<!-- Copyright 2016 The Chromium Authors. All rights reserved. + Use of this source code is governed by a BSD-style license that can be + found in the LICENSE file. +--> + +# Devil: Persistent Device List + +## What is it? + +A persistent device list that stores all expected devices between builds. It +is used by the perf test runner in order to properly shard tests by device +affinity. This is important because the same performance test can yield +meaningfully different results when run on different devices. + +## Bots + +The list is usually located at one of these locations: + + - `/b/build/site_config/.known_devices`. + - `~/.android`. + +Look at recipes listed below in order to find more up to date location. + +## Local Runs + +The persistent device list is unnecessary for local runs. It is only used on the +bots that upload data to the perf dashboard. + +## Where it is used + +The persistent device list is used in performance test recipes via +[api.chromium\_tests.steps.DynamicPerfTests](https://cs.chromium.org/chromium/build/scripts/slave/recipe_modules/chromium_tests/steps.py?q=DynamicPerfTests). +For example, the [android/perf](https://cs.chromium.org/chromium/build/scripts/slave/recipes/android/perf.py) recipe uses it like this: + +```python +dynamic_perf_tests = api.chromium_tests.steps.DynamicPerfTests( + builder['perf_id'], 'android', None, + known_devices_file=builder.get('known_devices_file', None)) +dynamic_perf_tests.run(api, None) +``` + |