From 5ad62731e62b9eb8d13f6e66dd1b57deaebdee11 Mon Sep 17 00:00:00 2001 From: Chris Craik Date: Fri, 16 Sep 2016 15:18:07 -0700 Subject: Update to latest catapult (be85e5fd) bug:31252260 bug:31335915 bug:31340203 Test: ./catapult/systrace/bin/run_tests - systrace tests pass Change-Id: I844b83711c56f3ea8a6d0bf254a1187b62c33f81 --- catapult/devil/bin/run_py_devicetests | 17 +- catapult/devil/devil/android/device_test_case.py | 54 ++ catapult/devil/devil/android/device_utils.py | 6 + .../devil/devil/android/device_utils_devicetest.py | 21 +- .../devil/android/perf/perf_control_devicetest.py | 9 +- .../android/sdk/adb_compatibility_devicetest.py | 8 +- catapult/devil/devil/android/sdk/adb_wrapper.py | 4 +- .../devil/android/sdk/adb_wrapper_devicetest.py | 16 +- catapult/devil/devil/android/sdk/keyevent.py | 47 ++ catapult/devil/devil/android/settings.py | 275 ++++++++++ .../devil/devil/android/tools/device_recovery.py | 35 +- .../devil/devil/android/tools/device_status.py | 18 +- catapult/devil/devil/android/tools/keyboard.py | 129 +++++ .../devil/devil/android/tools/provision_devices.py | 578 +++++++++++++++++++++ .../devil/devil/android/tools/script_common.py | 7 +- catapult/devil/devil/devil_env.py | 39 +- catapult/devil/devil/utils/markdown.py | 74 +++ .../devil/devil/utils/timeout_retry_unittest.py | 2 +- 18 files changed, 1263 insertions(+), 76 deletions(-) create mode 100644 catapult/devil/devil/android/device_test_case.py create mode 100644 catapult/devil/devil/android/settings.py create mode 100755 catapult/devil/devil/android/tools/keyboard.py create mode 100755 catapult/devil/devil/android/tools/provision_devices.py create mode 100755 catapult/devil/devil/utils/markdown.py (limited to 'catapult/devil') diff --git a/catapult/devil/bin/run_py_devicetests b/catapult/devil/bin/run_py_devicetests index c23839f3..656bedf2 100755 --- a/catapult/devil/bin/run_py_devicetests +++ b/catapult/devil/bin/run_py_devicetests @@ -10,14 +10,23 @@ _CATAPULT_PATH = os.path.abspath(os.path.join( os.path.dirname(__file__), '..', '..')) _DEVIL_PATH = os.path.abspath(os.path.join( os.path.dirname(__file__), '..')) +_TYP_PATH = os.path.abspath(os.path.join(_CATAPULT_PATH, 'third_party', 'typ')) -sys.path.append(_CATAPULT_PATH) -from catapult_build import run_with_typ +sys.path.append(_TYP_PATH) +import typ + +sys.path.append(_DEVIL_PATH) +from devil.android import device_test_case def main(): - sys.argv.extend(['--suffixes', '*_devicetest.py', '-j', '1']) - return run_with_typ.Run(top_level_dir=_DEVIL_PATH) + runner = typ.Runner() + runner.setup_fn = device_test_case.PrepareDevices + return runner.main( + coverage_source=[_DEVIL_PATH], + jobs=1, + suffixes=['*_devicetest.py'], + top_level_dir=_DEVIL_PATH) if __name__ == '__main__': sys.exit(main()) diff --git a/catapult/devil/devil/android/device_test_case.py b/catapult/devil/devil/android/device_test_case.py new file mode 100644 index 00000000..b995fa6f --- /dev/null +++ b/catapult/devil/devil/android/device_test_case.py @@ -0,0 +1,54 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import threading +import unittest + +from devil.android import device_errors +from devil.android import device_utils + +_devices_lock = threading.Lock() +_devices_condition = threading.Condition(_devices_lock) +_devices = set() + + +def PrepareDevices(*_args): + + raw_devices = device_utils.DeviceUtils.HealthyDevices() + live_devices = [] + for d in raw_devices: + try: + d.WaitUntilFullyBooted(timeout=5, retries=0) + live_devices.append(str(d)) + except (device_errors.CommandFailedError, + device_errors.CommandTimeoutError): + pass + with _devices_lock: + _devices.update(set(live_devices)) + + if not _devices: + raise Exception('No live devices attached.') + + +class DeviceTestCase(unittest.TestCase): + + def __init__(self, *args, **kwargs): + super(DeviceTestCase, self).__init__(*args, **kwargs) + self.serial = None + + #override + def setUp(self): + super(DeviceTestCase, self).setUp() + with _devices_lock: + while not _devices: + _devices_condition.wait(5) + self.serial = _devices.pop() + + #override + def tearDown(self): + super(DeviceTestCase, self).tearDown() + with _devices_lock: + _devices.add(self.serial) + _devices_condition.notify() + diff --git a/catapult/devil/devil/android/device_utils.py b/catapult/devil/devil/android/device_utils.py index 93aaaf92..1a996500 100644 --- a/catapult/devil/devil/android/device_utils.py +++ b/catapult/devil/devil/android/device_utils.py @@ -489,6 +489,12 @@ class DeviceUtils(object): apks = [] for line in output: if not line.startswith('package:'): + # KitKat on x86 may show following warnings that is safe to ignore. + if (line.startswith('WARNING: linker: libdvm.so has text relocations.') + and version_codes.KITKAT <= self.build_version_sdk + and self.build_version_sdk <= version_codes.KITKAT_WATCH + and self.product_cpu_abi == 'x86'): + continue raise device_errors.CommandFailedError( 'pm path returned: %r' % '\n'.join(output), str(self)) apks.append(line[len('package:'):]) diff --git a/catapult/devil/devil/android/device_utils_devicetest.py b/catapult/devil/devil/android/device_utils_devicetest.py index 33c1fb93..54ab7dbf 100755 --- a/catapult/devil/devil/android/device_utils_devicetest.py +++ b/catapult/devil/devil/android/device_utils_devicetest.py @@ -17,6 +17,7 @@ if __name__ == '__main__': sys.path.append( os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', ))) +from devil.android import device_test_case from devil.android import device_utils from devil.android.sdk import adb_wrapper from devil.utils import cmd_helper @@ -29,12 +30,11 @@ _SUB_DIR1 = "sub1" _SUB_DIR2 = "sub2" -class DeviceUtilsPushDeleteFilesTest(unittest.TestCase): +class DeviceUtilsPushDeleteFilesTest(device_test_case.DeviceTestCase): def setUp(self): - devices = adb_wrapper.AdbWrapper.Devices() - assert devices, 'A device must be attached' - self.adb = devices[0] + super(DeviceUtilsPushDeleteFilesTest, self).setUp() + self.adb = adb_wrapper.AdbWrapper(self.serial) self.adb.WaitForDevice() self.device = device_utils.DeviceUtils( self.adb, default_timeout=10, default_retries=0) @@ -210,11 +210,16 @@ class DeviceUtilsPushDeleteFilesTest(unittest.TestCase): cmd_helper.RunCmd(['rm', '-rf', host_tmp_dir]) def testRestartAdbd(self): - old_adbd_pid = self.device.RunShellCommand( - ['ps', '|', 'grep', 'adbd'])[1].split()[1] + def get_adbd_pid(): + ps_output = self.device.RunShellCommand(['ps']) + for ps_line in ps_output: + if 'adbd' in ps_line: + return ps_line.split()[1] + self.fail('Unable to find adbd') + + old_adbd_pid = get_adbd_pid() self.device.RestartAdbd() - new_adbd_pid = self.device.RunShellCommand( - ['ps', '|', 'grep', 'adbd'])[1].split()[1] + new_adbd_pid = get_adbd_pid() self.assertNotEqual(old_adbd_pid, new_adbd_pid) diff --git a/catapult/devil/devil/android/perf/perf_control_devicetest.py b/catapult/devil/devil/android/perf/perf_control_devicetest.py index 71bf3fbc..b6458030 100644 --- a/catapult/devil/devil/android/perf/perf_control_devicetest.py +++ b/catapult/devil/devil/android/perf/perf_control_devicetest.py @@ -9,19 +9,18 @@ import unittest sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) +from devil.android import device_test_case from devil.android import device_utils from devil.android.perf import perf_control -class TestPerfControl(unittest.TestCase): +class TestPerfControl(device_test_case.DeviceTestCase): def setUp(self): + super(TestPerfControl, self).setUp() if not os.getenv('BUILDTYPE'): os.environ['BUILDTYPE'] = 'Debug' - - devices = device_utils.DeviceUtils.HealthyDevices(blacklist=None) - self.assertGreater(len(devices), 0, 'No device attached!') - self._device = devices[0] + self._device = device_utils.DeviceUtils(self.serial) def testHighPerfMode(self): perf = perf_control.PerfControl(self._device) diff --git a/catapult/devil/devil/android/sdk/adb_compatibility_devicetest.py b/catapult/devil/devil/android/sdk/adb_compatibility_devicetest.py index 6f670168..49a4971e 100644 --- a/catapult/devil/devil/android/sdk/adb_compatibility_devicetest.py +++ b/catapult/devil/devil/android/sdk/adb_compatibility_devicetest.py @@ -17,6 +17,7 @@ _CATAPULT_BASE_DIR = os.path.abspath(os.path.join( sys.path.append(os.path.join(_CATAPULT_BASE_DIR, 'devil')) from devil import devil_env from devil.android import device_errors +from devil.android import device_test_case from devil.android.sdk import adb_wrapper from devil.utils import cmd_helper from devil.utils import timeout_retry @@ -37,7 +38,7 @@ def _hostAdbPids(): if name == 'adb'] -class AdbCompatibilityTest(unittest.TestCase): +class AdbCompatibilityTest(device_test_case.DeviceTestCase): @classmethod def setUpClass(cls): @@ -97,10 +98,7 @@ class AdbCompatibilityTest(unittest.TestCase): def getTestInstance(self): """Creates a real AdbWrapper instance for testing.""" - devices = adb_wrapper.AdbWrapper.Devices() - if not devices: - self.skipTest('No test device available.') - return adb_wrapper.AdbWrapper(devices[0]) + return adb_wrapper.AdbWrapper(self.serial) def testShell(self): under_test = self.getTestInstance() diff --git a/catapult/devil/devil/android/sdk/adb_wrapper.py b/catapult/devil/devil/android/sdk/adb_wrapper.py index 57e6396a..cbc5ed5c 100644 --- a/catapult/devil/devil/android/sdk/adb_wrapper.py +++ b/catapult/devil/devil/android/sdk/adb_wrapper.py @@ -28,6 +28,8 @@ with devil_env.SysPath(devil_env.DEPENDENCY_MANAGER_PATH): import dependency_manager # pylint: disable=import-error +ADB_KEYS_FILE = '/data/misc/adb/adb_keys' + DEFAULT_TIMEOUT = 30 DEFAULT_RETRIES = 2 @@ -129,7 +131,7 @@ class AdbWrapper(object): up a new ADB shell for each command. Example of use: - with pshell as PersistentShell('123456789'): + with PersistentShell('123456789') as pshell: pshell.RunCommand('which ls') pshell.RunCommandAndClose('echo TEST') ''' diff --git a/catapult/devil/devil/android/sdk/adb_wrapper_devicetest.py b/catapult/devil/devil/android/sdk/adb_wrapper_devicetest.py index 9a38c6cd..d97d56a1 100755 --- a/catapult/devil/devil/android/sdk/adb_wrapper_devicetest.py +++ b/catapult/devil/devil/android/sdk/adb_wrapper_devicetest.py @@ -11,16 +11,16 @@ import tempfile import time import unittest +from devil.android import device_test_case from devil.android import device_errors from devil.android.sdk import adb_wrapper -class TestAdbWrapper(unittest.TestCase): +class TestAdbWrapper(device_test_case.DeviceTestCase): def setUp(self): - devices = adb_wrapper.AdbWrapper.Devices() - assert devices, 'A device must be attached' - self._adb = devices[0] + super(TestAdbWrapper, self).setUp() + self._adb = adb_wrapper.AdbWrapper(self.serial) self._adb.WaitForDevice() @staticmethod @@ -94,7 +94,13 @@ class TestAdbWrapper(unittest.TestCase): self._adb.WaitForDevice() self.assertEqual(self._adb.GetState(), 'device') print 'waiting for package manager...' - while 'package:' not in self._adb.Shell('pm path android'): + while True: + try: + android_path = self._adb.Shell('pm path android') + except device_errors.AdbShellCommandFailedError: + android_path = None + if android_path and 'package:' in android_path: + break time.sleep(1) def testRootRemount(self): diff --git a/catapult/devil/devil/android/sdk/keyevent.py b/catapult/devil/devil/android/sdk/keyevent.py index 732a7dc9..40f9416c 100644 --- a/catapult/devil/devil/android/sdk/keyevent.py +++ b/catapult/devil/devil/android/sdk/keyevent.py @@ -8,7 +8,54 @@ http://developer.android.com/reference/android/view/KeyEvent.html """ KEYCODE_BACK = 4 + +KEYCODE_0 = 7 +KEYCODE_1 = 8 +KEYCODE_2 = 9 +KEYCODE_3 = 10 +KEYCODE_4 = 11 +KEYCODE_5 = 12 +KEYCODE_6 = 13 +KEYCODE_7 = 14 +KEYCODE_8 = 15 +KEYCODE_9 = 16 + KEYCODE_DPAD_RIGHT = 22 + +KEYCODE_A = 29 +KEYCODE_B = 30 +KEYCODE_C = 31 +KEYCODE_D = 32 +KEYCODE_E = 33 +KEYCODE_F = 34 +KEYCODE_G = 35 +KEYCODE_H = 36 +KEYCODE_I = 37 +KEYCODE_J = 38 +KEYCODE_K = 39 +KEYCODE_L = 40 +KEYCODE_M = 41 +KEYCODE_N = 42 +KEYCODE_O = 43 +KEYCODE_P = 44 +KEYCODE_Q = 45 +KEYCODE_R = 46 +KEYCODE_S = 47 +KEYCODE_T = 48 +KEYCODE_U = 49 +KEYCODE_V = 50 +KEYCODE_W = 51 +KEYCODE_X = 52 +KEYCODE_Y = 53 +KEYCODE_Z = 54 + +KEYCODE_PERIOD = 56 + +KEYCODE_SPACE = 62 + KEYCODE_ENTER = 66 +KEYCODE_DEL = 67 + KEYCODE_MENU = 82 + KEYCODE_APP_SWITCH = 187 diff --git a/catapult/devil/devil/android/settings.py b/catapult/devil/devil/android/settings.py new file mode 100644 index 00000000..2249c2a5 --- /dev/null +++ b/catapult/devil/devil/android/settings.py @@ -0,0 +1,275 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import logging + +_LOCK_SCREEN_SETTINGS_PATH = '/data/system/locksettings.db' +_ALTERNATE_LOCK_SCREEN_SETTINGS_PATH = ( + '/data/data/com.android.providers.settings/databases/settings.db') +PASSWORD_QUALITY_UNSPECIFIED = '0' +_COMPATIBLE_BUILD_TYPES = ['userdebug', 'eng'] + + +ENABLE_LOCATION_SETTINGS = [ + # Note that setting these in this order is required in order for all of + # them to take and stick through a reboot. + ('com.google.settings/partner', [ + ('use_location_for_services', 1), + ]), + ('settings/secure', [ + # Ensure Geolocation is enabled and allowed for tests. + ('location_providers_allowed', 'gps,network'), + ]), + ('com.google.settings/partner', [ + ('network_location_opt_in', 1), + ]) +] + +DISABLE_LOCATION_SETTINGS = [ + ('com.google.settings/partner', [ + ('use_location_for_services', 0), + ]), + ('settings/secure', [ + # Ensure Geolocation is disabled. + ('location_providers_allowed', ''), + ]), +] + +ENABLE_MOCK_LOCATION_SETTINGS = [ + ('settings/secure', [ + ('mock_location', 1), + ]), +] + +DISABLE_MOCK_LOCATION_SETTINGS = [ + ('settings/secure', [ + ('mock_location', 0), + ]), +] + +DETERMINISTIC_DEVICE_SETTINGS = [ + ('settings/global', [ + ('assisted_gps_enabled', 0), + + # Disable "auto time" and "auto time zone" to avoid network-provided time + # to overwrite the device's datetime and timezone synchronized from host + # when running tests later. See b/6569849. + ('auto_time', 0), + ('auto_time_zone', 0), + + ('development_settings_enabled', 1), + + # Flag for allowing ActivityManagerService to send ACTION_APP_ERROR intents + # on application crashes and ANRs. If this is disabled, the crash/ANR dialog + # will never display the "Report" button. + # Type: int ( 0 = disallow, 1 = allow ) + ('send_action_app_error', 0), + + ('stay_on_while_plugged_in', 3), + + ('verifier_verify_adb_installs', 0), + ]), + ('settings/secure', [ + ('allowed_geolocation_origins', + 'http://www.google.co.uk http://www.google.com'), + + # Ensure that we never get random dialogs like "Unfortunately the process + # android.process.acore has stopped", which steal the focus, and make our + # automation fail (because the dialog steals the focus then mistakenly + # receives the injected user input events). + ('anr_show_background', 0), + + ('lockscreen.disabled', 1), + + ('screensaver_enabled', 0), + + ('skip_first_use_hints', 1), + ]), + ('settings/system', [ + # Don't want devices to accidentally rotate the screen as that could + # affect performance measurements. + ('accelerometer_rotation', 0), + + ('lockscreen.disabled', 1), + + # Turn down brightness and disable auto-adjust so that devices run cooler. + ('screen_brightness', 5), + ('screen_brightness_mode', 0), + + ('user_rotation', 0), + ]), +] + +NETWORK_DISABLED_SETTINGS = [ + ('settings/global', [ + ('airplane_mode_on', 1), + ('wifi_on', 0), + ]), +] + + +class ContentSettings(dict): + + """A dict interface to interact with device content settings. + + System properties are key/value pairs as exposed by adb shell content. + """ + + def __init__(self, table, device): + super(ContentSettings, self).__init__() + self._table = table + self._device = device + + @staticmethod + def _GetTypeBinding(value): + if isinstance(value, bool): + return 'b' + if isinstance(value, float): + return 'f' + if isinstance(value, int): + return 'i' + if isinstance(value, long): + return 'l' + if isinstance(value, str): + return 's' + raise ValueError('Unsupported type %s' % type(value)) + + def iteritems(self): + # Example row: + # 'Row: 0 _id=13, name=logging_id2, value=-1fccbaa546705b05' + for row in self._device.RunShellCommand( + 'content query --uri content://%s' % self._table, as_root=True): + fields = row.split(', ') + key = None + value = None + for field in fields: + k, _, v = field.partition('=') + if k == 'name': + key = v + elif k == 'value': + value = v + if not key: + continue + if not value: + value = '' + yield key, value + + def __getitem__(self, key): + return self._device.RunShellCommand( + 'content query --uri content://%s --where "name=\'%s\'" ' + '--projection value' % (self._table, key), as_root=True).strip() + + def __setitem__(self, key, value): + if key in self: + self._device.RunShellCommand( + 'content update --uri content://%s ' + '--bind value:%s:%s --where "name=\'%s\'"' % ( + self._table, + self._GetTypeBinding(value), value, key), + as_root=True) + else: + self._device.RunShellCommand( + 'content insert --uri content://%s ' + '--bind name:%s:%s --bind value:%s:%s' % ( + self._table, + self._GetTypeBinding(key), key, + self._GetTypeBinding(value), value), + as_root=True) + + def __delitem__(self, key): + self._device.RunShellCommand( + 'content delete --uri content://%s ' + '--bind name:%s:%s' % ( + self._table, + self._GetTypeBinding(key), key), + as_root=True) + + +def ConfigureContentSettings(device, desired_settings): + """Configures device content setings from a list. + + Many settings are documented at: + http://developer.android.com/reference/android/provider/Settings.Global.html + http://developer.android.com/reference/android/provider/Settings.Secure.html + http://developer.android.com/reference/android/provider/Settings.System.html + + Many others are undocumented. + + Args: + device: A DeviceUtils instance for the device to configure. + desired_settings: A list of (table, [(key: value), ...]) for all + settings to configure. + """ + for table, key_value in desired_settings: + settings = ContentSettings(table, device) + for key, value in key_value: + settings[key] = value + logging.info('\n%s %s', table, (80 - len(table)) * '-') + for key, value in sorted(settings.iteritems()): + logging.info('\t%s: %s', key, value) + + +def SetLockScreenSettings(device): + """Sets lock screen settings on the device. + + On certain device/Android configurations we need to disable the lock screen in + a different database. Additionally, the password type must be set to + DevicePolicyManager.PASSWORD_QUALITY_UNSPECIFIED. + Lock screen settings are stored in sqlite on the device in: + /data/system/locksettings.db + + IMPORTANT: The first column is used as a primary key so that all rows with the + same value for that column are removed from the table prior to inserting the + new values. + + Args: + device: A DeviceUtils instance for the device to configure. + + Raises: + Exception if the setting was not properly set. + """ + if device.build_type not in _COMPATIBLE_BUILD_TYPES: + logging.warning('Unable to disable lockscreen on %s builds.', + device.build_type) + return + + def get_lock_settings(table): + return [(table, 'lockscreen.disabled', '1'), + (table, 'lockscreen.password_type', PASSWORD_QUALITY_UNSPECIFIED), + (table, 'lockscreen.password_type_alternate', + PASSWORD_QUALITY_UNSPECIFIED)] + + if device.FileExists(_LOCK_SCREEN_SETTINGS_PATH): + db = _LOCK_SCREEN_SETTINGS_PATH + locksettings = get_lock_settings('locksettings') + columns = ['name', 'user', 'value'] + generate_values = lambda k, v: [k, '0', v] + elif device.FileExists(_ALTERNATE_LOCK_SCREEN_SETTINGS_PATH): + db = _ALTERNATE_LOCK_SCREEN_SETTINGS_PATH + locksettings = get_lock_settings('secure') + get_lock_settings('system') + columns = ['name', 'value'] + generate_values = lambda k, v: [k, v] + else: + logging.warning('Unable to find database file to set lock screen settings.') + return + + for table, key, value in locksettings: + # Set the lockscreen setting for default user '0' + values = generate_values(key, value) + + cmd = """begin transaction; +delete from '%(table)s' where %(primary_key)s='%(primary_value)s'; +insert into '%(table)s' (%(columns)s) values (%(values)s); +commit transaction;""" % { + 'table': table, + 'primary_key': columns[0], + 'primary_value': values[0], + 'columns': ', '.join(columns), + 'values': ', '.join(["'%s'" % value for value in values]) + } + output_msg = device.RunShellCommand('sqlite3 %s "%s"' % (db, cmd), + as_root=True) + if output_msg: + logging.info(' '.join(output_msg)) + diff --git a/catapult/devil/devil/android/tools/device_recovery.py b/catapult/devil/devil/android/tools/device_recovery.py index e6456a70..a7844e26 100755 --- a/catapult/devil/devil/android/tools/device_recovery.py +++ b/catapult/devil/devil/android/tools/device_recovery.py @@ -22,7 +22,8 @@ from devil.android import device_errors from devil.android import device_utils from devil.android.tools import device_status from devil.utils import lsusb -from devil.utils import reset_usb +# TODO(jbudorick): Resolve this after experimenting w/ disabling the USB reset. +from devil.utils import reset_usb # pylint: disable=unused-import from devil.utils import run_tests_helper @@ -101,7 +102,7 @@ def RecoverDevice(device, blacklist, should_reboot=lambda device: True): reason='reboot_timeout') -def RecoverDevices(devices, blacklist): +def RecoverDevices(devices, blacklist, enable_usb_reset=False): """Attempts to recover any inoperable devices in the provided list. Args: @@ -140,7 +141,13 @@ def RecoverDevices(devices, blacklist): KillAllAdb() for serial in should_restart_usb: try: - reset_usb.reset_android_usb(serial) + # TODO(crbug.com/642194): Resetting may be causing more harm + # (specifically, kernel panics) than it does good. + if enable_usb_reset: + reset_usb.reset_android_usb(serial) + else: + logging.warning('USB reset disabled for %s (crbug.com/642914)', + serial) except IOError: logging.exception('Unable to reset USB for %s.', serial) if blacklist: @@ -163,27 +170,19 @@ def main(): parser.add_argument('--known-devices-file', action='append', default=[], dest='known_devices_files', help='Path to known device lists.') + parser.add_argument('--enable-usb-reset', action='store_true', + help='Reset USB if necessary.') parser.add_argument('-v', '--verbose', action='count', default=1, help='Log more information.') args = parser.parse_args() run_tests_helper.SetLogLevel(args.verbose) - devil_dynamic_config = { - 'config_type': 'BaseConfig', - 'dependencies': {}, - } - + devil_dynamic_config = devil_env.EmptyConfig() if args.adb_path: - devil_dynamic_config['dependencies'].update({ - 'adb': { - 'file_info': { - devil_env.GetPlatform(): { - 'local_paths': [args.adb_path] - } - } - } - }) + devil_dynamic_config['dependencies'].update( + devil_env.LocalConfigItem( + 'adb', devil_env.GetPlatform(), args.adb_path)) devil_env.config.Initialize(configs=[devil_dynamic_config]) blacklist = (device_blacklist.Blacklist(args.blacklist_file) @@ -195,7 +194,7 @@ def main(): devices = [device_utils.DeviceUtils(s) for s in expected_devices.union(usb_devices)] - RecoverDevices(devices, blacklist) + RecoverDevices(devices, blacklist, enable_usb_reset=args.enable_usb_reset) if __name__ == '__main__': diff --git a/catapult/devil/devil/android/tools/device_status.py b/catapult/devil/devil/android/tools/device_status.py index 7e169634..dbfccb6e 100755 --- a/catapult/devil/devil/android/tools/device_status.py +++ b/catapult/devil/devil/android/tools/device_status.py @@ -260,22 +260,12 @@ def main(): run_tests_helper.SetLogLevel(args.verbose) - - devil_dynamic_config = { - 'config_type': 'BaseConfig', - 'dependencies': {}, - } + devil_dynamic_config = devil_env.EmptyConfig() if args.adb_path: - devil_dynamic_config['dependencies'].update({ - 'adb': { - 'file_info': { - devil_env.GetPlatform(): { - 'local_paths': [args.adb_path] - } - } - } - }) + devil_dynamic_config['dependencies'].update( + devil_env.LocalConfigItem( + 'adb', devil_env.GetPlatform(), args.adb_path)) devil_env.config.Initialize(configs=[devil_dynamic_config]) blacklist = (device_blacklist.Blacklist(args.blacklist_file) diff --git a/catapult/devil/devil/android/tools/keyboard.py b/catapult/devil/devil/android/tools/keyboard.py new file mode 100755 index 00000000..31daf59e --- /dev/null +++ b/catapult/devil/devil/android/tools/keyboard.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Use your keyboard as your phone's keyboard. Experimental.""" + +import argparse +import copy +import os +import sys +import termios +import tty + +if __name__ == '__main__': + sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..', '..'))) +from devil import base_error +from devil.android.sdk import keyevent +from devil.android.tools import script_common +from devil.utils import run_tests_helper + + +_KEY_MAPPING = { + '\x08': keyevent.KEYCODE_DEL, + '\x0a': keyevent.KEYCODE_ENTER, + ' ': keyevent.KEYCODE_SPACE, + '.': keyevent.KEYCODE_PERIOD, + '0': keyevent.KEYCODE_0, + '1': keyevent.KEYCODE_1, + '2': keyevent.KEYCODE_2, + '3': keyevent.KEYCODE_3, + '4': keyevent.KEYCODE_4, + '5': keyevent.KEYCODE_5, + '6': keyevent.KEYCODE_6, + '7': keyevent.KEYCODE_7, + '8': keyevent.KEYCODE_8, + '9': keyevent.KEYCODE_9, + 'a': keyevent.KEYCODE_A, + 'b': keyevent.KEYCODE_B, + 'c': keyevent.KEYCODE_C, + 'd': keyevent.KEYCODE_D, + 'e': keyevent.KEYCODE_E, + 'f': keyevent.KEYCODE_F, + 'g': keyevent.KEYCODE_G, + 'h': keyevent.KEYCODE_H, + 'i': keyevent.KEYCODE_I, + 'j': keyevent.KEYCODE_J, + 'k': keyevent.KEYCODE_K, + 'l': keyevent.KEYCODE_L, + 'm': keyevent.KEYCODE_M, + 'n': keyevent.KEYCODE_N, + 'o': keyevent.KEYCODE_O, + 'p': keyevent.KEYCODE_P, + 'q': keyevent.KEYCODE_Q, + 'r': keyevent.KEYCODE_R, + 's': keyevent.KEYCODE_S, + 't': keyevent.KEYCODE_T, + 'u': keyevent.KEYCODE_U, + 'v': keyevent.KEYCODE_V, + 'w': keyevent.KEYCODE_W, + 'x': keyevent.KEYCODE_X, + 'y': keyevent.KEYCODE_Y, + 'z': keyevent.KEYCODE_Z, + '\x7f': keyevent.KEYCODE_DEL, +} + + +def Keyboard(device, stream_itr): + try: + for c in stream_itr: + k = _KEY_MAPPING.get(c) + if k: + device.SendKeyEvent(k) + else: + print + print '(No mapping for character 0x%x)' % ord(c) + except KeyboardInterrupt: + pass + + +def AddArguments(parser): + parser.add_argument('-d', '--device', action='append', dest='devices', + metavar='DEVICE', help='device serial') + parser.add_argument('-v', '--verbose', action='count', help='print more') + + +class MultipleDevicesError(base_error.BaseError): + def __init__(self, devices): + super(MultipleDevicesError, self).__init__( + 'More than one device found: %s' % ', '.join(str(d) for d in devices)) + + +def main(raw_args): + parser = argparse.ArgumentParser( + description="Use your keyboard as your phone's keyboard.") + AddArguments(parser) + args = parser.parse_args(raw_args) + + run_tests_helper.SetLogLevel(args.verbose) + + devices = script_common.GetDevices(args.devices, None) + if len(devices) > 1: + raise MultipleDevicesError(devices) + + def next_char(): + while True: + yield sys.stdin.read(1) + + try: + fd = sys.stdin.fileno() + + # See man 3 termios for more info on what this is doing. + old_attrs = termios.tcgetattr(fd) + new_attrs = copy.deepcopy(old_attrs) + new_attrs[tty.LFLAG] = new_attrs[tty.LFLAG] & ~(termios.ICANON) + new_attrs[tty.CC][tty.VMIN] = 1 + new_attrs[tty.CC][tty.VTIME] = 0 + termios.tcsetattr(fd, termios.TCSAFLUSH, new_attrs) + + Keyboard(devices[0], next_char()) + finally: + termios.tcsetattr(fd, termios.TCSAFLUSH, old_attrs) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/catapult/devil/devil/android/tools/provision_devices.py b/catapult/devil/devil/android/tools/provision_devices.py new file mode 100755 index 00000000..d9bff5dc --- /dev/null +++ b/catapult/devil/devil/android/tools/provision_devices.py @@ -0,0 +1,578 @@ +#!/usr/bin/env python +# +# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Provisions Android devices with settings required for bots. + +Usage: + ./provision_devices.py [-d ] +""" + +import argparse +import datetime +import json +import logging +import os +import posixpath +import re +import sys +import time + +# Import _strptime before threaded code. datetime.datetime.strptime is +# threadsafe except for the initial import of the _strptime module. +# See crbug.com/584730 and https://bugs.python.org/issue7980. +import _strptime # pylint: disable=unused-import + +if __name__ == '__main__': + sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', '..', '..'))) + +from devil import devil_env +from devil.android import battery_utils +from devil.android import device_blacklist +from devil.android import device_errors +from devil.android import device_temp_file +from devil.android import device_utils +from devil.android import settings +from devil.android.constants import chrome +from devil.android.sdk import adb_wrapper +from devil.android.sdk import keyevent +from devil.android.sdk import version_codes +from devil.android.tools import script_common +from devil.constants import exit_codes +from devil.utils import run_tests_helper +from devil.utils import timeout_retry + +_SYSTEM_WEBVIEW_PATHS = ['/system/app/webview', '/system/app/WebViewGoogle'] +_CHROME_PACKAGE_REGEX = re.compile('.*chrom.*') +_TOMBSTONE_REGEX = re.compile('tombstone.*') + + +class _DEFAULT_TIMEOUTS(object): + # L can take a while to reboot after a wipe. + LOLLIPOP = 600 + PRE_LOLLIPOP = 180 + + HELP_TEXT = '{}s on L, {}s on pre-L'.format(LOLLIPOP, PRE_LOLLIPOP) + + +class ProvisionStep(object): + + def __init__(self, cmd, reboot=False): + self.cmd = cmd + self.reboot = reboot + + +def ProvisionDevices( + devices, + blacklist_file, + adb_key_files=None, + disable_location=False, + disable_mock_location=False, + disable_network=False, + disable_system_chrome=False, + emulators=False, + enable_java_debug=False, + max_battery_temp=None, + min_battery_level=None, + output_device_blacklist=None, + reboot_timeout=None, + remove_system_webview=False, + wipe=True): + blacklist = (device_blacklist.Blacklist(blacklist_file) + if blacklist_file + else None) + devices = script_common.GetDevices(devices, blacklist) + devices = [d for d in devices + if not emulators or d.adb.is_emulator] + parallel_devices = device_utils.DeviceUtils.parallel(devices) + + steps = [] + if wipe: + steps += [ProvisionStep(lambda d: Wipe(d, adb_key_files), reboot=True)] + steps += [ProvisionStep( + lambda d: SetProperties(d, enable_java_debug, disable_location, + disable_mock_location), + reboot=not emulators)] + + if disable_network: + steps.append(ProvisionStep(DisableNetwork)) + + if disable_system_chrome: + steps.append(ProvisionStep(DisableSystemChrome)) + + if max_battery_temp: + steps.append(ProvisionStep( + lambda d: WaitForTemperature(d, max_battery_temp))) + + if min_battery_level: + steps.append(ProvisionStep( + lambda d: WaitForCharge(d, min_battery_level))) + + if remove_system_webview: + steps.append(ProvisionStep(RemoveSystemWebView)) + + steps.append(ProvisionStep(SetDate)) + steps.append(ProvisionStep(CheckExternalStorage)) + + parallel_devices.pMap(ProvisionDevice, steps, blacklist, reboot_timeout) + + blacklisted_devices = blacklist.Read() if blacklist else [] + if output_device_blacklist: + with open(output_device_blacklist, 'w') as f: + json.dump(blacklisted_devices, f) + if all(d in blacklisted_devices for d in devices): + raise device_errors.NoDevicesError + return 0 + + +def ProvisionDevice(device, steps, blacklist, reboot_timeout=None): + try: + if not reboot_timeout: + if device.build_version_sdk >= version_codes.LOLLIPOP: + reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP + else: + reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP + + for step in steps: + try: + device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0) + except device_errors.CommandTimeoutError: + logging.error('Device did not finish booting. Will try to reboot.') + device.Reboot(timeout=reboot_timeout) + step.cmd(device) + if step.reboot: + device.Reboot(False, retries=0) + device.adb.WaitForDevice() + + except device_errors.CommandTimeoutError: + logging.exception('Timed out waiting for device %s. Adding to blacklist.', + str(device)) + if blacklist: + blacklist.Extend([str(device)], reason='provision_timeout') + + except device_errors.CommandFailedError: + logging.exception('Failed to provision device %s. Adding to blacklist.', + str(device)) + if blacklist: + blacklist.Extend([str(device)], reason='provision_failure') + + +def Wipe(device, adb_key_files=None): + if (device.IsUserBuild() or + device.build_version_sdk >= version_codes.MARSHMALLOW): + WipeChromeData(device) + + package = "com.google.android.gms" + version_name = device.GetApplicationVersion(package) + logging.info("Version name for %s is %s", package, version_name) + else: + WipeDevice(device, adb_key_files) + + +def WipeChromeData(device): + """Wipes chrome specific data from device + + (1) uninstall any app whose name matches *chrom*, except + com.android.chrome, which is the chrome stable package. Doing so also + removes the corresponding dirs under /data/data/ and /data/app/ + (2) remove any dir under /data/app-lib/ whose name matches *chrom* + (3) remove any files under /data/tombstones/ whose name matches "tombstone*" + (4) remove /data/local.prop if there is any + (5) remove /data/local/chrome-command-line if there is any + (6) remove anything under /data/local/.config/ if the dir exists + (this is telemetry related) + (7) remove anything under /data/local/tmp/ + + Arguments: + device: the device to wipe + """ + try: + if device.IsUserBuild(): + _UninstallIfMatch(device, _CHROME_PACKAGE_REGEX, + chrome.PACKAGE_INFO['chrome_stable'].package) + device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(), + check_return=True) + device.RunShellCommand('rm -rf /data/local/tmp/*', check_return=True) + else: + device.EnableRoot() + _UninstallIfMatch(device, _CHROME_PACKAGE_REGEX, + chrome.PACKAGE_INFO['chrome_stable'].package) + _WipeUnderDirIfMatch(device, '/data/app-lib/', _CHROME_PACKAGE_REGEX) + _WipeUnderDirIfMatch(device, '/data/tombstones/', _TOMBSTONE_REGEX) + + _WipeFileOrDir(device, '/data/local.prop') + _WipeFileOrDir(device, '/data/local/chrome-command-line') + _WipeFileOrDir(device, '/data/local/.config/') + _WipeFileOrDir(device, '/data/local/tmp/') + device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(), + check_return=True) + except device_errors.CommandFailedError: + logging.exception('Possible failure while wiping the device. ' + 'Attempting to continue.') + + +def _UninstallIfMatch(device, pattern, app_to_keep): + installed_packages = device.RunShellCommand(['pm', 'list', 'packages']) + installed_system_packages = [ + pkg.split(':')[1] for pkg in device.RunShellCommand(['pm', 'list', + 'packages', '-s'])] + for package_output in installed_packages: + package = package_output.split(":")[1] + if pattern.match(package) and not package == app_to_keep: + if not device.IsUserBuild() or package not in installed_system_packages: + device.Uninstall(package) + + +def _WipeUnderDirIfMatch(device, path, pattern): + for filename in device.ListDirectory(path): + if pattern.match(filename): + _WipeFileOrDir(device, posixpath.join(path, filename)) + + +def _WipeFileOrDir(device, path): + if device.PathExists(path): + device.RunShellCommand(['rm', '-rf', path], check_return=True) + + +def WipeDevice(device, adb_key_files): + """Wipes data from device, keeping only the adb_keys for authorization. + + After wiping data on a device that has been authorized, adb can still + communicate with the device, but after reboot the device will need to be + re-authorized because the adb keys file is stored in /data/misc/adb/. + Thus, adb_keys file is rewritten so the device does not need to be + re-authorized. + + Arguments: + device: the device to wipe + """ + try: + device.EnableRoot() + device_authorized = device.FileExists(adb_wrapper.ADB_KEYS_FILE) + if device_authorized: + adb_keys = device.ReadFile(adb_wrapper.ADB_KEYS_FILE, + as_root=True).splitlines() + device.RunShellCommand(['wipe', 'data'], + as_root=True, check_return=True) + device.adb.WaitForDevice() + + if device_authorized: + adb_keys_set = set(adb_keys) + for adb_key_file in adb_key_files or []: + try: + with open(adb_key_file, 'r') as f: + adb_public_keys = f.readlines() + adb_keys_set.update(adb_public_keys) + except IOError: + logging.warning('Unable to find adb keys file %s.', adb_key_file) + _WriteAdbKeysFile(device, '\n'.join(adb_keys_set)) + except device_errors.CommandFailedError: + logging.exception('Possible failure while wiping the device. ' + 'Attempting to continue.') + + +def _WriteAdbKeysFile(device, adb_keys_string): + dir_path = posixpath.dirname(adb_wrapper.ADB_KEYS_FILE) + device.RunShellCommand(['mkdir', '-p', dir_path], + as_root=True, check_return=True) + device.RunShellCommand(['restorecon', dir_path], + as_root=True, check_return=True) + device.WriteFile(adb_wrapper.ADB_KEYS_FILE, adb_keys_string, as_root=True) + device.RunShellCommand(['restorecon', adb_wrapper.ADB_KEYS_FILE], + as_root=True, check_return=True) + + +def SetProperties(device, enable_java_debug, disable_location, + disable_mock_location): + try: + device.EnableRoot() + except device_errors.CommandFailedError as e: + logging.warning(str(e)) + + if not device.IsUserBuild(): + _ConfigureLocalProperties(device, enable_java_debug) + else: + logging.warning('Cannot configure properties in user builds.') + settings.ConfigureContentSettings( + device, settings.DETERMINISTIC_DEVICE_SETTINGS) + if disable_location: + settings.ConfigureContentSettings( + device, settings.DISABLE_LOCATION_SETTINGS) + else: + settings.ConfigureContentSettings( + device, settings.ENABLE_LOCATION_SETTINGS) + + if disable_mock_location: + settings.ConfigureContentSettings( + device, settings.DISABLE_MOCK_LOCATION_SETTINGS) + else: + settings.ConfigureContentSettings( + device, settings.ENABLE_MOCK_LOCATION_SETTINGS) + + settings.SetLockScreenSettings(device) + + # Some device types can momentarily disappear after setting properties. + device.adb.WaitForDevice() + + +def DisableNetwork(device): + settings.ConfigureContentSettings( + device, settings.NETWORK_DISABLED_SETTINGS) + if device.build_version_sdk >= version_codes.MARSHMALLOW: + # Ensure that NFC is also switched off. + device.RunShellCommand(['svc', 'nfc', 'disable'], + as_root=True, check_return=True) + + +def DisableSystemChrome(device): + # The system chrome version on the device interferes with some tests. + device.RunShellCommand(['pm', 'disable', 'com.android.chrome'], + check_return=True) + + +def RemoveSystemWebView(device): + if any(device.PathExists(p) for p in _SYSTEM_WEBVIEW_PATHS): + logging.info('System WebView exists and needs to be removed') + if device.HasRoot(): + # Disabled Marshmallow's Verity security feature + if device.build_version_sdk >= version_codes.MARSHMALLOW: + device.adb.DisableVerity() + device.Reboot() + device.WaitUntilFullyBooted() + device.EnableRoot() + + # This is required, e.g., to replace the system webview on a device. + device.adb.Remount() + device.RunShellCommand(['stop'], check_return=True) + device.RunShellCommand(['rm', '-rf'] + _SYSTEM_WEBVIEW_PATHS, + check_return=True) + device.RunShellCommand(['start'], check_return=True) + else: + logging.warning('Cannot remove system webview from a non-rooted device') + else: + logging.info('System WebView already removed') + + + +def _ConfigureLocalProperties(device, java_debug=True): + """Set standard readonly testing device properties prior to reboot.""" + local_props = [ + 'persist.sys.usb.config=adb', + 'ro.monkey=1', + 'ro.test_harness=1', + 'ro.audio.silent=1', + 'ro.setupwizard.mode=DISABLED', + ] + if java_debug: + local_props.append( + '%s=all' % device_utils.DeviceUtils.JAVA_ASSERT_PROPERTY) + local_props.append('debug.checkjni=1') + try: + device.WriteFile( + device.LOCAL_PROPERTIES_PATH, + '\n'.join(local_props), as_root=True) + # Android will not respect the local props file if it is world writable. + device.RunShellCommand( + ['chmod', '644', device.LOCAL_PROPERTIES_PATH], + as_root=True, check_return=True) + except device_errors.CommandFailedError: + logging.exception('Failed to configure local properties.') + + +def FinishProvisioning(device): + # The lockscreen can't be disabled on user builds, so send a keyevent + # to unlock it. + if device.IsUserBuild(): + device.SendKeyEvent(keyevent.KEYCODE_MENU) + + +def WaitForCharge(device, min_battery_level): + battery = battery_utils.BatteryUtils(device) + try: + battery.ChargeDeviceToLevel(min_battery_level) + except device_errors.DeviceChargingError: + device.Reboot() + battery.ChargeDeviceToLevel(min_battery_level) + + +def WaitForTemperature(device, max_battery_temp): + try: + battery = battery_utils.BatteryUtils(device) + battery.LetBatteryCoolToTemperature(max_battery_temp) + except device_errors.CommandFailedError: + logging.exception('Unable to let battery cool to specified temperature.') + + +def SetDate(device): + def _set_and_verify_date(): + if device.build_version_sdk >= version_codes.MARSHMALLOW: + date_format = '%m%d%H%M%Y.%S' + set_date_command = ['date', '-u'] + get_date_command = ['date', '-u'] + else: + date_format = '%Y%m%d.%H%M%S' + set_date_command = ['date', '-s'] + get_date_command = ['date'] + + # TODO(jbudorick): This is wrong on pre-M devices -- get/set are + # dealing in local time, but we're setting based on GMT. + strgmtime = time.strftime(date_format, time.gmtime()) + set_date_command.append(strgmtime) + device.RunShellCommand(set_date_command, as_root=True, check_return=True) + + get_date_command.append('+"%Y%m%d.%H%M%S"') + device_time = device.RunShellCommand( + get_date_command, as_root=True, single_line=True).replace('"', '') + device_time = datetime.datetime.strptime(device_time, "%Y%m%d.%H%M%S") + correct_time = datetime.datetime.strptime(strgmtime, date_format) + tdelta = (correct_time - device_time).seconds + if tdelta <= 1: + logging.info('Date/time successfully set on %s', device) + return True + else: + logging.error('Date mismatch. Device: %s Correct: %s', + device_time.isoformat(), correct_time.isoformat()) + return False + + # Sometimes the date is not set correctly on the devices. Retry on failure. + if device.IsUserBuild(): + # TODO(bpastene): Figure out how to set the date & time on user builds. + pass + else: + if not timeout_retry.WaitFor( + _set_and_verify_date, wait_period=1, max_tries=2): + raise device_errors.CommandFailedError( + 'Failed to set date & time.', device_serial=str(device)) + + +def LogDeviceProperties(device): + props = device.RunShellCommand('getprop', check_return=True) + for prop in props: + logging.info(' %s', prop) + + +def CheckExternalStorage(device): + """Checks that storage is writable and if not makes it writable. + + Arguments: + device: The device to check. + """ + try: + with device_temp_file.DeviceTempFile( + device.adb, suffix='.sh', dir=device.GetExternalStoragePath()) as f: + device.WriteFile(f.name, 'test') + except device_errors.CommandFailedError: + logging.info('External storage not writable. Remounting / as RW') + device.RunShellCommand(['mount', '-o', 'remount,rw', '/'], + check_return=True, as_root=True) + device.EnableRoot() + with device_temp_file.DeviceTempFile( + device.adb, suffix='.sh', dir=device.GetExternalStoragePath()) as f: + device.WriteFile(f.name, 'test') + + +def main(raw_args): + # Recommended options on perf bots: + # --disable-network + # TODO(tonyg): We eventually want network on. However, currently radios + # can cause perfbots to drain faster than they charge. + # --min-battery-level 95 + # Some perf bots run benchmarks with USB charging disabled which leads + # to gradual draining of the battery. We must wait for a full charge + # before starting a run in order to keep the devices online. + + parser = argparse.ArgumentParser( + description='Provision Android devices with settings required for bots.') + parser.add_argument( + '-d', '--device', metavar='SERIAL', action='append', dest='devices', + help='the serial number of the device to be provisioned ' + '(the default is to provision all devices attached)') + parser.add_argument( + '--adb-path', + help='Absolute path to the adb binary to use.') + parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') + parser.add_argument( + '--skip-wipe', action='store_true', default=False, + help="don't wipe device data during provisioning") + parser.add_argument( + '--reboot-timeout', metavar='SECS', type=int, + help='when wiping the device, max number of seconds to' + ' wait after each reboot ' + '(default: %s)' % _DEFAULT_TIMEOUTS.HELP_TEXT) + parser.add_argument( + '--min-battery-level', type=int, metavar='NUM', + help='wait for the device to reach this minimum battery' + ' level before trying to continue') + parser.add_argument( + '--disable-location', action='store_true', + help='disable Google location services on devices') + parser.add_argument( + '--disable-mock-location', action='store_true', default=False, + help='Set ALLOW_MOCK_LOCATION to false') + parser.add_argument( + '--disable-network', action='store_true', + help='disable network access on devices') + parser.add_argument( + '--disable-java-debug', action='store_false', + dest='enable_java_debug', default=True, + help='disable Java property asserts and JNI checking') + parser.add_argument( + '--disable-system-chrome', action='store_true', + help='Disable the system chrome from devices.') + parser.add_argument( + '--remove-system-webview', action='store_true', + help='Remove the system webview from devices.') + parser.add_argument( + '--adb-key-files', type=str, nargs='+', + help='list of adb keys to push to device') + parser.add_argument( + '-v', '--verbose', action='count', default=1, + help='Log more information.') + parser.add_argument( + '--max-battery-temp', type=int, metavar='NUM', + help='Wait for the battery to have this temp or lower.') + parser.add_argument( + '--output-device-blacklist', + help='Json file to output the device blacklist.') + parser.add_argument( + '--emulators', action='store_true', + help='provision only emulators and ignore usb devices') + args = parser.parse_args(raw_args) + + run_tests_helper.SetLogLevel(args.verbose) + + devil_dynamic_config = devil_env.EmptyConfig() + if args.adb_path: + devil_dynamic_config['dependencies'].update( + devil_env.LocalConfigItem( + 'adb', devil_env.GetPlatform(), args.adb_path)) + + devil_env.config.Initialize(configs=[devil_dynamic_config]) + + try: + return ProvisionDevices( + args.devices, + args.blacklist_file, + adb_key_files=args.adb_key_files, + disable_location=args.disable_location, + disable_mock_location=args.disable_mock_location, + disable_network=args.disable_network, + disable_system_chrome=args.disable_system_chrome, + emulators=args.emulators, + enable_java_debug=args.enable_java_debug, + max_battery_temp=args.max_battery_temp, + min_battery_level=args.min_battery_level, + output_device_blacklist=args.output_device_blacklist, + reboot_timeout=args.reboot_timeout, + remove_system_webview=args.remove_system_webview, + wipe=not args.skip_wipe) + except (device_errors.DeviceUnreachableError, device_errors.NoDevicesError): + return exit_codes.INFRA + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/catapult/devil/devil/android/tools/script_common.py b/catapult/devil/devil/android/tools/script_common.py index eb91cdcb..8e462c35 100644 --- a/catapult/devil/devil/android/tools/script_common.py +++ b/catapult/devil/devil/android/tools/script_common.py @@ -8,9 +8,10 @@ from devil.android import device_utils def GetDevices(requested_devices, blacklist_file): - blacklist = (device_blacklist.Blacklist(blacklist_file) - if blacklist_file - else None) + if not isinstance(blacklist_file, device_blacklist.Blacklist): + blacklist = (device_blacklist.Blacklist(blacklist_file) + if blacklist_file + else None) devices = device_utils.DeviceUtils.HealthyDevices(blacklist) if not devices: diff --git a/catapult/devil/devil/devil_env.py b/catapult/devil/devil/devil_env.py index b54e6f5b..6b647666 100644 --- a/catapult/devil/devil/devil_env.py +++ b/catapult/devil/devil/devil_env.py @@ -47,24 +47,39 @@ _LEGACY_ENVIRONMENT_VARIABLES = { } -def _GetEnvironmentVariableConfig(): - path_config = ( - (os.environ.get(k), v) - for k, v in _LEGACY_ENVIRONMENT_VARIABLES.iteritems()) +def EmptyConfig(): return { 'config_type': 'BaseConfig', - 'dependencies': { - c['dependency_name']: { - 'file_info': { - c['platform']: { - 'local_paths': [p], - }, + 'dependencies': {} + } + + +def LocalConfigItem(dependency_name, dependency_platform, dependency_path): + if isinstance(dependency_path, basestring): + dependency_path = [dependency_path] + return { + dependency_name: { + 'file_info': { + dependency_platform: { + 'local_paths': dependency_path }, - } for p, c in path_config if p + }, }, } +def _GetEnvironmentVariableConfig(): + env_config = EmptyConfig() + path_config = ( + (os.environ.get(k), v) + for k, v in _LEGACY_ENVIRONMENT_VARIABLES.iteritems()) + path_config = ((p, c) for p, c in path_config if p) + for p, c in path_config: + env_config['dependencies'].update( + LocalConfigItem(c['dependency_name'], c['platform'], p)) + return env_config + + class _Environment(object): def __init__(self): @@ -76,7 +91,7 @@ class _Environment(object): This uses all configurations provided via |configs| and |config_files| to determine the locations of devil's dependencies. Configurations should - all take the form described by catapult_base.dependency_manager.BaseConfig. + all take the form described by py_utils.dependency_manager.BaseConfig. If no configurations are provided, a default one will be used if available. Args: diff --git a/catapult/devil/devil/utils/markdown.py b/catapult/devil/devil/utils/markdown.py new file mode 100755 index 00000000..cb2dc2bd --- /dev/null +++ b/catapult/devil/devil/utils/markdown.py @@ -0,0 +1,74 @@ +# Copyright 2016 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse + +class MarkdownHelpFormatter(argparse.HelpFormatter): + """A really bare-bones argparse help formatter that generates valid markdown. + + This will generate something like: + + usage + + # **section heading**: + + ## **--argument-one** + + ``` + argument-one help text + ``` + + """ + + #override + def _format_usage(self, usage, actions, groups, prefix): + usage_text = super(MarkdownHelpFormatter, self)._format_usage( + usage, actions, groups, prefix) + return '\n```\n%s\n```\n\n' % usage_text + + #override + def format_help(self): + self._root_section.heading = '# %s' % self._prog + return super(MarkdownHelpFormatter, self).format_help() + + #override + def start_section(self, heading): + super(MarkdownHelpFormatter, self).start_section('## **%s**' % heading) + + #override + def _format_action(self, action): + lines = [] + action_header = self._format_action_invocation(action) + lines.append('### **%s** ' % action_header) + if action.help: + lines.append('') + lines.append('```') + help_text = self._expand_help(action) + lines.extend(self._split_lines(help_text, 80)) + lines.append('```') + lines.extend(['', '']) + return '\n'.join(lines) + + +class MarkdownHelpAction(argparse.Action): + def __init__(self, option_strings, + dest=argparse.SUPPRESS, default=argparse.SUPPRESS, + **kwargs): + super(MarkdownHelpAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + parser.formatter_class = MarkdownHelpFormatter + parser.print_help() + parser.exit() + + +def add_md_help_argument(parser): + parser.add_argument('--md-help', action=MarkdownHelpAction, + help='print Markdown-formatted help text and exit.') + diff --git a/catapult/devil/devil/utils/timeout_retry_unittest.py b/catapult/devil/devil/utils/timeout_retry_unittest.py index 84982889..0eeb31a4 100755 --- a/catapult/devil/devil/utils/timeout_retry_unittest.py +++ b/catapult/devil/devil/utils/timeout_retry_unittest.py @@ -40,7 +40,7 @@ class TestRun(unittest.TestCase): time.sleep(1) self.assertRaises( - reraiser_thread.TimeoutError, timeout_retry.Run, _sleep, .0001, 1, + reraiser_thread.TimeoutError, timeout_retry.Run, _sleep, .01, 1, error_log_func=logging.debug) self.assertEqual(tries[0], 2) -- cgit v1.2.3