#!/usr/bin/env python # # Copyright (C) 2016 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """Runs the test suite over a set of devices.""" from __future__ import print_function import argparse import distutils.spawn import os import re import shutil import signal import site import subprocess import sys import ndk.debug import ndk.notify import ndk.paths THIS_DIR = os.path.realpath(os.path.dirname(__file__)) class Device(object): def __init__(self, serial, name, version, build_id, abis, is_emulator): self.serial = serial self.name = name self.version = version self.build_id = build_id self.abis = abis self.is_emulator = is_emulator def __str__(self): return 'android-{} {} {} {}'.format( self.version, self.name, self.serial, self.build_id) class DeviceFleet(object): def __init__(self): self.devices = { # Earliest supported target. 15: { 'armeabi': None, 'armeabi-v7a': None, }, # Earliest supported target with any real number of users. 16: { 'armeabi': None, 'armeabi-v7a': None, 'x86': None, }, # Latest target. 25: { 'armeabi': None, 'armeabi-v7a': None, 'arm64-v8a': None, 'x86': None, 'x86_64': None, }, } def add_device(self, device): if device.version not in self.devices: print('Ignoring device for unwanted API level: {}'.format(device)) return same_version = self.devices[device.version] for abi, current_device in same_version.iteritems(): # This device can't fulfill this ABI. if abi not in device.abis: continue # Never houdini. if abi.startswith('armeabi') and 'x86' in device.abis: continue # Anything is better than nothing. if current_device is None: self.devices[device.version][abi] = device continue # The emulator images have actually been changed over time, so the # devices are more trustworthy. if current_device.is_emulator and not device.is_emulator: self.devices[device.version][abi] = device def get_device(self, version, abi): return self.devices[version][abi] def get_missing(self): missing = [] for version, abis in self.devices.iteritems(): for abi, device in abis.iteritems(): if device is None: missing.append('android-{} {}'.format(version, abi)) return missing def get_versions(self): return self.devices.keys() def get_abis(self, version): return self.devices[version].keys() def get_device_abis(properties): # 64-bit devices list their ABIs differently than 32-bit devices. Check all # the possible places for stashing ABI info and merge them. abi_properties = [ 'ro.product.cpu.abi', 'ro.product.cpu.abi2', 'ro.product.cpu.abilist', ] abis = set() for abi_prop in abi_properties: if abi_prop in properties: abis.update(properties[abi_prop].split(',')) return sorted(list(abis)) def get_device_details(serial): import adb # pylint: disable=import-error props = adb.get_device(serial).get_props() name = props['ro.product.name'] version = int(props['ro.build.version.sdk']) supported_abis = get_device_abis(props) build_id = props['ro.build.id'] is_emulator = props.get('ro.build.characteristics') == 'emulator' return Device(serial, name, version, build_id, supported_abis, is_emulator) def find_devices(): """Detects connected devices and returns a set for testing. We get a list of devices by scanning the output of `adb devices`. We want to run the tests for the cross product of the following configurations: ABIs: {armeabi, armeabi-v7a, arm64-v8a, mips, mips64, x86, x86_64} Platform versions: {android-10, android-16, android-21} Toolchains: {clang, gcc} Note that not all ABIs are available for every platform version. There are no 64-bit ABIs before android-21, and there were no MIPS ABIs for android-10. """ if distutils.spawn.find_executable('adb') is None: raise RuntimeError('Could not find adb.') # We could get the device name from `adb devices -l`, but we need to # getprop to find other details anyway, and older devices don't report # their names properly (nakasi on android-16, for example). p = subprocess.Popen(['adb', 'devices'], stdout=subprocess.PIPE) out, _ = p.communicate() if p.returncode != 0: raise RuntimeError('Failed to get list of devices from adb.') # The first line of `adb devices` just says "List of attached devices", so # skip that. fleet = DeviceFleet() for line in out.split('\n')[1:]: if not line.strip(): continue serial, _ = re.split(r'\s+', line, maxsplit=1) if 'offline' in line: print('Ignoring offline device: ' + serial) continue if 'unauthorized' in line: print('Ignoring unauthorized device: ' + serial) continue device = get_device_details(serial) print('Found device {}'.format(device)) fleet.add_device(device) return fleet def parse_args(): parser = argparse.ArgumentParser() parser.add_argument( 'ndk', metavar='NDK', type=os.path.realpath, nargs='?', help='NDK to validate. Defaults to ../out/android-ndk-$RELEASE.') parser.add_argument( '--filter', help='Only run tests that match the given pattern.') parser.add_argument( '--log-dir', type=os.path.realpath, default='test-logs', help='Directory to store test logs.') return parser.parse_args() def get_aggregate_results(details): tests = {} for config, results in details.iteritems(): for suite, test_results in results.iteritems(): for test in test_results: if test.failed(): name = '.'.join([suite, test.test_name]) if name not in tests: tests[name] = [] tests[name].append((config, test)) return tests def print_aggregate_details(details, use_color): tests = get_aggregate_results(details) for test_name, configs in tests.iteritems(): # We might be printing a lot of crap here, so let's be obvious about # where each test starts. print('BEGIN TEST RESULT: ' + test_name) print('=' * 80) for config, result in configs: print('FAILED {}:'.format(config)) print(result.to_string(colored=use_color)) def main(): if sys.platform != 'win32': ndk.debug.register_debug_handler(signal.SIGUSR1) ndk.debug.register_trace_handler(signal.SIGUSR2) args = parse_args() os.chdir(THIS_DIR) if args.ndk is None: args.ndk = ndk.paths.get_install_path() # We need to do this here rather than at the top because we load the module # from a path that is given on the command line. We load it from the NDK # given on the command line so this script can be run even without a full # platform checkout. site.addsitedir(os.path.join(args.ndk, 'python-packages')) ndk_build_path = os.path.join(args.ndk, 'ndk-build') if os.name == 'nt': ndk_build_path += '.cmd' if not os.path.exists(ndk_build_path): sys.exit(ndk_build_path + ' does not exist.') fleet = find_devices() print('Test configuration:') for version in sorted(fleet.get_versions()): print('\tandroid-{}:'.format(version)) for abi in sorted(fleet.get_abis(version)): print('\t\t{}: {}'.format(abi, fleet.get_device(version, abi))) missing_configs = fleet.get_missing() if len(missing_configs): print('Missing configurations: {}'.format(', '.join(missing_configs))) if os.path.exists(args.log_dir): shutil.rmtree(args.log_dir) os.makedirs(args.log_dir) use_color = sys.stdin.isatty() and os.name != 'nt' with ndk.paths.temp_dir_in_out('validate-out') as out_dir: import tests.runners good, details = tests.runners.run_for_fleet( args.ndk, fleet, out_dir, args.log_dir, args.filter, use_color) print_aggregate_details(details, use_color) subject = 'NDK Testing {}!'.format('Passed' if good else 'Failed') ndk.notify.toast(subject) sys.exit(not good) if __name__ == '__main__': main()