From 7c091cb35feb4df35a6ef5674274d4fb5614d5cd Mon Sep 17 00:00:00 2001 From: Roman Lavrov Date: Tue, 19 Apr 2022 21:12:35 -0400 Subject: Use adb directly (instead of catapult) in gold tests. Android detected by angle_perftests apk presence relative to pwd. All of the restricted_traces/*/*.json files are copied but those are small. Only the necessary .angledata.gz files are copied. angle_system_info_test is also handled via adb. Bug: angleproject:6854 Change-Id: I7a89ff57fcdd8ce5dc63a5e3a8f5c0132f766894 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/3595241 Reviewed-by: Jamie Madill Commit-Queue: Roman Lavrov --- src/tests/py_utils/android_helper.py | 210 +++++++++++++++++++++ src/tests/restricted_traces/BUILD.gn | 1 + .../restricted_trace_gold_tests.py | 76 ++++++-- 3 files changed, 270 insertions(+), 17 deletions(-) create mode 100644 src/tests/py_utils/android_helper.py diff --git a/src/tests/py_utils/android_helper.py b/src/tests/py_utils/android_helper.py new file mode 100644 index 0000000000..e3648c3a14 --- /dev/null +++ b/src/tests/py_utils/android_helper.py @@ -0,0 +1,210 @@ +# Copyright 2022 The ANGLE Project Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import contextlib +import glob +import json +import logging +import os +import posixpath +import random +import subprocess +import tarfile +import tempfile +import time + +import angle_path_util + + +def _ApkPath(suite_name): + return os.path.join('%s_apk' % suite_name, '%s-debug.apk' % suite_name) + + +def ApkFileExists(suite_name): + return os.path.exists(_ApkPath(suite_name)) + + +def _Run(cmd): + logging.debug('Executing command: %s', cmd) + startupinfo = None + if hasattr(subprocess, 'STARTUPINFO'): + # Prevent console window popping up on Windows + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + startupinfo.wShowWindow = subprocess.SW_HIDE + output = subprocess.check_output(cmd, startupinfo=startupinfo) + return output + + +class Adb(object): + + def __init__(self, adb_path=None): + if not adb_path: + adb_path = os.path.join(angle_path_util.ANGLE_ROOT_DIR, + 'third_party/android_sdk/public/platform-tools/adb') + self._adb_path = adb_path + + def Run(self, args): + return _Run([self._adb_path] + args) + + def Shell(self, cmd): + return _Run([self._adb_path, 'shell', cmd]) + + +def _GetAdbRoot(adb): + adb.Run(['root']) + + for _ in range(20): + time.sleep(0.5) + try: + id_out = adb.Shell(adb_path, 'id').decode('ascii') + if 'uid=0(root)' in id_out: + return + except Exception: + continue + raise Exception("adb root failed") + + +def _ReadDeviceFile(adb, device_path): + out_wc = adb.Shell('cat %s | wc -c' % device_path) + expected_size = int(out_wc.decode('ascii').strip()) + out = adb.Run(['exec-out', 'cat %s' % device_path]) + assert len(out) == expected_size, 'exec-out mismatch' + return out + + +def _RemoveDeviceFile(adb, device_path): + adb.Shell('rm -f ' + device_path + ' || true') # ignore errors + + +def _AddRestrictedTracesJson(adb): + adb.Shell('mkdir -p /sdcard/chromium_tests_root/') + + def add(tar, fn): + assert (fn.startswith('../../')) + tar.add(fn, arcname=fn.replace('../../', '')) + + with _TempLocalFile() as tempfile_path: + with tarfile.open(tempfile_path, 'w', format=tarfile.GNU_FORMAT) as tar: + for f in glob.glob('../../src/tests/restricted_traces/*/*.json', recursive=True): + add(tar, f) + add(tar, '../../src/tests/restricted_traces/restricted_traces.json') + adb.Run(['push', tempfile_path, '/sdcard/chromium_tests_root/t.tar']) + + adb.Shell('r=/sdcard/chromium_tests_root; tar -xf $r/t.tar -C $r/ && rm $r/t.tar') + + +def PrepareTestSuite(adb, suite_name): + apk_path = _ApkPath(suite_name) + logging.info('Installing apk path=%s size=%s' % (apk_path, os.path.getsize(apk_path))) + + adb.Run(['install', '-r', '-d', apk_path]) + + permissions = [ + 'android.permission.CAMERA', 'android.permission.CHANGE_CONFIGURATION', + 'android.permission.READ_EXTERNAL_STORAGE', 'android.permission.RECORD_AUDIO', + 'android.permission.WRITE_EXTERNAL_STORAGE' + ] + adb.Shell('p=com.android.angle.test;' + 'for q in %s;do pm grant "$p" "$q";done;' % ' '.join(permissions)) + + if suite_name == 'angle_perftests': + _AddRestrictedTracesJson(adb) + + +def PrepareRestrictedTraces(adb, traces): + start = time.time() + total_size = 0 + for trace in traces: + path_from_root = 'src/tests/restricted_traces/' + trace + '/' + trace + '.angledata.gz' + local_path = '../../' + path_from_root + total_size += os.path.getsize(local_path) + adb.Run(['push', local_path, '/sdcard/chromium_tests_root/' + path_from_root]) + + logging.info('Pushed %d trace files (%.1fMB) in %.1fs', len(traces), total_size / 1e6, + time.time() - start) + + +def _RandomHex(): + return hex(random.randint(0, 2**64))[2:] + + +@contextlib.contextmanager +def _TempDeviceDir(adb): + path = '/sdcard/Download/temp_dir-%s' % _RandomHex() + adb.Shell('mkdir -p ' + path) + try: + yield path + finally: + adb.Shell('rm -rf ' + path) + + +@contextlib.contextmanager +def _TempDeviceFile(adb): + path = '/sdcard/Download/temp_file-%s' % _RandomHex() + try: + yield path + finally: + adb.Shell('rm -f ' + path) + + +@contextlib.contextmanager +def _TempLocalFile(): + fd, path = tempfile.mkstemp() + os.close(fd) + try: + yield path + finally: + os.remove(path) + + +def _RunInstrumentation(adb, flags): + with _TempDeviceFile(adb) as temp_device_file: + cmd = ' '.join([ + 'p=com.android.angle.test;', + 'ntr=org.chromium.native_test.NativeTestInstrumentationTestRunner;', + 'am instrument -w', + '-e $ntr.NativeTestActivity "$p".AngleUnitTestActivity', + '-e $ntr.ShardNanoTimeout 2400000000000', + '-e org.chromium.native_test.NativeTest.CommandLineFlags "%s"' % ' '.join(flags), + '-e $ntr.StdoutFile ' + temp_device_file, + '"$p"/org.chromium.build.gtest_apk.NativeTestInstrumentationTestRunner', + ]) + + adb.Shell(cmd) + return _ReadDeviceFile(adb, temp_device_file) + + +def AngleSystemInfo(adb, args): + PrepareTestSuite(adb, 'angle_system_info_test') + + with _TempDeviceDir(adb) as temp_dir: + _RunInstrumentation(adb, args + ['--render-test-output-dir=' + temp_dir]) + output_file = posixpath.join(temp_dir, 'angle_system_info.json') + return json.loads(_ReadDeviceFile(adb, output_file)) + + +def ListTests(adb): + out_lines = _RunInstrumentation(adb, ["--list-tests"]).decode('ascii').split('\n') + + start = out_lines.index('Tests list:') + end = out_lines.index('End tests list.') + return out_lines[start + 1:end] + + +def _PullDir(adb, device_dir, local_dir): + files = adb.Shell('ls -1 %s' % device_dir).decode('ascii').split('\n') + for f in files: + f = f.strip() + if f: + adb.Run(['pull', posixpath.join(device_dir, f), posixpath.join(local_dir, f)]) + + +def RunTests(adb, test_suite, args, stdoutfile, output_dir): + with _TempDeviceDir(adb) as temp_dir: + output = _RunInstrumentation(adb, args + ['--render-test-output-dir=' + temp_dir]) + with open(stdoutfile, 'wb') as f: + f.write(output) + logging.info(output.decode()) + _PullDir(adb, temp_dir, output_dir) diff --git a/src/tests/restricted_traces/BUILD.gn b/src/tests/restricted_traces/BUILD.gn index 1108deb607..320addff0f 100644 --- a/src/tests/restricted_traces/BUILD.gn +++ b/src/tests/restricted_traces/BUILD.gn @@ -78,6 +78,7 @@ group("angle_restricted_trace_gold_tests") { data = [ "restricted_trace_gold_tests.py", "restricted_traces.json", + "../py_utils/android_helper.py", "../py_utils/angle_path_util.py", "../py_utils/skia_gold/", "//build/skia_gold_common/", diff --git a/src/tests/restricted_traces/restricted_trace_gold_tests.py b/src/tests/restricted_traces/restricted_trace_gold_tests.py index c19c7d0b4f..2708f5e339 100755 --- a/src/tests/restricted_traces/restricted_trace_gold_tests.py +++ b/src/tests/restricted_traces/restricted_trace_gold_tests.py @@ -30,6 +30,7 @@ def _AddToPathIfNeeded(path): _AddToPathIfNeeded(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'py_utils'))) +import android_helper import angle_path_util from skia_gold import angle_skia_gold_properties from skia_gold import angle_skia_gold_session_manager @@ -120,11 +121,43 @@ def add_skia_gold_args(parser): 'pre-authenticated. Meant for testing locally instead of on the bots.') -def run_wrapper(args, cmd, env, stdoutfile=None): +def _adb_if_android(args): + if android_helper.ApkFileExists(args.test_suite): + return android_helper.Adb() + + return None + + +def run_wrapper(test_suite, cmd_args, args, env, stdoutfile, output_dir=None): + cmd = [get_binary_name(test_suite)] + cmd_args + if output_dir: + cmd += ['--render-test-output-dir=%s' % output_dir] + if args.xvfb: return xvfb.run_executable(cmd, env, stdoutfile=stdoutfile) else: - return test_env.run_command_with_output(cmd, env=env, stdoutfile=stdoutfile) + adb = _adb_if_android(args) + if adb: + try: + android_helper.RunTests(adb, test_suite, cmd_args, stdoutfile, output_dir) + return 0 + except Exception as e: + logging.exception(e) + return 1 + else: + return test_env.run_command_with_output(cmd, env=env, stdoutfile=stdoutfile) + + +def run_angle_system_info_test(sysinfo_args, args, env): + with temporary_dir() as temp_dir: + tempfile_path = os.path.join(temp_dir, 'stdout') + sysinfo_args += ['--render-test-output-dir=' + temp_dir] + + if run_wrapper('angle_system_info_test', sysinfo_args, args, env, tempfile_path): + raise Exception('Error getting system info.') + + with open(os.path.join(temp_dir, 'angle_system_info.json')) as f: + return json.load(f) def to_hex(num): @@ -159,17 +192,16 @@ def get_skia_gold_keys(args, env): logging.exception('get_skia_gold_keys may only be called once') get_skia_gold_keys.called = True - with temporary_dir() as temp_dir: - binary = get_binary_name('angle_system_info_test') - sysinfo_args = [binary, '--vulkan', '-v', '--render-test-output-dir=' + temp_dir] - if args.swiftshader: - sysinfo_args.append('--swiftshader') - tempfile_path = os.path.join(temp_dir, 'stdout') - if run_wrapper(args, sysinfo_args, env, tempfile_path): - raise Exception('Error getting system info.') + sysinfo_args = ['--vulkan', '-v'] + if args.swiftshader: + sysinfo_args.append('--swiftshader') - with open(os.path.join(temp_dir, 'angle_system_info.json')) as f: - json_data = json.load(f) + adb = _adb_if_android(args) + if adb: + json_data = android_helper.AngleSystemInfo(adb, sysinfo_args) + logging.info(json_data) + else: + json_data = run_angle_system_info_test(sysinfo_args, args, env) if len(json_data.get('gpus', [])) == 0 or not 'activeGPUIndex' in json_data: raise Exception('Error getting system info.') @@ -291,6 +323,10 @@ def _get_gtest_filter_for_batch(args, batch): def _run_tests(args, tests, extra_flags, env, screenshot_dir, results, test_results): keys = get_skia_gold_keys(args, env) + adb = _adb_if_android(args) + if adb: + android_helper.PrepareTestSuite(adb, args.test_suite) + with temporary_dir('angle_skia_gold_') as skia_gold_temp_dir: gold_properties = angle_skia_gold_properties.ANGLESkiaGoldProperties(args) gold_session_manager = angle_skia_gold_session_manager.ANGLESkiaGoldSessionManager( @@ -314,6 +350,9 @@ def _run_tests(args, tests, extra_flags, env, screenshot_dir, results, test_resu batches = _get_batches(traces, args.batch_size) for batch in batches: + if adb: + android_helper.PrepareRestrictedTraces(adb, batch) + for iteration in range(0, args.flaky_retries + 1): with common.temporary_file() as tempfile_path: # This is how we signal early exit @@ -324,16 +363,19 @@ def _run_tests(args, tests, extra_flags, env, screenshot_dir, results, test_resu logging.info('Test run failed, running retry #%d...' % iteration) gtest_filter = _get_gtest_filter_for_batch(args, batch) - cmd = [ - get_binary_name(args.test_suite), + cmd_args = [ gtest_filter, - '--render-test-output-dir=%s' % screenshot_dir, '--one-frame-only', '--verbose-logging', '--enable-all-trace-tests', ] + extra_flags - batch_result = PASS if run_wrapper(args, cmd, env, - tempfile_path) == 0 else FAIL + batch_result = PASS if run_wrapper( + args.test_suite, + cmd_args, + args, + env, + tempfile_path, + output_dir=screenshot_dir) == 0 else FAIL with open(tempfile_path) as f: test_output = f.read() + '\n' -- cgit v1.2.3