diff options
Diffstat (limited to 'public/actions')
| -rw-r--r-- | public/actions/base_device_factory.py | 36 | ||||
| -rw-r--r-- | public/actions/common_operations.py | 142 | ||||
| -rw-r--r-- | public/actions/common_operations_test.py | 93 | ||||
| -rw-r--r-- | public/actions/create_cuttlefish_action.py | 299 | ||||
| -rw-r--r-- | public/actions/create_cuttlefish_action_test.py | 187 | ||||
| -rw-r--r-- | public/actions/create_goldfish_action.py | 21 | ||||
| -rw-r--r-- | public/actions/create_goldfish_action_test.py | 7 | ||||
| -rw-r--r-- | public/actions/gce_device_factory.py | 7 | ||||
| -rw-r--r-- | public/actions/remote_host_cf_device_factory.py | 244 | ||||
| -rw-r--r-- | public/actions/remote_host_cf_device_factory_test.py | 189 | ||||
| -rw-r--r-- | public/actions/remote_host_gf_device_factory.py | 506 | ||||
| -rw-r--r-- | public/actions/remote_host_gf_device_factory_test.py | 188 | ||||
| -rw-r--r-- | public/actions/remote_instance_cf_device_factory.py | 185 | ||||
| -rw-r--r-- | public/actions/remote_instance_cf_device_factory_test.py | 144 | ||||
| -rw-r--r-- | public/actions/remote_instance_fvp_device_factory.py | 2 | ||||
| -rw-r--r-- | public/actions/remote_instance_fvp_device_factory_test.py | 3 |
16 files changed, 1346 insertions, 907 deletions
diff --git a/public/actions/base_device_factory.py b/public/actions/base_device_factory.py index 847f19c3..8ac52720 100644 --- a/public/actions/base_device_factory.py +++ b/public/actions/base_device_factory.py @@ -50,6 +50,42 @@ class BaseDeviceFactory(): return # pylint: disable=no-self-use + def GetAdbPorts(self): + """Get ADB ports of the created devices. + + Subclasses should define this function if their ADB ports are not + constant. + + Returns: + The port numbers as a list of integers. + """ + return [None] + + # pylint: disable=no-self-use + def GetFastbootPorts(self): + """Get Fastboot ports of the created devices. + + Subclasses should define this function if their ADB ports are not + constant. + + Returns: + The port numbers as a list of integers. + """ + return [None] + + # pylint: disable=no-self-use + def GetVncPorts(self): + """Get VNC ports of the created devices. + + Subclasses should define this function if they support VNC and their + VNC ports are not constant. + + Returns: + The port numbers as a list of integers. + """ + return [None] + + # pylint: disable=no-self-use def GetBuildInfoDict(self): """Get build info dictionary. diff --git a/public/actions/common_operations.py b/public/actions/common_operations.py index 2b64ec54..f7823121 100644 --- a/public/actions/common_operations.py +++ b/public/actions/common_operations.py @@ -13,12 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Common operations between managing GCE and Cuttlefish devices. - -This module provides the common operations between managing GCE (device_driver) -and Cuttlefish (create_cuttlefish_action) devices. Should not be called -directly. -""" +"""Common operations to create remote devices.""" import logging import os @@ -109,16 +104,19 @@ class DevicePool: for _ in range(num): instance = self._device_factory.CreateInstance() ip = self._compute_client.GetInstanceIP(instance) - time_info = self._compute_client.execution_time if hasattr( - self._compute_client, "execution_time") else {} + time_info = { + stage: round(exec_time, 2) for stage, exec_time in + getattr(self._compute_client, "execution_time", {}).items()} stage = self._compute_client.stage if hasattr( self._compute_client, "stage") else 0 openwrt = self._compute_client.openwrt if hasattr( self._compute_client, "openwrt") else False + gce_hostname = self._compute_client.gce_hostname if hasattr( + self._compute_client, "gce_hostname") else None self.devices.append( avd.AndroidVirtualDevice(ip=ip, instance_name=instance, time_info=time_info, stage=stage, - openwrt=openwrt)) + openwrt=openwrt, gce_hostname=gce_hostname)) @utils.TimeExecute(function_description="Waiting for AVD(s) to boot up", result_evaluator=utils.BootEvaluator) @@ -208,24 +206,10 @@ def _GetErrorType(error): return constants.GCE_QUOTA_ERROR return constants.ACLOUD_UNKNOWN_ERROR -def _GetAdbPort(avd_type, base_instance_num): - """Get Adb port according to avd_type and device offset. - - Args: - avd_type: String, the AVD type(cuttlefish, goldfish...). - base_instance_num: int, device offset. - - Returns: - int, adb port. - """ - if avd_type in utils.AVD_PORT_DICT: - return utils.AVD_PORT_DICT[avd_type].adb_port + base_instance_num - 1 - return None - -# pylint: disable=too-many-locals,unused-argument,too-many-branches +# pylint: disable=too-many-locals,unused-argument,too-many-branches,too-many-statements def CreateDevices(command, cfg, device_factory, num, avd_type, - report_internal_ip=False, autoconnect=False, - serial_log_file=None, client_adb_port=None, + report_internal_ip=False, autoconnect=False, serial_log_file=None, + client_adb_port=None, client_fastboot_port=None, boot_timeout_secs=None, unlock_screen=False, wait_for_boot=True, connect_webrtc=False, ssh_private_key_path=None, @@ -247,6 +231,7 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, serial_log_file: String, the file path to tar the serial logs. autoconnect: Boolean, whether to auto connect to device. client_adb_port: Integer, Specify port for adb forwarding. + client_fastboot_port: Integer, Specify port for fastboot forwarding. boot_timeout_secs: Integer, boot timeout secs. unlock_screen: Boolean, whether to unlock screen after invoke vnc client. wait_for_boot: Boolean, True to check serial log include boot up @@ -288,15 +273,16 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, for device in device_pool.devices: ip = (device.ip.internal if report_internal_ip else device.ip.external) - base_instance_num = 1 - if constants.BASE_INSTANCE_NUM in device_pool._compute_client.dict_report: - base_instance_num = device_pool._compute_client.dict_report[constants.BASE_INSTANCE_NUM] - adb_port = _GetAdbPort( - avd_type, - base_instance_num - ) + extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel + # TODO(b/154175542): Report multiple devices. + vnc_ports = device_factory.GetVncPorts() + adb_ports = device_factory.GetAdbPorts() + fastboot_ports = device_factory.GetFastbootPorts() + if not vnc_ports[0] and not adb_ports[0] and not fastboot_ports[0]: + vnc_ports[0], adb_ports[0], fastboot_ports[0] = utils.AVD_PORT_DICT[avd_type] + device_dict = { - "ip": ip + (":" + str(adb_port) if adb_port else ""), + "ip": ip + (":" + str(adb_ports[0]) if adb_ports[0] else ""), "instance_name": device.instance_name } if device.build_info: @@ -305,35 +291,45 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, device_dict.update(device.time_info) if device.openwrt: device_dict.update(device_factory.GetOpenWrtInfoDict()) + if device.gce_hostname: + device_dict[constants.GCE_HOSTNAME] = device.gce_hostname + logger.debug( + "To connect with hostname, erase the extra_args_ssh_tunnel: %s", + extra_args_ssh_tunnel) + extra_args_ssh_tunnel="" if autoconnect and reporter.status == report.Status.SUCCESS: - forwarded_ports = utils.AutoConnect( - ip_addr=ip, - rsa_key_file=(ssh_private_key_path or - cfg.ssh_private_key_path), - target_vnc_port=utils.AVD_PORT_DICT[avd_type].vnc_port, - target_adb_port=adb_port, - ssh_user=ssh_user, - client_adb_port=client_adb_port, - extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) - device_dict[constants.VNC_PORT] = forwarded_ports.vnc_port - device_dict[constants.ADB_PORT] = forwarded_ports.adb_port - device_dict[constants.DEVICE_SERIAL] = ( - constants.REMOTE_INSTANCE_ADB_SERIAL % - forwarded_ports.adb_port) - if unlock_screen: - AdbTools(forwarded_ports.adb_port).AutoUnlockScreen() + forwarded_ports = _EstablishDeviceConnections( + device.gce_hostname or ip, + vnc_ports, adb_ports, fastboot_ports, + client_adb_port, client_fastboot_port, ssh_user, + ssh_private_key_path=(ssh_private_key_path or + cfg.ssh_private_key_path), + extra_args_ssh_tunnel=extra_args_ssh_tunnel, + unlock_screen=unlock_screen) + if forwarded_ports: + forwarded_port = forwarded_ports[0] + device_dict[constants.VNC_PORT] = forwarded_port.vnc_port + device_dict[constants.ADB_PORT] = forwarded_port.adb_port + device_dict[constants.FASTBOOT_PORT] = forwarded_port.fastboot_port + device_dict[constants.DEVICE_SERIAL] = ( + constants.REMOTE_INSTANCE_ADB_SERIAL % + forwarded_port.adb_port) if connect_webrtc and reporter.status == report.Status.SUCCESS: webrtc_local_port = utils.PickFreePort() device_dict[constants.WEBRTC_PORT] = webrtc_local_port utils.EstablishWebRTCSshTunnel( - ip_addr=ip, + ip_addr=device.gce_hostname or ip, webrtc_local_port=webrtc_local_port, rsa_key_file=(ssh_private_key_path or cfg.ssh_private_key_path), ssh_user=ssh_user, - extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) + extra_args_ssh_tunnel=extra_args_ssh_tunnel) if device.instance_name in logs: device_dict[constants.LOGS] = logs[device.instance_name] + if hasattr(device_factory, 'GetFetchCvdWrapperLogIfExist'): + fetch_cvd_wrapper_log = device_factory.GetFetchCvdWrapperLogIfExist() + if fetch_cvd_wrapper_log: + device_dict["fetch_cvd_wrapper_log"] = fetch_cvd_wrapper_log if device.instance_name in failures: reporter.SetErrorType(constants.ACLOUD_BOOT_UP_ERROR) if device.stage: @@ -347,3 +343,45 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, reporter.AddError(str(e)) reporter.SetStatus(report.Status.FAIL) return reporter + + +def _EstablishDeviceConnections(ip, vnc_ports, adb_ports, fastboot_ports, + client_adb_port, client_fastboot_port, + ssh_user, ssh_private_key_path, + extra_args_ssh_tunnel, unlock_screen): + """Establish the adb and vnc connections. + + Create the ssh tunnels with adb ports and vnc ports. Then unlock the device + screen via the adb port. + + Args: + ip: String, the IPv4 address. + vnc_ports: List of integer, the vnc ports. + adb_ports: List of integer, the adb ports. + fastboot_ports: List of integer, the fastboot ports. + client_adb_port: Integer, Specify port for adb forwarding. + client_fastboot_port: Integer, Specify port for fastboot forwarding. + ssh_user: String, the user name for SSH tunneling. + ssh_private_key_path: String, the private key for SSH tunneling. + extra_args_ssh_tunnel: String, extra args for ssh tunnel connection. + unlock_screen: Boolean, whether to unlock screen after invoking vnc client. + + Returns: + A list of namedtuple of (vnc_port, adb_port) + """ + forwarded_ports = [] + for vnc_port, adb_port, fastboot_port in zip(vnc_ports, adb_ports, fastboot_ports): + forwarded_port = utils.AutoConnect( + ip_addr=ip, + rsa_key_file=ssh_private_key_path, + target_vnc_port=vnc_port, + target_adb_port=adb_port, + target_fastboot_port=fastboot_port, + ssh_user=ssh_user, + client_adb_port=client_adb_port, + client_fastboot_port=client_fastboot_port, + extra_args_ssh_tunnel=extra_args_ssh_tunnel) + forwarded_ports.append(forwarded_port) + if unlock_screen: + AdbTools(forwarded_port.adb_port).AutoUnlockScreen() + return forwarded_ports diff --git a/public/actions/common_operations_test.py b/public/actions/common_operations_test.py index b06a60cd..45a3d314 100644 --- a/public/actions/common_operations_test.py +++ b/public/actions/common_operations_test.py @@ -36,6 +36,7 @@ from acloud.public.actions import common_operations class CommonOperationsTest(driver_test_lib.BaseDriverTest): """Test Common Operations.""" + maxDiff = None IP = ssh.IP(external="127.0.0.1", internal="10.0.0.1") INSTANCE = "fake-instance" CMD = "test-cmd" @@ -56,6 +57,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "AndroidBuildClient", return_value=self.build_client) self.compute_client = mock.MagicMock() + self.compute_client.gce_hostname = None self.Patch( android_compute_client, "AndroidComputeClient", @@ -68,11 +70,9 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): self.device_factory, "GetComputeClient", return_value=self.compute_client) - self.Patch(self.device_factory, "GetBuildInfoDict", - return_value={"branch": self.BRANCH, - "build_id": self.BUILD_ID, - "build_target": self.BUILD_TARGET, - "gcs_bucket_build_id": self.BUILD_ID}) + self.Patch(self.device_factory, "GetVncPorts", return_value=[6444]) + self.Patch(self.device_factory, "GetAdbPorts", return_value=[6520]) + self.Patch(self.device_factory, "GetFastbootPorts", return_value=[7520]) self.Patch(self.device_factory, "GetBuildInfoDict", return_value={"branch": self.BRANCH, "build_id": self.BUILD_ID, @@ -80,6 +80,9 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "gcs_bucket_build_id": self.BUILD_ID}) self.Patch(self.device_factory, "GetLogs", return_value={self.INSTANCE: self.LOGS}) + self.Patch( + self.device_factory, + "GetFetchCvdWrapperLogIfExist", return_value={}) @staticmethod def _CreateCfg(): @@ -113,7 +116,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): self.assertEqual( _report.data, {"devices": [{ - "ip": self.IP.external, + "ip": self.IP.external + ":6520", "instance_name": self.INSTANCE, "branch": self.BRANCH, "build_id": self.BUILD_ID, @@ -122,9 +125,9 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "logs": self.LOGS }]}) - def testCreateDevicesWithAdbPort(self): + def testCreateDevicesWithAdbAndFastbootPorts(self): """Test Create Devices with adb port for cuttlefish avd type.""" - forwarded_ports = mock.Mock(adb_port=12345, vnc_port=56789) + forwarded_ports = mock.Mock(adb_port=12345, fastboot_port=54321, vnc_port=56789) mock_auto_connect = self.Patch(utils, "AutoConnect", return_value=forwarded_ports) cfg = self._CreateCfg() @@ -132,12 +135,13 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): self.device_factory, 1, "cuttlefish", autoconnect=True, - client_adb_port=12345) + client_adb_port=12345, + client_fastboot_port=54321) mock_auto_connect.assert_called_with( ip_addr="127.0.0.1", rsa_key_file="cfg/private/key", - target_vnc_port=6444, target_adb_port=6520, - ssh_user=constants.GCE_USER, client_adb_port=12345, + target_vnc_port=6444, target_adb_port=6520, target_fastboot_port=7520, + ssh_user=constants.GCE_USER, client_adb_port=12345, client_fastboot_port=54321, extra_args_ssh_tunnel="extra args") self.assertEqual(_report.command, self.CMD) self.assertEqual(_report.status, report.Status.SUCCESS) @@ -149,6 +153,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "branch": self.BRANCH, "build_id": self.BUILD_ID, "adb_port": 12345, + "fastboot_port": 54321, "device_serial": "127.0.0.1:12345", "vnc_port": 56789, "build_target": self.BUILD_TARGET, @@ -156,6 +161,37 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "logs": self.LOGS }]}) + def testCreateDevicesMultipleDevices(self): + """Test Create Devices with multiple cuttlefish devices.""" + forwarded_ports_1 = mock.Mock(adb_port=12345, vnc_port=56789) + forwarded_ports_2 = mock.Mock(adb_port=23456, vnc_port=67890) + self.Patch(self.device_factory, "GetVncPorts", return_value=[6444, 6445]) + self.Patch(self.device_factory, "GetAdbPorts", return_value=[6520, 6521]) + self.Patch(self.device_factory, "GetFastbootPorts", return_value=[7520, 7521]) + self.Patch(utils, "PickFreePort", return_value=12345) + mock_auto_connect = self.Patch( + utils, "AutoConnect", side_effects=[forwarded_ports_1, + forwarded_ports_2]) + cfg = self._CreateCfg() + _report = common_operations.CreateDevices(self.CMD, cfg, + self.device_factory, 1, + "cuttlefish", + autoconnect=True, + client_adb_port=None) + self.assertEqual(2, mock_auto_connect.call_count) + mock_auto_connect.assert_any_call( + ip_addr="127.0.0.1", rsa_key_file="cfg/private/key", + target_vnc_port=6444, target_adb_port=6520, target_fastboot_port=7520, + ssh_user=constants.GCE_USER, client_adb_port=None, client_fastboot_port=None, + extra_args_ssh_tunnel="extra args") + mock_auto_connect.assert_any_call( + ip_addr="127.0.0.1", rsa_key_file="cfg/private/key", + target_vnc_port=6444, target_adb_port=6520, target_fastboot_port=7520, + ssh_user=constants.GCE_USER, client_adb_port=None, client_fastboot_port=None, + extra_args_ssh_tunnel="extra args") + self.assertEqual(_report.command, self.CMD) + self.assertEqual(_report.status, report.Status.SUCCESS) + def testCreateDevicesInternalIP(self): """Test Create Devices and report internal IP.""" cfg = self._CreateCfg() @@ -168,7 +204,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): self.assertEqual( _report.data, {"devices": [{ - "ip": self.IP.internal, + "ip": self.IP.internal + ":6520", "instance_name": self.INSTANCE, "branch": self.BRANCH, "build_id": self.BUILD_ID, @@ -179,7 +215,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): def testCreateDevicesWithSshParameters(self): """Test Create Devices with ssh user and key.""" - forwarded_ports = mock.Mock(adb_port=12345, vnc_port=56789) + forwarded_ports = mock.Mock(adb_port=12345, fastboot_port=54321, vnc_port=56789) mock_auto_connect = self.Patch(utils, "AutoConnect", return_value=forwarded_ports) mock_establish_webrtc = self.Patch(utils, "EstablishWebRTCSshTunnel") @@ -192,8 +228,9 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): mock_auto_connect.assert_called_with( ip_addr="127.0.0.1", rsa_key_file="private/key", - target_vnc_port=6444, target_adb_port=6520, ssh_user="user", - client_adb_port=None, extra_args_ssh_tunnel="extra args") + target_vnc_port=6444, target_adb_port=6520, target_fastboot_port=7520, + ssh_user="user", client_adb_port=None, client_fastboot_port=None, + extra_args_ssh_tunnel="extra args") mock_establish_webrtc.assert_called_with( ip_addr="127.0.0.1", rsa_key_file="private/key", ssh_user="user", extra_args_ssh_tunnel="extra args", @@ -231,6 +268,32 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): expected_result = constants.GCE_QUOTA_ERROR self.assertEqual(common_operations._GetErrorType(error), expected_result) + def testCreateDevicesWithFetchCvdWrapper(self): + """Test Create Devices with FetchCvdWrapper.""" + self.Patch( + self.device_factory, + "GetFetchCvdWrapperLogIfExist", return_value={"fetch_log": "abc"}) + cfg = self._CreateCfg() + _report = common_operations.CreateDevices(self.CMD, cfg, + self.device_factory, 1, + constants.TYPE_CF) + self.assertEqual(_report.command, self.CMD) + self.assertEqual(_report.status, report.Status.SUCCESS) + self.assertEqual( + _report.data, + {"devices": [{ + "ip": self.IP.external + ":6520", + "instance_name": self.INSTANCE, + "branch": self.BRANCH, + "build_id": self.BUILD_ID, + "build_target": self.BUILD_TARGET, + "gcs_bucket_build_id": self.BUILD_ID, + "logs": self.LOGS, + "fetch_cvd_wrapper_log": { + "fetch_log": "abc" + }, + }]}) + if __name__ == "__main__": unittest.main() diff --git a/public/actions/create_cuttlefish_action.py b/public/actions/create_cuttlefish_action.py deleted file mode 100644 index 33545550..00000000 --- a/public/actions/create_cuttlefish_action.py +++ /dev/null @@ -1,299 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2018 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Create cuttlefish instances. - -TODO: This module now just contains the skeleton but not the actual logic. - Need to fill in the actuall logic. -""" - -import logging - -from acloud.public.actions import common_operations -from acloud.public.actions import base_device_factory -from acloud.internal import constants -from acloud.internal.lib import android_build_client -from acloud.internal.lib import auth -from acloud.internal.lib import cvd_compute_client -from acloud.internal.lib import cvd_compute_client_multi_stage -from acloud.internal.lib import utils - - -logger = logging.getLogger(__name__) - - -class CuttlefishDeviceFactory(base_device_factory.BaseDeviceFactory): - """A class that can produce a cuttlefish device. - - Attributes: - cfg: An AcloudConfig instance. - build_target: String,Target name. - build_id: String, Build id, e.g. "2263051", "P2804227" - kernel_build_id: String, Kernel build id. - gpu: String, GPU to attach to the device or None. e.g. "nvidia-tesla-k80" - """ - - LOG_FILES = ["/home/vsoc-01/cuttlefish_runtime/kernel.log", - "/home/vsoc-01/cuttlefish_runtime/logcat", - "/home/vsoc-01/cuttlefish_runtime/cuttlefish_config.json"] - - #pylint: disable=too-many-locals - def __init__(self, cfg, build_target, build_id, branch=None, - kernel_build_id=None, kernel_branch=None, - kernel_build_target=None, system_branch=None, - system_build_id=None, system_build_target=None, - bootloader_branch=None, bootloader_build_id=None, - bootloader_build_target=None, boot_timeout_secs=None, - ins_timeout_secs=None, report_internal_ip=None, gpu=None): - - self.credentials = auth.CreateCredentials(cfg) - - if cfg.enable_multi_stage: - compute_client = cvd_compute_client_multi_stage.CvdComputeClient( - cfg, self.credentials, boot_timeout_secs, ins_timeout_secs, - report_internal_ip, gpu) - else: - compute_client = cvd_compute_client.CvdComputeClient( - cfg, self.credentials) - super().__init__(compute_client) - - # Private creation parameters - self._cfg = cfg - self._build_target = build_target - self._build_id = build_id - self._branch = branch - self._kernel_build_id = kernel_build_id - self._blank_data_disk_size_gb = cfg.extra_data_disk_size_gb - self._extra_scopes = cfg.extra_scopes - - # Configure clients for interaction with GCE/Build servers - self._build_client = android_build_client.AndroidBuildClient( - self.credentials) - - # Get build_info namedtuple for platform, kernel, system build - self.build_info = self._build_client.GetBuildInfo( - build_target, build_id, branch) - self.kernel_build_info = self._build_client.GetBuildInfo( - kernel_build_target or cfg.kernel_build_target, kernel_build_id, - kernel_branch) - self.system_build_info = self._build_client.GetBuildInfo( - system_build_target or build_target, system_build_id, system_branch) - self.bootloader_build_info = self._build_client.GetBuildInfo( - bootloader_build_target, bootloader_build_id, bootloader_branch) - - def GetBuildInfoDict(self): - """Get build info dictionary. - - Returns: - A build info dictionary. - """ - build_info_dict = { - key: val for key, val in utils.GetDictItems(self.build_info) if val} - - build_info_dict.update( - {"kernel_%s" % key: val - for key, val in utils.GetDictItems(self.kernel_build_info) if val} - ) - build_info_dict.update( - {"system_%s" % key: val - for key, val in utils.GetDictItems(self.system_build_info) if val} - ) - build_info_dict.update( - {"bootloader_%s" % key: val - for key, val in utils.GetDictItems(self.bootloader_build_info) if val} - ) - return build_info_dict - - def GetFailures(self): - """Get failures from all devices. - - Returns: - A dictionary that contains all the failures. - The key is the name of the instance that fails to boot, - and the value is an errors.DeviceBootError object. - """ - return self._compute_client.all_failures - - @staticmethod - def _GetGcsBucketBuildId(build_id, release_id): - """Get GCS Bucket Build Id. - - Args: - build_id: The incremental build id. For example 5325535. - release_id: The release build id, None if not a release build. - For example AAAA.190220.001. - - Returns: - GCS bucket build id. For example: AAAA.190220.001-5325535 - """ - return "-".join([release_id, build_id]) if release_id else build_id - - def CreateInstance(self): - """Creates singe configured cuttlefish device. - - Override method from parent class. - - Returns: - A string, representing instance name. - """ - - # Create host instances for cuttlefish device. Currently one host instance - # has one cuttlefish device. In the future, these logics should be modified - # to support multiple cuttlefish devices per host instance. - instance = self._compute_client.GenerateInstanceName( - build_id=self.build_info.build_id, build_target=self._build_target) - - if self._cfg.enable_multi_stage: - remote_build_id = self.build_info.build_id - else: - remote_build_id = self._GetGcsBucketBuildId( - self.build_info.build_id, self.build_info.release_build_id) - - if self._cfg.enable_multi_stage: - remote_system_build_id = self.system_build_info.build_id - else: - remote_system_build_id = self._GetGcsBucketBuildId( - self.system_build_info.build_id, self.system_build_info.release_build_id) - - host_image_name = self._compute_client.GetHostImageName( - self._cfg.stable_host_image_name, - self._cfg.stable_host_image_family, - self._cfg.stable_host_image_project) - # Create an instance from Stable Host Image - self._compute_client.CreateInstance( - instance=instance, - image_name=host_image_name, - image_project=self._cfg.stable_host_image_project, - build_target=self.build_info.build_target, - branch=self.build_info.branch, - build_id=remote_build_id, - kernel_branch=self.kernel_build_info.branch, - kernel_build_id=self.kernel_build_info.build_id, - kernel_build_target=self.kernel_build_info.build_target, - blank_data_disk_size_gb=self._blank_data_disk_size_gb, - extra_scopes=self._extra_scopes, - system_build_target=self.system_build_info.build_target, - system_branch=self.system_build_info.branch, - system_build_id=remote_system_build_id, - bootloader_build_target=self.bootloader_build_info.build_target, - bootloader_branch=self.bootloader_build_info.branch, - bootloader_build_id=self.bootloader_build_info.build_id) - - return instance - - -#pylint: disable=too-many-locals -def CreateDevices(cfg, - build_target=None, - build_id=None, - branch=None, - kernel_build_id=None, - kernel_branch=None, - kernel_build_target=None, - system_branch=None, - system_build_id=None, - system_build_target=None, - bootloader_branch=None, - bootloader_build_id=None, - bootloader_build_target=None, - gpu=None, - num=1, - serial_log_file=None, - autoconnect=False, - report_internal_ip=False, - boot_timeout_secs=None, - ins_timeout_secs=None): - """Create one or multiple Cuttlefish devices. - - Args: - cfg: An AcloudConfig instance. - build_target: String, Target name. - build_id: String, Build id, e.g. "2263051", "P2804227" - branch: Branch name, a string, e.g. aosp_master - kernel_build_id: String, Kernel build id. - kernel_branch: String, Kernel branch name. - kernel_build_target: String, Kernel build target name. - system_branch: Branch name to consume the system.img from, a string. - system_build_id: System branch build id, a string. - system_build_target: System image build target, a string. - bootloader_branch: String of the bootloader branch name. - bootloader_build_id: String of the bootloader build id. - bootloader_build_target: String of the bootloader target name. - gpu: String, GPU to attach to the device or None. e.g. "nvidia-tesla-k80" - num: Integer, Number of devices to create. - serial_log_file: String, A path to a tar file where serial output should - be saved to. - autoconnect: Boolean, Create ssh tunnel(s) and adb connect after device - creation. - report_internal_ip: Boolean to report the internal ip instead of - external ip. - boot_timeout_secs: Integer, the maximum time in seconds used to wait - for the AVD to boot. - ins_timeout_secs: Integer, the maximum time in seconds used to wait for - the instance ready. - - Returns: - A Report instance. - """ - client_adb_port = None - unlock_screen = False - wait_for_boot = True - logger.info( - "Creating a cuttlefish device in project %s, " - "build_target: %s, " - "build_id: %s, " - "branch: %s, " - "kernel_build_id: %s, " - "kernel_branch: %s, " - "kernel_build_target: %s, " - "system_branch: %s, " - "system_build_id: %s, " - "system_build_target: %s, " - "bootloader_branch: %s, " - "bootloader_build_id: %s, " - "bootloader_build_target: %s, " - "gpu: %s" - "num: %s, " - "serial_log_file: %s, " - "autoconnect: %s, " - "report_internal_ip: %s", cfg.project, build_target, - build_id, branch, kernel_build_id, kernel_branch, kernel_build_target, - system_branch, system_build_id, system_build_target, bootloader_branch, - bootloader_build_id, bootloader_build_target, gpu, num, serial_log_file, - autoconnect, report_internal_ip) - # If multi_stage enable, launch_cvd don't write serial log to instance. So - # it doesn't go WaitForBoot function. - if cfg.enable_multi_stage: - wait_for_boot = False - device_factory = CuttlefishDeviceFactory( - cfg, build_target, build_id, branch=branch, - kernel_build_id=kernel_build_id, kernel_branch=kernel_branch, - kernel_build_target=kernel_build_target, system_branch=system_branch, - system_build_id=system_build_id, - system_build_target=system_build_target, - bootloader_branch=bootloader_branch, - bootloader_build_id=bootloader_build_id, - bootloader_build_target=bootloader_build_target, - boot_timeout_secs=boot_timeout_secs, - ins_timeout_secs=ins_timeout_secs, - report_internal_ip=report_internal_ip, - gpu=gpu) - return common_operations.CreateDevices("create_cf", cfg, device_factory, - num, constants.TYPE_CF, - report_internal_ip, autoconnect, - serial_log_file, client_adb_port, - boot_timeout_secs, unlock_screen, - wait_for_boot) diff --git a/public/actions/create_cuttlefish_action_test.py b/public/actions/create_cuttlefish_action_test.py deleted file mode 100644 index 0ddef21e..00000000 --- a/public/actions/create_cuttlefish_action_test.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2018 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for create_cuttlefish_action. - -Tests for acloud.public.actions.create_cuttlefish_action. -""" - -import uuid -import unittest - -from unittest import mock - -from acloud.internal.lib import android_build_client -from acloud.internal.lib import android_compute_client -from acloud.internal.lib import auth -from acloud.internal.lib import cvd_compute_client -from acloud.internal.lib import cvd_compute_client_multi_stage -from acloud.internal.lib import driver_test_lib -from acloud.internal.lib import ssh -from acloud.public.actions import create_cuttlefish_action - - -class CreateCuttlefishActionTest(driver_test_lib.BaseDriverTest): - """Test create_cuttlefish_action.""" - - IP = ssh.IP(external="127.0.0.1", internal="10.0.0.1") - INSTANCE = "fake-instance" - IMAGE = "fake-image" - BRANCH = "fake-branch" - BUILD_ID = "12345" - BUILD_TARGET = "fake-build-target" - KERNEL_BRANCH = "fake-kernel-branch" - KERNEL_BUILD_ID = "54321" - KERNEL_BUILD_TARGET = "kernel" - SYSTEM_BRANCH = "fake-system-branch" - SYSTEM_BUILD_ID = "23456" - SYSTEM_BUILD_TARGET = "fake-system-build-target" - BOOTLOADER_BRANCH = "fake-bootloader-branch" - BOOTLOADER_BUILD_ID = "34567" - BOOTLOADER_BUILD_TARGET = "fake-bootloader-build-target" - STABLE_HOST_IMAGE_NAME = "fake-stable-host-image-name" - STABLE_HOST_IMAGE_PROJECT = "fake-stable-host-image-project" - EXTRA_DATA_DISK_GB = 4 - EXTRA_SCOPES = ["scope1", "scope2"] - DEFAULT_ADB_PORT = 6520 - - def setUp(self): - """Set up the test.""" - super().setUp() - self.build_client = mock.MagicMock() - self.Patch( - android_build_client, - "AndroidBuildClient", - return_value=self.build_client) - self.compute_client = mock.MagicMock() - self.compute_client.openwrt = False - self.Patch( - cvd_compute_client, - "CvdComputeClient", - return_value=self.compute_client) - self.Patch( - cvd_compute_client_multi_stage, - "CvdComputeClient", - return_value=self.compute_client) - self.Patch( - android_compute_client, - "AndroidComputeClient", - return_value=self.compute_client) - self.Patch(auth, "CreateCredentials", return_value=mock.MagicMock()) - - def _CreateCfg(self): - """A helper method that creates a mock configuration object.""" - cfg = mock.MagicMock() - cfg.service_account_name = "fake@service.com" - cfg.service_account_private_key_path = "/fake/path/to/key" - cfg.zone = "fake_zone" - cfg.disk_image_name = "fake_image.tar.gz" - cfg.disk_image_mime_type = "fake/type" - cfg.ssh_private_key_path = "" - cfg.ssh_public_key_path = "" - cfg.stable_host_image_name = self.STABLE_HOST_IMAGE_NAME - cfg.stable_host_image_project = self.STABLE_HOST_IMAGE_PROJECT - cfg.extra_data_disk_size_gb = self.EXTRA_DATA_DISK_GB - cfg.kernel_build_target = self.KERNEL_BUILD_TARGET - cfg.extra_scopes = self.EXTRA_SCOPES - cfg.enable_multi_stage = False - return cfg - - def testCreateDevices(self): - """Test CreateDevices.""" - cfg = self._CreateCfg() - - # Mock uuid - fake_uuid = mock.MagicMock(hex="1234") - self.Patch(uuid, "uuid4", return_value=fake_uuid) - - # Mock compute client methods - self.compute_client.GetInstanceIP.return_value = self.IP - self.compute_client.GenerateImageName.return_value = self.IMAGE - self.compute_client.GenerateInstanceName.return_value = self.INSTANCE - self.compute_client.GetHostImageName.return_value = self.STABLE_HOST_IMAGE_NAME - - # Mock build client method - self.build_client.GetBuildInfo.side_effect = [ - android_build_client.BuildInfo( - self.BRANCH, self.BUILD_ID, self.BUILD_TARGET, None), - android_build_client.BuildInfo( - self.KERNEL_BRANCH, self.KERNEL_BUILD_ID, - self.KERNEL_BUILD_TARGET, None), - android_build_client.BuildInfo( - self.SYSTEM_BRANCH, self.SYSTEM_BUILD_ID, - self.SYSTEM_BUILD_TARGET, None), - android_build_client.BuildInfo( - self.BOOTLOADER_BRANCH, self.BOOTLOADER_BUILD_ID, - self.BOOTLOADER_BUILD_TARGET, None)] - - # Call CreateDevices - report = create_cuttlefish_action.CreateDevices( - cfg, self.BUILD_TARGET, self.BUILD_ID, branch=self.BRANCH, - kernel_build_id=self.KERNEL_BUILD_ID, - system_build_target=self.SYSTEM_BUILD_TARGET, - system_branch=self.SYSTEM_BRANCH, - system_build_id=self.SYSTEM_BUILD_ID, - bootloader_build_target=self.BOOTLOADER_BUILD_TARGET, - bootloader_branch=self.BOOTLOADER_BRANCH, - bootloader_build_id=self.BOOTLOADER_BUILD_ID) - - # Verify - self.compute_client.CreateInstance.assert_called_with( - instance=self.INSTANCE, - image_name=self.STABLE_HOST_IMAGE_NAME, - image_project=self.STABLE_HOST_IMAGE_PROJECT, - build_target=self.BUILD_TARGET, - branch=self.BRANCH, - build_id=self.BUILD_ID, - kernel_branch=self.KERNEL_BRANCH, - kernel_build_id=self.KERNEL_BUILD_ID, - kernel_build_target=self.KERNEL_BUILD_TARGET, - system_branch=self.SYSTEM_BRANCH, - system_build_id=self.SYSTEM_BUILD_ID, - system_build_target=self.SYSTEM_BUILD_TARGET, - bootloader_branch=self.BOOTLOADER_BRANCH, - bootloader_build_id=self.BOOTLOADER_BUILD_ID, - bootloader_build_target=self.BOOTLOADER_BUILD_TARGET, - blank_data_disk_size_gb=self.EXTRA_DATA_DISK_GB, - extra_scopes=self.EXTRA_SCOPES) - - self.assertEqual(report.data, { - "devices": [ - { - "branch": self.BRANCH, - "build_id": self.BUILD_ID, - "build_target": self.BUILD_TARGET, - "kernel_branch": self.KERNEL_BRANCH, - "kernel_build_id": self.KERNEL_BUILD_ID, - "kernel_build_target": self.KERNEL_BUILD_TARGET, - "system_branch": self.SYSTEM_BRANCH, - "system_build_id": self.SYSTEM_BUILD_ID, - "system_build_target": self.SYSTEM_BUILD_TARGET, - "bootloader_branch": self.BOOTLOADER_BRANCH, - "bootloader_build_id": self.BOOTLOADER_BUILD_ID, - "bootloader_build_target": self.BOOTLOADER_BUILD_TARGET, - "instance_name": self.INSTANCE, - "ip": self.IP.external + ":" + str(self.DEFAULT_ADB_PORT), - }, - ], - }) - self.assertEqual(report.command, "create_cf") - self.assertEqual(report.status, "SUCCESS") - - -if __name__ == "__main__": - unittest.main() diff --git a/public/actions/create_goldfish_action.py b/public/actions/create_goldfish_action.py index 069b9d93..d4617433 100644 --- a/public/actions/create_goldfish_action.py +++ b/public/actions/create_goldfish_action.py @@ -159,6 +159,7 @@ class GoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): build_id=self.build_info.build_id, emulator_branch=self.emulator_build_info.branch, emulator_build_id=self.emulator_build_info.build_id, + emulator_build_target=self.emulator_build_info.build_target, kernel_branch=self.kernel_build_info.branch, kernel_build_id=self.kernel_build_info.build_id, kernel_build_target=self.kernel_build_info.build_target, @@ -234,6 +235,7 @@ def CreateDevices(avd_spec=None, build_id=None, emulator_build_id=None, emulator_branch=None, + emulator_build_target=None, kernel_build_id=None, kernel_branch=None, kernel_build_target=None, @@ -254,7 +256,8 @@ def CreateDevices(avd_spec=None, build_id: String, Build id, e.g. "2263051", "P2804227" branch: String, Branch name for system image. emulator_build_id: String, emulator build id. - emulator_branch: String, Emulator branch name. + emulator_branch: String, emulator branch name. + emulator_build_target: String, emulator build target. gpu: String, GPU to attach to the device or None. e.g. "nvidia-k80" kernel_build_id: Kernel build id, a string. kernel_branch: Kernel branch name, a string. @@ -283,6 +286,7 @@ def CreateDevices(avd_spec=None, branch = avd_spec.remote_image[constants.BUILD_BRANCH] num = avd_spec.num emulator_build_id = avd_spec.emulator_build_id + emulator_build_target = avd_spec.emulator_build_target gpu = avd_spec.gpu serial_log_file = avd_spec.serial_log_file autoconnect = avd_spec.autoconnect @@ -290,12 +294,15 @@ def CreateDevices(avd_spec=None, client_adb_port = avd_spec.client_adb_port boot_timeout_secs = avd_spec.boot_timeout_secs + if not emulator_build_target: + emulator_build_target = cfg.emulator_build_target + # If emulator_build_id and emulator_branch is None, retrieve emulator # build id from platform build emulator-info.txt artifact # Example: require version-emulator=5292001 if not emulator_build_id and not emulator_branch: logger.info("emulator_build_id not provided. " - "Try to get %s from build %s/%s.", _EMULATOR_INFO_FILENAME, + "Attempting to get %s from build %s/%s.", _EMULATOR_INFO_FILENAME, build_id, build_target) emulator_build_id = _FetchBuildIdFromFile(cfg, build_target, @@ -311,7 +318,7 @@ def CreateDevices(avd_spec=None, # Example: version-sysimage-git_pi-dev-sdk_gphone_x86_64-userdebug=4833817 if not build_id and not branch: build_id = _FetchBuildIdFromFile(cfg, - cfg.emulator_build_target, + emulator_build_target, emulator_build_id, _SYSIMAGE_INFO_FILENAME) @@ -320,16 +327,16 @@ def CreateDevices(avd_spec=None, "in %s" % _SYSIMAGE_INFO_FILENAME) logger.info( "Creating a goldfish device in project %s, build_target: %s, " - "build_id: %s, emulator_bid: %s, kernel_build_id: %s, " + "build_id: %s, emulator_bid: %s, emulator_branch: %s, kernel_build_id: %s, " "kernel_branch: %s, kernel_build_target: %s, GPU: %s, num: %s, " "serial_log_file: %s, " "autoconnect: %s", cfg.project, build_target, build_id, - emulator_build_id, kernel_build_id, kernel_branch, kernel_build_target, - gpu, num, serial_log_file, autoconnect) + emulator_build_id, emulator_branch, kernel_build_id, kernel_branch, + kernel_build_target, gpu, num, serial_log_file, autoconnect) device_factory = GoldfishDeviceFactory( cfg, build_target, build_id, - cfg.emulator_build_target, + emulator_build_target, emulator_build_id, gpu=gpu, avd_spec=avd_spec, tags=tags, branch=branch, diff --git a/public/actions/create_goldfish_action_test.py b/public/actions/create_goldfish_action_test.py index dbc47b5b..b784be44 100644 --- a/public/actions/create_goldfish_action_test.py +++ b/public/actions/create_goldfish_action_test.py @@ -62,6 +62,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): return_value=self.build_client) self.compute_client = mock.MagicMock() self.compute_client.openwrt = False + self.compute_client.gce_hostname = None self.Patch( goldfish_compute_client, "GoldfishComputeClient", @@ -144,6 +145,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, @@ -201,6 +203,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, @@ -266,6 +269,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, @@ -321,6 +325,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, @@ -379,6 +384,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, @@ -434,6 +440,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, diff --git a/public/actions/gce_device_factory.py b/public/actions/gce_device_factory.py index 5502f296..42970178 100644 --- a/public/actions/gce_device_factory.py +++ b/public/actions/gce_device_factory.py @@ -51,7 +51,7 @@ class GCEDeviceFactory(base_device_factory.BaseDeviceFactory): super().__init__(compute_client) self._ssh = None - def _CreateGceInstance(self): + def CreateGceInstance(self): """Create a single configured GCE instance. build_target: The format is like "aosp_cf_x86_phone". We only get info @@ -87,14 +87,15 @@ class GCEDeviceFactory(base_device_factory.BaseDeviceFactory): instance=instance, image_name=host_image_name, image_project=self._cfg.stable_host_image_project, - blank_data_disk_size_gb=self._cfg.extra_data_disk_size_gb, avd_spec=self._avd_spec) ip = self._compute_client.GetInstanceIP(instance) + self._all_failures = self._compute_client.all_failures self._ssh = ssh.Ssh(ip=ip, user=constants.GCE_USER, ssh_private_key_path=self._cfg.ssh_private_key_path, extra_args_ssh_tunnel=self._cfg.extra_args_ssh_tunnel, - report_internal_ip=self._report_internal_ip) + report_internal_ip=self._report_internal_ip, + gce_hostname=self._compute_client.gce_hostname) return instance def GetFailures(self): diff --git a/public/actions/remote_host_cf_device_factory.py b/public/actions/remote_host_cf_device_factory.py index 4b4181e7..1ba46174 100644 --- a/public/actions/remote_host_cf_device_factory.py +++ b/public/actions/remote_host_cf_device_factory.py @@ -16,11 +16,14 @@ cuttlefish instances on a remote host.""" import glob +import json import logging import os +import posixpath as remote_path import shutil import subprocess import tempfile +import time from acloud import errors from acloud.internal import constants @@ -79,19 +82,42 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: A string, representing instance name. """ + init_remote_host_timestart = time.time() instance = self._InitRemotehost() + self._compute_client.execution_time[constants.TIME_GCE] = ( + time.time() - init_remote_host_timestart) + + process_artifacts_timestart = time.time() image_args = self._ProcessRemoteHostArtifacts() + self._compute_client.execution_time[constants.TIME_ARTIFACT] = ( + time.time() - process_artifacts_timestart) + + launch_cvd_timestart = time.time() failures = self._compute_client.LaunchCvd( - instance, - self._avd_spec, - self._avd_spec.cfg.extra_data_disk_size_gb, - boot_timeout_secs=self._avd_spec.boot_timeout_secs, - extra_args=image_args) + instance, self._avd_spec, self._GetInstancePath(), image_args) + self._compute_client.execution_time[constants.TIME_LAUNCH] = ( + time.time() - launch_cvd_timestart) + self._all_failures.update(failures) self._FindLogFiles( instance, instance in failures and not self._avd_spec.no_pull_log) return instance + def _GetInstancePath(self, relative_path=""): + """Append a relative path to the remote base directory. + + Args: + relative_path: The remote relative path. + + Returns: + The remote base directory if relative_path is empty. + The remote path under the base directory otherwise. + """ + base_dir = cvd_utils.GetRemoteHostBaseDir( + self._avd_spec.base_instance_num) + return (remote_path.join(base_dir, relative_path) if relative_path else + base_dir) + def _InitRemotehost(self): """Initialize remote host. @@ -116,8 +142,9 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: build_id = self._avd_spec.remote_image[constants.BUILD_ID] - instance = self._compute_client.FormatRemoteHostInstanceName( - self._avd_spec.remote_host, build_id, build_target) + instance = cvd_utils.FormatRemoteHostInstanceName( + self._avd_spec.remote_host, self._avd_spec.base_instance_num, + build_id, build_target) ip = ssh.IP(ip=self._avd_spec.remote_host) self._ssh = ssh.Ssh( ip=ip, @@ -127,7 +154,7 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): extra_args_ssh_tunnel=self._avd_spec.cfg.extra_args_ssh_tunnel, report_internal_ip=self._avd_spec.report_internal_ip) self._compute_client.InitRemoteHost( - self._ssh, ip, self._avd_spec.host_user) + self._ssh, ip, self._avd_spec.host_user, self._GetInstancePath()) return instance def _ProcessRemoteHostArtifacts(self): @@ -143,21 +170,120 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): A list of strings, the launch_cvd arguments. """ self._compute_client.SetStage(constants.STAGE_ARTIFACT) + self._ssh.Run(f"mkdir -p {self._GetInstancePath()}") if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: cvd_utils.UploadArtifacts( - self._ssh, + self._ssh, self._GetInstancePath(), self._local_image_artifact or self._avd_spec.local_image_dir, self._cvd_host_package_artifact) else: try: artifacts_path = tempfile.mkdtemp() logger.debug("Extracted path of artifacts: %s", artifacts_path) - self._DownloadArtifacts(artifacts_path) - self._UploadRemoteImageArtifacts(artifacts_path) + if self._avd_spec.remote_fetch: + # TODO: Check fetch cvd wrapper file is valid. + if self._avd_spec.fetch_cvd_wrapper: + self._UploadFetchCvd(artifacts_path) + self._DownloadArtifactsByFetchWrapper() + else: + self._UploadFetchCvd(artifacts_path) + self._DownloadArtifactsRemotehost() + else: + self._DownloadArtifacts(artifacts_path) + self._UploadRemoteImageArtifacts(artifacts_path) finally: shutil.rmtree(artifacts_path) - return cvd_utils.UploadExtraImages(self._ssh, self._avd_spec) + return cvd_utils.UploadExtraImages(self._ssh, self._GetInstancePath(), + self._avd_spec) + + def _GetRemoteFetchCredentialArg(self): + """Get the credential source argument for remote fetch_cvd. + + Remote fetch_cvd uses the service account key uploaded by + _UploadFetchCvd if it is available. Otherwise, fetch_cvd uses the + token extracted from the local credential file. + + Returns: + A string, the credential source argument. + """ + cfg = self._avd_spec.cfg + if cfg.service_account_json_private_key_path: + return "-credential_source=" + self._GetInstancePath( + constants.FETCH_CVD_CREDENTIAL_SOURCE) + + return self._compute_client.build_api.GetFetchCertArg( + os.path.join(_HOME_FOLDER, cfg.creds_cache_file)) + + @utils.TimeExecute( + function_description="Downloading artifacts on remote host by fetch cvd wrapper.") + def _DownloadArtifactsByFetchWrapper(self): + """Generate fetch_cvd args and run fetch cvd wrapper on remote host to download artifacts. + + Fetch cvd wrapper will fetch from cluster cached artifacts, and fallback to fetch_cvd if + the artifacts not exist. + """ + fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( + self._avd_spec.remote_image, + self._avd_spec.system_build_info, + self._avd_spec.kernel_build_info, + self._avd_spec.boot_build_info, + self._avd_spec.bootloader_build_info, + self._avd_spec.ota_build_info) + + fetch_cvd_args = self._avd_spec.fetch_cvd_wrapper.split(',') + [ + f"-directory={self._GetInstancePath()}", + f"-fetch_cvd_path={self._GetInstancePath(constants.FETCH_CVD)}", + self._GetRemoteFetchCredentialArg()] + fetch_cvd_args.extend(fetch_cvd_build_args) + + ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) + cmd = (f"{ssh_cmd} -- " + " ".join(fetch_cvd_args)) + logger.debug("cmd:\n %s", cmd) + ssh.ShellCmdWithRetry(cmd) + + @utils.TimeExecute(function_description="Downloading artifacts on remote host") + def _DownloadArtifactsRemotehost(self): + """Generate fetch_cvd args and run fetch_cvd on remote host to download artifacts.""" + fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( + self._avd_spec.remote_image, + self._avd_spec.system_build_info, + self._avd_spec.kernel_build_info, + self._avd_spec.boot_build_info, + self._avd_spec.bootloader_build_info, + self._avd_spec.ota_build_info) + + fetch_cvd_args = [self._GetInstancePath(constants.FETCH_CVD), + f"-directory={self._GetInstancePath()}", + self._GetRemoteFetchCredentialArg()] + fetch_cvd_args.extend(fetch_cvd_build_args) + + ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) + cmd = (f"{ssh_cmd} -- " + " ".join(fetch_cvd_args)) + logger.debug("cmd:\n %s", cmd) + ssh.ShellCmdWithRetry(cmd) + + @utils.TimeExecute(function_description="Download and upload fetch_cvd") + def _UploadFetchCvd(self, extract_path): + """Download fetch_cvd, duplicate service account json private key when available and upload + to remote host. + + Args: + extract_path: String, a path include extracted files. + """ + cfg = self._avd_spec.cfg + is_arm_img = (cvd_utils.IsArmImage(self._avd_spec.remote_image) + and self._avd_spec.remote_fetch) + fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) + self._compute_client.build_api.DownloadFetchcvd( + fetch_cvd, self._avd_spec.fetch_cvd_version, is_arm_img) + # Duplicate fetch_cvd API key when available + if cfg.service_account_json_private_key_path: + shutil.copyfile( + cfg.service_account_json_private_key_path, + os.path.join(extract_path, constants.FETCH_CVD_CREDENTIAL_SOURCE)) + + self._UploadRemoteImageArtifacts(extract_path) @utils.TimeExecute(function_description="Downloading Android Build artifact") def _DownloadArtifacts(self, extract_path): @@ -173,28 +299,18 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): errors.GetRemoteImageError: Fails to download rom images. """ cfg = self._avd_spec.cfg - build_id = self._avd_spec.remote_image[constants.BUILD_ID] - build_branch = self._avd_spec.remote_image[constants.BUILD_BRANCH] - build_target = self._avd_spec.remote_image[constants.BUILD_TARGET] # Download images with fetch_cvd fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) - self._compute_client.build_api.DownloadFetchcvd(fetch_cvd, - cfg.fetch_cvd_version) + self._compute_client.build_api.DownloadFetchcvd( + fetch_cvd, self._avd_spec.fetch_cvd_version) fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( - build_id, build_branch, build_target, - self._avd_spec.system_build_info.get(constants.BUILD_ID), - self._avd_spec.system_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.system_build_info.get(constants.BUILD_TARGET), - self._avd_spec.kernel_build_info.get(constants.BUILD_ID), - self._avd_spec.kernel_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.kernel_build_info.get(constants.BUILD_TARGET), - self._avd_spec.bootloader_build_info.get(constants.BUILD_ID), - self._avd_spec.bootloader_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.bootloader_build_info.get(constants.BUILD_TARGET), - self._avd_spec.ota_build_info.get(constants.BUILD_ID), - self._avd_spec.ota_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.ota_build_info.get(constants.BUILD_TARGET)) + self._avd_spec.remote_image, + self._avd_spec.system_build_info, + self._avd_spec.kernel_build_info, + self._avd_spec.boot_build_info, + self._avd_spec.bootloader_build_info, + self._avd_spec.ota_build_info) creds_cache_file = os.path.join(_HOME_FOLDER, cfg.creds_cache_file) fetch_cvd_cert_arg = self._compute_client.build_api.GetFetchCertArg( creds_cache_file) @@ -223,7 +339,8 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): # TODO(b/182259589): Refactor upload image command into a function. cmd = (f"tar -cf - --lzop -S -C {images_dir} " f"{' '.join(artifact_files)} | " - f"{ssh_cmd} -- tar -xf - --lzop -S") + f"{ssh_cmd} -- " + f"tar -xf - --lzop -S -C {self._GetInstancePath()}") logger.debug("cmd:\n %s", cmd) ssh.ShellCmdWithRetry(cmd) @@ -235,11 +352,22 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): download: Whether to download the files to a temporary directory and show messages to the user. """ - self._all_logs[instance] = [cvd_utils.TOMBSTONES] - log_files = pull.GetAllLogFilePaths(self._ssh) - self._all_logs[instance].extend(cvd_utils.ConvertRemoteLogs(log_files)) + logs = [] + if (self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE and + self._avd_spec.remote_fetch): + logs.append( + cvd_utils.GetRemoteFetcherConfigJson(self._GetInstancePath())) + logs.extend(cvd_utils.FindRemoteLogs( + self._ssh, + self._GetInstancePath(), + self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance)) + self._all_logs[instance] = logs if download: + # To avoid long download time, fetch from the first device only. + log_files = pull.GetAllLogFilePaths( + self._ssh, self._GetInstancePath(constants.REMOTE_LOG_FOLDER)) error_log_folder = pull.PullLogs(self._ssh, log_files, instance) self._compute_client.ExtendReportData(constants.ERROR_LOG_FOLDER, error_log_folder) @@ -265,6 +393,33 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): return None return cvd_utils.GetRemoteBuildInfoDict(self._avd_spec) + def GetAdbPorts(self): + """Get ADB ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetAdbPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + + def GetFastbootPorts(self): + """Get Fastboot ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetFastbootPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + + def GetVncPorts(self): + """Get VNC ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetVncPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + def GetFailures(self): """Get failures from all devices. @@ -282,3 +437,24 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): A dictionary that maps instance names to lists of report.LogFile. """ return self._all_logs + + def GetFetchCvdWrapperLogIfExist(self): + """Get FetchCvdWrapper log if exist. + + Returns: + A dictionary that includes FetchCvdWrapper logs. + """ + if not self._avd_spec.fetch_cvd_wrapper: + return {} + path = os.path.join(self._GetInstancePath(), "fetch_cvd_wrapper_log.json") + ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) + " cat " + path + proc = subprocess.run(ssh_cmd, shell=True, capture_output=True, + check=False) + if proc.stderr: + logger.debug("`%s` stderr: %s", ssh_cmd, proc.stderr.decode()) + if proc.stdout: + try: + return json.loads(proc.stdout) + except ValueError as e: + return {"status": "FETCH_WRAPPER_REPORT_PARSE_ERROR"} + return {} diff --git a/public/actions/remote_host_cf_device_factory_test.py b/public/actions/remote_host_cf_device_factory_test.py index 4b9c3e68..76670888 100644 --- a/public/actions/remote_host_cf_device_factory_test.py +++ b/public/actions/remote_host_cf_device_factory_test.py @@ -36,11 +36,11 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): def _CreateMockAvdSpec(): """Create a mock AvdSpec with necessary attributes.""" mock_cfg = mock.Mock(spec=[], - extra_data_disk_size_gb=10, ssh_private_key_path="/mock/id_rsa", extra_args_ssh_tunnel="extra args", fetch_cvd_version="123456", - creds_cache_file="credential") + creds_cache_file="credential", + service_account_json_private_key_path="/mock/key") return mock.Mock(spec=[], remote_image={ "branch": "aosp-android12-gsi", @@ -48,6 +48,7 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): "build_target": "aosp_cf_x86_64_phone-userdebug"}, system_build_info={}, kernel_build_info={}, + boot_build_info={}, bootloader_build_info={}, ota_build_info={}, remote_host="192.0.2.100", @@ -60,6 +61,11 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): boot_timeout_secs=100, gpu="auto", no_pull_log=False, + remote_fetch=False, + fetch_cvd_wrapper=None, + base_instance_num=None, + num_avds_per_instance=None, + fetch_cvd_version="123456", cfg=mock_cfg) @mock.patch("acloud.public.actions.remote_host_cf_device_factory." @@ -74,34 +80,42 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_avd_spec = self._CreateMockAvdSpec() mock_avd_spec.image_source = constants.IMAGE_SRC_LOCAL mock_avd_spec.local_image_dir = "/mock/img" + mock_avd_spec.base_instance_num = 2 + mock_avd_spec.num_avds_per_instance = 3 + mock_ssh_obj = mock.Mock() + mock_ssh.Ssh.return_value = mock_ssh_obj factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( mock_avd_spec, cvd_host_package_artifact="/mock/cvd.tar.gz") mock_client_obj = factory.GetComputeClient() - mock_client_obj.FormatRemoteHostInstanceName.return_value = "inst" mock_client_obj.LaunchCvd.return_value = {"inst": "failure"} log = {"path": "/log.txt"} - tombstones = {"path": "/tombstones"} - mock_cvd_utils.TOMBSTONES = tombstones + mock_cvd_utils.GetRemoteHostBaseDir.return_value = "acloud_cf_2" + mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" mock_cvd_utils.UploadExtraImages.return_value = ["extra"] - mock_cvd_utils.ConvertRemoteLogs.return_value = [log] + mock_cvd_utils.FindRemoteLogs.return_value = [log] self.assertEqual("inst", factory.CreateInstance()) - mock_ssh.Ssh.assert_called_once() mock_client_obj.InitRemoteHost.assert_called_once() + mock_cvd_utils.GetRemoteHostBaseDir.assert_called_with(2) + mock_ssh_obj.Run.assert_called_with("mkdir -p acloud_cf_2") mock_cvd_utils.UploadArtifacts.assert_called_with( - mock.ANY, "/mock/img", "/mock/cvd.tar.gz") + mock.ANY, "acloud_cf_2", "/mock/img", "/mock/cvd.tar.gz") + mock_cvd_utils.FindRemoteLogs.assert_called_with( + mock.ANY, "acloud_cf_2", 2, 3) mock_client_obj.LaunchCvd.assert_called_with( - "inst", - mock_avd_spec, - mock_avd_spec.cfg.extra_data_disk_size_gb, - boot_timeout_secs=mock_avd_spec.boot_timeout_secs, - extra_args=["extra"]) + "inst", mock_avd_spec, "acloud_cf_2", ["extra"]) mock_pull.GetAllLogFilePaths.assert_called_once() mock_pull.PullLogs.assert_called_once() + factory.GetAdbPorts() + mock_cvd_utils.GetAdbPorts.assert_called_with(2, 3) + factory.GetFastbootPorts() + mock_cvd_utils.GetFastbootPorts.assert_called_with(2, 3) + factory.GetVncPorts() + mock_cvd_utils.GetVncPorts.assert_called_with(2, 3) self.assertEqual({"inst": "failure"}, factory.GetFailures()) - self.assertEqual({"inst": [tombstones, log]}, factory.GetLogs()) + self.assertDictEqual({"inst": [log]}, factory.GetLogs()) @mock.patch("acloud.public.actions.remote_host_cf_device_factory." "cvd_compute_client_multi_stage") @@ -114,24 +128,38 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): """Test CreateInstance with local image zip.""" mock_avd_spec = self._CreateMockAvdSpec() mock_avd_spec.image_source = constants.IMAGE_SRC_LOCAL + mock_ssh_obj = mock.Mock() + mock_ssh.Ssh.return_value = mock_ssh_obj factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( mock_avd_spec, local_image_artifact="/mock/img.zip", cvd_host_package_artifact="/mock/cvd.tar.gz") mock_client_obj = factory.GetComputeClient() - mock_client_obj.FormatRemoteHostInstanceName.return_value = "inst" mock_client_obj.LaunchCvd.return_value = {} + mock_cvd_utils.GetRemoteHostBaseDir.return_value = "acloud_cf_1" + mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" + mock_cvd_utils.FindRemoteLogs.return_value = [] + self.assertEqual("inst", factory.CreateInstance()) - mock_ssh.Ssh.assert_called_once() + mock_cvd_utils.GetRemoteHostBaseDir.assert_called_with(None) mock_client_obj.InitRemoteHost.assert_called_once() + mock_ssh_obj.Run.assert_called_with("mkdir -p acloud_cf_1") mock_cvd_utils.UploadArtifacts.assert_called_with( - mock.ANY, "/mock/img.zip", "/mock/cvd.tar.gz") + mock.ANY, "acloud_cf_1", "/mock/img.zip", "/mock/cvd.tar.gz") + mock_cvd_utils.FindRemoteLogs.assert_called_with( + mock.ANY, "acloud_cf_1", None, None) mock_client_obj.LaunchCvd.assert_called() - mock_pull.GetAllLogFilePaths.assert_called_once() + mock_pull.GetAllLogFilePaths.assert_not_called() mock_pull.PullLogs.assert_not_called() + factory.GetAdbPorts() + mock_cvd_utils.GetAdbPorts.assert_called_with(None, None) + factory.GetFastbootPorts() + mock_cvd_utils.GetFastbootPorts.assert_called_with(None, None) + factory.GetVncPorts() + mock_cvd_utils.GetVncPorts.assert_called_with(None, None) self.assertFalse(factory.GetFailures()) - self.assertEqual(1, len(factory.GetLogs()["inst"])) + self.assertDictEqual({"inst": []}, factory.GetLogs()) @mock.patch("acloud.public.actions.remote_host_cf_device_factory." "cvd_compute_client_multi_stage") @@ -143,7 +171,7 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): @mock.patch("acloud.public.actions.remote_host_cf_device_factory.glob") @mock.patch("acloud.public.actions.remote_host_cf_device_factory.pull") def testCreateInstanceWithRemoteImages(self, mock_pull, mock_glob, - mock_check_call, _mock_cvd_utils, + mock_check_call, mock_cvd_utils, mock_ssh, _mock_client): """Test CreateInstance with remote images.""" mock_avd_spec = self._CreateMockAvdSpec() @@ -155,24 +183,135 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( mock_avd_spec) + mock_cvd_utils.GetRemoteHostBaseDir.return_value = "acloud_cf_1" + mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" + mock_cvd_utils.FindRemoteLogs.return_value = [] + mock_client_obj = factory.GetComputeClient() - mock_client_obj.FormatRemoteHostInstanceName.return_value = "inst" mock_client_obj.LaunchCvd.return_value = {} self.assertEqual("inst", factory.CreateInstance()) - mock_ssh.Ssh.assert_called_once() mock_client_obj.InitRemoteHost.assert_called_once() + mock_ssh_obj.Run.assert_called_with("mkdir -p acloud_cf_1") mock_check_call.assert_called_once() mock_ssh.ShellCmdWithRetry.assert_called_once() self.assertRegex(mock_ssh.ShellCmdWithRetry.call_args[0][0], r"^tar -cf - --lzop -S -C \S+ super\.img \| " - r"/mock/ssh -- tar -xf - --lzop -S$") + r"/mock/ssh -- tar -xf - --lzop -S -C acloud_cf_1$") mock_client_obj.LaunchCvd.assert_called() - mock_pull.GetAllLogFilePaths.assert_called_once() + mock_pull.GetAllLogFilePaths.assert_not_called() mock_pull.PullLogs.assert_not_called() self.assertFalse(factory.GetFailures()) - self.assertEqual(1, len(factory.GetLogs()["inst"])) + self.assertDictEqual({"inst": []}, factory.GetLogs()) + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_compute_client_multi_stage") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.ssh") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_utils") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.glob") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.shutil") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.pull") + def testCreateInstanceWithRemoteFetch(self, mock_pull, mock_shutil, + mock_glob, mock_cvd_utils, mock_ssh, + _mock_client): + """Test CreateInstance with remotely fetched images.""" + mock_avd_spec = self._CreateMockAvdSpec() + mock_avd_spec.remote_fetch = True + mock_ssh_obj = mock.Mock() + mock_ssh.Ssh.return_value = mock_ssh_obj + mock_ssh_obj.GetBaseCmd.return_value = "/mock/ssh" + mock_glob.glob.return_value = ["/mock/fetch_cvd"] + factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( + mock_avd_spec) + + log = {"path": "/log.txt"} + mock_cvd_utils.GetRemoteHostBaseDir.return_value = "acloud_cf_1" + mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" + mock_cvd_utils.FindRemoteLogs.return_value = [] + mock_cvd_utils.GetRemoteFetcherConfigJson.return_value = log + + mock_client_obj = factory.GetComputeClient() + mock_client_obj.LaunchCvd.return_value = {} + mock_client_obj.build_api.GetFetchBuildArgs.return_value = ["-test"] + + self.assertEqual("inst", factory.CreateInstance()) + mock_client_obj.InitRemoteHost.assert_called_once() + mock_ssh_obj.Run.assert_called_with("mkdir -p acloud_cf_1") + mock_client_obj.build_api.DownloadFetchcvd.assert_called_once() + mock_shutil.copyfile.assert_called_with("/mock/key", mock.ANY) + self.assertRegex(mock_ssh.ShellCmdWithRetry.call_args_list[0][0][0], + r"^tar -cf - --lzop -S -C \S+ fetch_cvd \| " + r"/mock/ssh -- tar -xf - --lzop -S -C acloud_cf_1$") + self.assertRegex(mock_ssh.ShellCmdWithRetry.call_args_list[1][0][0], + r"^/mock/ssh -- acloud_cf_1/fetch_cvd " + r"-directory=acloud_cf_1 " + r"-credential_source=acloud_cf_1/credential_key.json " + r"-test$") + mock_client_obj.LaunchCvd.assert_called() + mock_pull.GetAllLogFilePaths.assert_not_called() + mock_pull.PullLogs.assert_not_called() + self.assertFalse(factory.GetFailures()) + self.assertDictEqual({"inst": [log]}, factory.GetLogs()) + + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_compute_client_multi_stage") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.ssh") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_utils") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.glob") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.shutil") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.pull") + def testCreateInstanceWithFetchCvdWrapper(self, mock_pull, mock_shutil, + mock_glob, mock_cvd_utils, mock_ssh, + _mock_client): + """Test CreateInstance with remotely fetched images.""" + mock_avd_spec = self._CreateMockAvdSpec() + mock_avd_spec.remote_fetch = True + mock_avd_spec.fetch_cvd_wrapper = ( + r"GOOGLE_APPLICATION_CREDENTIALS=/fake_key.json," + r"CACHE_CONFIG=/home/shared/cache.properties," + r"java,-jar,/home/shared/FetchCvdWrapper.jar" + ) + mock_ssh_obj = mock.Mock() + mock_ssh.Ssh.return_value = mock_ssh_obj + mock_ssh_obj.GetBaseCmd.return_value = "/mock/ssh" + mock_glob.glob.return_value = ["/mock/fetch_cvd"] + factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( + mock_avd_spec) + + log = {"path": "/log.txt"} + mock_cvd_utils.GetRemoteHostBaseDir.return_value = "acloud_cf_1" + mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" + mock_cvd_utils.FindRemoteLogs.return_value = [] + mock_cvd_utils.GetRemoteFetcherConfigJson.return_value = log + + mock_client_obj = factory.GetComputeClient() + mock_client_obj.LaunchCvd.return_value = {} + mock_client_obj.build_api.GetFetchBuildArgs.return_value = ["-test"] + + self.assertEqual("inst", factory.CreateInstance()) + mock_client_obj.InitRemoteHost.assert_called_once() + mock_ssh_obj.Run.assert_called_with("mkdir -p acloud_cf_1") + mock_client_obj.build_api.DownloadFetchcvd.assert_called_once() + mock_shutil.copyfile.assert_called_with("/mock/key", mock.ANY) + self.assertRegex(mock_ssh.ShellCmdWithRetry.call_args_list[0][0][0], + r"^tar -cf - --lzop -S -C \S+ fetch_cvd \| " + r"/mock/ssh -- tar -xf - --lzop -S -C acloud_cf_1$") + self.assertRegex(mock_ssh.ShellCmdWithRetry.call_args_list[1][0][0], + r"^/mock/ssh -- " + r"GOOGLE_APPLICATION_CREDENTIALS=/fake_key.json " + r"CACHE_CONFIG=/home/shared/cache.properties " + r"java -jar /home/shared/FetchCvdWrapper.jar " + r"-directory=acloud_cf_1 " + r"-fetch_cvd_path=acloud_cf_1/fetch_cvd " + r"-credential_source=acloud_cf_1/credential_key.json " + r"-test$") + mock_client_obj.LaunchCvd.assert_called() + mock_pull.GetAllLogFilePaths.assert_not_called() + mock_pull.PullLogs.assert_not_called() + self.assertFalse(factory.GetFailures()) + self.assertDictEqual({"inst": [log]}, factory.GetLogs()) if __name__ == "__main__": unittest.main() diff --git a/public/actions/remote_host_gf_device_factory.py b/public/actions/remote_host_gf_device_factory.py index 5cf82503..c4b44c4d 100644 --- a/public/actions/remote_host_gf_device_factory.py +++ b/public/actions/remote_host_gf_device_factory.py @@ -21,17 +21,20 @@ import os import posixpath as remote_path import re import shutil +import subprocess import tempfile +import time import zipfile from acloud import errors +from acloud.create import create_common from acloud.internal import constants from acloud.internal.lib import android_build_client from acloud.internal.lib import auth -from acloud.internal.lib import goldfish_remote_host_client from acloud.internal.lib import goldfish_utils from acloud.internal.lib import emulator_console from acloud.internal.lib import ota_tools +from acloud.internal.lib import remote_host_client from acloud.internal.lib import utils from acloud.internal.lib import ssh from acloud.public import report @@ -45,32 +48,41 @@ _SDK_REPO_IMAGE_ZIP_NAME_FORMAT = ("sdk-repo-linux-system-images-" _EXTRA_IMAGE_ZIP_NAME_FORMAT = "emu-extra-linux-system-images-%(build_id)s.zip" _IMAGE_ZIP_NAME_FORMAT = "%(build_target)s-img-%(build_id)s.zip" _OTA_TOOLS_ZIP_NAME = "otatools.zip" -_SYSTEM_IMAGE_NAME = "system.img" - _EMULATOR_INFO_NAME = "emulator-info.txt" _EMULATOR_VERSION_PATTERN = re.compile(r"require\s+version-emulator=" r"(?P<build_id>\w+)") _EMULATOR_ZIP_NAME_FORMAT = "sdk-repo-%(os)s-emulator-%(build_id)s.zip" _EMULATOR_BIN_DIR_NAMES = ("bin64", "qemu") _EMULATOR_BIN_NAME = "emulator" -# Remote paths -_REMOTE_WORKING_DIR = "acloud_gf" -_REMOTE_ARTIFACT_DIR = remote_path.join(_REMOTE_WORKING_DIR, "artifact") -_REMOTE_IMAGE_DIR = remote_path.join(_REMOTE_WORKING_DIR, "image") -_REMOTE_KERNEL_PATH = remote_path.join(_REMOTE_WORKING_DIR, "kernel") -_REMOTE_RAMDISK_PATH = remote_path.join(_REMOTE_WORKING_DIR, "mixed_ramdisk") -_REMOTE_EMULATOR_DIR = remote_path.join(_REMOTE_WORKING_DIR, "emulator") -_REMOTE_INSTANCE_DIR = remote_path.join(_REMOTE_WORKING_DIR, "instance") -_REMOTE_LOGCAT_PATH = os.path.join(_REMOTE_INSTANCE_DIR, "logcat.txt") -_REMOTE_STDOUTERR_PATH = os.path.join(_REMOTE_INSTANCE_DIR, "kernel.log") +_SDK_REPO_EMULATOR_DIR_NAME = "emulator" +# Files in temporary artifact directory. +_DOWNLOAD_DIR_NAME = "download" +_OTA_TOOLS_DIR_NAME = "ota_tools" +_SYSTEM_IMAGE_NAME = "system.img" +# Base directory of an instance. +_REMOTE_INSTANCE_DIR_FORMAT = "acloud_gf_%d" +# Relative paths in a base directory. +_REMOTE_IMAGE_ZIP_PATH = "image.zip" +_REMOTE_EMULATOR_ZIP_PATH = "emulator.zip" +_REMOTE_IMAGE_DIR = "image" +_REMOTE_KERNEL_PATH = "kernel" +_REMOTE_RAMDISK_PATH = "mixed_ramdisk" +_REMOTE_EMULATOR_DIR = "emulator" +_REMOTE_RUNTIME_DIR = "instance" +_REMOTE_LOGCAT_PATH = remote_path.join(_REMOTE_RUNTIME_DIR, "logcat.txt") +_REMOTE_STDOUT_PATH = remote_path.join(_REMOTE_RUNTIME_DIR, "kernel.log") +_REMOTE_STDERR_PATH = remote_path.join(_REMOTE_RUNTIME_DIR, "emu_stderr.txt") # Runtime parameters _EMULATOR_DEFAULT_CONSOLE_PORT = 5554 _DEFAULT_BOOT_TIMEOUT_SECS = 150 +# Error messages +_MISSING_EMULATOR_MSG = ("No emulator zip. Specify " + "--emulator-build-id, or --emulator-zip.") ArtifactPaths = collections.namedtuple( "ArtifactPaths", - ["image_zip", "emulator_zip", "ota_tools_zip", - "system_image_zip", "boot_image"]) + ["image_zip", "emulator_zip", "ota_tools_dir", + "system_image", "boot_image"]) RemotePaths = collections.namedtuple( "RemotePaths", @@ -82,6 +94,9 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Attributes: avd_spec: AVDSpec object that tells us what we're going to create. + android_build_client: An AndroidBuildClient that is lazily initialized. + temp_artifact_dir: The temporary artifact directory that is lazily + initialized during PrepareArtifacts. ssh: Ssh object that executes commands on the remote host. failures: A dictionary the maps instance names to error.DeviceBootError objects. @@ -90,6 +105,8 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): def __init__(self, avd_spec): """Initialize the attributes and the compute client.""" self._avd_spec = avd_spec + self._android_build_client = None + self._temp_artifact_dir = None self._ssh = ssh.Ssh( ip=ssh.IP(ip=self._avd_spec.remote_host), user=self._ssh_user, @@ -99,7 +116,32 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): self._failures = {} self._logs = {} super().__init__(compute_client=( - goldfish_remote_host_client.GoldfishRemoteHostClient())) + remote_host_client.RemoteHostClient(avd_spec.remote_host))) + + @property + def _build_api(self): + """Initialize android_build_client.""" + if not self._android_build_client: + credentials = auth.CreateCredentials(self._avd_spec.cfg) + self._android_build_client = android_build_client.AndroidBuildClient( + credentials) + return self._android_build_client + + @property + def _artifact_dir(self): + """Initialize temp_artifact_dir.""" + if not self._temp_artifact_dir: + self._temp_artifact_dir = tempfile.mkdtemp("host_gf") + logger.info("Create temporary artifact directory: %s", + self._temp_artifact_dir) + return self._temp_artifact_dir + + @property + def _download_dir(self): + """Get the directory where the artifacts are downloaded.""" + if self._avd_spec.image_download_dir: + return self._avd_spec.image_download_dir + return os.path.join(self._artifact_dir, _DOWNLOAD_DIR_NAME) @property def _ssh_user(self): @@ -114,38 +156,69 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): def _ssh_extra_args(self): return self._avd_spec.cfg.extra_args_ssh_tunnel + def _GetConsolePort(self): + """Calculate the console port from the instance number. + + By convention, the console port is an even number, and the adb port is + the console port + 1. The first instance uses port 5554 and 5555. The + second instance uses 5556 and 5557, and so on. + """ + return (_EMULATOR_DEFAULT_CONSOLE_PORT + + ((self._avd_spec.base_instance_num or 1) - 1) * 2) + + def _GetInstancePath(self, relative_path): + """Append a relative path to the instance directory.""" + return remote_path.join( + _REMOTE_INSTANCE_DIR_FORMAT % + (self._avd_spec.base_instance_num or 1), + relative_path) + def CreateInstance(self): """Create a goldfish instance on the remote host. Returns: The instance name. """ - self._InitRemoteHost() - remote_paths = self._PrepareArtifacts() - - instance_name = goldfish_remote_host_client.FormatInstanceName( + instance_name = goldfish_utils.FormatRemoteHostInstanceName( self._avd_spec.remote_host, - _EMULATOR_DEFAULT_CONSOLE_PORT, + self._GetConsolePort(), self._avd_spec.remote_image) - self._logs[instance_name] = [ - report.LogFile(_REMOTE_STDOUTERR_PATH, - constants.LOG_TYPE_KERNEL_LOG), - report.LogFile(_REMOTE_LOGCAT_PATH, constants.LOG_TYPE_LOGCAT)] + + client = self.GetComputeClient() + timed_stage = constants.TIME_GCE + start_time = time.time() try: + client.SetStage(constants.STAGE_SSH_CONNECT) + self._InitRemoteHost() + + start_time = client.RecordTime(timed_stage, start_time) + timed_stage = constants.TIME_ARTIFACT + client.SetStage(constants.STAGE_ARTIFACT) + remote_paths = self._PrepareArtifacts() + + start_time = client.RecordTime(timed_stage, start_time) + timed_stage = constants.TIME_LAUNCH + client.SetStage(constants.STAGE_BOOT_UP) + self._logs[instance_name] = self._GetEmulatorLogs() self._StartEmulator(remote_paths) self._WaitForEmulator() - except errors.DeviceBootError as e: + except (errors.DriverError, subprocess.CalledProcessError) as e: + # Catch the generic runtime error and CalledProcessError which is + # raised by the ssh module. self._failures[instance_name] = e + finally: + client.RecordTime(timed_stage, start_time) + return instance_name def _InitRemoteHost(self): - """Remove existing instance and working directory.""" + """Remove the existing instance and the instance directory.""" # Disable authentication for emulator console. self._ssh.Run("""'echo -n "" > .emulator_console_auth_token'""") try: with emulator_console.RemoteEmulatorConsole( self._avd_spec.remote_host, - _EMULATOR_DEFAULT_CONSOLE_PORT, + self._GetConsolePort(), self._ssh_user, self._ssh_private_key_path, self._ssh_extra_args) as console: @@ -154,7 +227,7 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): except errors.DeviceConnectionError as e: logger.info("Did not kill existing emulator: %s", str(e)) # Delete instance files. - self._ssh.Run("rm -rf %s" % _REMOTE_WORKING_DIR) + self._ssh.Run(f"rm -rf {self._GetInstancePath('')}") def _PrepareArtifacts(self): """Prepare artifacts on remote host. @@ -165,21 +238,13 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: An object of RemotePaths. """ - if self._avd_spec.image_download_dir: - temp_download_dir = None - download_dir = self._avd_spec.image_download_dir - else: - temp_download_dir = tempfile.mkdtemp() - download_dir = temp_download_dir - logger.info("--image-download-dir is not specified. Create " - "temporary download directory: %s", download_dir) - try: - artifact_paths = self._RetrieveArtifacts(download_dir) + artifact_paths = self._RetrieveArtifacts() return self._UploadArtifacts(artifact_paths) finally: - if temp_download_dir: - shutil.rmtree(temp_download_dir, ignore_errors=True) + if self._temp_artifact_dir: + shutil.rmtree(self._temp_artifact_dir, ignore_errors=True) + self._temp_artifact_dir = None @staticmethod def _InferEmulatorZipName(build_target, build_id): @@ -191,7 +256,7 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Args: build_target: The emulator build target name, e.g., - "sdk_tools_linux", "aarch64_sdk_tools_mac". + "emulator-linux_x64_nolocationui", "aarch64_sdk_tools_mac". build_id: A string, the emulator build ID. Returns: @@ -210,14 +275,11 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): return _EMULATOR_ZIP_NAME_FORMAT % {"os": os_name, "build_id": build_id} - @staticmethod - def _RetrieveArtifact(download_dir, build_api, build_target, build_id, + def _RetrieveArtifact(self, build_target, build_id, resource_id): """Retrieve an artifact from cache or Android Build API. Args: - download_dir: The cache directory. - build_api: An AndroidBuildClient object. build_target: A string, the build target of the artifact. e.g., "sdk_phone_x86_64-userdebug". build_id: A string, the build ID of the artifact. @@ -227,7 +289,7 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: The path to the artifact in download_dir. """ - local_path = os.path.join(download_dir, build_id, build_target, + local_path = os.path.join(self._download_dir, build_id, build_target, resource_id) if os.path.isfile(local_path): logger.info("Skip downloading existing artifact: %s", local_path) @@ -236,129 +298,187 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): complete = False try: os.makedirs(os.path.dirname(local_path), exist_ok=True) - build_api.DownloadArtifact(build_target, build_id, resource_id, - local_path, build_api.LATEST) + self._build_api.DownloadArtifact( + build_target, build_id, resource_id, local_path, + self._build_api.LATEST) complete = True finally: if not complete and os.path.isfile(local_path): os.remove(local_path) return local_path - def _RetrieveEmulatorBuildID(self, download_dir, build_api, build_target, - build_id): - """Retrieve required emulator build from a goldfish image build.""" - emulator_info_path = self._RetrieveArtifact(download_dir, build_api, - build_target, build_id, - _EMULATOR_INFO_NAME) - with open(emulator_info_path, 'r') as emulator_info: - for line in emulator_info: - match = _EMULATOR_VERSION_PATTERN.fullmatch(line.strip()) - if match: - logger.info("Found emulator build ID: %s", line) - return match.group("build_id") - return None - @utils.TimeExecute(function_description="Download Android Build artifacts") - def _RetrieveArtifacts(self, download_dir): + def _RetrieveArtifacts(self): """Retrieve goldfish images and tools from cache or Android Build API. - Args: - download_dir: The cache directory. - Returns: An object of ArtifactPaths. Raises: errors.GetRemoteImageError: Fails to download rom images. + errors.GetLocalImageError: Fails to validate local image zip. + errors.GetSdkRepoPackageError: Fails to retrieve emulator zip. """ - credentials = auth.CreateCredentials(self._avd_spec.cfg) - build_api = android_build_client.AndroidBuildClient(credentials) # Device images. + if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: + image_zip_path = self._RetrieveDeviceImageZip() + elif self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: + image_zip_path = self._avd_spec.local_image_artifact + if not image_zip_path or not zipfile.is_zipfile(image_zip_path): + raise errors.GetLocalImageError( + f"{image_zip_path or self._avd_spec.local_image_dir} is " + "not an SDK repository zip.") + else: + raise errors.CreateError( + f"Unknown image source: {self._avd_spec.image_source}") + + # Emulator tools. + emu_zip_path = (self._avd_spec.emulator_zip or + self._RetrieveEmulatorZip()) + if not emu_zip_path: + raise errors.GetSdkRepoPackageError(_MISSING_EMULATOR_MSG) + + # System image. + if self._avd_spec.local_system_image: + system_image_path = create_common.FindSystemImage( + self._avd_spec.local_system_image) + else: + system_image_path = self._RetrieveSystemImage() + + # Boot image. + if self._avd_spec.local_kernel_image: + boot_image_path = create_common.FindBootImage( + self._avd_spec.local_kernel_image) + else: + boot_image_path = self._RetrieveBootImage() + + # OTA tools. + ota_tools_dir = None + if system_image_path or boot_image_path: + if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: + ota_tools_dir = self._RetrieveOtaTools() + else: + ota_tools_dir = ota_tools.FindOtaToolsDir( + self._avd_spec.local_tool_dirs + + create_common.GetNonEmptyEnvVars( + constants.ENV_ANDROID_SOONG_HOST_OUT, + constants.ENV_ANDROID_HOST_OUT)) + + return ArtifactPaths(image_zip_path, emu_zip_path, ota_tools_dir, + system_image_path, boot_image_path) + + def _RetrieveDeviceImageZip(self): + """Retrieve device image zip from cache or Android Build API. + + Returns: + The path to the device image zip in download_dir. + """ build_id = self._avd_spec.remote_image.get(constants.BUILD_ID) build_target = self._avd_spec.remote_image.get(constants.BUILD_TARGET) image_zip_name_format = (_EXTRA_IMAGE_ZIP_NAME_FORMAT if self._ShouldMixDiskImage() else _SDK_REPO_IMAGE_ZIP_NAME_FORMAT) - image_zip_path = self._RetrieveArtifact( - download_dir, build_api, build_target, build_id, + return self._RetrieveArtifact( + build_target, build_id, image_zip_name_format % {"build_id": build_id}) - # Emulator tools. - emu_build_id = self._avd_spec.emulator_build_id - if not emu_build_id: - emu_build_id = self._RetrieveEmulatorBuildID( - download_dir, build_api, build_target, build_id) - if not emu_build_id: - raise errors.GetRemoteImageError( - "No emulator build ID in command line or " - "emulator-info.txt.") + def _RetrieveEmulatorBuildID(self): + """Retrieve required emulator build from a goldfish image build. + + Returns: + A string, the emulator build ID. + None if the build info is empty. + """ + build_id = self._avd_spec.remote_image.get(constants.BUILD_ID) + build_target = self._avd_spec.remote_image.get(constants.BUILD_TARGET) + if build_id and build_target: + emu_info_path = self._RetrieveArtifact(build_target, build_id, + _EMULATOR_INFO_NAME) + with open(emu_info_path, "r", encoding="utf-8") as emu_info: + for line in emu_info: + match = _EMULATOR_VERSION_PATTERN.fullmatch(line.strip()) + if match: + logger.info("Found emulator build ID: %s", line) + return match.group("build_id") + return None + def _RetrieveEmulatorZip(self): + """Retrieve emulator zip from cache or Android Build API. + + Returns: + The path to the emulator zip in download_dir. + None if this method cannot determine the emulator build ID. + """ + emu_build_id = (self._avd_spec.emulator_build_id or + self._RetrieveEmulatorBuildID()) + if not emu_build_id: + return None emu_build_target = (self._avd_spec.emulator_build_target or self._avd_spec.cfg.emulator_build_target) emu_zip_name = self._InferEmulatorZipName(emu_build_target, emu_build_id) - emu_zip_path = self._RetrieveArtifact(download_dir, build_api, - emu_build_target, emu_build_id, - emu_zip_name) - - system_image_zip_path = self._RetrieveSystemImageZip( - download_dir, build_api) - boot_image_path = self._RetrieveBootImage(download_dir, build_api) - # Retrieve OTA tools from the goldfish build which contains - # mk_combined_img. - ota_tools_zip_path = ( - self._RetrieveArtifact(download_dir, build_api, build_target, - build_id, _OTA_TOOLS_ZIP_NAME) - if system_image_zip_path or boot_image_path else None) - - return ArtifactPaths(image_zip_path, emu_zip_path, - ota_tools_zip_path, system_image_zip_path, - boot_image_path) - - def _RetrieveSystemImageZip(self, download_dir, build_api): - """Retrieve system image zip if system build info is not empty. + return self._RetrieveArtifact(emu_build_target, emu_build_id, + emu_zip_name) - Args: - download_dir: The download cache directory. - build_api: An AndroidBuildClient object. + def _RetrieveSystemImage(self): + """Retrieve and unzip system image if system build info is not empty. Returns: - The path to the system image zip in download_dir. + The path to the temporary system image. None if the system build info is empty. """ build_id = self._avd_spec.system_build_info.get(constants.BUILD_ID) build_target = self._avd_spec.system_build_info.get( constants.BUILD_TARGET) - if build_id and build_target: - image_zip_name = _IMAGE_ZIP_NAME_FORMAT % { - "build_target": build_target.split("-", 1)[0], - "build_id": build_id} - return self._RetrieveArtifact( - download_dir, build_api, build_target, build_id, - image_zip_name) - return None - - def _RetrieveBootImage(self, download_dir, build_api): - """Retrieve boot image if kernel build info is not empty. - - Args: - download_dir: The download cache directory. - build_api: An AndroidBuildClient object. + if not build_id or not build_target: + return None + image_zip_name = _IMAGE_ZIP_NAME_FORMAT % { + "build_target": build_target.split("-", 1)[0], + "build_id": build_id} + image_zip_path = self._RetrieveArtifact(build_target, build_id, + image_zip_name) + logger.debug("Unzip %s from %s to %s.", + _SYSTEM_IMAGE_NAME, image_zip_path, self._artifact_dir) + with zipfile.ZipFile(image_zip_path, "r") as zip_file: + zip_file.extract(_SYSTEM_IMAGE_NAME, self._artifact_dir) + return os.path.join(self._artifact_dir, _SYSTEM_IMAGE_NAME) + + def _RetrieveBootImage(self): + """Retrieve boot image if boot build info is not empty. Returns: The path to the boot image in download_dir. - None if the kernel build info is empty. + None if the boot build info is empty. """ - build_id = self._avd_spec.kernel_build_info.get(constants.BUILD_ID) - build_target = self._avd_spec.kernel_build_info.get( + build_id = self._avd_spec.boot_build_info.get(constants.BUILD_ID) + build_target = self._avd_spec.boot_build_info.get( constants.BUILD_TARGET) - image_name = self._avd_spec.kernel_build_info.get( + image_name = self._avd_spec.boot_build_info.get( constants.BUILD_ARTIFACT) if build_id and build_target and image_name: - return self._RetrieveArtifact( - download_dir, build_api, build_target, build_id, image_name) + return self._RetrieveArtifact(build_target, build_id, image_name) return None + def _RetrieveOtaTools(self): + """Retrieve and unzip OTA tools. + + This method retrieves OTA tools from the goldfish build which contains + mk_combined_img. + + Returns: + The path to the temporary OTA tools directory. + """ + build_id = self._avd_spec.remote_image.get(constants.BUILD_ID) + build_target = self._avd_spec.remote_image.get(constants.BUILD_TARGET) + zip_path = self._RetrieveArtifact(build_target, build_id, + _OTA_TOOLS_ZIP_NAME) + ota_tools_dir = os.path.join(self._artifact_dir, _OTA_TOOLS_DIR_NAME) + logger.debug("Unzip %s to %s.", zip_path, ota_tools_dir) + os.mkdir(ota_tools_dir) + with zipfile.ZipFile(zip_path, "r") as zip_file: + zip_file.extractall(ota_tools_dir) + return ota_tools_dir + @staticmethod def _GetSubdirNameInZip(zip_path): """Get the name of the only subdirectory in a zip. @@ -367,12 +487,12 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): subdirectory. This class needs to find out the subdirectory name in order to construct the remote commands. - For example, in sdk-repo-*-emulator-*.zip, all files are in - "emulator/". The zip entries are: + For example, in a sdk-repo-linux-system-images-*.zip for arm64, all + files are in "arm64-v8a/". The zip entries are: - emulator/NOTICE.txt - emulator/emulator - emulator/lib64/libc++.so + arm64-v8a/NOTICE.txt + arm64-v8a/system.img + arm64-v8a/data/local.prop ... This method scans the entries and returns the common subdirectory name. @@ -388,7 +508,7 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): zip_path, " ".join(entries)) return "" - def _UploadArtifacts(self, artifacts_paths): + def _UploadArtifacts(self, artifact_paths): """Process and upload all images and tools to the remote host. Args: @@ -398,38 +518,33 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): An object of RemotePaths. """ remote_emulator_dir, remote_image_dir = self._UploadDeviceImages( - artifacts_paths.emulator_zip, artifacts_paths.image_zip) + artifact_paths.emulator_zip, artifact_paths.image_zip) remote_kernel_path = None remote_ramdisk_path = None - if artifacts_paths.boot_image or artifacts_paths.system_image_zip: + if artifact_paths.boot_image or artifact_paths.system_image: with tempfile.TemporaryDirectory("host_gf") as temp_dir: - ota_tools_dir = os.path.join(temp_dir, "ota_tools") - logger.debug("Unzip %s.", artifacts_paths.ota_tools_zip) - with zipfile.ZipFile(artifacts_paths.ota_tools_zip, - "r") as zip_file: - zip_file.extractall(ota_tools_dir) - ota = ota_tools.OtaTools(ota_tools_dir) + ota = ota_tools.OtaTools(artifact_paths.ota_tools_dir) image_dir = os.path.join(temp_dir, "images") - logger.debug("Unzip %s.", artifacts_paths.image_zip) - with zipfile.ZipFile(artifacts_paths.image_zip, + logger.debug("Unzip %s.", artifact_paths.image_zip) + with zipfile.ZipFile(artifact_paths.image_zip, "r") as zip_file: zip_file.extractall(image_dir) image_dir = os.path.join( image_dir, - self._GetSubdirNameInZip(artifacts_paths.image_zip)) + self._GetSubdirNameInZip(artifact_paths.image_zip)) - if artifacts_paths.system_image_zip: + if artifact_paths.system_image: self._MixAndUploadDiskImage( remote_image_dir, image_dir, - artifacts_paths.system_image_zip, ota) + artifact_paths.system_image, ota) - if artifacts_paths.boot_image: + if artifact_paths.boot_image: remote_kernel_path, remote_ramdisk_path = ( self._MixAndUploadKernelImages( - image_dir, artifacts_paths.boot_image, ota)) + image_dir, artifact_paths.boot_image, ota)) return RemotePaths(remote_image_dir, remote_emulator_dir, remote_kernel_path, remote_ramdisk_path) @@ -444,8 +559,9 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: Boolean, whether a mixed disk image is required. """ - return (self._avd_spec.system_build_info.get(constants.BUILD_ID) and - self._avd_spec.system_build_info.get(constants.BUILD_TARGET)) + return self._avd_spec.local_system_image or ( + self._avd_spec.system_build_info.get(constants.BUILD_ID) and + self._avd_spec.system_build_info.get(constants.BUILD_TARGET)) @utils.TimeExecute( function_description="Processing and uploading tools and images") @@ -459,24 +575,22 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: The remote paths to the extracted emulator tools and images. """ - self._ssh.Run("mkdir -p " + - " ".join([_REMOTE_INSTANCE_DIR, _REMOTE_ARTIFACT_DIR, - _REMOTE_EMULATOR_DIR, _REMOTE_IMAGE_DIR])) - self._ssh.ScpPushFile(emulator_zip_path, _REMOTE_ARTIFACT_DIR) - self._ssh.ScpPushFile(image_zip_path, _REMOTE_ARTIFACT_DIR) - - self._ssh.Run("unzip -d %s %s" % ( - _REMOTE_EMULATOR_DIR, - remote_path.join(_REMOTE_ARTIFACT_DIR, - os.path.basename(emulator_zip_path)))) - self._ssh.Run("unzip -d %s %s" % ( - _REMOTE_IMAGE_DIR, - remote_path.join(_REMOTE_ARTIFACT_DIR, - os.path.basename(image_zip_path)))) + remote_emulator_dir = self._GetInstancePath(_REMOTE_EMULATOR_DIR) + remote_image_dir = self._GetInstancePath(_REMOTE_IMAGE_DIR) + remote_emulator_zip_path = self._GetInstancePath( + _REMOTE_EMULATOR_ZIP_PATH) + remote_image_zip_path = self._GetInstancePath(_REMOTE_IMAGE_ZIP_PATH) + self._ssh.Run(f"mkdir -p {remote_emulator_dir} {remote_image_dir}") + self._ssh.ScpPushFile(emulator_zip_path, remote_emulator_zip_path) + self._ssh.ScpPushFile(image_zip_path, remote_image_zip_path) + + self._ssh.Run(f"unzip -d {remote_emulator_dir} " + f"{remote_emulator_zip_path}") + self._ssh.Run(f"unzip -d {remote_image_dir} {remote_image_zip_path}") remote_emulator_subdir = remote_path.join( - _REMOTE_EMULATOR_DIR, self._GetSubdirNameInZip(emulator_zip_path)) + remote_emulator_dir, _SDK_REPO_EMULATOR_DIR_NAME) remote_image_subdir = remote_path.join( - _REMOTE_IMAGE_DIR, self._GetSubdirNameInZip(image_zip_path)) + remote_image_dir, self._GetSubdirNameInZip(image_zip_path)) # TODO(b/141898893): In Android build environment, emulator gets build # information from $ANDROID_PRODUCT_OUT/system/build.prop. # If image_dir is an extacted SDK repository, the file is at @@ -493,30 +607,22 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): return remote_emulator_subdir, remote_image_subdir def _MixAndUploadDiskImage(self, remote_image_dir, image_dir, - system_image_zip_path, ota): + system_image_path, ota): """Mix emulator images with a system image and upload them. Args: remote_image_dir: The remote directory where the mixed disk image is uploaded. image_dir: The directory containing emulator images. - system_image_zip_path: The path to the zip containing the system - image. + system_image_path: The path to the system image. ota: An instance of ota_tools.OtaTools. Returns: The remote path to the mixed disk image. """ with tempfile.TemporaryDirectory("host_gf_disk") as temp_dir: - logger.debug("Unzip %s.", system_image_zip_path) - with zipfile.ZipFile(system_image_zip_path, "r") as zip_file: - zip_file.extract(_SYSTEM_IMAGE_NAME, temp_dir) - mixed_image = goldfish_utils.MixWithSystemImage( - os.path.join(temp_dir, "mix_disk"), - image_dir, - os.path.join(temp_dir, _SYSTEM_IMAGE_NAME), - ota) + temp_dir, image_dir, system_image_path, ota) # TODO(b/142228085): Use -system instead of overwriting the file. remote_disk_image_path = os.path.join( @@ -536,14 +642,25 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: The remote paths to the kernel image and the ramdisk image. """ + remote_kernel_path = self._GetInstancePath(_REMOTE_KERNEL_PATH) + remote_ramdisk_path = self._GetInstancePath(_REMOTE_RAMDISK_PATH) with tempfile.TemporaryDirectory("host_gf_kernel") as temp_dir: kernel_path, ramdisk_path = goldfish_utils.MixWithBootImage( temp_dir, image_dir, boot_image_path, ota) - self._ssh.ScpPushFile(kernel_path, _REMOTE_KERNEL_PATH) - self._ssh.ScpPushFile(ramdisk_path, _REMOTE_RAMDISK_PATH) + self._ssh.ScpPushFile(kernel_path, remote_kernel_path) + self._ssh.ScpPushFile(ramdisk_path, remote_ramdisk_path) - return _REMOTE_KERNEL_PATH, _REMOTE_RAMDISK_PATH + return remote_kernel_path, remote_ramdisk_path + + def _GetEmulatorLogs(self): + """Return the logs created by the remote emulator command.""" + return [report.LogFile(self._GetInstancePath(_REMOTE_STDOUT_PATH), + constants.LOG_TYPE_KERNEL_LOG), + report.LogFile(self._GetInstancePath(_REMOTE_STDERR_PATH), + constants.LOG_TYPE_TEXT), + report.LogFile(self._GetInstancePath(_REMOTE_LOGCAT_PATH), + constants.LOG_TYPE_LOGCAT)] @utils.TimeExecute(function_description="Start emulator") def _StartEmulator(self, remote_paths): @@ -561,16 +678,16 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): remote_bin_paths.append(remote_emulator_bin_path) self._ssh.Run("chmod -R +x %s" % " ".join(remote_bin_paths)) + remote_runtime_dir = self._GetInstancePath(_REMOTE_RUNTIME_DIR) + self._ssh.Run(f"mkdir -p {remote_runtime_dir}") env = {constants.ENV_ANDROID_PRODUCT_OUT: remote_paths.image_dir, - constants.ENV_ANDROID_TMP: _REMOTE_INSTANCE_DIR, - constants.ENV_ANDROID_BUILD_TOP: _REMOTE_INSTANCE_DIR} - adb_port = _EMULATOR_DEFAULT_CONSOLE_PORT + 1 + constants.ENV_ANDROID_TMP: remote_runtime_dir, + constants.ENV_ANDROID_BUILD_TOP: remote_runtime_dir} cmd = ["nohup", remote_emulator_bin_path, "-verbose", "-show-kernel", "-read-only", "-ports", - str(_EMULATOR_DEFAULT_CONSOLE_PORT) + "," + str(adb_port), + str(self._GetConsolePort()) + "," + str(self.GetAdbPorts()[0]), "-no-window", - "-logcat-output", _REMOTE_LOGCAT_PATH, - "-stdouterr-file", _REMOTE_STDOUTERR_PATH] + "-logcat-output", self._GetInstancePath(_REMOTE_LOGCAT_PATH)] if remote_paths.kernel: cmd.extend(("-kernel", remote_paths.kernel)) @@ -586,12 +703,13 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): cmd.extend(("-qemu", "-append", "androidboot.verifiedbootstate=orange")) - # Emulator doesn't create -stdouterr-file automatically. + # Emulator does not support -stdouterr-file on macOS. self._ssh.Run( - "'export {env} ; touch {stdouterr} ; {cmd} &'".format( + "'export {env} ; {cmd} 1> {stdout} 2> {stderr} &'".format( env=" ".join(k + "=~/" + v for k, v in env.items()), - stdouterr=_REMOTE_STDOUTERR_PATH, - cmd=" ".join(cmd))) + cmd=" ".join(cmd), + stdout=self._GetInstancePath(_REMOTE_STDOUT_PATH), + stderr=self._GetInstancePath(_REMOTE_STDERR_PATH))) @utils.TimeExecute(function_description="Wait for emulator") def _WaitForEmulator(self): @@ -602,7 +720,7 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): errors.DeviceBootTimeoutError if boot times out. """ ip_addr = self._avd_spec.remote_host - console_port = _EMULATOR_DEFAULT_CONSOLE_PORT + console_port = self._GetConsolePort() poll_timeout_secs = (self._avd_spec.boot_timeout_secs or _DEFAULT_BOOT_TIMEOUT_SECS) try: @@ -633,6 +751,16 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): self._avd_spec.remote_image.items() if val} return build_info_dict + def GetAdbPorts(self): + """Get ADB ports of the created devices. + + This class does not support --num-avds-per-instance. + + Returns: + The port numbers as a list of integers. + """ + return [self._GetConsolePort() + 1] + def GetFailures(self): """Get Failures from all devices. diff --git a/public/actions/remote_host_gf_device_factory_test.py b/public/actions/remote_host_gf_device_factory_test.py index 8321eb32..f0d5a9cd 100644 --- a/public/actions/remote_host_gf_device_factory_test.py +++ b/public/actions/remote_host_gf_device_factory_test.py @@ -44,14 +44,15 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): constants.BUILD_TARGET: "sdk_arm64-sdk", } _ARM64_INSTANCE_NAME = ( - "host-goldfish-192.0.2.1-5554-123456-sdk_arm64-sdk") + "host-goldfish-192.0.2.1-5556-123456-sdk_arm64-sdk") _CFG_ATTRS = { "ssh_private_key_path": "cfg_key_path", "extra_args_ssh_tunnel": "extra args", - "emulator_build_target": "sdk_tools_linux", + "emulator_build_target": "emulator-linux_x64_nolocationui", } _AVD_SPEC_ATTRS = { "cfg": None, + "image_source": constants.IMAGE_SRC_REMOTE, "remote_image": _X86_64_BUILD_INFO, "image_download_dir": None, "host_user": "user", @@ -59,33 +60,42 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): "host_ssh_private_key_path": None, "emulator_build_id": None, "emulator_build_target": None, + "emulator_zip": None, "system_build_info": {}, - "kernel_build_info": {}, + "boot_build_info": {}, + "local_image_artifact": None, + "local_kernel_image": None, + "local_system_image": None, + "local_tool_dirs": [], + "base_instance_num": None, "boot_timeout_secs": None, "hw_customize": False, "hw_property": {}, "gpu": "auto", } - _LOGS = [{"path": "acloud_gf/instance/kernel.log", "type": "KERNEL_LOG"}, - {"path": "acloud_gf/instance/logcat.txt", "type": "LOGCAT"}] + _LOGS = [{"path": "acloud_gf_1/instance/kernel.log", "type": "KERNEL_LOG"}, + {"path": "acloud_gf_1/instance/emu_stderr.txt", "type": "TEXT"}, + {"path": "acloud_gf_1/instance/logcat.txt", "type": "LOGCAT"}] _SSH_COMMAND = ( - "'export ANDROID_PRODUCT_OUT=~/acloud_gf/image/x86_64 " - "ANDROID_TMP=~/acloud_gf/instance " - "ANDROID_BUILD_TOP=~/acloud_gf/instance ; " - "touch acloud_gf/instance/kernel.log ; " - "nohup acloud_gf/emulator/x86_64/emulator -verbose " + "'export ANDROID_PRODUCT_OUT=~/acloud_gf_1/image/x86_64 " + "ANDROID_TMP=~/acloud_gf_1/instance " + "ANDROID_BUILD_TOP=~/acloud_gf_1/instance ; " + "nohup acloud_gf_1/emulator/emulator/emulator -verbose " "-show-kernel -read-only -ports 5554,5555 -no-window " - "-logcat-output acloud_gf/instance/logcat.txt " - "-stdouterr-file acloud_gf/instance/kernel.log -gpu auto &'" + "-logcat-output acloud_gf_1/instance/logcat.txt -gpu auto " + "1> acloud_gf_1/instance/kernel.log " + "2> acloud_gf_1/instance/emu_stderr.txt &'" ) def setUp(self): super().setUp() self._mock_ssh = mock.Mock() self.Patch(gf_factory.ssh, "Ssh", return_value=self._mock_ssh) - self.Patch(gf_factory.goldfish_remote_host_client, - "GoldfishRemoteHostClient") - self.Patch(gf_factory.auth, "CreateCredentials") + self._mock_remote_host_client = mock.Mock() + self.Patch(gf_factory.remote_host_client, "RemoteHostClient", + return_value=self._mock_remote_host_client) + self._mock_create_credentials = self.Patch( + gf_factory.auth, "CreateCredentials") # Emulator console self._mock_console = mock.MagicMock() self._mock_console.__enter__.return_value = self._mock_console @@ -133,10 +143,10 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): else: self._CreateImageZip(local_path) elif resource_id == "emulator-info.txt": - with open(local_path, "w") as file: + with open(local_path, "w", encoding="utf-8") as file: file.write(self._EMULATOR_INFO) else: - with open(local_path, "w") as file: + with open(local_path, "w", encoding="utf-8") as file: pass def testCreateInstanceWithCfg(self): @@ -147,11 +157,14 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.assertEqual(self._X86_64_INSTANCE_NAME, instance_name) self.assertEqual(self._X86_64_BUILD_INFO, factory.GetBuildInfoDict()) + self.assertEqual([5555], factory.GetAdbPorts()) + self.assertEqual([None], factory.GetFastbootPorts()) + self.assertEqual([None], factory.GetVncPorts()) self.assertEqual({}, factory.GetFailures()) self.assertEqual({instance_name: self._LOGS}, factory.GetLogs()) # Artifacts. self._mock_android_build_client.DownloadArtifact.assert_any_call( - "sdk_tools_linux", "111111", + "emulator-linux_x64_nolocationui", "111111", "sdk-repo-linux-emulator-111111.zip", mock.ANY, mock.ANY) self._mock_android_build_client.DownloadArtifact.assert_any_call( "sdk_x86_64-sdk", "123456", @@ -167,6 +180,18 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.assertEqual(self._mock_console.Ping.call_count, self._mock_console.Reconnect.call_count + 1) self._mock_console.Reconnect.assert_called() + # RemoteHostClient. + self._mock_remote_host_client.RecordTime.assert_has_calls([ + mock.call(constants.TIME_GCE, mock.ANY), + mock.call(constants.TIME_ARTIFACT, mock.ANY), + mock.call(constants.TIME_LAUNCH, mock.ANY)]) + self.assertEqual(3, + self._mock_remote_host_client.RecordTime.call_count) + self._mock_remote_host_client.SetStage.assert_has_calls([ + mock.call(constants.STAGE_SSH_CONNECT), + mock.call(constants.STAGE_ARTIFACT), + mock.call(constants.STAGE_BOOT_UP)]) + self.assertEqual(3, self._mock_remote_host_client.SetStage.call_count) def testCreateInstanceWithAvdSpec(self): """Test RemoteHostGoldfishDeviceFactory with command options.""" @@ -174,11 +199,12 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): self._mock_avd_spec.host_ssh_private_key_path = "key_path" self._mock_avd_spec.emulator_build_id = "999999" self._mock_avd_spec.emulator_build_target = "aarch64_sdk_tools_mac" + self._mock_avd_spec.base_instance_num = 2 self._mock_avd_spec.boot_timeout_secs = 1 self._mock_avd_spec.hw_customize = True self._mock_avd_spec.hw_property = {"disk": "4096"} - self._mock_android_build_client.DownloadArtifact.side_effect = ( - AssertionError("DownloadArtifact should not be called.")) + self._mock_create_credentials.side_effect = AssertionError( + "CreateCredentials should not be called.") # All artifacts are cached. with tempfile.TemporaryDirectory() as download_dir: self._mock_avd_spec.image_download_dir = download_dir @@ -200,6 +226,8 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.assertEqual(self._ARM64_INSTANCE_NAME, instance_name) self.assertEqual(self._ARM64_BUILD_INFO, factory.GetBuildInfoDict()) + self.assertEqual([5557], factory.GetAdbPorts()) + self.assertEqual([None], factory.GetVncPorts()) self.assertEqual({}, factory.GetFailures()) @mock.patch("acloud.public.actions.remote_host_gf_device_factory." @@ -215,7 +243,7 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): factory = gf_factory.RemoteHostGoldfishDeviceFactory( self._mock_avd_spec) - instance_name = factory.CreateInstance() + factory.CreateInstance() # Artifacts. self._mock_android_build_client.DownloadArtifact.assert_any_call( "sdk_x86_64-sdk", "123456", @@ -231,19 +259,21 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): # Images. mock_gf_utils.MixWithSystemImage.assert_called_once() self._mock_ssh.ScpPushFile.assert_called_with( - "/mixed/disk", "acloud_gf/image/x86_64/system-qemu.img") + "/mixed/disk", "acloud_gf_1/image/x86_64/system-qemu.img") - self.assertEqual(self._X86_64_INSTANCE_NAME, instance_name) + mock_gf_utils.FormatRemoteHostInstanceName.assert_called() self.assertEqual(self._X86_64_BUILD_INFO, factory.GetBuildInfoDict()) + self.assertEqual([5555], factory.GetAdbPorts()) + self.assertEqual([None], factory.GetVncPorts()) self.assertEqual({}, factory.GetFailures()) @mock.patch("acloud.public.actions.remote_host_gf_device_factory." "goldfish_utils") - def testCreateInstanceWithKernelBuild(self, mock_gf_utils): - """Test RemoteHostGoldfishDeviceFactory with kernel build.""" - self._mock_avd_spec.kernel_build_info = { + def testCreateInstanceWithBootBuild(self, mock_gf_utils): + """Test RemoteHostGoldfishDeviceFactory with boot build.""" + self._mock_avd_spec.boot_build_info = { constants.BUILD_ID: "111111", - constants.BUILD_TARGET: "aosp_x86_64-userdebug", + constants.BUILD_TARGET: "gki_x86_64-userdebug", constants.BUILD_ARTIFACT: "boot-5.10.img"} mock_gf_utils.ConvertAvdSpecToArgs.return_value = ["-gpu", "auto"] mock_gf_utils.MixWithBootImage.return_value = ( @@ -251,13 +281,13 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): factory = gf_factory.RemoteHostGoldfishDeviceFactory( self._mock_avd_spec) - instance_name = factory.CreateInstance() + factory.CreateInstance() # Artifacts. self._mock_android_build_client.DownloadArtifact.assert_any_call( "sdk_x86_64-sdk", "123456", "sdk-repo-linux-system-images-123456.zip", mock.ANY, mock.ANY) self._mock_android_build_client.DownloadArtifact.assert_any_call( - "aosp_x86_64-userdebug", "111111", + "gki_x86_64-userdebug", "111111", "boot-5.10.img", mock.ANY, mock.ANY) self._mock_android_build_client.DownloadArtifact.assert_any_call( "sdk_x86_64-sdk", "123456", @@ -267,15 +297,101 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): # Images. mock_gf_utils.MixWithBootImage.assert_called_once() self._mock_ssh.ScpPushFile.assert_any_call( - "/path/to/kernel", "acloud_gf/kernel") + "/path/to/kernel", "acloud_gf_1/kernel") self._mock_ssh.ScpPushFile.assert_any_call( - "/path/to/ramdisk", "acloud_gf/mixed_ramdisk") + "/path/to/ramdisk", "acloud_gf_1/mixed_ramdisk") - self.assertEqual(self._X86_64_INSTANCE_NAME, instance_name) + mock_gf_utils.FormatRemoteHostInstanceName.assert_called() self.assertEqual(self._X86_64_BUILD_INFO, factory.GetBuildInfoDict()) + self.assertEqual([5555], factory.GetAdbPorts()) + self.assertEqual([None], factory.GetVncPorts()) self.assertEqual({}, factory.GetFailures()) - def testCreateInstanceError(self): + @mock.patch("acloud.public.actions.remote_host_gf_device_factory." + "ota_tools") + @mock.patch("acloud.public.actions.remote_host_gf_device_factory." + "goldfish_utils") + def testCreateInstanceWithLocalFiles(self, mock_gf_utils, mock_ota_tools): + """Test RemoteHostGoldfishDeviceFactory with local files.""" + with tempfile.TemporaryDirectory() as temp_dir: + emulator_zip_path = os.path.join(temp_dir, "emulator.zip") + self._CreateSdkRepoZip(emulator_zip_path) + image_zip_path = os.path.join(temp_dir, "image.zip") + self._CreateSdkRepoZip(image_zip_path) + boot_image_path = os.path.join(temp_dir, "boot.img") + self.CreateFile(boot_image_path, b"ANDROID!") + system_image_path = os.path.join(temp_dir, "system.img") + self.CreateFile(system_image_path) + self._mock_avd_spec.emulator_zip = emulator_zip_path + self._mock_avd_spec.image_source = constants.IMAGE_SRC_LOCAL + self._mock_avd_spec.remote_image = {} + self._mock_avd_spec.local_image_artifact = image_zip_path + self._mock_avd_spec.local_kernel_image = boot_image_path + self._mock_avd_spec.local_system_image = system_image_path + self._mock_avd_spec.local_tool_dirs.append("/otatools") + mock_gf_utils.ConvertAvdSpecToArgs.return_value = ["-gpu", "auto"] + mock_gf_utils.MixWithBootImage.return_value = ( + "/path/to/kernel", "/path/to/ramdisk") + self._mock_create_credentials.side_effect = AssertionError( + "CreateCredentials should not be called.") + + factory = gf_factory.RemoteHostGoldfishDeviceFactory( + self._mock_avd_spec) + factory.CreateInstance() + + mock_gf_utils.MixWithBootImage.assert_called_once() + mock_gf_utils.MixWithSystemImage.assert_called_once() + mock_ota_tools.FindOtaToolsDir.assert_called_once() + self.assertEqual("/otatools", + mock_ota_tools.FindOtaToolsDir.call_args[0][0][0]) + + mock_gf_utils.FormatRemoteHostInstanceName.assert_called() + self.assertEqual({}, factory.GetBuildInfoDict()) + self.assertEqual([5555], factory.GetAdbPorts()) + self.assertEqual([None], factory.GetVncPorts()) + self.assertEqual({}, factory.GetFailures()) + + def testCreateInstanceInitError(self): + """Test RemoteHostGoldfishDeviceFactory with SSH error.""" + self._mock_ssh.Run.side_effect = errors.DeviceConnectionError + + factory = gf_factory.RemoteHostGoldfishDeviceFactory( + self._mock_avd_spec) + factory.CreateInstance() + + failures = factory.GetFailures() + self.assertIsInstance(failures.get(self._X86_64_INSTANCE_NAME), + errors.DeviceConnectionError) + self.assertEqual({}, factory.GetLogs()) + self._mock_remote_host_client.RecordTime.assert_called_once_with( + constants.TIME_GCE, mock.ANY) + self._mock_remote_host_client.SetStage.assert_called_once_with( + constants.STAGE_SSH_CONNECT) + + def testCreateInstanceDownloadError(self): + """Test RemoteHostGoldfishDeviceFactory with download error.""" + self._mock_android_build_client.DownloadArtifact.side_effect = ( + errors.DriverError) + + factory = gf_factory.RemoteHostGoldfishDeviceFactory( + self._mock_avd_spec) + factory.CreateInstance() + + failures = factory.GetFailures() + self.assertIsInstance(failures.get(self._X86_64_INSTANCE_NAME), + errors.DriverError) + self.assertEqual({}, factory.GetLogs()) + self._mock_remote_host_client.RecordTime.assert_has_calls([ + mock.call(constants.TIME_GCE, mock.ANY), + mock.call(constants.TIME_ARTIFACT, mock.ANY)]) + self.assertEqual(2, + self._mock_remote_host_client.RecordTime.call_count) + self._mock_remote_host_client.SetStage.assert_has_calls([ + mock.call(constants.STAGE_SSH_CONNECT), + mock.call(constants.STAGE_ARTIFACT)]) + self.assertEqual(2, self._mock_remote_host_client.SetStage.call_count) + + def testCreateInstanceBootError(self): """Test RemoteHostGoldfishDeviceFactory with boot error.""" self._mock_console.Reconnect.side_effect = ( errors.DeviceConnectionError) @@ -289,6 +405,9 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): errors.DeviceBootError) self.assertEqual({self._X86_64_INSTANCE_NAME: self._LOGS}, factory.GetLogs()) + self.assertEqual(3, + self._mock_remote_host_client.RecordTime.call_count) + self.assertEqual(3, self._mock_remote_host_client.SetStage.call_count) def testCreateInstanceTimeout(self): """Test RemoteHostGoldfishDeviceFactory with timeout.""" @@ -310,6 +429,9 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): errors.DeviceBootTimeoutError) self.assertEqual({self._X86_64_INSTANCE_NAME: self._LOGS}, factory.GetLogs()) + self.assertEqual(3, + self._mock_remote_host_client.RecordTime.call_count) + self.assertEqual(3, self._mock_remote_host_client.SetStage.call_count) if __name__ == "__main__": diff --git a/public/actions/remote_instance_cf_device_factory.py b/public/actions/remote_instance_cf_device_factory.py index cf447743..072cb39e 100644 --- a/public/actions/remote_instance_cf_device_factory.py +++ b/public/actions/remote_instance_cf_device_factory.py @@ -16,15 +16,21 @@ device factory.""" import logging +import os +import tempfile +from acloud.create import create_common from acloud.internal import constants from acloud.internal.lib import cvd_utils +from acloud.internal.lib import ota_tools +from acloud.internal.lib import utils from acloud.public.actions import gce_device_factory from acloud.pull import pull logger = logging.getLogger(__name__) _SCREEN_CONSOLE_COMMAND = "screen ~/cuttlefish_runtime/console" +_MIXED_SUPER_IMAGE_NAME = "mixed_super.img" class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): @@ -54,18 +60,14 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): Returns: A string, representing instance name. """ - instance = self._CreateGceInstance() + instance = self.CreateGceInstance() # If instance is failed, no need to go next step. if instance in self.GetFailures(): return instance try: image_args = self._ProcessArtifacts() failures = self._compute_client.LaunchCvd( - instance, - self._avd_spec, - self._cfg.extra_data_disk_size_gb, - boot_timeout_secs=self._avd_spec.boot_timeout_secs, - extra_args=image_args) + instance, self._avd_spec, cvd_utils.GCE_BASE_DIR, image_args) for failing_instance, error_msg in failures.items(): self._SetFailures(failing_instance, error_msg) except Exception as e: @@ -88,45 +90,115 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): Returns: A list of strings, the launch_cvd arguments. """ - if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: + avd_spec = self._avd_spec + if avd_spec.image_source == constants.IMAGE_SRC_LOCAL: cvd_utils.UploadArtifacts( self._ssh, - self._local_image_artifact or self._avd_spec.local_image_dir, + cvd_utils.GCE_BASE_DIR, + self._local_image_artifact or avd_spec.local_image_dir, self._cvd_host_package_artifact) - elif self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: - self._compute_client.UpdateFetchCvd() - self._FetchBuild(self._avd_spec) + elif avd_spec.image_source == constants.IMAGE_SRC_REMOTE: + self._compute_client.UpdateFetchCvd(avd_spec.fetch_cvd_version) + self._compute_client.FetchBuild( + avd_spec.remote_image, + avd_spec.system_build_info, + avd_spec.kernel_build_info, + avd_spec.boot_build_info, + avd_spec.bootloader_build_info, + avd_spec.ota_build_info) + + launch_cvd_args = [] + if avd_spec.local_system_image or avd_spec.local_vendor_image: + with tempfile.TemporaryDirectory() as temp_dir: + super_image_path = os.path.join(temp_dir, + _MIXED_SUPER_IMAGE_NAME) + self._CreateMixedSuperImage( + super_image_path, self._GetLocalTargetFilesDir(temp_dir)) + launch_cvd_args += cvd_utils.UploadSuperImage( + self._ssh, cvd_utils.GCE_BASE_DIR, super_image_path) - if self._avd_spec.mkcert and self._avd_spec.connect_webrtc: + if avd_spec.mkcert and avd_spec.connect_webrtc: self._compute_client.UpdateCertificate() - if self._avd_spec.extra_files: - self._compute_client.UploadExtraFiles(self._avd_spec.extra_files) + if avd_spec.extra_files: + self._compute_client.UploadExtraFiles(avd_spec.extra_files) + + launch_cvd_args += cvd_utils.UploadExtraImages( + self._ssh, cvd_utils.GCE_BASE_DIR, avd_spec) + return launch_cvd_args - return cvd_utils.UploadExtraImages(self._ssh, self._avd_spec) + @utils.TimeExecute(function_description="Downloading target_files archive") + def _DownloadTargetFiles(self, download_dir): + avd_spec = self._avd_spec + build_id = avd_spec.remote_image[constants.BUILD_ID] + build_target = avd_spec.remote_image[constants.BUILD_TARGET] + create_common.DownloadRemoteArtifact( + avd_spec.cfg, build_target, build_id, + cvd_utils.GetMixBuildTargetFilename(build_target, build_id), + download_dir, decompress=True) + + def _GetLocalTargetFilesDir(self, temp_dir): + """Return a directory of extracted target_files or local images. + + Args: + temp_dir: Temporary directory to store downloaded build artifacts + and extracted target_files archive. + """ + avd_spec = self._avd_spec + if avd_spec.image_source == constants.IMAGE_SRC_LOCAL: + if self._local_image_artifact: + target_files_dir = os.path.join(temp_dir, "local_images") + os.makedirs(target_files_dir, exist_ok=True) + utils.Decompress(self._local_image_artifact, target_files_dir) + else: + target_files_dir = os.path.abspath(avd_spec.local_image_dir) + else: # must be IMAGE_SRC_REMOTE + target_files_dir = os.path.join(temp_dir, "remote_images") + os.makedirs(target_files_dir, exist_ok=True) + self._DownloadTargetFiles(target_files_dir) + return target_files_dir - def _FetchBuild(self, avd_spec): - """Download CF artifacts from android build. + def _CreateMixedSuperImage(self, super_image_path, target_files_dir): + """Create a mixed super image from device images and local system image. Args: - avd_spec: AVDSpec object that tells us what we're going to create. + super_image_path: Path to the output mixed super image. + target_files_dir: Path to extracted target_files directory + containing device images and misc_info.txt. """ - self._compute_client.FetchBuild( - avd_spec.remote_image[constants.BUILD_ID], - avd_spec.remote_image[constants.BUILD_BRANCH], - avd_spec.remote_image[constants.BUILD_TARGET], - avd_spec.system_build_info[constants.BUILD_ID], - avd_spec.system_build_info[constants.BUILD_BRANCH], - avd_spec.system_build_info[constants.BUILD_TARGET], - avd_spec.kernel_build_info[constants.BUILD_ID], - avd_spec.kernel_build_info[constants.BUILD_BRANCH], - avd_spec.kernel_build_info[constants.BUILD_TARGET], - avd_spec.bootloader_build_info[constants.BUILD_ID], - avd_spec.bootloader_build_info[constants.BUILD_BRANCH], - avd_spec.bootloader_build_info[constants.BUILD_TARGET], - avd_spec.ota_build_info[constants.BUILD_ID], - avd_spec.ota_build_info[constants.BUILD_BRANCH], - avd_spec.ota_build_info[constants.BUILD_TARGET]) + avd_spec = self._avd_spec + misc_info_path = cvd_utils.FindMiscInfo(target_files_dir) + image_dir = cvd_utils.FindImageDir(target_files_dir) + ota = ota_tools.FindOtaTools( + avd_spec.local_tool_dirs + + create_common.GetNonEmptyEnvVars( + constants.ENV_ANDROID_SOONG_HOST_OUT, + constants.ENV_ANDROID_HOST_OUT)) + + system_image_path=None + vendor_image_path=None + vendor_dlkm_image_path=None + odm_image_path=None + odm_dlkm_image_path=None + + if avd_spec.local_system_image: + system_image_path = create_common.FindSystemImage( + avd_spec.local_system_image) + + if avd_spec.local_vendor_image: + vendor_image_paths = cvd_utils.FindVendorImages( + avd_spec.local_vendor_image) + vendor_image_path = vendor_image_paths.vendor + vendor_dlkm_image_path = vendor_image_paths.vendor_dlkm + odm_image_path = vendor_image_paths.odm + odm_dlkm_image_path = vendor_image_paths.odm_dlkm + + ota.MixSuperImage(super_image_path, misc_info_path, image_dir, + system_image=system_image_path, + vendor_image=vendor_image_path, + vendor_dlkm_image=vendor_dlkm_image_path, + odm_image=odm_image_path, + odm_dlkm_image=odm_dlkm_image_path) def _FindLogFiles(self, instance, download): """Find and pull all log files from instance. @@ -136,13 +208,21 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): download: Whether to download the files to a temporary directory and show messages to the user. """ - self._all_logs[instance] = [cvd_utils.TOMBSTONES, - cvd_utils.HOST_KERNEL_LOG] + logs = [cvd_utils.HOST_KERNEL_LOG] if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: - self._all_logs[instance].append(cvd_utils.FETCHER_CONFIG_JSON) - log_files = pull.GetAllLogFilePaths(self._ssh) - self._all_logs[instance].extend(cvd_utils.ConvertRemoteLogs(log_files)) + logs.append( + cvd_utils.GetRemoteFetcherConfigJson(cvd_utils.GCE_BASE_DIR)) + logs.extend(cvd_utils.FindRemoteLogs( + self._ssh, + cvd_utils.GCE_BASE_DIR, + self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance)) + self._all_logs[instance] = logs + if download: + # To avoid long download time, fetch from the first device only. + log_files = pull.GetAllLogFilePaths(self._ssh, + constants.REMOTE_LOG_FOLDER) error_log_folder = pull.PullLogs(self._ssh, log_files, instance) self._compute_client.ExtendReportData(constants.ERROR_LOG_FOLDER, error_log_folder) @@ -158,6 +238,33 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): return {"ssh_command": self._compute_client.GetSshConnectCmd(), "screen_command": _SCREEN_CONSOLE_COMMAND} + def GetAdbPorts(self): + """Get ADB ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetAdbPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + + def GetFastbootPorts(self): + """Get Fastboot ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetFastbootPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + + def GetVncPorts(self): + """Get VNC ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetVncPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + def GetBuildInfoDict(self): """Get build info dictionary. diff --git a/public/actions/remote_instance_cf_device_factory_test.py b/public/actions/remote_instance_cf_device_factory_test.py index f0c52534..eaad9f80 100644 --- a/public/actions/remote_instance_cf_device_factory_test.py +++ b/public/actions/remote_instance_cf_device_factory_test.py @@ -15,6 +15,7 @@ import glob import os +import tempfile import unittest import uuid @@ -25,6 +26,7 @@ from acloud.internal import constants from acloud.internal.lib import android_build_client from acloud.internal.lib import auth from acloud.internal.lib import cvd_compute_client_multi_stage +from acloud.internal.lib import cvd_utils from acloud.internal.lib import driver_test_lib from acloud.internal.lib import utils from acloud.list import list as list_instances @@ -41,6 +43,8 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.Patch(android_build_client.AndroidBuildClient, "InitResourceHandle") self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "InitResourceHandle") self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "LaunchCvd") + self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "UpdateFetchCvd") + self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "FetchBuild") self.Patch(list_instances, "GetInstancesFromInstanceNames", return_value=mock.MagicMock()) self.Patch(list_instances, "ChooseOneRemoteInstance", return_value=mock.MagicMock()) self.Patch(utils, "GetBuildEnvironmentVariable", @@ -48,14 +52,12 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.Patch(glob, "glob", return_vale=["fake.img"]) # pylint: disable=protected-access + @staticmethod @mock.patch.object(cvd_compute_client_multi_stage.CvdComputeClient, "UpdateCertificate") - @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_FetchBuild") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." "cvd_utils") - def testProcessArtifacts(self, mock_cvd_utils, mock_download, - mock_uploadca): + def testProcessArtifacts(mock_cvd_utils, mock_uploadca): """test ProcessArtifacts.""" # Test image source type is local. args = mock.MagicMock() @@ -64,6 +66,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.flavor = "phone" args.local_image = constants.FIND_IN_BUILD_ENV args.local_system_image = None + args.local_vendor_image = None args.launch_args = None args.autoconnect = constants.INS_KEY_WEBRTC avd_spec_local_img = avd_spec.AVDSpec(args) @@ -78,7 +81,8 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_uploadca.assert_called_once() mock_uploadca.reset_mock() mock_cvd_utils.UploadArtifacts.assert_called_once_with( - mock.ANY, fake_image_name, fake_host_package_name) + mock.ANY, mock_cvd_utils.GCE_BASE_DIR, fake_image_name, + fake_host_package_name) mock_cvd_utils.UploadExtraImages.assert_called_once() # given autoconnect to vnc should not upload certificates @@ -103,11 +107,13 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.kernel_branch = "kernel_branch" args.kernel_build_target = "kernel_target" avd_spec_remote_img = avd_spec.AVDSpec(args) - self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "UpdateFetchCvd") factory_remote_img = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( avd_spec_remote_img) factory_remote_img._ProcessArtifacts() - mock_download.assert_called_once() + + compute_client = factory_remote_img.GetComputeClient() + compute_client.UpdateFetchCvd.assert_called_once() + compute_client.FetchBuild.assert_called_once() # pylint: disable=protected-access @mock.patch.dict(os.environ, {constants.ENV_BUILD_TARGET:'fake-target'}) @@ -121,6 +127,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.local_image = constants.FIND_IN_BUILD_ENV args.local_system_image = None args.adb_port = None + args.fastboot_port = None args.launch_args = None fake_avd_spec = avd_spec.AVDSpec(args) fake_avd_spec.cfg.enable_multi_stage = True @@ -138,7 +145,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec, fake_image_name, fake_host_package_name) - self.assertEqual(factory._CreateGceInstance(), "ins-1234-userbuild-aosp-cf-x86-phone") + self.assertEqual(factory.CreateGceInstance(), "ins-1234-userbuild-aosp-cf-x86-phone") # Can't get target name from zip file name. fake_image_name = "/fake/aosp_cf_x86_phone.username.zip" @@ -146,7 +153,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec, fake_image_name, fake_host_package_name) - self.assertEqual(factory._CreateGceInstance(), "ins-1234-userbuild-fake-target") + self.assertEqual(factory.CreateGceInstance(), "ins-1234-userbuild-fake-target") # No image zip path, it uses local build images. fake_image_name = "" @@ -154,7 +161,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec, fake_image_name, fake_host_package_name) - self.assertEqual(factory._CreateGceInstance(), "ins-1234-userbuild-fake-target") + self.assertEqual(factory.CreateGceInstance(), "ins-1234-userbuild-fake-target") def testReuseInstanceNameMultiStage(self): """Test reuse instance name.""" @@ -165,6 +172,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.local_image = constants.FIND_IN_BUILD_ENV args.local_system_image = None args.adb_port = None + args.fastboot_port = None args.launch_args = None fake_avd_spec = avd_spec.AVDSpec(args) fake_avd_spec.cfg.enable_multi_stage = True @@ -180,7 +188,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec, fake_image_name, fake_host_package_name) - self.assertEqual(factory._CreateGceInstance(), "fake-1234-userbuild-fake-target") + self.assertEqual(factory.CreateGceInstance(), "fake-1234-userbuild-fake-target") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." "cvd_utils") @@ -196,6 +204,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.local_image = "fake_local_image" args.local_system_image = None args.adb_port = None + args.fastboot_port = None args.cheeps_betty_image = None args.launch_args = None avd_spec_local_image = avd_spec.AVDSpec(args) @@ -220,7 +229,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_cvd_utils.GetRemoteBuildInfoDict.assert_called() @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_CreateGceInstance") + "CreateGceInstance") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory.pull") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." "cvd_utils") @@ -236,8 +245,12 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec.image_source = constants.IMAGE_SRC_LOCAL fake_avd_spec._instance_name_to_reuse = None fake_avd_spec.no_pull_log = False + fake_avd_spec.base_instance_num = None + fake_avd_spec.num_avds_per_instance = None + fake_avd_spec.local_system_image = None + fake_avd_spec.local_vendor_image = None - mock_cvd_utils.ConvertRemoteLogs.return_value = [{"path": "/logcat"}] + mock_cvd_utils.FindRemoteLogs.return_value = [{"path": "/logcat"}] mock_cvd_utils.UploadExtraImages.return_value = [ "-boot_image", "/boot/img"] @@ -252,17 +265,93 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): factory.CreateInstance() mock_create_gce_instance.assert_called_once() mock_cvd_utils.UploadArtifacts.assert_called_once() + mock_cvd_utils.FindRemoteLogs.assert_called_with( + mock.ANY, mock_cvd_utils.GCE_BASE_DIR, None, None) compute_client.LaunchCvd.assert_called_once() - self.assertEqual( - ["-boot_image", "/boot/img"], - compute_client.LaunchCvd.call_args[1].get("extra_args")) - mock_pull.GetAllLogFilePaths.assert_called_once() + self.assertIn(["-boot_image", "/boot/img"], + compute_client.LaunchCvd.call_args[0]) + mock_pull.GetAllLogFilePaths.assert_called_once_with( + mock.ANY, constants.REMOTE_LOG_FOLDER) mock_pull.PullLogs.assert_called_once() + + factory.GetAdbPorts() + mock_cvd_utils.GetAdbPorts.assert_called_with(None, None) + factory.GetVncPorts() + mock_cvd_utils.GetVncPorts.assert_called_with(None, None) + factory.GetFastbootPorts() + mock_cvd_utils.GetFastbootPorts.assert_called_with(None, None) self.assertEqual({"instance": "failure"}, factory.GetFailures()) - self.assertEqual(3, len(factory.GetLogs().get("instance"))) + self.assertEqual(2, len(factory.GetLogs().get("instance"))) + + @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." + "ota_tools") + def testLocalSystemAndVendorImageCreateInstance(self, mock_ota_tools): + """Test CreateInstance with local system image.""" + with tempfile.TemporaryDirectory() as temp_dir: + local_image_dir = os.path.join(temp_dir, "cf") + misc_info_path = os.path.join(local_image_dir, "misc_info.txt") + local_system_image_dir = os.path.join(temp_dir, "system") + local_system_image_path = os.path.join( + local_system_image_dir, "system.img") + local_vendor_image_dir = os.path.join(temp_dir, "vendor") + local_vendor_image_path = os.path.join( + local_vendor_image_dir, "vendor.img") + local_vendor_dlkm_image_path = os.path.join( + local_vendor_image_dir, "vendor_dlkm.img") + local_odm_image_path = os.path.join( + local_vendor_image_dir, "odm.img") + local_odm_dlkm_image_path = os.path.join( + local_vendor_image_dir, "odm_dlkm.img") + self.CreateFile(misc_info_path, b"key=value") + self.CreateFile(local_system_image_path) + self.CreateFile(local_vendor_image_path) + self.CreateFile(local_vendor_dlkm_image_path) + self.CreateFile(local_odm_image_path) + self.CreateFile(local_odm_dlkm_image_path) + + self.Patch(cvd_utils, "UploadArtifacts") + self.Patch(cvd_utils, "UploadSuperImage") + self.Patch(cvd_utils, "UploadExtraImages") + self.Patch(cvd_compute_client_multi_stage, "CvdComputeClient") + self.Patch( + remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, + "CreateGceInstance", return_value="instance") + self.Patch( + remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, + "_FindLogFiles") + mock_ota_tools_object = mock_ota_tools.FindOtaTools.return_value + + args = mock.MagicMock() + args.config_file = "" + args.avd_type = constants.TYPE_CF + args.flavor = "phone" + args.local_image = local_image_dir + args.local_system_image = local_system_image_dir + args.local_vendor_image = local_vendor_image_dir + args.local_tool = ["/ota/tools/dir"] + args.launch_args = None + args.no_pull_log = True + avd_spec_local_img = avd_spec.AVDSpec(args) + factory_local_img = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( + avd_spec_local_img) + compute_client = factory_local_img.GetComputeClient() + compute_client.LaunchCvd.return_value = {} + + factory_local_img.CreateInstance() + + cvd_utils.UploadArtifacts.assert_called_once_with( + mock.ANY, mock.ANY, local_image_dir, mock.ANY) + mock_ota_tools_object.MixSuperImage.assert_called_once_with( + mock.ANY, misc_info_path, local_image_dir, + system_image=local_system_image_path, + vendor_image=local_vendor_image_path, + vendor_dlkm_image=local_vendor_dlkm_image_path, + odm_image=local_odm_image_path, + odm_dlkm_image=local_odm_dlkm_image_path) + cvd_utils.UploadSuperImage.assert_called_once() @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_CreateGceInstance") + "CreateGceInstance") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory.pull") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." "cvd_utils") @@ -278,8 +367,12 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec.image_source = constants.IMAGE_SRC_REMOTE fake_avd_spec.host_user = None fake_avd_spec.no_pull_log = True + fake_avd_spec.base_instance_num = 2 + fake_avd_spec.num_avds_per_instance = 3 + fake_avd_spec.local_system_image = None + fake_avd_spec.local_vendor_image = None - mock_cvd_utils.ConvertRemoteLogs.return_value = [{"path": "/logcat"}] + mock_cvd_utils.FindRemoteLogs.return_value = [{"path": "/logcat"}] mock_cvd_utils.UploadExtraImages.return_value = [] factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( @@ -289,10 +382,17 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): factory.CreateInstance() compute_client.FetchBuild.assert_called_once() - mock_pull.GetAllLogFilePaths.assert_called_once() + mock_cvd_utils.FindRemoteLogs.assert_called_with( + mock.ANY, mock_cvd_utils.GCE_BASE_DIR, 2, 3) + mock_pull.GetAllLogFilePaths.assert_not_called() mock_pull.PullLogs.assert_not_called() + + factory.GetAdbPorts() + mock_cvd_utils.GetAdbPorts.assert_called_with(2, 3) + factory.GetVncPorts() + mock_cvd_utils.GetVncPorts.assert_called_with(2, 3) self.assertFalse(factory.GetFailures()) - self.assertEqual(4, len(factory.GetLogs().get("instance"))) + self.assertEqual(3, len(factory.GetLogs().get("instance"))) def testGetOpenWrtInfoDict(self): """Test GetOpenWrtInfoDict.""" diff --git a/public/actions/remote_instance_fvp_device_factory.py b/public/actions/remote_instance_fvp_device_factory.py index c60235da..690fc2fe 100644 --- a/public/actions/remote_instance_fvp_device_factory.py +++ b/public/actions/remote_instance_fvp_device_factory.py @@ -34,7 +34,7 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): Returns: The instance. """ - instance = self._CreateGceInstance() + instance = self.CreateGceInstance() if instance in self.GetFailures(): return instance diff --git a/public/actions/remote_instance_fvp_device_factory_test.py b/public/actions/remote_instance_fvp_device_factory_test.py index 862c7c03..86c9ba17 100644 --- a/public/actions/remote_instance_fvp_device_factory_test.py +++ b/public/actions/remote_instance_fvp_device_factory_test.py @@ -46,7 +46,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): @staticmethod @mock.patch.object( remote_instance_fvp_device_factory.RemoteInstanceDeviceFactory, - "_CreateGceInstance") + "CreateGceInstance") @mock.patch.object(ssh, "ShellCmdWithRetry") @mock.patch.dict(os.environ, { constants.ENV_BUILD_TARGET:'fvp', @@ -65,6 +65,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.local_image = "fake_local_image" args.local_system_image = None args.adb_port = None + args.fastboot_port = None args.launch_args = None avd_spec_local_image = avd_spec.AVDSpec(args) factory = remote_instance_fvp_device_factory.RemoteInstanceDeviceFactory( |