diff options
| author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 05:24:07 +0000 |
|---|---|---|
| committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 05:24:07 +0000 |
| commit | 85583c1b5b5e26cbf277263af7a40d08dcf803bf (patch) | |
| tree | 370e854504795f12e4df3a64e9b50225fd034993 | |
| parent | c77f2bfb00905e14b31d001e29abe30f47440b32 (diff) | |
| parent | 3c062c519750095a0dc61dcf036f15d55610917e (diff) | |
| download | acloud-android14-mainline-uwb-release.tar.gz | |
Snap for 10453563 from 3c062c519750095a0dc61dcf036f15d55610917e to mainline-uwb-releaseaml_uwb_341513070aml_uwb_341511050aml_uwb_341310300aml_uwb_341310030aml_uwb_341111010aml_uwb_341011000android14-mainline-uwb-release
Change-Id: I8d2ba6778f9188683a33409ad631b07d53a9a439
97 files changed, 4796 insertions, 2894 deletions
@@ -29,29 +29,11 @@ license { ], } -python_defaults { - name: "acloud_default", - pkg_path: "acloud", - version: { - py2: { - enabled: false, - embedded_launcher: false, - libs: [ - "py-pyopenssl", - ] - }, - py3: { - enabled: true, - embedded_launcher: false, - }, - }, -} - python_binary_host { name: "acloud", // Make acloud's built name to acloud-dev default build python3 binary. stem: "acloud-dev", - defaults: ["acloud_default"], + pkg_path: "acloud", main: "public/acloud_main.py", srcs: [ "public/acloud_main.py", @@ -88,8 +70,8 @@ python_binary_host { python_test_host { name: "acloud_test", + pkg_path: "acloud", main: "acloud_test.py", - defaults: ["acloud_default"], data: [ "public/data/default.config", ], @@ -125,13 +107,15 @@ python_test_host { "general-tests", ], test_options: { - unit_test: true, + // TODO(b/270225397) + unit_test: false, + tags: ["no-remote"], } } python_library_host { name: "acloud_public", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "public/*.py", "public/actions/*.py", @@ -145,7 +129,7 @@ python_library_host { python_library_host { name: "acloud_internal", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "internal/*.py", "internal/lib/*.py", @@ -157,7 +141,7 @@ python_library_host { python_library_host { name: "acloud_proto", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "internal/proto/*.proto", ], @@ -168,7 +152,7 @@ python_library_host { python_library_host{ name: "acloud_setup", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "setup/*.py", ], @@ -179,7 +163,7 @@ python_library_host{ python_library_host{ name: "acloud_create", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "create/*.py", ], @@ -187,7 +171,7 @@ python_library_host{ python_library_host{ name: "acloud_delete", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "delete/*.py", ], @@ -195,7 +179,7 @@ python_library_host{ python_library_host{ name: "acloud_list", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "list/*.py", ], @@ -203,7 +187,7 @@ python_library_host{ python_library_host{ name: "acloud_reconnect", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "reconnect/*.py", ], @@ -211,7 +195,7 @@ python_library_host{ python_library_host{ name: "acloud_pull", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "pull/*.py", ], @@ -219,7 +203,7 @@ python_library_host{ python_library_host{ name: "acloud_powerwash", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "powerwash/*.py", ], @@ -227,7 +211,7 @@ python_library_host{ python_library_host{ name: "acloud_restart", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "restart/*.py", ], @@ -235,7 +219,7 @@ python_library_host{ python_library_host{ name: "acloud_hostcleanup", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "hostcleanup/*.py", ], @@ -243,13 +227,12 @@ python_library_host{ python_library_host{ name: "acloud_metrics", - defaults: ["acloud_default"], + pkg_path: "acloud", srcs: [ "metrics/*.py", ], libs: [ "asuite_cc_client", - "asuite_metrics", ], } @@ -1,3 +1,4 @@ herbertxue@google.com +hsinyichen@google.com kevcheng@google.com samchiu@google.com diff --git a/acloud_test.py b/acloud_test.py index a8137bd6..1950de14 100644 --- a/acloud_test.py +++ b/acloud_test.py @@ -19,7 +19,6 @@ from importlib import import_module import logging import os import sys -import sysconfig import unittest @@ -37,17 +36,6 @@ logger = logging.getLogger(ACLOUD_LOGGER) logger.setLevel(logging.CRITICAL) logger.addHandler(logging.FileHandler("/dev/null")) -if sys.version_info.major == 3: - sys.path.insert(0, os.path.dirname(sysconfig.get_paths()['purelib'])) - -# (b/219847353) Move googleapiclient to the last position of sys.path when -# existed. -for lib in sys.path: - if 'googleapiclient' in lib: - sys.path.remove(lib) - sys.path.append(lib) - break - def GetTestModules(): """Return list of testable modules. @@ -60,10 +48,11 @@ def GetTestModules(): List of strings (the testable module import path). """ testable_modules = [] - base_path = os.path.dirname(os.path.realpath(__file__)) + package = os.path.dirname(os.path.realpath(__file__)) + base_path = os.path.dirname(package) # Get list of all python files that end in _test.py (except for __file__). - for dirpath, _, files in os.walk(base_path): + for dirpath, _, files in os.walk(package): for f in files: if f.endswith("_test.py") and f != os.path.basename(__file__): # Now transform it into a relative import path. diff --git a/create/avd_spec.py b/create/avd_spec.py index 9eff3fc1..4f23dfaf 100644 --- a/create/avd_spec.py +++ b/create/avd_spec.py @@ -102,6 +102,7 @@ class AVDSpec(): # Let's define the private class vars here and then process the user # args afterwards. self._client_adb_port = args.adb_port + self._client_fastboot_port = args.fastboot_port self._autoconnect = None self._cvd_host_package = None self._instance_name_to_reuse = None @@ -120,6 +121,7 @@ class AVDSpec(): self._local_instance_dir = None self._local_kernel_image = None self._local_system_image = None + self._local_vendor_image = None self._local_tool_dirs = None self._image_download_dir = None self._num_of_instances = None @@ -128,15 +130,17 @@ class AVDSpec(): self._mkcert = None self._oxygen = None self._openwrt = None - self._remote_image = None - self._system_build_info = None - self._kernel_build_info = None - self._ota_build_info = None - self._bootloader_build_info = None + self._remote_image = {} + self._system_build_info = {} + self._kernel_build_info = {} + self._boot_build_info = {} + self._ota_build_info = {} + self._bootloader_build_info = {} self._hw_property = None self._hw_customize = False self._remote_host = None self._gce_metadata = None + self._gce_only = None self._host_user = None self._host_ssh_private_key_path = None self._gpu = None @@ -144,6 +148,12 @@ class AVDSpec(): self._base_instance_num = None self._stable_host_image_name = None self._use_launch_cvd = None + self._remote_fetch = None + self._webrtc_device_id = None + self._connect_hostname = None + self._fetch_cvd_wrapper = None + self._fetch_cvd_version = None + # Create config instance for android_build_client to query build api. self._cfg = config.GetAcloudConfig(args) # Reporting args. @@ -151,6 +161,7 @@ class AVDSpec(): # emulator_* are only used for goldfish avd_type. self._emulator_build_id = None self._emulator_build_target = None + self._emulator_zip = None # Fields only used for cheeps type. self._stable_cheeps_host_image_name = None @@ -245,6 +256,10 @@ class AVDSpec(): self._local_system_image = self._GetLocalImagePath( args.local_system_image) + if args.local_vendor_image is not None: + self._local_vendor_image = self._GetLocalImagePath( + args.local_vendor_image) + self.image_download_dir = ( args.image_download_dir if args.image_download_dir else tempfile.gettempdir()) @@ -358,10 +373,13 @@ class AVDSpec(): self._use_launch_cvd = args.use_launch_cvd self._serial_log_file = args.serial_log_file self._emulator_build_id = args.emulator_build_id - self._emulator_build_target = args.emulator_build_target + self._emulator_build_target = (args.emulator_build_target + or self._cfg.emulator_build_target) + self._emulator_zip = args.emulator_zip self._gpu = args.gpu self._disk_type = (args.disk_type or self._cfg.disk_type) self._base_instance_num = args.base_instance_num + self._gce_only = args.gce_only self._gce_metadata = create_common.ParseKeyValuePairArgs(args.gce_metadata) self._stable_host_image_name = ( args.stable_host_image_name or self._cfg.stable_host_image_name) @@ -378,6 +396,11 @@ class AVDSpec(): self._ins_timeout_secs = args.ins_timeout_secs self._launch_args = " ".join( list(filter(None, [self._cfg.launch_args, args.launch_args]))) + self._remote_fetch = args.remote_fetch + self._webrtc_device_id = args.webrtc_device_id + self._connect_hostname = args.connect_hostname or self._cfg.connect_hostname + self._fetch_cvd_wrapper = args.fetch_cvd_wrapper + self._fetch_cvd_version = self._GetFetchCVDVersion(args) if args.reuse_gce: if args.reuse_gce != constants.SELECT_ONE_GCE_INSTANCE: @@ -388,6 +411,21 @@ class AVDSpec(): instance = list_instance.ChooseOneRemoteInstance(self._cfg) self._instance_name_to_reuse = instance.name + def _GetFetchCVDVersion(self, args): + """Get the fetch_cvd version. + + Acloud will get the LKGB of fetch_cvd if no version specified. + + Args: + args: Namespace object from argparse.parse_args. + + Returns: + The build id of fetch_cvd. + """ + if args.fetch_cvd_build_id: + return args.fetch_cvd_build_id + return constants.LKGB + @staticmethod def _GetFlavorFromString(flavor_string): """Get flavor name from flavor string. @@ -425,11 +463,11 @@ class AVDSpec(): elif self._avd_type == constants.TYPE_FVP: self._ProcessFVPLocalImageArgs() elif self._avd_type == constants.TYPE_GF: - self._local_image_dir = self._GetLocalImagePath( - args.local_image) - if not os.path.isdir(self._local_image_dir): - raise errors.GetLocalImageError("%s is not a directory." % - args.local_image) + local_image_path = self._GetLocalImagePath(args.local_image) + if os.path.isdir(local_image_path): + self._local_image_dir = local_image_path + else: + self._local_image_artifact = local_image_path elif self._avd_type == constants.TYPE_GCE: self._local_image_artifact = self._GetGceLocalImagePath( args.local_image) @@ -577,7 +615,6 @@ class AVDSpec(): Args: args: Namespace object from argparse.parse_args. """ - self._remote_image = {} self._remote_image[constants.BUILD_BRANCH] = args.branch if not self._remote_image[constants.BUILD_BRANCH]: self._remote_image[constants.BUILD_BRANCH] = self._GetBuildBranch( @@ -616,8 +653,11 @@ class AVDSpec(): constants.BUILD_TARGET: args.ota_build_target} self._kernel_build_info = {constants.BUILD_ID: args.kernel_build_id, constants.BUILD_BRANCH: args.kernel_branch, - constants.BUILD_TARGET: args.kernel_build_target, - constants.BUILD_ARTIFACT: args.kernel_artifact} + constants.BUILD_TARGET: args.kernel_build_target} + self._boot_build_info = {constants.BUILD_ID: args.boot_build_id, + constants.BUILD_BRANCH: args.boot_branch, + constants.BUILD_TARGET: args.boot_build_target, + constants.BUILD_ARTIFACT: args.boot_artifact} self._bootloader_build_info = { constants.BUILD_ID: args.bootloader_build_id, constants.BUILD_BRANCH: args.bootloader_branch, @@ -787,6 +827,11 @@ class AVDSpec(): return self._local_system_image @property + def local_vendor_image(self): + """Return local vendor image path.""" + return self._local_vendor_image + + @property def local_tool_dirs(self): """Return a list of local tool directories.""" return self._local_tool_dirs @@ -810,7 +855,15 @@ class AVDSpec(): def connect_adb(self): """Auto-connect to adb. - Return: Boolean, whether autoconnect is enabled. + Return: Boolean, whether adb autoconnect is enabled. + """ + return self._autoconnect is not False + + @property + def connect_fastboot(self): + """Auto-connect to fastboot. + + Return: Boolean, whether fastboot autoconnect is enabled. """ return self._autoconnect is not False @@ -841,6 +894,27 @@ class AVDSpec(): return self._remote_image @property + def remote_fetch(self): + """Fetch cvd in remote host. + + Return: Boolean, whether fetch cvd in remote host. + """ + return self._remote_fetch is True + + @property + def fetch_cvd_wrapper(self): + """use fetch_cvd wrapper + + Return: Boolean, whether fetch cvd in remote host. + """ + return self._fetch_cvd_wrapper + + @property + def fetch_cvd_version(self): + """Return fetch_cvd_version.""" + return self._fetch_cvd_version + + @property def num(self): """Return num of instances.""" return self._num_of_instances @@ -866,6 +940,11 @@ class AVDSpec(): return self._kernel_build_info @property + def boot_build_info(self): + """Return boot build info.""" + return self._boot_build_info + + @property def bootloader_build_info(self): """Return bootloader build info.""" return self._bootloader_build_info @@ -921,11 +1000,21 @@ class AVDSpec(): return self._emulator_build_target @property + def emulator_zip(self): + """Return emulator_zip.""" + return self._emulator_zip + + @property def client_adb_port(self): """Return the client adb port.""" return self._client_adb_port @property + def client_fastboot_port(self): + """Return the client fastboot port.""" + return self._client_fastboot_port + + @property def stable_host_image_name(self): """Return the Cuttlefish host image name.""" return self._stable_host_image_name @@ -1022,6 +1111,11 @@ class AVDSpec(): return self._gce_metadata @property + def gce_only(self): + """Return gce_only.""" + return self._gce_only + + @property def oxygen(self): """Return oxygen.""" return self._oxygen @@ -1055,3 +1149,13 @@ class AVDSpec(): def force_sync(self): """Return force_sync.""" return self._force_sync + + @property + def webrtc_device_id(self): + """Return webrtc_device_id.""" + return self._webrtc_device_id + + @property + def connect_hostname(self): + """Return connect_hostname""" + return self._connect_hostname diff --git a/create/avd_spec_test.py b/create/avd_spec_test.py index 71d2405f..5fe7ca74 100644 --- a/create/avd_spec_test.py +++ b/create/avd_spec_test.py @@ -46,6 +46,7 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.args.config_file = "" self.args.build_target = "fake_build_target" self.args.adb_port = None + self.args.fastboot_port = None self.args.launch_args = None self.Patch(list_instances, "ChooseOneRemoteInstance", return_value=mock.MagicMock()) self.Patch(list_instances, "GetInstancesFromInstanceNames", return_value=mock.MagicMock()) @@ -120,9 +121,11 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): # Specified --local-*-image with dirs. self.args.local_kernel_image = expected_image_dir self.args.local_system_image = expected_image_dir + self.args.local_vendor_image = expected_image_dir self.AvdSpec._ProcessImageArgs(self.args) self.assertEqual(self.AvdSpec.local_kernel_image, expected_image_dir) self.assertEqual(self.AvdSpec.local_system_image, expected_image_dir) + self.assertEqual(self.AvdSpec.local_vendor_image, expected_image_dir) # Specified --local-*-image with files. self.args.local_kernel_image = expected_image_file @@ -134,12 +137,14 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): # Specified --local-*-image without args. self.args.local_kernel_image = constants.FIND_IN_BUILD_ENV self.args.local_system_image = constants.FIND_IN_BUILD_ENV + self.args.local_vendor_image = constants.FIND_IN_BUILD_ENV with mock.patch("acloud.create.avd_spec.utils." "GetBuildEnvironmentVariable", return_value=expected_image_dir): self.AvdSpec._ProcessImageArgs(self.args) self.assertEqual(self.AvdSpec.local_kernel_image, expected_image_dir) self.assertEqual(self.AvdSpec.local_system_image, expected_image_dir) + self.assertEqual(self.AvdSpec.local_vendor_image, expected_image_dir) def testProcessAutoconnect(self): """Test process autoconnect.""" @@ -390,7 +395,10 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.args.kernel_branch = "kernel_branch" self.args.kernel_build_target = "kernel_build_target" self.args.kernel_build_id = "kernel_build_id" - self.args.kernel_artifact = "kernel_artifact" + self.args.boot_branch = "boot_branch" + self.args.boot_build_target = "boot_build_target" + self.args.boot_build_id = "boot_build_id" + self.args.boot_artifact = "boot_artifact" self.AvdSpec._ProcessRemoteBuildArgs(self.args) self.assertEqual( {constants.BUILD_BRANCH: "system_branch", @@ -400,10 +408,15 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.assertEqual( {constants.BUILD_BRANCH: "kernel_branch", constants.BUILD_TARGET: "kernel_build_target", - constants.BUILD_ID: "kernel_build_id", - constants.BUILD_ARTIFACT: "kernel_artifact"}, + constants.BUILD_ID: "kernel_build_id"}, self.AvdSpec.kernel_build_info) self.assertEqual( + {constants.BUILD_BRANCH: "boot_branch", + constants.BUILD_TARGET: "boot_build_target", + constants.BUILD_ID: "boot_build_id", + constants.BUILD_ARTIFACT: "boot_artifact"}, + self.AvdSpec.boot_build_info) + self.assertEqual( {constants.BUILD_BRANCH: "ota_branch", constants.BUILD_TARGET: "ota_build_target", constants.BUILD_ID: "ota_build_id"}, @@ -487,6 +500,7 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.AvdSpec._ProcessMiscArgs(self.args) self.assertEqual(self.AvdSpec.autoconnect, False) self.assertEqual(self.AvdSpec.connect_adb, False) + self.assertEqual(self.AvdSpec.connect_fastboot, False) self.assertEqual(self.AvdSpec.connect_vnc, False) self.assertEqual(self.AvdSpec.connect_webrtc, False) @@ -494,6 +508,7 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.AvdSpec._ProcessMiscArgs(self.args) self.assertEqual(self.AvdSpec.autoconnect, True) self.assertEqual(self.AvdSpec.connect_adb, True) + self.assertEqual(self.AvdSpec.connect_fastboot, True) self.assertEqual(self.AvdSpec.connect_vnc, True) self.assertEqual(self.AvdSpec.connect_webrtc, False) @@ -501,6 +516,15 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.AvdSpec._ProcessMiscArgs(self.args) self.assertEqual(self.AvdSpec.autoconnect, True) self.assertEqual(self.AvdSpec.connect_adb, True) + self.assertEqual(self.AvdSpec.connect_fastboot, True) + self.assertEqual(self.AvdSpec.connect_vnc, False) + self.assertEqual(self.AvdSpec.connect_webrtc, False) + + self.args.autoconnect = constants.INS_KEY_FASTBOOT + self.AvdSpec._ProcessMiscArgs(self.args) + self.assertEqual(self.AvdSpec.autoconnect, True) + self.assertEqual(self.AvdSpec.connect_adb, True) + self.assertEqual(self.AvdSpec.connect_fastboot, True) self.assertEqual(self.AvdSpec.connect_vnc, False) self.assertEqual(self.AvdSpec.connect_webrtc, False) @@ -508,6 +532,7 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.AvdSpec._ProcessMiscArgs(self.args) self.assertEqual(self.AvdSpec.autoconnect, True) self.assertEqual(self.AvdSpec.connect_adb, True) + self.assertEqual(self.AvdSpec.connect_fastboot, True) self.assertEqual(self.AvdSpec.connect_vnc, False) self.assertEqual(self.AvdSpec.connect_webrtc, True) @@ -532,6 +557,23 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.AvdSpec._ProcessMiscArgs(self.args) self.assertEqual(self.args.cheeps_features, ['a', 'b', 'c']) + # Verify connect_hostname + self.mock_config.connect_hostname = True + self.AvdSpec._ProcessMiscArgs(self.args) + self.assertTrue(self.AvdSpec.connect_hostname) + self.args.connect_hostname = True + self.mock_config.connect_hostname = False + self.assertTrue(self.AvdSpec.connect_hostname) + + # Verify fetch_cvd_version + self.args.fetch_cvd_build_id = None + self.AvdSpec._ProcessMiscArgs(self.args) + self.assertEqual(self.AvdSpec.fetch_cvd_version, "LKGB") + + self.args.fetch_cvd_build_id = "23456" + self.AvdSpec._ProcessMiscArgs(self.args) + self.assertEqual(self.AvdSpec.fetch_cvd_version, "23456") + if __name__ == "__main__": unittest.main() diff --git a/create/cheeps_remote_image_remote_instance.py b/create/cheeps_remote_image_remote_instance.py index 7d2ddcbd..af03ddeb 100644 --- a/create/cheeps_remote_image_remote_instance.py +++ b/create/cheeps_remote_image_remote_instance.py @@ -60,6 +60,7 @@ class CheepsRemoteImageRemoteInstance(base_avd_create.BaseAVDCreate): autoconnect=avd_spec.autoconnect, avd_type=constants.TYPE_CHEEPS, client_adb_port=avd_spec.client_adb_port, + client_fastboot_port=avd_spec.client_fastboot_port, boot_timeout_secs=avd_spec.boot_timeout_secs) # Launch vnc client if we're auto-connecting. diff --git a/create/cheeps_remote_image_remote_instance_test.py b/create/cheeps_remote_image_remote_instance_test.py index f5dcae4c..90ac24ca 100644 --- a/create/cheeps_remote_image_remote_instance_test.py +++ b/create/cheeps_remote_image_remote_instance_test.py @@ -37,6 +37,7 @@ class CheepsRemoteImageRemoteInstanceTest(driver_test_lib.BaseDriverTest): return_value=self.build_client) self.compute_client = mock.MagicMock() self.compute_client.openwrt = False + self.compute_client.gce_hostname = None self.Patch( cheeps_compute_client, "CheepsComputeClient", diff --git a/create/create.py b/create/create.py index fdf739f1..d4104af8 100644 --- a/create/create.py +++ b/create/create.py @@ -33,7 +33,7 @@ from acloud.create import cheeps_remote_image_remote_instance from acloud.create import gce_local_image_remote_instance from acloud.create import gce_remote_image_remote_instance from acloud.create import goldfish_local_image_local_instance -from acloud.create import goldfish_remote_image_remote_host +from acloud.create import goldfish_remote_host from acloud.create import goldfish_remote_image_remote_instance from acloud.create import local_image_local_instance from acloud.create import local_image_remote_instance @@ -82,7 +82,9 @@ _CREATOR_CLASS_DICT = { (constants.TYPE_GF, constants.IMAGE_SRC_LOCAL, constants.INSTANCE_TYPE_LOCAL): goldfish_local_image_local_instance.GoldfishLocalImageLocalInstance, (constants.TYPE_GF, constants.IMAGE_SRC_REMOTE, constants.INSTANCE_TYPE_HOST): - goldfish_remote_image_remote_host.GoldfishRemoteImageRemoteHost, + goldfish_remote_host.GoldfishRemoteHost, + (constants.TYPE_GF, constants.IMAGE_SRC_LOCAL, constants.INSTANCE_TYPE_HOST): + goldfish_remote_host.GoldfishRemoteHost, # FVP types (constants.TYPE_FVP, constants.IMAGE_SRC_LOCAL, constants.INSTANCE_TYPE_REMOTE): local_image_remote_instance.LocalImageRemoteInstance, diff --git a/create/create_args.py b/create/create_args.py index aa3292dc..533d88ff 100644 --- a/create/create_args.py +++ b/create/create_args.py @@ -60,12 +60,13 @@ def AddCommonCreateArgs(parser): dest="autoconnect", required=False, choices=[constants.INS_KEY_VNC, constants.INS_KEY_ADB, - constants.INS_KEY_WEBRTC], - help="Determines to establish a tunnel forwarding adb/vnc and " - "launch VNC/webrtc. Establish a tunnel forwarding adb and vnc " + constants.INS_KEY_FASTBOOT, constants.INS_KEY_WEBRTC], + help="Determines to establish a tunnel forwarding adb/fastboot/vnc and " + "launch VNC/webrtc. Establish a tunnel forwarding adb, fastboot and vnc " "then launch vnc if --autoconnect vnc is provided. Establish a " - "tunnel forwarding adb if --autoconnect adb is provided. " - "Establish a tunnel forwarding adb and auto-launch on the browser " + "tunnel forwarding adb and fastboot if --autoconnect adb is provided. Enstablish a " + "tunnel forwarding adb and fastboot if --autoconnect fastboot is provided. " + "Establish a tunnel forwarding adb, fastboot and auto-launch on the browser " "if --autoconnect webrtc is provided. For local goldfish " "instance, create a window.") parser.add_argument( @@ -199,12 +200,31 @@ def AddCommonCreateArgs(parser): default="kernel", help="Kernel build target, specify if different from 'kernel'") parser.add_argument( - "--kernel-artifact", + "--boot-build-id", type=str, - dest="kernel_artifact", + dest="boot_build_id", required=False, - help="Goldfish remote host only. The name of the boot image to be " - "retrieved from Android build, e.g., boot-5.10.img.") + help="Boot image build ID, e.g., 8747889, 8748012.") + parser.add_argument( + "--boot-branch", + type=str, + dest="boot_branch", + required=False, + help="Boot image branch, e.g., aosp-gki13-boot-release, aosp-master.") + parser.add_argument( + "--boot-build-target", + type=str, + dest="boot_build_target", + required=False, + help="Boot image build target, " + "e.g., gki_x86_64-userdebug, aosp_cf_x86_64_phone-userdebug.") + parser.add_argument( + "--boot-artifact", + type=str, + dest="boot_artifact", + required=False, + help="The name of the boot image to be retrieved from Android build, " + "e.g., boot-5.10.img, boot.img.") parser.add_argument( "--ota-branch", type=str, @@ -253,12 +273,25 @@ def AddCommonCreateArgs(parser): help="'cuttlefish only' Add extra args to launch_cvd command.", required=False) parser.add_argument( + "--pet-name", + "--webrtc_device_id", + type=str, + dest="webrtc_device_id", + help="'cuttlefish only' Give the pet name of the instance.", + required=False) + parser.add_argument( "--gce-metadata", type=str, dest="gce_metadata", default=None, help="'GCE instance only' Record data into GCE instance metadata with " "key-value pair format. e.g. id:12,name:unknown.") + parser.add_argument( + "--fetch_cvd-build-id", + type=str, + dest="fetch_cvd_build_id", + required=False, + help="'cuttlefish only' Build id of fetch_cvd, e.g. 2145099, P2804227") # TODO(146314062): Remove --multi-stage-launch after infra don't use this # args. parser.add_argument( @@ -302,18 +335,35 @@ def AddCommonCreateArgs(parser): help="GPU accelerator to use if any. e.g. nvidia-tesla-k80. For local " "instances, this arg without assigning any value is to enable " "local gpu support.") - # Hide following args for users, it is only used in infra. - parser.add_argument( - "--local-instance-dir", - dest="local_instance_dir", - required=False, - help=argparse.SUPPRESS) parser.add_argument( "--num-avds-per-instance", + "--num-instances", + "--num_instances", type=int, dest="num_avds_per_instance", required=False, default=1, + help="'cuttlefish only' Create multiple cuttlefish AVDs in one local " + "instance.") + parser.add_argument( + "--connect-hostname", + action="store_true", + dest="connect_hostname", + required=False, + default=False, + help="Ssh connects to the GCE instance with hostname.") + parser.add_argument( + "--gce-only", + action="store_true", + dest="gce_only", + required=False, + default=False, + help="Only create the GCE instance. It won't create virtual devices.") + # Hide following args for users, it is only used in infra. + parser.add_argument( + "--local-instance-dir", + dest="local_instance_dir", + required=False, help=argparse.SUPPRESS) parser.add_argument( "--oxygen", @@ -402,6 +452,28 @@ def AddCommonCreateArgs(parser): dest="bootloader_build_target", help=argparse.SUPPRESS, required=False) + parser.add_argument( + "--fetch_cvd_build_id", + type=str, + dest="fetch_cvd_build_id", + help=argparse.SUPPRESS, + required=False) + parser.add_argument( + "--remote-fetch", + action="store_true", + dest="remote_fetch", + required=False, + default=None, + help="'cuttlefish only' Fetch artifacts in remote host.") + parser.add_argument( + "--fetch-cvd-wrapper", + dest="fetch_cvd_wrapper", + type=str, + required=False, + help="'cuttlefish only' Fetch artifacts in remote host by a" + " provided static executable fetch cvd wrapper file. " + " (Still in experiment, this flag only works on lab hosts" + " with special setup.)") def GetCreateArgParser(subparser): @@ -437,6 +509,13 @@ def GetCreateArgParser(subparser): required=False, help="Specify port for adb forwarding.") create_parser.add_argument( + "--fastboot-port", "-f", + type=int, + default=None, + dest="fastboot_port", + required=False, + help="Specify port for fastboot forwarding.") + create_parser.add_argument( "--base-instance-num", type=int, default=None, @@ -475,11 +554,12 @@ def GetCreateArgParser(subparser): dest="local_kernel_image", nargs="?", required=False, - help="Use the locally built kernel image for the AVD. Look for " - "boot.img or boot-*.img if the argument is a directory. Look for the " - "image in $ANDROID_PRODUCT_OUT if no argument is provided. e.g., " - "--local-kernel-image, --local-kernel-image /path/to/dir, or " - "--local-kernel-image /path/to/img") + help="Use the locally built kernel and ramdisk for the AVD. Look " + "for boot.img, vendor_boot.img, kernel, initramfs.img, etc. if the " + "argument is a directory. Look for the images in $ANDROID_PRODUCT_OUT " + "if no argument is provided. e.g., --local-kernel-image, " + "--local-kernel-image /path/to/dir, or --local-kernel-image " + "/path/to/boot.img") create_parser.add_argument( "--local-system-image", const=constants.FIND_IN_BUILD_ENV, @@ -492,6 +572,18 @@ def GetCreateArgParser(subparser): "e.g., --local-system-image, --local-system-image /path/to/dir, or " "--local-system-image /path/to/img") create_parser.add_argument( + "--local-vendor-image", + const=constants.FIND_IN_BUILD_ENV, + type=str, + dest="local_vendor_image", + nargs="?", + required=False, + help="'cuttlefish only' Use the locally built vendor images for the " + "AVD. Look for vendor.img, vendor_dlkm.img, odm.img, and odm_dlkm.img " + "if the argument is a directory. Look for the images in " + "$ANDROID_PRODUCT_OUT if no argument is provided. e.g., " + "--local-vendor-image, or --local-vendor-image /path/to/dir") + create_parser.add_argument( "--local-tool", type=str, dest="local_tool", @@ -604,7 +696,7 @@ def GetCreateArgParser(subparser): # Arguments for goldfish type. create_parser.add_argument( "--emulator-build-id", - type=int, + type=str, dest="emulator_build_id", required=False, help="'goldfish only' Emulator build ID used to run the images. " @@ -614,7 +706,13 @@ def GetCreateArgParser(subparser): dest="emulator_build_target", required=False, help="'goldfish remote host only' Emulator build target used to run " - "the images. e.g. sdk_tools_linux.") + "the images. e.g. emulator-linux_x64_nolocationui.") + create_parser.add_argument( + "--emulator-zip", + dest="emulator_zip", + required=False, + help="'goldfish remote host only' Emulator zip used to run the " + "images. e.g., /path/sdk-repo-linux-emulator-1234567.zip.") # Arguments for cheeps type. create_parser.add_argument( @@ -768,24 +866,32 @@ def _VerifyGoldfishArgs(args): goldfish_only_flags = [ args.emulator_build_id, args.emulator_build_target, - args.kernel_artifact + args.emulator_zip ] if args.avd_type != constants.TYPE_GF and any(goldfish_only_flags): raise errors.UnsupportedCreateArgs( - "--emulator-* and --kernel-artifact are only valid with " - "avd_type == %s" % constants.TYPE_GF) + f"--emulator-* is only valid with avd_type == {constants.TYPE_GF}") # Exclude kernel_build_target because the default value isn't empty. remote_kernel_flags = [ args.kernel_build_id, args.kernel_branch, - args.kernel_artifact, ] - if (args.avd_type == constants.TYPE_GF and any(remote_kernel_flags) and - not all(remote_kernel_flags)): + if args.avd_type == constants.TYPE_GF and any(remote_kernel_flags): + raise errors.UnsupportedCreateArgs( + "--kernel-* is not supported for goldfish.") + + remote_boot_flags = [ + args.boot_build_id, + args.boot_build_target, + args.boot_branch, + args.boot_artifact, + ] + if (args.avd_type == constants.TYPE_GF and any(remote_boot_flags) and + not all(remote_boot_flags)): raise errors.UnsupportedCreateArgs( - "Either none or all of --kernel-branch, --kernel-build-target, " - "--kernel-build-id, and --kernel-artifact must be specified for " + "Either none or all of --boot-branch, --boot-build-target, " + "--boot-build-id, and --boot-artifact must be specified for " "goldfish.") remote_system_flags = [ @@ -799,13 +905,12 @@ def _VerifyGoldfishArgs(args): "Either none or all of --system-branch, --system-build-target, " "and --system-build-id must be specified for goldfish.") - remote_host_only_flags = ([args.emulator_build_target] + - remote_kernel_flags + remote_system_flags) + remote_host_only_flags = remote_boot_flags + remote_system_flags if args.avd_type == constants.TYPE_GF and args.remote_host is None and any( remote_host_only_flags): raise errors.UnsupportedCreateArgs( - "--kernel-*, --system-*, and --emulator-build-target for goldfish " - "are only supported for remote host.") + "--boot-* and --system-* for goldfish are only supported for " + "remote host.") def VerifyArgs(args): @@ -833,22 +938,30 @@ def VerifyArgs(args): "--system-* args are not supported for AVD type: %s" % args.avd_type) - if args.num > 1 and args.adb_port: - raise errors.UnsupportedMultiAdbPort( - "--adb-port is not supported for multi-devices.") + if args.num > 1: + if args.adb_port is not None: + raise errors.UnsupportedMultiAdbPort( + "--adb-port is not supported for multi-devices.") - if args.num > 1 and args.local_instance is not None: - raise errors.UnsupportedCreateArgs( - "--num is not supported for local instance.") + if args.fastboot_port is not None: + raise errors.UnsupportedMultiAdbPort( + "--fastboot-port is not supported for multi-devices.") + + if args.local_instance is not None: + raise errors.UnsupportedCreateArgs( + "--num is not supported for local instance.") if args.local_instance is None and args.gpu == _DEFAULT_GPU: raise errors.UnsupportedCreateArgs( "Please assign one gpu model for GCE instance. Reference: " "https://cloud.google.com/compute/docs/gpus") - if args.adb_port: + if args.adb_port is not None: utils.CheckPortFree(args.adb_port) + if args.fastboot_port is not None: + utils.CheckPortFree(args.fastboot_port) + hw_properties = create_common.ParseKeyValuePairArgs(args.hw_property) for key in hw_properties: if key not in constants.HW_PROPERTIES: diff --git a/create/create_args_test.py b/create/create_args_test.py index 788955f4..a6c34c0c 100644 --- a/create/create_args_test.py +++ b/create/create_args_test.py @@ -30,6 +30,7 @@ def _CreateArgs(): flavor=None, num=1, adb_port=None, + fastboot_port=None, hw_property=None, stable_cheeps_host_image_name=None, stable_cheeps_host_image_project=None, @@ -44,7 +45,10 @@ def _CreateArgs(): kernel_branch=None, kernel_build_id=None, kernel_build_target="kernel", - kernel_artifact=None, + boot_branch=None, + boot_build_id=None, + boot_build_target=None, + boot_artifact=None, system_branch=None, system_build_id=None, system_build_target=None, @@ -54,6 +58,7 @@ def _CreateArgs(): host_ssh_private_key_path=None, emulator_build_id=None, emulator_build_target=None, + emulator_zip=None, avd_type=constants.TYPE_CF, autoconnect=constants.INS_KEY_WEBRTC) return mock_args @@ -73,7 +78,7 @@ class CreateArgsTest(driver_test_lib.BaseDriverTest): """test goldfish arguments.""" # emulator_build_id with wrong avd_type. mock_args = _CreateArgs() - mock_args.emulator_build_id = 123456 + mock_args.emulator_build_id = "123456" self.assertRaises(errors.UnsupportedCreateArgs, create_args.VerifyArgs, mock_args) # Valid emulator_build_id. @@ -82,22 +87,25 @@ class CreateArgsTest(driver_test_lib.BaseDriverTest): # emulator_build_target with wrong avd_type. mock_args.avd_type = constants.TYPE_CF mock_args.emulator_build_id = None - mock_args.emulator_build_target = "sdk_tools_linux" + mock_args.emulator_build_target = "emulator-linux_x64_nolocationui" mock_args.remote_host = "192.0.2.2" self.assertRaises(errors.UnsupportedCreateArgs, create_args.VerifyArgs, mock_args) - # emulator_build_target without remote_host. + mock_args.emulator_build_target = None + # Incomplete system build info. mock_args.avd_type = constants.TYPE_GF - mock_args.emulator_build_target = "sdk_tools_linux" - mock_args.remote_host = None + mock_args.system_build_target = "aosp_x86_64-userdebug" + mock_args.remote_host = "192.0.2.2" self.assertRaises(errors.UnsupportedCreateArgs, create_args.VerifyArgs, mock_args) - # Incomplete system build info. - mock_args.emulator_build_target = None - mock_args.system_build_target = "aosp_x86_64-userdebug" + mock_args.system_build_target = None + # Incomplete boot build info. + mock_args.avd_type = constants.TYPE_GF + mock_args.boot_build_target = "gki_x86_64-userdebug" mock_args.remote_host = "192.0.2.2" self.assertRaises(errors.UnsupportedCreateArgs, create_args.VerifyArgs, mock_args) + mock_args.boot_build_target = None # System build info without remote_host. mock_args.system_branch = "aosp-master" mock_args.system_build_target = "aosp_x86_64-userdebug" @@ -106,14 +114,14 @@ class CreateArgsTest(driver_test_lib.BaseDriverTest): self.assertRaises(errors.UnsupportedCreateArgs, create_args.VerifyArgs, mock_args) # Valid build info. - mock_args.emulator_build_target = "sdk_tools_linux" + mock_args.emulator_build_target = "emulator-linux_x64_nolocationui" mock_args.system_branch = "aosp-master" mock_args.system_build_target = "aosp_x86_64-userdebug" mock_args.system_build_id = "123456" - mock_args.kernel_branch = "aosp-master" - mock_args.kernel_build_target = "aosp_x86_64-userdebug" - mock_args.kernel_build_id = "123456" - mock_args.kernel_artifact = "boot-5.10.img" + mock_args.boot_branch = "aosp-master" + mock_args.boot_build_target = "aosp_x86_64-userdebug" + mock_args.boot_build_id = "123456" + mock_args.boot_artifact = "boot-5.10.img" mock_args.remote_host = "192.0.2.2" create_args.VerifyArgs(mock_args) diff --git a/create/create_common.py b/create/create_common.py index a5694d7b..00b5e23c 100644 --- a/create/create_common.py +++ b/create/create_common.py @@ -30,6 +30,16 @@ from acloud.internal.lib import utils logger = logging.getLogger(__name__) +# The boot image name pattern supports the following cases: +# - Cuttlefish ANDROID_PRODUCT_OUT directory conatins boot.img. +# - In Android 12, the officially released GKI (Generic Kernel Image) name is +# boot-<kernel version>.img. +# - In Android 13, the name is boot.img. +_BOOT_IMAGE_NAME_PATTERN = r"boot(-[\d.]+)?\.img" +_SYSTEM_IMAGE_NAME_PATTERN = r"system\.img" + +_ANDROID_BOOT_IMAGE_MAGIC = b"ANDROID!" + # Store the file path to upload to the remote instance. ExtraFile = collections.namedtuple("ExtraFile", ["source", "target"]) @@ -136,10 +146,11 @@ def GetCvdHostPackage(package_path=None): dirs_to_check.append(dist_dir) for path in dirs_to_check: - cvd_host_package = os.path.join(path, constants.CVD_HOST_PACKAGE) - if os.path.exists(cvd_host_package): - logger.debug("cvd host package: %s", cvd_host_package) - return cvd_host_package + for name in [constants.CVD_HOST_TARBALL, constants.CVD_HOST_PACKAGE]: + cvd_host_package = os.path.join(path, name) + if os.path.exists(cvd_host_package): + logger.debug("cvd host package: %s", cvd_host_package) + return cvd_host_package raise errors.GetCvdLocalHostPackageError( "Can't find the cvd host package (Try lunching a cuttlefish target" " like aosp_cf_x86_64_phone-userdebug and running 'm'): \n%s" % @@ -166,15 +177,45 @@ def FindLocalImage(path, default_name_pattern, raise_error=True): re.fullmatch(default_name_pattern, name)] if not names: if raise_error: - raise errors.GetLocalImageError("No image in %s." % path) + raise errors.GetLocalImageError(f"No image in {path}.") return None if len(names) != 1: - raise errors.GetLocalImageError("More than one image in %s: %s" % - (path, " ".join(names))) + raise errors.GetLocalImageError( + f"More than one image in {path}: {' '.join(names)}") path = os.path.join(path, names[0]) if os.path.isfile(path): return path - raise errors.GetLocalImageError("%s is not a file." % path) + raise errors.GetLocalImageError(f"{path} is not a file.") + + +def _IsBootImage(image_path): + """Check if a file is an Android boot image by reading the magic bytes. + + Args: + image_path: The file path. + + Returns: + A boolean, whether the file is a boot image. + """ + if not os.path.isfile(image_path): + return False + with open(image_path, "rb") as image_file: + return image_file.read(8) == _ANDROID_BOOT_IMAGE_MAGIC + + +def FindBootImage(path, raise_error=True): + """Find a boot image file in the given path.""" + boot_image_path = FindLocalImage(path, _BOOT_IMAGE_NAME_PATTERN, + raise_error) + if boot_image_path and not _IsBootImage(boot_image_path): + raise errors.GetLocalImageError( + f"{boot_image_path} is not a boot image.") + return boot_image_path + + +def FindSystemImage(path): + """Find a system image file in a given path.""" + return FindLocalImage(path, _SYSTEM_IMAGE_NAME_PATTERN, raise_error=True) def DownloadRemoteArtifact(cfg, build_target, build_id, artifact, extract_path, diff --git a/create/create_common_test.py b/create/create_common_test.py index 14503e6a..1704d0cb 100644 --- a/create/create_common_test.py +++ b/create/create_common_test.py @@ -110,9 +110,9 @@ class CreateCommonTest(driver_test_lib.BaseDriverTest): self.Patch(os.environ, "get", return_value="/fake_dir2") self.Patch(utils, "GetDistDir", return_value="/fake_dir1") - # First and 2nd path are host out dirs, 3rd path is dist dir. self.Patch(os.path, "exists", - side_effect=[False, False, True]) + side_effect=lambda path: + path == "/fake_dir1/cvd-host_package.tar.gz") # Find cvd host in dist dir. self.assertEqual( @@ -123,10 +123,17 @@ class CreateCommonTest(driver_test_lib.BaseDriverTest): self.Patch(os.environ, "get", return_value="/fake_dir2") self.Patch(utils, "GetDistDir", return_value=None) with mock.patch("os.path.exists") as exists: - exists.return_value = True + exists.side_effect = lambda path: \ + path == "/fake_dir2/cvd-host_package.tar.gz" self.assertEqual( create_common.GetCvdHostPackage(), "/fake_dir2/cvd-host_package.tar.gz") + with mock.patch("os.path.exists") as exists: + exists.side_effect = lambda path: \ + path == "/fake_dir2/cvd-host_package" + self.assertEqual( + create_common.GetCvdHostPackage(), + "/fake_dir2/cvd-host_package") # Find cvd host in specified path. package_path = "/tool_dir/cvd-host_package.tar.gz" @@ -161,6 +168,29 @@ class CreateCommonTest(driver_test_lib.BaseDriverTest): with self.assertRaises(errors.GetLocalImageError): create_common.FindLocalImage("/dir", "name.?", raise_error=False) + def testFindBootImage(self): + """Test FindBootImage.""" + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(errors.GetLocalImageError): + create_common.FindBootImage(temp_dir) + + boot_image_path = os.path.join(temp_dir, "boot.img") + self.CreateFile(boot_image_path, b"invalid") + with self.assertRaises(errors.GetLocalImageError): + create_common.FindBootImage(temp_dir) + os.remove(boot_image_path) + + boot_image_path = os.path.join(temp_dir, "boot.img") + self.CreateFile(boot_image_path, b"ANDROID!") + self.assertEqual(boot_image_path, + create_common.FindBootImage(temp_dir)) + os.remove(boot_image_path) + + boot_image_path = os.path.join(temp_dir, "boot-5.10.img") + self.CreateFile(boot_image_path, b"ANDROID!") + self.assertEqual(boot_image_path, + create_common.FindBootImage(temp_dir)) + @mock.patch.object(utils, "Decompress") def testDownloadRemoteArtifact(self, mock_decompress): """Test Download cuttlefish package.""" diff --git a/create/goldfish_local_image_local_instance.py b/create/goldfish_local_image_local_instance.py index b885afcd..0ad37cad 100644 --- a/create/goldfish_local_image_local_instance.py +++ b/create/goldfish_local_image_local_instance.py @@ -59,10 +59,6 @@ logger = logging.getLogger(__name__) _EMULATOR_BIN_NAME = "emulator" _EMULATOR_BIN_DIR_NAMES = ("bin64", "qemu") _SDK_REPO_EMULATOR_DIR_NAME = "emulator" -# The pattern corresponds to the officially released GKI (Generic Kernel -# Image). The names are boot-<kernel version>.img. Emulator has no boot.img. -_BOOT_IMAGE_NAME_PATTERN = r"boot-[\d.]+\.img" -_SYSTEM_IMAGE_NAME_PATTERN = r"system\.img" _NON_MIXED_BACKUP_IMAGE_EXT = ".bak-non-mixed" _BUILD_PROP_FILE_NAME = "build.prop" # Timeout @@ -405,14 +401,10 @@ class GoldfishLocalImageLocalInstance(base_avd_create.BaseAVDCreate): A pair of strings, the paths to kernel image and ramdisk image. """ # Find generic boot image. - try: - boot_image_path = create_common.FindLocalImage( - kernel_search_path, _BOOT_IMAGE_NAME_PATTERN) - logger.info("Found boot image: %s", boot_image_path) - except errors.GetLocalImageError: - boot_image_path = None - + boot_image_path = create_common.FindBootImage(kernel_search_path, + raise_error=False) if boot_image_path: + logger.info("Found boot image: %s", boot_image_path) return goldfish_utils.MixWithBootImage( os.path.join(instance_dir, "mix_kernel"), self._FindImageDir(image_dir), @@ -456,8 +448,7 @@ class GoldfishLocalImageLocalInstance(base_avd_create.BaseAVDCreate): image_dir = self._FindImageDir(avd_spec.local_image_dir) mixed_image = goldfish_utils.MixWithSystemImage( os.path.join(instance_dir, "mix_disk"), image_dir, - create_common.FindLocalImage(avd_spec.local_system_image, - _SYSTEM_IMAGE_NAME_PATTERN), + create_common.FindSystemImage(avd_spec.local_system_image), ota_tools.FindOtaTools(ota_tools_search_paths)) # TODO(b/142228085): Use -system instead of modifying image_dir. diff --git a/create/goldfish_local_image_local_instance_test.py b/create/goldfish_local_image_local_instance_test.py index 925b659e..3444b79d 100644 --- a/create/goldfish_local_image_local_instance_test.py +++ b/create/goldfish_local_image_local_instance_test.py @@ -22,9 +22,10 @@ from unittest import mock from acloud import errors import acloud.create.goldfish_local_image_local_instance as instance_module +from acloud.internal.lib import driver_test_lib -class GoldfishLocalImageLocalInstance(unittest.TestCase): +class GoldfishLocalImageLocalInstance(driver_test_lib.BaseDriverTest): """Test GoldfishLocalImageLocalInstance methods.""" def setUp(self): @@ -401,7 +402,7 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): image_subdir = os.path.join(self._image_dir, "x86") boot_image_path = os.path.join(self._temp_dir, "kernel_images", "boot-5.10.img") - self._CreateEmptyFile(boot_image_path) + self.CreateFile(boot_image_path, b"ANDROID!") self._CreateEmptyFile(os.path.join(image_subdir, "system.img")) self._CreateEmptyFile(os.path.join(image_subdir, "build.prop")) diff --git a/create/goldfish_remote_image_remote_host.py b/create/goldfish_remote_host.py index 5f32f764..80c19c22 100644 --- a/create/goldfish_remote_image_remote_host.py +++ b/create/goldfish_remote_host.py @@ -11,10 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -r"""GoldfishRemoteImageRemoteHost class. +r"""GoldfishRemoteHost class. Create class that is responsible for creating a goldfish instance with remote -images on a remote host. +or local images on a remote host. """ import logging @@ -28,8 +28,8 @@ from acloud.public.actions import remote_host_gf_device_factory logger = logging.getLogger(__name__) -class GoldfishRemoteImageRemoteHost(base_avd_create.BaseAVDCreate): - """Create remote-image-remote-host goldfish.""" +class GoldfishRemoteHost(base_avd_create.BaseAVDCreate): + """Create goldfish on a remote host.""" @utils.TimeExecute(function_description="Total time: ", print_before_call=False, print_status=False) @@ -60,6 +60,7 @@ class GoldfishRemoteImageRemoteHost(base_avd_create.BaseAVDCreate): autoconnect=avd_spec.autoconnect, serial_log_file=avd_spec.serial_log_file, client_adb_port=avd_spec.client_adb_port, + client_fastboot_port=None, boot_timeout_secs=avd_spec.boot_timeout_secs, unlock_screen=avd_spec.unlock_screen, wait_for_boot=False, connect_webrtc=avd_spec.connect_webrtc, diff --git a/create/goldfish_remote_image_remote_host_test.py b/create/goldfish_remote_host_test.py index 35c9a2b0..f5d56b8c 100644 --- a/create/goldfish_remote_image_remote_host_test.py +++ b/create/goldfish_remote_host_test.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for GoldfishRemoteImageRemoteHost.""" +"""Tests for GoldfishRemoteHost.""" import unittest @@ -24,12 +24,12 @@ from acloud.internal.lib import driver_test_lib from acloud.public.actions import common_operations from acloud.public.actions import remote_host_gf_device_factory -class GoldfishRemoteImageRemoteHostTest(driver_test_lib.BaseDriverTest): - """Test GoldfishRemoteImageRemoteHost method.""" +class GoldfishRemoteHostTest(driver_test_lib.BaseDriverTest): + """Test GoldfishRemoteHost method.""" # pylint: disable=no-member def testRun(self): - """Test Create AVD of goldfish remote image remote host.""" + """Test Create AVD of goldfish remote host.""" args = mock.MagicMock() args.skip_pre_run_check = True spec = mock.MagicMock() diff --git a/create/local_image_local_instance.py b/create/local_image_local_instance.py index c5913fb4..e9b15d58 100644 --- a/create/local_image_local_instance.py +++ b/create/local_image_local_instance.py @@ -26,13 +26,14 @@ The user can optionally specify the folder by --local-instance-dir and the instance id by --local-instance. The adb port and vnc port of local instance will be decided according to -instance id. The rule of adb port will be '6520 + [instance id] - 1' and the vnc -port will be '6444 + [instance id] - 1'. +instance id. The rule of adb port will be '6520 + [instance id] - 1' and the +vnc port will be '6444 + [instance id] - 1'. e.g: If instance id = 3 the adb port will be 6522 and vnc port will be 6446. -To delete the local instance, we will call stop_cvd with the environment variable -[CUTTLEFISH_CONFIG_FILE] which is pointing to the runtime cuttlefish json. +To delete the local instance, we will call stop_cvd with the environment +variable [CUTTLEFISH_CONFIG_FILE] which is pointing to the runtime cuttlefish +json. To run this program outside of a build environment, the following setup is required. @@ -50,7 +51,6 @@ required. """ import collections -import glob import logging import os import re @@ -74,15 +74,13 @@ from acloud.setup import mkcert logger = logging.getLogger(__name__) -_SYSTEM_IMAGE_NAME_PATTERN = r"system\.img" -_MISC_INFO_FILE_NAME = "misc_info.txt" -_TARGET_FILES_IMAGES_DIR_NAME = "IMAGES" -_TARGET_FILES_META_DIR_NAME = "META" +_SUPER_IMAGE_NAME = "super.img" _MIXED_SUPER_IMAGE_NAME = "mixed_super.img" _CMD_CVD_START = " start" +_CMD_CVD_VERSION = " version" _CMD_LAUNCH_CVD_ARGS = ( " -daemon -config=%s -system_image_dir %s -instance_dir %s " - "-undefok=report_anonymous_usage_stats,config " + "-undefok=report_anonymous_usage_stats,config,proxy_fastboot " "-report_anonymous_usage_stats=y") _CMD_LAUNCH_CVD_HW_ARGS = " -cpus %s -x_res %s -y_res %s -dpi %s -memory_mb %s" _CMD_LAUNCH_CVD_DISK_ARGS = ( @@ -92,9 +90,16 @@ _CMD_LAUNCH_CVD_VNC_ARG = " -start_vnc_server=true" _CMD_LAUNCH_CVD_SUPER_IMAGE_ARG = " -super_image=%s" _CMD_LAUNCH_CVD_BOOT_IMAGE_ARG = " -boot_image=%s" _CMD_LAUNCH_CVD_VENDOR_BOOT_IMAGE_ARG = " -vendor_boot_image=%s" +_CMD_LAUNCH_CVD_KERNEL_IMAGE_ARG = " -kernel_path=%s" +_CMD_LAUNCH_CVD_INITRAMFS_IMAGE_ARG = " -initramfs_path=%s" +_CMD_LAUNCH_CVD_VBMETA_IMAGE_ARG = " -vbmeta_image=%s" _CMD_LAUNCH_CVD_NO_ADB_ARG = " -run_adb_connector=false" +# Supported since U. +_CMD_LAUNCH_CVD_NO_FASTBOOT_ARG = " -proxy_fastboot=false" +_CMD_LAUNCH_CVD_INSTANCE_NUMS_ARG = " -instance_nums=%s" # Connect the OpenWrt device via console file. _CMD_LAUNCH_CVD_CONSOLE_ARG = " -console=true" +_CMD_LAUNCH_CVD_WEBRTC_DEIVE_ID = " -webrtc_device_id=%s" _CONFIG_RE = re.compile(r"^config=(?P<config>.+)") _CONSOLE_NAME = "console" # Files to store the output when launching cvds. @@ -113,8 +118,8 @@ _INSTANCES_IN_USE_MSG = ("All instances are in use. Try resetting an instance " "and %d. Alternatively, to run 'acloud delete --all' " % _MAX_INSTANCE_ID) _CONFIRM_RELAUNCH = ("\nCuttlefish AVD[id:%d] is already running. \n" - "Enter 'y' to terminate current instance and launch a new " - "instance, enter anything else to exit out[y/N]: ") + "Enter 'y' to terminate current instance and launch a " + "new instance, enter anything else to exit out[y/N]: ") # The first two fields of this named tuple are image folder and CVD host # package folder which are essential for local instances. The following fields @@ -122,7 +127,9 @@ _CONFIRM_RELAUNCH = ("\nCuttlefish AVD[id:%d] is already running. \n" ArtifactPaths = collections.namedtuple( "ArtifactPaths", ["image_dir", "host_bins", "host_artifacts", "misc_info", "ota_tools_dir", - "system_image", "boot_image", "vendor_boot_image"]) + "system_image", "boot_image", "vendor_boot_image", "kernel_image", + "initramfs_image", "vendor_image", "vendor_dlkm_image", "odm_image", + "odm_dlkm_image"]) class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): @@ -154,29 +161,51 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): artifact_paths = self.GetImageArtifactsPath(avd_spec) try: - ins_id, ins_lock = self._SelectAndLockInstance(avd_spec) + ins_ids, ins_locks = self._SelectAndLockInstances(avd_spec) except errors.CreateError as e: result_report.UpdateFailure(str(e)) return result_report try: - if not self._CheckRunningCvd(ins_id, no_prompts): - # Mark as in-use so that it won't be auto-selected again. - ins_lock.SetInUse(True) - sys.exit(constants.EXIT_BY_USER) + for ins_id, ins_lock in zip(ins_ids, ins_locks): + if not self._CheckRunningCvd(ins_id, no_prompts): + # Mark as in-use so that it won't be auto-selected again. + ins_lock.SetInUse(True) + sys.exit(constants.EXIT_BY_USER) - result_report = self._CreateInstance(ins_id, artifact_paths, + result_report = self._CreateInstance(ins_ids, artifact_paths, avd_spec, no_prompts) - # The infrastructure is able to delete the instance only if the - # instance name is reported. This method changes the state to - # in-use after creating the report. - ins_lock.SetInUse(True) + # Set the state to in-use if the instances start successfully. + # Failing instances are not set to in-use so that the user can + # restart them with the same IDs. + if result_report.status == report.Status.SUCCESS: + for ins_lock in ins_locks: + ins_lock.SetInUse(True) return result_report finally: - ins_lock.Unlock() + for ins_lock in ins_locks: + ins_lock.Unlock() - @staticmethod - def _SelectAndLockInstance(avd_spec): + def _SelectAndLockInstances(self, avd_spec): + """Select the ids and lock these instances. + + Args: + avd_spec: AVCSpec for the device. + + Returns: + The instance ids and the LocalInstanceLock that are locked. + """ + main_id, main_lock = self._SelectAndLockInstance(avd_spec) + ins_ids = [main_id] + ins_locks = [main_lock] + for _ in range(2, avd_spec.num_avds_per_instance + 1): + ins_id, ins_lock = self._SelectOneFreeInstance() + ins_ids.append(ins_id) + ins_locks.append(ins_lock) + logger.info("Selected instance ids: %s", ins_ids) + return ins_ids, ins_locks + + def _SelectAndLockInstance(self, avd_spec): """Select an id and lock the instance. Args: @@ -196,21 +225,32 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): return ins_id, ins_lock raise errors.CreateError("Instance %d is locked by another " "process." % ins_id) + return self._SelectOneFreeInstance() + + @staticmethod + def _SelectOneFreeInstance(): + """Select one free id and lock the instance. + Returns: + The instance id and the LocalInstanceLock that is locked by this + process. + + Raises: + errors.CreateError if fails to select or lock the instance. + """ for ins_id in range(1, _MAX_INSTANCE_ID + 1): ins_lock = instance.GetLocalInstanceLock(ins_id) if ins_lock.LockIfNotInUse(timeout_secs=0): - logger.info("Selected instance id: %d", ins_id) return ins_id, ins_lock raise errors.CreateError(_INSTANCES_IN_USE_MSG) - #pylint: disable=too-many-locals,too-many-statements - def _CreateInstance(self, local_instance_id, artifact_paths, avd_spec, + # pylint: disable=too-many-locals,too-many-statements + def _CreateInstance(self, instance_ids, artifact_paths, avd_spec, no_prompts): """Create a CVD instance. Args: - local_instance_id: Integer of instance id. + instance_ids: List of integer of instance ids. artifact_paths: ArtifactPaths object. avd_spec: AVDSpec for the instance. no_prompts: Boolean, True to skip all prompts. @@ -218,6 +258,7 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): Returns: A Report instance. """ + local_instance_id = instance_ids[0] webrtc_port = self.GetWebrtcSigServerPort(local_instance_id) if avd_spec.connect_webrtc: utils.ReleasePort(webrtc_port) @@ -225,22 +266,40 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): cvd_home_dir = instance.GetLocalInstanceHomeDir(local_instance_id) create_common.PrepareLocalInstanceDir(cvd_home_dir, avd_spec) super_image_path = None - if artifact_paths.system_image: - super_image_path = self._MixSuperImage(cvd_home_dir, - artifact_paths) + vbmeta_image_path = None + if artifact_paths.system_image or artifact_paths.vendor_image: + super_image_path = os.path.join(cvd_home_dir, + _MIXED_SUPER_IMAGE_NAME) + ota = ota_tools.OtaTools(artifact_paths.ota_tools_dir) + ota.MixSuperImage( + super_image_path, artifact_paths.misc_info, + artifact_paths.image_dir, + system_image=artifact_paths.system_image, + vendor_image=artifact_paths.vendor_image, + vendor_dlkm_image=artifact_paths.vendor_dlkm_image, + odm_image=artifact_paths.odm_image, + odm_dlkm_image=artifact_paths.odm_dlkm_image) + if artifact_paths.vendor_image: + vbmeta_image_path = os.path.join(cvd_home_dir, + "disabled_vbmeta.img") + ota.MakeDisabledVbmetaImage(vbmeta_image_path) runtime_dir = instance.GetLocalInstanceRuntimeDir(local_instance_id) # TODO(b/168171781): cvd_status of list/delete via the symbolic. self.PrepareLocalCvdToolsLink(cvd_home_dir, artifact_paths.host_bins) if avd_spec.mkcert and avd_spec.connect_webrtc: self._TrustCertificatesForWebRTC(artifact_paths.host_artifacts) + if not avd_spec.use_launch_cvd: + self._LogCvdVersion(artifact_paths.host_bins) hw_property = None if avd_spec.hw_customize: hw_property = avd_spec.hw_property config = self._GetConfigFromAndroidInfo( - os.path.join(artifact_paths.image_dir, constants.ANDROID_INFO_FILE)) + os.path.join(artifact_paths.image_dir, + constants.ANDROID_INFO_FILE)) cmd = self.PrepareLaunchCVDCmd(hw_property, avd_spec.connect_adb, + avd_spec.connect_fastboot, artifact_paths, runtime_dir, avd_spec.connect_webrtc, @@ -249,7 +308,10 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): avd_spec.launch_args, config or avd_spec.flavor, avd_spec.openwrt, - avd_spec.use_launch_cvd) + avd_spec.use_launch_cvd, + instance_ids, + avd_spec.webrtc_device_id, + vbmeta_image_path) result_report = report.Report(command="create") instance_name = instance.GetLocalInstanceName(local_instance_id) @@ -258,16 +320,16 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): artifact_paths.host_artifacts, cvd_home_dir, (avd_spec.boot_timeout_secs or constants.DEFAULT_CF_BOOT_TIMEOUT)) - logs = self._FindLogs(local_instance_id) + logs = cvd_utils.FindLocalLogs(runtime_dir, local_instance_id) except errors.LaunchCVDFail as launch_error: - logs = self._FindLogs(local_instance_id) + logs = cvd_utils.FindLocalLogs(runtime_dir, local_instance_id) err_msg = ("Cannot create cuttlefish instance: %s\n" "For more detail: %s/launcher.log" % (launch_error, runtime_dir)) if constants.ERROR_MSG_WEBRTC_NOT_SUPPORT in str(launch_error): err_msg = ( - "WEBRTC is not supported in current build. Please try VNC such " - "as '$acloud create --autoconnect vnc'") + "WEBRTC is not supported in current build. Please try VNC " + "such as '$acloud create --autoconnect vnc'") result_report.SetStatus(report.Status.BOOT_FAIL) result_report.SetErrorType(constants.ACLOUD_BOOT_UP_ERROR) result_report.AddDeviceBootFailure( @@ -286,7 +348,8 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): result_report.SetStatus(report.Status.SUCCESS) result_report.AddDevice(instance_name, constants.LOCALHOST, active_ins.adb_port, active_ins.vnc_port, - webrtc_port, logs=logs, update_data=update_data) + webrtc_port, logs=logs, + update_data=update_data) # Launch vnc client if we're auto-connecting. if avd_spec.connect_vnc: utils.LaunchVNCFromReport(result_report, avd_spec, no_prompts) @@ -330,70 +393,85 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): @staticmethod def _FindCvdHostArtifactsPath(search_paths): - """Return the directory that contains CVD host artifacts (in particular webrtc).""" + """Return the directory that contains CVD host artifacts (in particular + webrtc). + """ for search_path in search_paths: - if os.path.isfile(os.path.join(search_path, "usr/share/webrtc/certs", "server.crt")): + if os.path.isfile(os.path.join(search_path, + "usr/share/webrtc/certs", + "server.crt")): return search_path raise errors.GetCvdLocalHostPackageError( - "CVD host webrtc artifacts are not found. Please run `make hosttar`, or " - "set --local-tool to an extracted CVD host package.") + "CVD host webrtc artifacts are not found. Please run " + "`make hosttar`, or set --local-tool to an extracted CVD host " + "package.") @staticmethod - def FindMiscInfo(image_dir): - """Find misc info in build output dir or extracted target files. + def _VerifyExtractedImgZip(image_dir): + """Verify that a path is build output dir or extracted img zip. - Args: - image_dir: The directory to search for misc info. + This method checks existence of super image. The file is in img zip + but not in target files zip. A cuttlefish instance requires a super + image if no system image or OTA tools are given. - Returns: - image_dir if the directory structure looks like an output directory - in build environment. - image_dir/META if it looks like extracted target files. + Args: + image_dir: The directory to be verified. Raises: - errors.CheckPathError if this method cannot find misc info. + errors.GetLocalImageError if the directory does not contain the + needed file. """ - misc_info_path = os.path.join(image_dir, _MISC_INFO_FILE_NAME) - if os.path.isfile(misc_info_path): - return misc_info_path - misc_info_path = os.path.join(image_dir, _TARGET_FILES_META_DIR_NAME, - _MISC_INFO_FILE_NAME) - if os.path.isfile(misc_info_path): - return misc_info_path - raise errors.CheckPathError( - "Cannot find %s in %s." % (_MISC_INFO_FILE_NAME, image_dir)) + if not os.path.isfile(os.path.join(image_dir, _SUPER_IMAGE_NAME)): + raise errors.GetLocalImageError( + f"Cannot find {_SUPER_IMAGE_NAME} in {image_dir}. The " + f"directory is expected to be an extracted img zip or " + f"{constants.ENV_ANDROID_PRODUCT_OUT}.") @staticmethod - def FindImageDir(image_dir): - """Find images in build output dir or extracted target files. + def _FindBootOrKernelImages(image_path): + """Find boot, vendor_boot, kernel, and initramfs images in a path. + + This method expects image_path to be: + - An output directory of a kernel build. It contains a kernel image and + initramfs.img. + - A generic boot image or its parent directory. The image name is + boot-*.img. The directory does not contain vendor_boot.img. + - An output directory of a cuttlefish build. It contains boot.img and + vendor_boot.img. Args: - image_dir: The directory to search for images. + image_path: A path to an image file or an image directory. Returns: - image_dir if the directory structure looks like an output directory - in build environment. - image_dir/IMAGES if it looks like extracted target files. + A tuple of strings, the paths to boot, vendor_boot, kernel, and + initramfs images. Each value can be None. Raises: - errors.GetLocalImageError if this method cannot find images. + errors.GetLocalImageError if image_path does not contain boot or + kernel images. """ - if glob.glob(os.path.join(image_dir, "*.img")): - return image_dir - subdir = os.path.join(image_dir, _TARGET_FILES_IMAGES_DIR_NAME) - if glob.glob(os.path.join(subdir, "*.img")): - return subdir - raise errors.GetLocalImageError( - "Cannot find images in %s." % image_dir) + kernel_image_path, initramfs_image_path = cvd_utils.FindKernelImages( + image_path) + if kernel_image_path and initramfs_image_path: + return None, None, kernel_image_path, initramfs_image_path + + boot_image_path, vendor_boot_image_path = cvd_utils.FindBootImages( + image_path) + if boot_image_path: + return boot_image_path, vendor_boot_image_path, None, None + + raise errors.GetLocalImageError(f"{image_path} is not a boot image or " + f"a directory containing images.") def GetImageArtifactsPath(self, avd_spec): """Get image artifacts path. This method will check if launch_cvd is exist and return the tuple path (image path and host bins path) where they are located respectively. - For remote image, RemoteImageLocalInstance will override this method and - return the artifacts path which is extracted and downloaded from remote. + For remote image, RemoteImageLocalInstance will override this method + and return the artifacts path which is extracted and downloaded from + remote. Args: avd_spec: AVDSpec object that tells us what we're going to create. @@ -415,23 +493,44 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): host_artifacts_path = self._FindCvdHostArtifactsPath(tool_dirs) if avd_spec.local_system_image: - misc_info_path = self.FindMiscInfo(image_dir) - image_dir = self.FindImageDir(image_dir) + misc_info_path = cvd_utils.FindMiscInfo(image_dir) + image_dir = cvd_utils.FindImageDir(image_dir) ota_tools_dir = os.path.abspath( ota_tools.FindOtaToolsDir(tool_dirs)) - system_image_path = create_common.FindLocalImage( - avd_spec.local_system_image, _SYSTEM_IMAGE_NAME_PATTERN) + system_image_path = create_common.FindSystemImage( + avd_spec.local_system_image) else: + self._VerifyExtractedImgZip(image_dir) misc_info_path = None ota_tools_dir = None system_image_path = None if avd_spec.local_kernel_image: - boot_image_path, vendor_boot_image_path = cvd_utils.FindBootImages( - avd_spec.local_kernel_image) + ( + boot_image_path, + vendor_boot_image_path, + kernel_image_path, + initramfs_image_path, + ) = self._FindBootOrKernelImages( + os.path.abspath(avd_spec.local_kernel_image)) else: boot_image_path = None vendor_boot_image_path = None + kernel_image_path = None + initramfs_image_path = None + + if avd_spec.local_vendor_image: + vendor_image_paths = cvd_utils.FindVendorImages( + avd_spec.local_vendor_image) + vendor_image_path = vendor_image_paths.vendor + vendor_dlkm_image_path = vendor_image_paths.vendor_dlkm + odm_image_path = vendor_image_paths.odm + odm_dlkm_image_path = vendor_image_paths.odm_dlkm + else: + vendor_image_path = None + vendor_dlkm_image_path = None + odm_image_path = None + odm_dlkm_image_path = None return ArtifactPaths(image_dir, host_bins_path, host_artifacts=host_artifacts_path, @@ -439,27 +538,13 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): ota_tools_dir=ota_tools_dir, system_image=system_image_path, boot_image=boot_image_path, - vendor_boot_image=vendor_boot_image_path) - - @staticmethod - def _MixSuperImage(output_dir, artifact_paths): - """Mix cuttlefish images and a system image into a super image. - - Args: - output_dir: The path to the output directory. - artifact_paths: ArtifactPaths object. - - Returns: - The path to the super image in output_dir. - """ - ota = ota_tools.OtaTools(artifact_paths.ota_tools_dir) - super_image_path = os.path.join(output_dir, _MIXED_SUPER_IMAGE_NAME) - ota.BuildSuperImage( - super_image_path, artifact_paths.misc_info, - lambda partition: ota_tools.GetImageForPartition( - partition, artifact_paths.image_dir, - system=artifact_paths.system_image)) - return super_image_path + vendor_boot_image=vendor_boot_image_path, + kernel_image=kernel_image_path, + initramfs_image=initramfs_image_path, + vendor_image=vendor_image_path, + vendor_dlkm_image=vendor_dlkm_image_path, + odm_image=odm_image_path, + odm_dlkm_image=odm_dlkm_image_path) @staticmethod def _GetConfigFromAndroidInfo(android_info_path): @@ -482,11 +567,14 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): return config_match.group("config") return None + # pylint: disable=too-many-branches @staticmethod - def PrepareLaunchCVDCmd(hw_property, connect_adb, artifact_paths, - runtime_dir, connect_webrtc, connect_vnc, - super_image_path, launch_args, config, - openwrt=False, use_launch_cvd=False): + def PrepareLaunchCVDCmd(hw_property, connect_adb, connect_fastboot, + artifact_paths, runtime_dir, connect_webrtc, + connect_vnc, super_image_path, launch_args, + config, openwrt=False, use_launch_cvd=False, + instance_ids=None, webrtc_device_id=None, + vbmeta_image_path=None): """Prepare launch_cvd command. Create the launch_cvd commands with all the required args and add @@ -496,6 +584,7 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): hw_property: dict object of hw property. artifact_paths: ArtifactPaths object. connect_adb: Boolean flag that enables adb_connector. + connect_fastboot: Boolean flag that enables fastboot_proxy. runtime_dir: String of runtime directory path. connect_webrtc: Boolean of connect_webrtc. connect_vnc: Boolean of connect_vnc. @@ -504,14 +593,17 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): config: String of config name. openwrt: Boolean of enable OpenWrt devices. use_launch_cvd: Boolean of using launch_cvd for old build cases. + instance_ids: List of integer of instance ids. + webrtc_device_id: String of webrtc device id. + vbmeta_image_path: String of vbmeta image path. Returns: String, cvd start cmd. """ bin_dir = os.path.join(artifact_paths.host_bins, "bin") - start_cvd_cmd = (os.path.join(bin_dir, constants.CMD_CVD) + - _CMD_CVD_START) - if use_launch_cvd: + cvd_path = os.path.join(bin_dir, constants.CMD_CVD) + start_cvd_cmd = cvd_path + _CMD_CVD_START + if use_launch_cvd or not os.path.isfile(cvd_path): start_cvd_cmd = os.path.join(bin_dir, constants.CMD_LAUNCH_CVD) launch_cvd_w_args = start_cvd_cmd + _CMD_LAUNCH_CVD_ARGS % ( config, artifact_paths.image_dir, runtime_dir) @@ -520,12 +612,16 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): hw_property["cpu"], hw_property["x_res"], hw_property["y_res"], hw_property["dpi"], hw_property["memory"]) if constants.HW_ALIAS_DISK in hw_property: - launch_cvd_w_args = (launch_cvd_w_args + _CMD_LAUNCH_CVD_DISK_ARGS % + launch_cvd_w_args = (launch_cvd_w_args + + _CMD_LAUNCH_CVD_DISK_ARGS % hw_property[constants.HW_ALIAS_DISK]) if not connect_adb: launch_cvd_w_args = launch_cvd_w_args + _CMD_LAUNCH_CVD_NO_ADB_ARG + if not connect_fastboot: + launch_cvd_w_args = launch_cvd_w_args + _CMD_LAUNCH_CVD_NO_FASTBOOT_ARG + if connect_webrtc: launch_cvd_w_args = launch_cvd_w_args + _CMD_LAUNCH_CVD_WEBRTC_ARGS @@ -547,9 +643,35 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): _CMD_LAUNCH_CVD_VENDOR_BOOT_IMAGE_ARG % artifact_paths.vendor_boot_image) + if artifact_paths.kernel_image: + launch_cvd_w_args = (launch_cvd_w_args + + _CMD_LAUNCH_CVD_KERNEL_IMAGE_ARG % + artifact_paths.kernel_image) + + if artifact_paths.initramfs_image: + launch_cvd_w_args = (launch_cvd_w_args + + _CMD_LAUNCH_CVD_INITRAMFS_IMAGE_ARG % + artifact_paths.initramfs_image) + + if vbmeta_image_path: + launch_cvd_w_args = (launch_cvd_w_args + + _CMD_LAUNCH_CVD_VBMETA_IMAGE_ARG % + vbmeta_image_path) + if openwrt: launch_cvd_w_args = launch_cvd_w_args + _CMD_LAUNCH_CVD_CONSOLE_ARG + if instance_ids and len(instance_ids) > 1: + launch_cvd_w_args = ( + launch_cvd_w_args + + _CMD_LAUNCH_CVD_INSTANCE_NUMS_ARG % + ",".join(map(str, instance_ids))) + + if webrtc_device_id: + launch_cvd_w_args = (launch_cvd_w_args + + _CMD_LAUNCH_CVD_WEBRTC_DEIVE_ID % + webrtc_device_id) + if launch_args: launch_cvd_w_args = launch_cvd_w_args + " " + launch_args @@ -573,7 +695,8 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): Returns: String of cvd_tools link path """ - cvd_tools_link_path = os.path.join(cvd_home_dir, constants.CVD_TOOLS_LINK_NAME) + cvd_tools_link_path = os.path.join(cvd_home_dir, + constants.CVD_TOOLS_LINK_NAME) if os.path.islink(cvd_tools_link_path): os.unlink(cvd_tools_link_path) os.symlink(host_bins_path, cvd_tools_link_path) @@ -602,6 +725,30 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): os.path.join(webrtc_certs_dir, cert_file_name)) @staticmethod + def _LogCvdVersion(host_bins_path): + """Log the version of the cvd server. + + Args: + host_bins_path: String of host package directory. + """ + cvd_path = os.path.join(host_bins_path, "bin", constants.CMD_CVD) + if not os.path.isfile(cvd_path): + logger.info("Skip logging cvd version as %s is not a file", + cvd_path) + return + + cmd = cvd_path + _CMD_CVD_VERSION + try: + proc = subprocess.run(cmd, shell=True, text=True, + capture_output=True, timeout=5, + check=False, cwd=host_bins_path) + logger.info("`%s` returned %d; stdout:\n%s", + cmd, proc.returncode, proc.stdout) + logger.info("`%s` stderr:\n%s", cmd, proc.stderr) + except subprocess.SubprocessError as e: + logger.error("`%s` failed: %s", cmd, e) + + @staticmethod def _CheckRunningCvd(local_instance_id, no_prompts=False): """Check if launch_cvd with the same instance id is running. @@ -644,8 +791,8 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): proc.terminate() @utils.TimeExecute(function_description="Waiting for AVD(s) to boot up") - def _LaunchCvd(self, cmd, local_instance_id, host_bins_path, host_artifacts_path, - cvd_home_dir, timeout): + def _LaunchCvd(self, cmd, local_instance_id, host_bins_path, + host_artifacts_path, cvd_home_dir, timeout): """Execute Launch CVD. Kick off the launch_cvd command and log the output. @@ -653,11 +800,13 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): Args: cmd: String, launch_cvd command. local_instance_id: Integer of instance id. - host_bins_path: String of host package directory containing binaries. + host_bins_path: String of host package directory containing + binaries. host_artifacts_path: String of host package directory containing other artifacts. cvd_home_dir: String, the home directory for the instance. - timeout: Integer, the number of seconds to wait for the AVD to boot up. + timeout: Integer, the number of seconds to wait for the AVD to + boot up. Raises: errors.LaunchCVDFail if launch_cvd times out or returns non-zero. @@ -670,6 +819,7 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str(local_instance_id) cvd_env[constants.ENV_CUTTLEFISH_CONFIG_FILE] = ( instance.GetLocalInstanceConfigPath(local_instance_id)) + cvd_env[constants.ENV_CVD_ACQUIRE_FILE_LOCK] = "false" stdout_file = os.path.join(cvd_home_dir, _STDOUT) stderr_file = os.path.join(cvd_home_dir, _STDERR) # Check the result of launch_cvd command. @@ -701,20 +851,3 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): split_stderr = stderr.splitlines()[-_MAX_REPORTED_ERROR_LINES:] raise errors.LaunchCVDFail( "%s Stderr:\n%s" % (error_msg, "\n".join(split_stderr))) - - @staticmethod - def _FindLogs(local_instance_id): - """Find log paths that will be written to report. - - Args: - local_instance_id: An integer, the instance id. - - Returns: - A list of report.LogFile. - """ - log_dir = instance.GetLocalInstanceLogDir(local_instance_id) - return [report.LogFile(os.path.join(log_dir, name), log_type) - for name, log_type in [ - ("launcher.log", constants.LOG_TYPE_TEXT), - ("kernel.log", constants.LOG_TYPE_KERNEL_LOG), - ("logcat", constants.LOG_TYPE_LOGCAT)]] diff --git a/create/local_image_local_instance_test.py b/create/local_image_local_instance_test.py index 7baf9711..28e757d3 100644 --- a/create/local_image_local_instance_test.py +++ b/create/local_image_local_instance_test.py @@ -30,6 +30,7 @@ from acloud.list import list as list_instance from acloud.internal import constants from acloud.internal.lib import driver_test_lib from acloud.internal.lib import utils +from acloud.public import report class LocalImageLocalInstanceTest(driver_test_lib.BaseDriverTest): @@ -37,37 +38,62 @@ class LocalImageLocalInstanceTest(driver_test_lib.BaseDriverTest): LAUNCH_CVD_CMD_WITH_DISK = """sg group1 <<EOF sg group2 -bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -blank_data_image_mb fake -data_policy always_create -start_vnc_server=true +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -blank_data_image_mb fake -data_policy always_create -start_vnc_server=true EOF""" LAUNCH_CVD_CMD_NO_DISK = """sg group1 <<EOF sg group2 -bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -start_vnc_server=true +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -start_vnc_server=true EOF""" LAUNCH_CVD_CMD_NO_DISK_WITH_GPU = """sg group1 <<EOF sg group2 -bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -start_vnc_server=true +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -start_vnc_server=true EOF""" LAUNCH_CVD_CMD_WITH_WEBRTC = """sg group1 <<EOF sg group2 -bin/cvd start -daemon -config=auto -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_webrtc=true +bin/cvd start -daemon -config=auto -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -start_webrtc=true EOF""" LAUNCH_CVD_CMD_WITH_MIXED_IMAGES = """sg group1 <<EOF sg group2 -bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_vnc_server=true -super_image=fake_super_image -boot_image=fake_boot_image -vendor_boot_image=fake_vendor_boot_image +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -start_vnc_server=true -super_image=fake_super_image -boot_image=fake_boot_image -vendor_boot_image=fake_vendor_boot_image +EOF""" + + LAUNCH_CVD_CMD_WITH_KERNEL_IMAGES = """sg group1 <<EOF +sg group2 +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -start_vnc_server=true -kernel_path=fake_kernel_image -initramfs_path=fake_initramfs_image +EOF""" + + LAUNCH_CVD_CMD_WITH_VBMETA_IMAGE = """sg group1 <<EOF +sg group2 +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -start_vnc_server=true -vbmeta_image=fake_vbmeta_image EOF""" LAUNCH_CVD_CMD_WITH_ARGS = """sg group1 <<EOF sg group2 -bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_vnc_server=true -setupwizard_mode=REQUIRED +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -start_vnc_server=true -setupwizard_mode=REQUIRED EOF""" LAUNCH_CVD_CMD_WITH_OPENWRT = """sg group1 <<EOF sg group2 -bin/launch_cvd -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_vnc_server=true -console=true +bin/launch_cvd -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -start_vnc_server=true -console=true +EOF""" + + LAUNCH_CVD_CMD_WITH_PET_NAME = """sg group1 <<EOF +sg group2 +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -start_vnc_server=true -webrtc_device_id=pet-name +EOF""" + + LAUNCH_CVD_CMD_WITH_NO_CVD = """sg group1 <<EOF +sg group2 +bin/launch_cvd -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -start_vnc_server=true +EOF""" + + LAUNCH_CVD_CMD_WITH_INS_IDS = """sg group1 <<EOF +sg group2 +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config,proxy_fastboot -report_anonymous_usage_stats=y -start_vnc_server=true -instance_nums=1,2 EOF""" _EXPECTED_DEVICES_IN_REPORT = [ @@ -77,11 +103,7 @@ EOF""" "adb_port": 6520, "vnc_port": 6444, "webrtc_port": 8443, - 'logs': [ - {'path': '/log/launcher.log', 'type': 'TEXT'}, - {'path': '/log/kernel.log', 'type': 'KERNEL_LOG'}, - {'path': '/log/logcat', 'type': 'LOGCAT'} - ], + 'logs': [{'path': '/log/launcher.log', 'type': 'TEXT'}], "screen_command": "screen /instances/cvd/console" } ] @@ -90,11 +112,7 @@ EOF""" { "instance_name": "local-instance-1", "ip": "0.0.0.0", - 'logs': [ - {'path': '/log/launcher.log', 'type': 'TEXT'}, - {'path': '/log/kernel.log', 'type': 'KERNEL_LOG'}, - {'path': '/log/logcat', 'type': 'LOGCAT'} - ] + 'logs': [{'path': '/log/launcher.log', 'type': 'TEXT'}], } ] @@ -119,15 +137,19 @@ EOF""" mock_utils.IsSupportedPlatform.return_value = True mock_get_image.return_value = local_image_local_instance.ArtifactPaths( "/image/path", "/host/bin/path", "host/usr/path", - None, None, None, None, None) + None, None, None, None, None, None, None, None, None, None, None) mock_check_running_cvd.return_value = True mock_avd_spec = mock.Mock() + mock_avd_spec.num_avds_per_instance = 1 + mock_avd_spec.local_instance_dir = None mock_lock = mock.Mock() mock_lock.Unlock.return_value = False mock_lock_instance.return_value = (1, mock_lock) + mock_report = mock.Mock() + mock_create.return_value = mock_report # Success - mock_create.return_value = mock.Mock() + mock_report.status = report.Status.SUCCESS self.local_image_local_instance._CreateAVD( mock_avd_spec, no_prompts=True) mock_lock_instance.assert_called_once() @@ -138,6 +160,17 @@ EOF""" mock_lock.SetInUse.reset_mock() mock_lock.Unlock.reset_mock() + # Failure with report + mock_report.status = report.Status.BOOT_FAIL + self.local_image_local_instance._CreateAVD( + mock_avd_spec, no_prompts=True) + mock_lock_instance.assert_called_once() + mock_lock.SetInUse.assert_not_called() + mock_lock.Unlock.assert_called_once() + + mock_lock_instance.reset_mock() + mock_lock.Unlock.reset_mock() + # Failure with no report mock_create.side_effect = ValueError("unit test") with self.assertRaises(ValueError): @@ -147,11 +180,25 @@ EOF""" mock_lock.SetInUse.assert_not_called() mock_lock.Unlock.assert_called_once() - # Failure with report - mock_lock_instance.side_effect = errors.CreateError("unit test") - report = self.local_image_local_instance._CreateAVD( - mock_avd_spec, no_prompts=True) - self.assertEqual(report.errors, ["unit test"]) + def testSelectAndLockInstances(self): + """test _SelectAndLockInstances.""" + mock_avd_spec = mock.Mock(num_avds_per_instance=1) + mock_main_lock = mock.Mock() + self.Patch(local_image_local_instance.LocalImageLocalInstance, + "_SelectAndLockInstance", return_value=(1, mock_main_lock)) + ins_ids, ins_locks = self.local_image_local_instance._SelectAndLockInstances( + mock_avd_spec) + self.assertEqual([1], ins_ids) + self.assertEqual([mock_main_lock], ins_locks) + + mock_avd_spec.num_avds_per_instance = 2 + mock_second_lock = mock.Mock() + self.Patch(local_image_local_instance.LocalImageLocalInstance, + "_SelectOneFreeInstance", return_value=(2, mock_second_lock)) + ins_ids, ins_locks = self.local_image_local_instance._SelectAndLockInstances( + mock_avd_spec) + self.assertEqual([1,2], ins_ids) + self.assertEqual([mock_main_lock, mock_second_lock], ins_locks) def testSelectAndLockInstance(self): """test _SelectAndLockInstance.""" @@ -183,29 +230,39 @@ EOF""" @mock.patch("acloud.create.local_image_local_instance.ota_tools") @mock.patch("acloud.create.local_image_local_instance.create_common") @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, + "_LogCvdVersion") + @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, "_LaunchCvd") @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, "PrepareLaunchCVDCmd") + @mock.patch("acloud.create.local_image_local_instance.cvd_utils") @mock.patch("acloud.create.local_image_local_instance.instance") - def testCreateInstance(self, mock_instance, + def testCreateInstance(self, mock_instance, mock_cvd_utils, _mock_prepare_cmd, mock_launch_cvd, - _mock_create_common, mock_ota_tools, _mock_utils, - _mock_trust_certs): + mock_log_cvd_version, _mock_create_common, + mock_ota_tools, _mock_utils, mock_trust_certs): """Test the report returned by _CreateInstance.""" mock_instance.GetLocalInstanceHomeDir.return_value = ( "/local-instance-1") mock_instance.GetLocalInstanceName.return_value = "local-instance-1" - mock_instance.GetLocalInstanceLogDir.return_value = "/log" + mock_instance.GetLocalInstanceRuntimeDir.return_value = ( + "/instances/cvd") mock_instance.GetLocalInstanceConfig.return_value = ( "/instances/cvd/config") + mock_cvd_utils.FindLocalLogs.return_value = [ + {'path': '/log/launcher.log', 'type': 'TEXT'}] artifact_paths = local_image_local_instance.ArtifactPaths( "/image/path", "/host/bin/path", "/host/usr/path", "/misc/info/path", "/ota/tools/dir", "/system/image/path", "/boot/image/path", - "/vendor_boot/image/path") + "/vendor_boot/image/path", "/kernel/image/path", + "/initramfs/image/path", "/vendor/image/path", + "/vendor_dlkm/image/path", "/odm/image/path", + "/odm_dlkm/image/path") mock_ota_tools_object = mock.Mock() mock_ota_tools.OtaTools.return_value = mock_ota_tools_object mock_avd_spec = mock.Mock( - unlock_screen=False, connect_webrtc=True, openwrt=True) + unlock_screen=False, connect_webrtc=True, openwrt=True, + use_launch_cvd=False) local_ins = mock.Mock( adb_port=6520, vnc_port=6444 @@ -217,35 +274,49 @@ EOF""" return_value=local_ins) self.Patch(os, "symlink") + ins_ids = [1] # Success - report = self.local_image_local_instance._CreateInstance( - 1, artifact_paths, mock_avd_spec, no_prompts=True) + result_report = self.local_image_local_instance._CreateInstance( + ins_ids, artifact_paths, mock_avd_spec, no_prompts=True) - self.assertEqual(report.data.get("devices"), + self.assertEqual(result_report.data.get("devices"), self._EXPECTED_DEVICES_IN_REPORT) mock_ota_tools.OtaTools.assert_called_with("/ota/tools/dir") - mock_ota_tools_object.BuildSuperImage.assert_called_with( - "/local-instance-1/mixed_super.img", "/misc/info/path", mock.ANY) + mock_ota_tools_object.MixSuperImage.assert_called_with( + "/local-instance-1/mixed_super.img", "/misc/info/path", + "/image/path", + system_image="/system/image/path", + vendor_image="/vendor/image/path", + vendor_dlkm_image="/vendor_dlkm/image/path", + odm_image="/odm/image/path", + odm_dlkm_image="/odm_dlkm/image/path") + mock_ota_tools_object.MakeDisabledVbmetaImage.assert_called_once() + mock_cvd_utils.FindLocalLogs.assert_called_with( + "/instances/cvd", 1) + mock_log_cvd_version.assert_called_with("/host/bin/path") # should call _TrustCertificatesForWebRTC - _mock_trust_certs.assert_called_once() - _mock_trust_certs.reset_mock() + mock_trust_certs.assert_called_once() + mock_trust_certs.reset_mock() # should not call _TrustCertificatesForWebRTC mock_avd_spec.connect_webrtc = False self.local_image_local_instance._CreateInstance( - 1, artifact_paths, mock_avd_spec, no_prompts=True) - self.assertEqual(_mock_create_common.call_count, 0) + ins_ids, artifact_paths, mock_avd_spec, no_prompts=True) + mock_trust_certs.assert_not_called() # Failure + mock_cvd_utils.reset_mock() mock_launch_cvd.side_effect = errors.LaunchCVDFail("unit test") - report = self.local_image_local_instance._CreateInstance( - 1, artifact_paths, mock_avd_spec, no_prompts=True) + result_report = self.local_image_local_instance._CreateInstance( + ins_ids, artifact_paths, mock_avd_spec, no_prompts=True) - self.assertEqual(report.data.get("devices_failing_boot"), + self.assertEqual(result_report.data.get("devices_failing_boot"), self._EXPECTED_DEVICES_IN_FAILED_REPORT) - self.assertIn("unit test", report.errors[0]) + self.assertIn("unit test", result_report.errors[0]) + mock_cvd_utils.FindLocalLogs.assert_called_with( + "/instances/cvd", 1) # pylint: disable=protected-access @mock.patch("acloud.create.local_image_local_instance.os.path.isfile") @@ -267,15 +338,13 @@ EOF""" @staticmethod def _CreateEmptyFile(path): - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w"): - pass + driver_test_lib.BaseDriverTest.CreateFile(path) @mock.patch("acloud.create.local_image_local_instance.ota_tools") def testGetImageArtifactsPath(self, mock_ota_tools): """Test GetImageArtifactsPath without system image dir.""" with tempfile.TemporaryDirectory() as temp_dir: - image_dir = "/unit/test" + image_dir = os.path.join(temp_dir, "image") cvd_dir = os.path.join(temp_dir, "cvd-host_package") self._CreateEmptyFile(os.path.join(cvd_dir, "bin", "launch_cvd")) self._CreateEmptyFile(os.path.join(cvd_dir, "usr/share/webrtc/certs", "server.crt")) @@ -284,24 +353,29 @@ EOF""" local_image_dir=image_dir, local_kernel_image=None, local_system_image=None, + local_vendor_image=None, local_tool_dirs=[cvd_dir]) + with self.assertRaisesRegex( + errors.GetLocalImageError, + r"The directory is expected to be an extracted img zip " + r"or ANDROID_PRODUCT_OUT\."): + self.local_image_local_instance.GetImageArtifactsPath( + mock_avd_spec) + + self._CreateEmptyFile(os.path.join(image_dir, "super.img")) + paths = self.local_image_local_instance.GetImageArtifactsPath( mock_avd_spec) mock_ota_tools.FindOtaToolsDir.assert_not_called() self.assertEqual(paths, (image_dir, cvd_dir, cvd_dir, - None, None, None, None, None)) + None, None, None, None, None, None, None, + None, None, None, None)) @mock.patch("acloud.create.local_image_local_instance.ota_tools") - @mock.patch("acloud.create.local_image_local_instance.cvd_utils") - def testGetImageFromBuildEnvironment(self, mock_cvd_utils, mock_ota_tools): + def testGetImageFromBuildEnvironment(self, mock_ota_tools): """Test GetImageArtifactsPath with files in build environment.""" - boot_image_path = "/mock/boot.img" - vendor_boot_image_path = "/mock/vendor_boot.img" - mock_cvd_utils.FindBootImages.return_value = (boot_image_path, - vendor_boot_image_path) - with tempfile.TemporaryDirectory() as temp_dir: image_dir = os.path.join(temp_dir, "image") cvd_dir = os.path.join(temp_dir, "cvd-host_package") @@ -309,6 +383,13 @@ EOF""" extra_image_dir = os.path.join(temp_dir, "extra_image") system_image_path = os.path.join(extra_image_dir, "system.img") misc_info_path = os.path.join(image_dir, "misc_info.txt") + boot_image_path = os.path.join(extra_image_dir, "boot.img") + vendor_boot_image_path = os.path.join(extra_image_dir, + "vendor_boot.img") + vendor_image_path = os.path.join(extra_image_dir, "vendor.img") + vendor_dlkm_image_path = os.path.join(extra_image_dir, "vendor_dlkm.img") + odm_image_path = os.path.join(extra_image_dir, "odm.img") + odm_dlkm_image_path = os.path.join(extra_image_dir, "odm_dlkm.img") self._CreateEmptyFile(os.path.join(image_dir, "vbmeta.img")) self._CreateEmptyFile(os.path.join(cvd_dir, "bin", "launch_cvd")) self._CreateEmptyFile(os.path.join(cvd_dir, "usr/share/webrtc/certs", "server.crt")) @@ -316,11 +397,18 @@ EOF""" self._CreateEmptyFile(os.path.join(extra_image_dir, "boot-debug.img")) self._CreateEmptyFile(misc_info_path) + self._CreateEmptyFile(vendor_image_path) + self._CreateEmptyFile(vendor_dlkm_image_path) + self._CreateEmptyFile(odm_image_path) + self._CreateEmptyFile(odm_dlkm_image_path) + self.CreateFile(boot_image_path, b"ANDROID!test_boot_image") + self.CreateFile(vendor_boot_image_path) mock_avd_spec = mock.Mock( local_image_dir=image_dir, local_kernel_image=extra_image_dir, local_system_image=extra_image_dir, + local_vendor_image=extra_image_dir, local_tool_dirs=[]) with mock.patch.dict("acloud.create.local_image_local_instance." @@ -332,38 +420,43 @@ EOF""" mock_avd_spec) mock_ota_tools.FindOtaToolsDir.assert_called_with([cvd_dir, "/cvd"]) - mock_cvd_utils.FindBootImages.asssert_called_with(extra_image_dir) self.assertEqual(paths, (image_dir, cvd_dir, cvd_dir, misc_info_path, cvd_dir, system_image_path, boot_image_path, - vendor_boot_image_path)) + vendor_boot_image_path, None, None, + vendor_image_path, vendor_dlkm_image_path, + odm_image_path, odm_dlkm_image_path)) @mock.patch("acloud.create.local_image_local_instance.ota_tools") - @mock.patch("acloud.create.local_image_local_instance.cvd_utils") - def testGetImageFromTargetFiles(self, mock_cvd_utils, mock_ota_tools): + def testGetImageFromTargetFiles(self, mock_ota_tools): """Test GetImageArtifactsPath with extracted target files.""" ota_tools_dir = "/mock_ota_tools" mock_ota_tools.FindOtaToolsDir.return_value = ota_tools_dir - boot_image_path = "/mock/boot.img" - mock_cvd_utils.FindBootImages.return_value = (boot_image_path, None) - with tempfile.TemporaryDirectory() as temp_dir: image_dir = os.path.join(temp_dir, "image") cvd_dir = os.path.join(temp_dir, "cvd-host_package") system_image_path = os.path.join(temp_dir, "system", "test.img") misc_info_path = os.path.join(image_dir, "META", "misc_info.txt") - - self._CreateEmptyFile(os.path.join(image_dir, "IMAGES", - "vbmeta.img")) - self._CreateEmptyFile(os.path.join(cvd_dir, "bin", "launch_cvd")) - self._CreateEmptyFile(os.path.join(cvd_dir, "usr/share/webrtc/certs", "server.crt")) - self._CreateEmptyFile(system_image_path) - self._CreateEmptyFile(misc_info_path) + kernel_image_dir = os.path.join(temp_dir, "kernel_image") + kernel_image_path = os.path.join(kernel_image_dir, "Image") + initramfs_image_path = os.path.join(kernel_image_dir, + "initramfs.img") + + self.CreateFile(os.path.join(kernel_image_dir, "boot.img")) + self.CreateFile(os.path.join(image_dir, "IMAGES", "vbmeta.img")) + self.CreateFile(os.path.join(cvd_dir, "bin", "launch_cvd")) + self.CreateFile(os.path.join(cvd_dir, "usr/share/webrtc/certs", + "server.crt")) + self.CreateFile(system_image_path) + self.CreateFile(misc_info_path) + self.CreateFile(kernel_image_path) + self.CreateFile(initramfs_image_path) mock_avd_spec = mock.Mock( local_image_dir=image_dir, - local_kernel_image=boot_image_path, + local_kernel_image=kernel_image_dir, local_system_image=system_image_path, + local_vendor_image=None, local_tool_dirs=[ota_tools_dir, cvd_dir]) with mock.patch.dict("acloud.create.local_image_local_instance." @@ -374,16 +467,17 @@ EOF""" mock_ota_tools.FindOtaToolsDir.assert_called_with( [ota_tools_dir, cvd_dir]) - mock_cvd_utils.FindBootImages.assert_called_with(boot_image_path) self.assertEqual(paths, (os.path.join(image_dir, "IMAGES"), cvd_dir, cvd_dir, misc_info_path, ota_tools_dir, system_image_path, - boot_image_path, None)) + None, None, kernel_image_path, initramfs_image_path, + None, None, None, None)) @mock.patch.object(utils, "CheckUserInGroups") def testPrepareLaunchCVDCmd(self, mock_usergroups): """test PrepareLaunchCVDCmd.""" mock_usergroups.return_value = False + self.Patch(os.path, "isfile", return_value=True) hw_property = {"cpu": "fake", "x_res": "fake", "y_res": "fake", "dpi":"fake", "memory": "fake", "disk": "fake"} constants.LIST_CF_USER_GROUPS = ["group1", "group2"] @@ -396,10 +490,16 @@ EOF""" ota_tools_dir=None, system_image=None, boot_image=None, - vendor_boot_image=None) + vendor_boot_image=None, + kernel_image=None, + initramfs_image=None, + vendor_image=None, + vendor_dlkm_image=None, + odm_image=None, + odm_dlkm_image=None) launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - hw_property, True, mock_artifact_paths, "fake_cvd_dir", False, + hw_property, True, True, mock_artifact_paths, "fake_cvd_dir", False, True, None, None, "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_DISK) @@ -407,43 +507,103 @@ EOF""" hw_property = {"cpu": "fake", "x_res": "fake", "y_res": "fake", "dpi": "fake", "memory": "fake"} launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - hw_property, True, mock_artifact_paths, "fake_cvd_dir", False, + hw_property, True, True, mock_artifact_paths, "fake_cvd_dir", False, True, None, None, "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_NO_DISK) # "gpu" is enabled with "default" launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - hw_property, True, mock_artifact_paths, "fake_cvd_dir", False, + hw_property, True, True, mock_artifact_paths, "fake_cvd_dir", False, True, None, None, "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_NO_DISK_WITH_GPU) # Following test with hw_property is None. launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - None, True, mock_artifact_paths, "fake_cvd_dir", True, False, + None, True, True, mock_artifact_paths, "fake_cvd_dir", True, False, None, None, "auto") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_WEBRTC) + # Mix super and boot images. mock_artifact_paths.boot_image = "fake_boot_image" mock_artifact_paths.vendor_boot_image = "fake_vendor_boot_image" launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - None, True, mock_artifact_paths, "fake_cvd_dir", False, True, + None, True, True, mock_artifact_paths, "fake_cvd_dir", False, True, "fake_super_image", None, "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_MIXED_IMAGES) mock_artifact_paths.boot_image = None mock_artifact_paths.vendor_boot_image = None + # Mix kernel images. + mock_artifact_paths.kernel_image = "fake_kernel_image" + mock_artifact_paths.initramfs_image = "fake_initramfs_image" + launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( + None, True, True, mock_artifact_paths, "fake_cvd_dir", False, True, + None, None, "phone") + self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_KERNEL_IMAGES) + mock_artifact_paths.kernel_image = None + mock_artifact_paths.initramfs_image = None + + # Specify vbmeta image. + launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( + None, True, True, mock_artifact_paths, "fake_cvd_dir", False, True, + None, None, "phone", vbmeta_image_path="fake_vbmeta_image" + ) + self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_VBMETA_IMAGE) + # Add args into launch command with "-setupwizard_mode=REQUIRED" launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - None, True, mock_artifact_paths, "fake_cvd_dir", False, True, + None, True, True, mock_artifact_paths, "fake_cvd_dir", False, True, None, "-setupwizard_mode=REQUIRED", "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_ARGS) # Test with "openwrt" and "use_launch_cvd" are enabled. launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - None, True, mock_artifact_paths, "fake_cvd_dir", False, True, + None, True, True, mock_artifact_paths, "fake_cvd_dir", False, True, None, None, "phone", openwrt=True, use_launch_cvd=True) self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_OPENWRT) + # Test with instance_ids + launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( + None, True, True, mock_artifact_paths, "fake_cvd_dir", False, True, + None, None, "phone", instance_ids=[1,2]) + self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_INS_IDS) + + # Test with "pet-name" + launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( + None, True, True, mock_artifact_paths, "fake_cvd_dir", False, True, + None, None, "phone", webrtc_device_id="pet-name") + self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_PET_NAME) + + # Test with "cvd" doesn't exist + self.Patch(os.path, "isfile", return_value=False) + launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( + None, True, True, mock_artifact_paths, "fake_cvd_dir", False, True, + None, None, "phone", openwrt=False, use_launch_cvd=False) + self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_NO_CVD) + + @mock.patch("acloud.create.local_image_local_instance.subprocess.run") + def testLogCvdVersion(self, mock_run): + """Test _LogCvdVersion.""" + with tempfile.TemporaryDirectory() as temp_dir: + # cvd does not exist in old versions. + self.local_image_local_instance._LogCvdVersion(temp_dir) + mock_run.assert_not_called() + + # cvd command completes. + mock_run.return_value = mock.Mock( + returncode=1, stdout=None, stderr="err") + cvd_path = os.path.join(temp_dir, "bin", "cvd") + self.CreateFile(cvd_path) + self.local_image_local_instance._LogCvdVersion(temp_dir) + mock_run.assert_called_once() + self.assertEqual(mock_run.call_args[0][0], f"{cvd_path} version") + + # cvd cannot run. + mock_run.reset_mock() + mock_run.side_effect = subprocess.SubprocessError + self.local_image_local_instance._LogCvdVersion(temp_dir) + mock_run.assert_called_once() + @mock.patch.object(utils, "GetUserAnswerYes") @mock.patch.object(list_instance, "GetActiveCVD") def testCheckRunningCvd(self, mock_cvd_running, mock_get_answer): @@ -475,11 +635,6 @@ EOF""" host_artifacts_path = "host_artifacts_path" cvd_home_dir = "fake_home" timeout = 100 - cvd_env = {} - cvd_env[constants.ENV_CVD_HOME] = cvd_home_dir - cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str(local_instance_id) - cvd_env[constants.ENV_ANDROID_SOONG_HOST_OUT] = host_artifacts_path - cvd_env[constants.ENV_ANDROID_HOST_OUT] = host_bins_path mock_proc = mock.Mock(returncode=0) mock_popen.return_value = mock_proc mock_proc.communicate.return_value = ("stdout", "stderr") diff --git a/create/local_image_remote_instance.py b/create/local_image_remote_instance.py index 58bbb0bb..b34026b3 100644 --- a/create/local_image_remote_instance.py +++ b/create/local_image_remote_instance.py @@ -64,7 +64,8 @@ class LocalImageRemoteInstance(base_avd_create.BaseAVDCreate): unlock_screen=avd_spec.unlock_screen, wait_for_boot=False, connect_webrtc=avd_spec.connect_webrtc, - client_adb_port=avd_spec.client_adb_port) + client_adb_port=avd_spec.client_adb_port, + client_fastboot_port=avd_spec.client_fastboot_port) if create_report.status == report.Status.SUCCESS: if avd_spec.connect_vnc: utils.LaunchVNCFromReport(create_report, avd_spec, no_prompts) diff --git a/create/remote_image_local_instance.py b/create/remote_image_local_instance.py index d0d0958f..4aaf4f57 100644 --- a/create/remote_image_local_instance.py +++ b/create/remote_image_local_instance.py @@ -30,6 +30,7 @@ from acloud.create import local_image_local_instance from acloud.internal import constants from acloud.internal.lib import android_build_client from acloud.internal.lib import auth +from acloud.internal.lib import cvd_utils from acloud.internal.lib import ota_tools from acloud.internal.lib import utils from acloud.setup import setup_common @@ -52,9 +53,34 @@ _HOME_FOLDER = os.path.expanduser("~") # for the downloaded image artifacts. _REQUIRED_SPACE = 10 -_SYSTEM_IMAGE_NAME_PATTERN = r"system\.img" _SYSTEM_MIX_IMAGE_DIR = "mix_image_{build_id}" -_DOWNLOAD_MIX_IMAGE_NAME = "{build_target}-target_files-{build_id}.zip" + + +def _ShouldClearFetchDir(fetch_dir, avd_spec, fetch_cvd_args_str, + fetch_cvd_args_file): + """Check if the previous fetch directory should be removed. + + The fetch directory would be removed when the user explicitly sets the + --force-sync flag, or when the fetch_cvd_args_str changed. + + Args: + fetch_dir: String, path to the fetch directory. + avd_spec: AVDSpec object that tells us what we're going to create. + fetch_cvd_args_str: String, args for current fetch_cvd command. + fetch_cvd_args_file: String, path to file of previous fetch_cvd args. + + Returns: + Boolean. True if the fetch directory should be removed. + """ + if not os.path.exists(fetch_dir): + return False + if avd_spec.force_sync: + return True + + if not os.path.exists(fetch_cvd_args_file): + return True + with open(fetch_cvd_args_file, "r") as f: + return fetch_cvd_args_str != f.read() @utils.TimeExecute(function_description="Downloading Android Build image") @@ -62,7 +88,11 @@ def DownloadAndProcessImageFiles(avd_spec): """Download the CF image artifacts and process them. To download rom images, Acloud would download the tool fetch_cvd that can - help process mixed build images. + help process mixed build images, and store the fetch_cvd_build_args in + FETCH_CVD_ARGS_FILE when the fetch command executes successfully. Next + time when this function is called with the same image_download_dir, the + FETCH_CVD_ARGS_FILE would be used to check whether this image_download_dir + can be reused or not. Args: avd_spec: AVDSpec object that tells us what we're going to create. @@ -75,7 +105,6 @@ def DownloadAndProcessImageFiles(avd_spec): """ cfg = avd_spec.cfg build_id = avd_spec.remote_image[constants.BUILD_ID] - build_branch = avd_spec.remote_image[constants.BUILD_BRANCH] build_target = avd_spec.remote_image[constants.BUILD_TARGET] extract_path = os.path.join( @@ -85,30 +114,29 @@ def DownloadAndProcessImageFiles(avd_spec): logger.debug("Extract path: %s", extract_path) - if avd_spec.force_sync and os.path.exists(extract_path): + build_api = ( + android_build_client.AndroidBuildClient(auth.CreateCredentials(cfg))) + fetch_cvd_build_args = build_api.GetFetchBuildArgs( + avd_spec.remote_image, + avd_spec.system_build_info, + avd_spec.kernel_build_info, + avd_spec.boot_build_info, + avd_spec.bootloader_build_info, + avd_spec.ota_build_info) + + fetch_cvd_args_str = " ".join(fetch_cvd_build_args) + fetch_cvd_args_file = os.path.join(extract_path, + constants.FETCH_CVD_ARGS_FILE) + if _ShouldClearFetchDir(extract_path, avd_spec, fetch_cvd_args_str, + fetch_cvd_args_file): shutil.rmtree(extract_path) + if not os.path.exists(extract_path): os.makedirs(extract_path) - build_api = ( - android_build_client.AndroidBuildClient(auth.CreateCredentials(cfg))) # Download rom images via fetch_cvd fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) - build_api.DownloadFetchcvd(fetch_cvd, cfg.fetch_cvd_version) - fetch_cvd_build_args = build_api.GetFetchBuildArgs( - build_id, build_branch, build_target, - avd_spec.system_build_info.get(constants.BUILD_ID), - avd_spec.system_build_info.get(constants.BUILD_BRANCH), - avd_spec.system_build_info.get(constants.BUILD_TARGET), - avd_spec.kernel_build_info.get(constants.BUILD_ID), - avd_spec.kernel_build_info.get(constants.BUILD_BRANCH), - avd_spec.kernel_build_info.get(constants.BUILD_TARGET), - avd_spec.bootloader_build_info.get(constants.BUILD_ID), - avd_spec.bootloader_build_info.get(constants.BUILD_BRANCH), - avd_spec.bootloader_build_info.get(constants.BUILD_TARGET), - avd_spec.ota_build_info.get(constants.BUILD_ID), - avd_spec.ota_build_info.get(constants.BUILD_BRANCH), - avd_spec.ota_build_info.get(constants.BUILD_TARGET)) + build_api.DownloadFetchcvd(fetch_cvd, avd_spec.fetch_cvd_version) creds_cache_file = os.path.join(_HOME_FOLDER, cfg.creds_cache_file) fetch_cvd_cert_arg = build_api.GetFetchCertArg(creds_cache_file) fetch_cvd_args = [fetch_cvd, "-directory=%s" % extract_path, @@ -120,6 +148,10 @@ def DownloadAndProcessImageFiles(avd_spec): except subprocess.CalledProcessError as e: raise errors.GetRemoteImageError("Fails to download images: %s" % e) + # Save the fetch cvd build args when the fetch command succeeds + with open(fetch_cvd_args_file, "w") as output: + output.write(fetch_cvd_args_str) + return extract_path @@ -160,21 +192,6 @@ def ConfirmDownloadRemoteImageDir(download_dir): return download_dir -def GetMixBuildTargetFilename(build_target, build_id): - """Get the mix build target filename. - - Args: - build_id: String, Build id, e.g. "2263051", "P2804227" - build_target: String, the build target, e.g. cf_x86_phone-userdebug - - Returns: - String, a file name, e.g. "cf_x86_phone-target_files-2263051.zip" - """ - return _DOWNLOAD_MIX_IMAGE_NAME.format( - build_target=build_target.split('-')[0], - build_id=build_id) - - class RemoteImageLocalInstance(local_image_local_instance.LocalImageLocalInstance): """Create class for a remote image local instance AVD. @@ -212,7 +229,15 @@ class RemoteImageLocalInstance(local_image_local_instance.LocalImageLocalInstanc % image_dir) mix_image_dir = None - if avd_spec.local_system_image: + misc_info_path = None + ota_tools_dir = None + system_image_path = None + vendor_image_path = None + vendor_dlkm_image_path = None + odm_image_path = None + odm_dlkm_image_path = None + host_bins_path = image_dir + if avd_spec.local_system_image or avd_spec.local_vendor_image: build_id = avd_spec.remote_image[constants.BUILD_ID] build_target = avd_spec.remote_image[constants.BUILD_TARGET] mix_image_dir =os.path.join( @@ -221,31 +246,51 @@ class RemoteImageLocalInstance(local_image_local_instance.LocalImageLocalInstanc os.makedirs(mix_image_dir) create_common.DownloadRemoteArtifact( avd_spec.cfg, build_target, build_id, - GetMixBuildTargetFilename(build_target, build_id), + cvd_utils.GetMixBuildTargetFilename(build_target, build_id), mix_image_dir, decompress=True) - misc_info_path = super().FindMiscInfo(mix_image_dir) - mix_image_dir = super().FindImageDir(mix_image_dir) + misc_info_path = cvd_utils.FindMiscInfo(mix_image_dir) + mix_image_dir = cvd_utils.FindImageDir(mix_image_dir) tool_dirs = (avd_spec.local_tool_dirs + create_common.GetNonEmptyEnvVars( constants.ENV_ANDROID_SOONG_HOST_OUT, constants.ENV_ANDROID_HOST_OUT)) ota_tools_dir = os.path.abspath( ota_tools.FindOtaToolsDir(tool_dirs)) - system_image_path = create_common.FindLocalImage( - avd_spec.local_system_image, _SYSTEM_IMAGE_NAME_PATTERN) - else: - misc_info_path = None - ota_tools_dir = None - system_image_path = None + + # When using local vendor image, use cvd in local-tool if it + # exists. Fall back to downloaded tools in case it's missing + + if avd_spec.local_vendor_image and avd_spec.local_tool_dirs: + try: + host_bins_path = self._FindCvdHostBinaries(tool_dirs) + except errors.GetCvdLocalHostPackageError: + logger.debug("fall back to downloaded cvd host binaries") + if avd_spec.local_system_image: + system_image_path = create_common.FindSystemImage( + avd_spec.local_system_image) + if avd_spec.local_vendor_image: + vendor_image_paths = cvd_utils.FindVendorImages( + avd_spec.local_vendor_image) + vendor_image_path = vendor_image_paths.vendor + vendor_dlkm_image_path = vendor_image_paths.vendor_dlkm + odm_image_path = vendor_image_paths.odm + odm_dlkm_image_path = vendor_image_paths.odm_dlkm + # This method does not set the optional fields because launch_cvd loads # the paths from the fetcher config in image_dir. return local_image_local_instance.ArtifactPaths( image_dir=mix_image_dir or image_dir, - host_bins=image_dir, + host_bins=host_bins_path, host_artifacts=image_dir, misc_info=misc_info_path, ota_tools_dir=ota_tools_dir, system_image=system_image_path, + vendor_image=vendor_image_path, + vendor_dlkm_image=vendor_dlkm_image_path, + odm_image=odm_image_path, + odm_dlkm_image=odm_dlkm_image_path, boot_image=None, - vendor_boot_image=None) + vendor_boot_image=None, + kernel_image=None, + initramfs_image=None) diff --git a/create/remote_image_local_instance_test.py b/create/remote_image_local_instance_test.py index 135fafac..18af6558 100644 --- a/create/remote_image_local_instance_test.py +++ b/create/remote_image_local_instance_test.py @@ -13,20 +13,22 @@ # limitations under the License. """Tests for remote_image_local_instance.""" -import unittest +import builtins from collections import namedtuple import os import shutil import subprocess +import unittest from unittest import mock from acloud import errors from acloud.create import create_common from acloud.create import remote_image_local_instance -from acloud.create import local_image_local_instance +from acloud.internal import constants from acloud.internal.lib import android_build_client from acloud.internal.lib import auth +from acloud.internal.lib import cvd_utils from acloud.internal.lib import driver_test_lib from acloud.internal.lib import ota_tools from acloud.internal.lib import utils @@ -58,6 +60,7 @@ class RemoteImageLocalInstanceTest(driver_test_lib.BaseDriverTest): mock_proc.return_value = "/unit/test" avd_spec = mock.MagicMock() avd_spec.local_system_image = None + avd_spec.local_vendor_image = None # raise errors.NoCuttlefishCommonInstalled self.Patch(setup_common, "PackageInstalled", return_value=False) self.assertRaises(errors.NoCuttlefishCommonInstalled, @@ -83,12 +86,15 @@ class RemoteImageLocalInstanceTest(driver_test_lib.BaseDriverTest): self.Patch(create_common, "DownloadRemoteArtifact") self.Patch(os.path, "exists", side_effect=[True, False]) self.Patch(create_common, "GetNonEmptyEnvVars") - self.Patch(local_image_local_instance.LocalImageLocalInstance, - "FindMiscInfo", return_value="/mix_image_1234/MISC") - self.Patch(local_image_local_instance.LocalImageLocalInstance, - "FindImageDir", return_value="/mix_image_1234/IMAGES") + self.Patch(cvd_utils, "FindMiscInfo", + return_value="/mix_image_1234/MISC") + self.Patch(cvd_utils, "FindImageDir", + return_value="/mix_image_1234/IMAGES") self.Patch(ota_tools, "FindOtaToolsDir", return_value="/ota_tools_dir") - self.Patch(create_common, "FindLocalImage", return_value="/system_image_path") + self.Patch(create_common, "FindSystemImage", + return_value="/system_image_path") + self.Patch(self.RemoteImageLocalInstance, "_FindCvdHostBinaries", + side_effect=errors.GetCvdLocalHostPackageError("not found")) paths = self.RemoteImageLocalInstance.GetImageArtifactsPath(avd_spec) create_common.DownloadRemoteArtifact.assert_called_with( avd_spec.cfg, "aosp_cf_x86_64_phone-userdebug", "1234", @@ -99,6 +105,28 @@ class RemoteImageLocalInstanceTest(driver_test_lib.BaseDriverTest): self.assertEqual(paths.host_bins, "/unit/test") self.assertEqual(paths.ota_tools_dir, "/ota_tools_dir") self.assertEqual(paths.system_image, "/system_image_path") + self.RemoteImageLocalInstance._FindCvdHostBinaries.assert_not_called() + + # local vendor image, local tool including host bins + avd_spec.local_vendor_image = "/test_local_vendor_image_dir" + vendor_image_paths = cvd_utils.VendorImagePaths( + "vendor.img", "vendor_dlkm.img", "odm.img", "odm_dlkm.img") + self.Patch(cvd_utils, "FindVendorImages", + return_value=vendor_image_paths) + self.Patch(os.path, "exists", side_effect=[True, False]) + self.Patch(self.RemoteImageLocalInstance, "_FindCvdHostBinaries", + return_value="/test_local_tool_dirs") + paths = self.RemoteImageLocalInstance.GetImageArtifactsPath(avd_spec) + self.assertEqual(paths.host_bins, "/test_local_tool_dirs") + + # local vendor image, local tool without host bins + avd_spec.local_vendor_image = "/test_local_vendor_image_dir" + self.Patch(os.path, "exists", side_effect=[True, False]) + self.Patch(self.RemoteImageLocalInstance, "_FindCvdHostBinaries", + side_effect=errors.GetCvdLocalHostPackageError("not found")) + paths = self.RemoteImageLocalInstance.GetImageArtifactsPath(avd_spec) + self.assertEqual(paths.host_bins, "/unit/test") + create_common.DownloadRemoteArtifact.reset_mock() self.Patch(os.path, "exists", side_effect=[True, True]) @@ -118,10 +146,15 @@ class RemoteImageLocalInstanceTest(driver_test_lib.BaseDriverTest): self.Patch(os.path, "exists", side_effect=[True, False]) self.Patch(os, "makedirs") self.Patch(subprocess, "check_call") - remote_image_local_instance.DownloadAndProcessImageFiles(avd_spec) + mock_open = self.Patch(builtins, "open") + fetch_dir = remote_image_local_instance.DownloadAndProcessImageFiles( + avd_spec) self.assertEqual(mock_rmtree.call_count, 1) self.assertEqual(self.build_client.GetFetchBuildArgs.call_count, 1) self.assertEqual(self.build_client.GetFetchCertArg.call_count, 1) + cvd_config_filename = os.path.join(fetch_dir, + constants.FETCH_CVD_ARGS_FILE) + mock_open.assert_called_once_with(cvd_config_filename, "w") def testConfirmDownloadRemoteImageDir(self): """Test confirm download remote image dir""" diff --git a/create/remote_image_remote_instance.py b/create/remote_image_remote_instance.py index 225df208..7cf5e153 100644 --- a/create/remote_image_remote_instance.py +++ b/create/remote_image_remote_instance.py @@ -60,6 +60,8 @@ class RemoteImageRemoteInstance(base_avd_create.BaseAVDCreate): """ if avd_spec.oxygen: return self._LeaseOxygenAVD(avd_spec) + if avd_spec.gce_only: + return self._CreateGceInstance(avd_spec) device_factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( avd_spec) create_report = common_operations.CreateDevices( @@ -71,7 +73,8 @@ class RemoteImageRemoteInstance(base_avd_create.BaseAVDCreate): unlock_screen=avd_spec.unlock_screen, wait_for_boot=False, connect_webrtc=avd_spec.connect_webrtc, - client_adb_port=avd_spec.client_adb_port) + client_adb_port=avd_spec.client_adb_port, + client_fastboot_port=avd_spec.client_fastboot_port) if create_report.status == report.Status.SUCCESS: if avd_spec.connect_vnc: utils.LaunchVNCFromReport(create_report, avd_spec, no_prompts) @@ -156,3 +159,27 @@ class RemoteImageRemoteInstance(base_avd_create.BaseAVDCreate): server_url = server_url_match.group("server_url") break return session_id, server_url + + @staticmethod + def _CreateGceInstance(avd_spec): + """Create the GCE instance. + + Args: + avd_spec: AVDSpec object. + + Returns: + A Report instance. + """ + device_factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( + avd_spec) + instance = device_factory.CreateGceInstance() + compute_client = device_factory.GetComputeClient() + ip = compute_client.GetInstanceIP(instance) + reporter = report.Report(command="create_cf") + reporter.SetStatus(report.Status.SUCCESS) + device_data = {"instance_name": instance, + "ip": ip.internal if avd_spec.report_internal_ip + else ip.external} + dict_devices = {_DEVICES: [device_data]} + reporter.UpdateData(dict_devices) + return reporter diff --git a/create/remote_image_remote_instance_test.py b/create/remote_image_remote_instance_test.py index 68af4f58..34387b97 100644 --- a/create/remote_image_remote_instance_test.py +++ b/create/remote_image_remote_instance_test.py @@ -62,6 +62,7 @@ class RemoteImageRemoteInstanceTest(driver_test_lib.BaseDriverTest): avd_spec.oxygen = False avd_spec.connect_webrtc = True avd_spec.connect_vnc = False + avd_spec.gce_only = False create_report = mock.Mock() create_report.status = report.Status.SUCCESS self.Patch(common_operations, "CreateDevices", diff --git a/delete/delete.py b/delete/delete.py index 98ee79e7..a993d926 100644 --- a/delete/delete.py +++ b/delete/delete.py @@ -25,10 +25,9 @@ import subprocess from acloud import errors from acloud.internal import constants -from acloud.internal.lib import cvd_compute_client_multi_stage from acloud.internal.lib import cvd_utils from acloud.internal.lib import emulator_console -from acloud.internal.lib import goldfish_remote_host_client +from acloud.internal.lib import goldfish_utils from acloud.internal.lib import oxygen_client from acloud.internal.lib import ssh from acloud.internal.lib import utils @@ -242,8 +241,7 @@ def DeleteHostGoldfishInstance(cfg, name, ssh_user, Returns: delete_report. """ - ip_addr, port = goldfish_remote_host_client.ParseEmulatorConsoleAddress( - name) + ip_addr, port = goldfish_utils.ParseRemoteHostConsoleAddress(name) try: with emulator_console.RemoteEmulatorConsole( ip_addr, port, @@ -264,8 +262,8 @@ def DeleteHostGoldfishInstance(cfg, name, ssh_user, @utils.TimeExecute(function_description=("Deleting remote host cuttlefish " "instance"), result_evaluator=utils.ReportEvaluator) -def CleanUpRemoteHost(cfg, remote_host, host_user, - host_ssh_private_key_path, delete_report): +def CleanUpRemoteHost(cfg, remote_host, host_user, host_ssh_private_key_path, + base_dir, delete_report): """Clean up the remote host. Args: @@ -274,6 +272,7 @@ def CleanUpRemoteHost(cfg, remote_host, host_user, host_user: String of user login into the instance. host_ssh_private_key_path: String of host key for logging in to the host. + base_dir: String, the base directory on the remote host. delete_report: A Report object. Returns: @@ -285,7 +284,7 @@ def CleanUpRemoteHost(cfg, remote_host, host_user, ssh_private_key_path=( host_ssh_private_key_path or cfg.ssh_private_key_path)) try: - cvd_utils.CleanUpRemoteCvd(ssh_obj, raise_error=True) + cvd_utils.CleanUpRemoteCvd(ssh_obj, base_dir, raise_error=True) delete_report.SetStatus(report.Status.SUCCESS) device_driver.AddDeletionResultToReport( delete_report, [remote_host], failed=[], @@ -322,11 +321,10 @@ def DeleteInstanceByNames(cfg, instances, host_user, local_names = set(name for name in instances if name.startswith(_LOCAL_INSTANCE_PREFIX)) remote_host_cf_names = set( - name for name in instances if - cvd_compute_client_multi_stage.CvdComputeClient.ParseRemoteHostAddress(name)) + name for name in instances if cvd_utils.ParseRemoteHostAddress(name)) remote_host_gf_names = set( name for name in instances if - goldfish_remote_host_client.ParseEmulatorConsoleAddress(name)) + goldfish_utils.ParseRemoteHostConsoleAddress(name)) remote_names = list(set(instances) - local_names - remote_host_cf_names - remote_host_gf_names) @@ -344,10 +342,10 @@ def DeleteInstanceByNames(cfg, instances, host_user, if remote_host_cf_names: for name in remote_host_cf_names: - ip_addr = cvd_compute_client_multi_stage.CvdComputeClient.ParseRemoteHostAddress( - name) + ip_addr, base_dir = cvd_utils.ParseRemoteHostAddress(name) CleanUpRemoteHost(cfg, ip_addr, host_user, - host_ssh_private_key_path, delete_report) + host_ssh_private_key_path, base_dir, + delete_report) if remote_host_gf_names: for name in remote_host_gf_names: @@ -422,7 +420,9 @@ def Run(args): if args.remote_host: delete_report = report.Report(command="delete") CleanUpRemoteHost(cfg, args.remote_host, args.host_user, - args.host_ssh_private_key_path, delete_report) + args.host_ssh_private_key_path, + cvd_utils.GetRemoteHostBaseDir(1), + delete_report) return delete_report instances = list_instances.GetLocalInstances() diff --git a/delete/delete_test.py b/delete/delete_test.py index 6baa8b91..454d4603 100644 --- a/delete/delete_test.py +++ b/delete/delete_test.py @@ -20,7 +20,6 @@ from unittest import mock from acloud import errors from acloud.delete import delete -from acloud.internal.lib import cvd_compute_client_multi_stage from acloud.internal.lib import driver_test_lib from acloud.internal.lib import oxygen_client from acloud.internal.lib import utils @@ -219,16 +218,16 @@ class DeleteTest(driver_test_lib.BaseDriverTest): cfg_attrs = {"ssh_private_key_path": "cfg_key_path"} mock_cfg = mock.Mock(spec_set=list(cfg_attrs.keys()), **cfg_attrs) delete_report = report.Report(command="delete") - delete.CleanUpRemoteHost(mock_cfg, "192.0.2.1", "vsoc-01", - None, delete_report) + delete.CleanUpRemoteHost(mock_cfg, "192.0.2.1", "vsoc-01", None, ".", + delete_report) mock_ssh.IP.assert_called_with(ip="192.0.2.1") mock_ssh.Ssh.assert_called_with( ip=mock_ssh_ip, user="vsoc-01", ssh_private_key_path="cfg_key_path") - mock_cvd_utils.CleanUpRemoteCvd.assert_called_with(mock_ssh_obj, - raise_error=True) + mock_cvd_utils.CleanUpRemoteCvd.assert_called_with( + mock_ssh_obj, ".", raise_error=True) self.assertEqual(delete_report.status, "SUCCESS") self.assertEqual(delete_report.data, { "deleted": [ @@ -246,15 +245,15 @@ class DeleteTest(driver_test_lib.BaseDriverTest): subprocess.CalledProcessError(cmd="test", returncode=1)) delete_report = report.Report(command="delete") - delete.CleanUpRemoteHost(mock_cfg, "192.0.2.2", "user", - "key_path", delete_report) + delete.CleanUpRemoteHost(mock_cfg, "192.0.2.2", "user", "key_path", + "acloud_cf_1", delete_report) mock_ssh.IP.assert_called_with(ip="192.0.2.2") mock_ssh.Ssh.assert_called_with( ip=mock_ssh_ip, user="user", ssh_private_key_path="key_path") - mock_cvd_utils.CleanUpRemoteCvd.assert_called_with(mock_ssh_obj, - raise_error=True) + mock_cvd_utils.CleanUpRemoteCvd.assert_called_with( + mock_ssh_obj, "acloud_cf_1", raise_error=True) self.assertEqual(delete_report.status, "FAIL") self.assertEqual(len(delete_report.errors), 1) @@ -281,12 +280,12 @@ class DeleteTest(driver_test_lib.BaseDriverTest): # Test delete remote host instances. instances = ["host-goldfish-192.0.2.1-5554-123456-sdk_x86_64-sdk", - "host-192.0.2.2-123456-aosp_cf_x86_64_phone"] + "host-192.0.2.2-3-123456-aosp_cf_x86_64_phone"] delete.DeleteInstanceByNames(cfg, instances, "user", "key") mock_delete_host_gf_ins.assert_called_with( cfg, instances[0], "user", "key", mock.ANY) mock_clean_up_remote_host.assert_called_with( - cfg, "192.0.2.2", "user", "key", mock.ANY) + cfg, "192.0.2.2", "user", "key", "acloud_cf_3", mock.ANY) # Test delete remote instances. instances = ["ins-id1-cf-x86-phone-userdebug", @@ -385,12 +384,11 @@ class DeleteTest(driver_test_lib.BaseDriverTest): args.remote_host = None args.local_only = True args.adb_port = None + args.fastboot_port = None args.all = True self.Patch(delete, "_ReleaseOxygenDevice") self.Patch(delete, "DeleteInstanceByNames") - self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, - "ParseRemoteHostAddress") self.Patch(delete, "CleanUpRemoteHost") fake_cfg = mock.MagicMock() fake_cfg.SupportRemoteInstance = mock.MagicMock() @@ -219,6 +219,10 @@ class LaunchCVDFail(CreateError): """Cuttlefish AVD launch failed.""" +class SshConnectFail(CreateError): + """Ssh connect to GCE instance failed.""" + + class SubprocessFail(CreateError): """Subprocess failed.""" diff --git a/internal/constants.py b/internal/constants.py index c0a87328..7883ba51 100755 --- a/internal/constants.py +++ b/internal/constants.py @@ -111,12 +111,14 @@ INSTANCE_NAME = "instance_name" GCE_USER = "vsoc-01" VNC_PORT = "vnc_port" ADB_PORT = "adb_port" +FASTBOOT_PORT = "fastboot_port" WEBRTC_PORT = "webrtc_port" DEVICE_SERIAL = "device_serial" LOGS = "logs" BASE_INSTANCE_NUM = "base_instance_num" # For cuttlefish remote instances CF_ADB_PORT = 6520 +CF_FASTBOOT_PORT = 7520 CF_VNC_PORT = 6444 # For cheeps remote instances CHEEPS_ADB_PORT = 9222 @@ -124,6 +126,8 @@ CHEEPS_VNC_PORT = 5900 # For gce_x86_phones remote instances GCE_ADB_PORT = 5555 GCE_VNC_PORT = 6444 +# For ssh connect with GCE hostname +GCE_HOSTNAME = "gce_hostname" # For goldfish remote instances GF_ADB_PORT = 5555 GF_VNC_PORT = 6444 @@ -131,6 +135,10 @@ GF_VNC_PORT = 6444 FVP_ADB_PORT = 5555 # Maximum port number MAX_PORT = 65535 +# Time info to write in report. +TIME_ARTIFACT = "fetch_artifact_time" +TIME_GCE = "gce_create_time" +TIME_LAUNCH = "launch_cvd_time" COMMAND_PS = ["ps", "aux"] CMD_CVD = "cvd" @@ -146,6 +154,7 @@ ENV_ANDROID_PRODUCT_OUT = "ANDROID_PRODUCT_OUT" ENV_ANDROID_SOONG_HOST_OUT = "ANDROID_SOONG_HOST_OUT" ENV_ANDROID_TMP = "ANDROID_TMP" ENV_BUILD_TARGET = "TARGET_PRODUCT" +ENV_CVD_ACQUIRE_FILE_LOCK = "CVD_ACQUIRE_FILE_LOCK" LOCALHOST = "0.0.0.0" LOCALHOST_ADB_SERIAL = LOCALHOST + ":%d" @@ -167,9 +176,11 @@ INS_KEY_STATUS = "status" INS_KEY_DISPLAY = "display" INS_KEY_IP = "ip" INS_KEY_ADB = "adb" +INS_KEY_FASTBOOT = "fastboot" INS_KEY_VNC = "vnc" INS_KEY_WEBRTC = "webrtc" INS_KEY_WEBRTC_PORT = "webrtc_port" +INS_KEY_WEBRTC_DEVICE_ID = "webrtc_device_id" INS_KEY_CREATETIME = "creationTimestamp" INS_KEY_AVD_TYPE = "avd_type" INS_KEY_AVD_FLAVOR = "flavor" @@ -183,7 +194,8 @@ ANDROID_INFO_FILE = "android-info.txt" CUTTLEFISH_CONFIG_FILE = "cuttlefish_config.json" TEMP_ARTIFACTS_FOLDER = "acloud_image_artifacts" -CVD_HOST_PACKAGE = "cvd-host_package.tar.gz" +CVD_HOST_PACKAGE = "cvd-host_package" +CVD_HOST_TARBALL = "cvd-host_package.tar.gz" # cvd tools symbolic link name of local instance. CVD_TOOLS_LINK_NAME = "host_bins" TOOL_NAME = "acloud" @@ -218,6 +230,7 @@ LOG_TYPE_DIR = "DIR" LOG_TYPE_KERNEL_LOG = "KERNEL_LOG" LOG_TYPE_LOGCAT = "LOGCAT" LOG_TYPE_TEXT = "TEXT" +LOG_TYPE_CUTTLEFISH_LOG = "CUTTLEFISH_LOG" # Stages for create progress STAGE_INIT = 0 @@ -244,9 +257,17 @@ ACLOUD_OXYGEN_RELEASE_ERROR = "ACLOUD_OXYGEN_RELEASE_ERROR" # Key words of error messages. ERROR_MSG_VNC_NOT_SUPPORT = "unknown command line flag 'start_vnc_server'" ERROR_MSG_WEBRTC_NOT_SUPPORT = "unknown command line flag 'start_webrtc'" +ERROR_MSG_SSO_INVALID = "stuck at SSO" # The name of download image tool. FETCH_CVD = "fetch_cvd" +FETCH_CVD_ARGS_FILE = "fetch-cvd-args.txt" +# Last known good build +LKGB = "LKGB" + +# The name of credential source key json file, a copy of +# cfg.service_account_json_private_key_path for remote host case. +FETCH_CVD_CREDENTIAL_SOURCE = "credential_key.json" # For setup and cleanup # Packages "devscripts" and "equivs" are required for "mk-build-deps". diff --git a/internal/lib/android_build_client.py b/internal/lib/android_build_client.py index d62ef97e..a5001acf 100644 --- a/internal/lib/android_build_client.py +++ b/internal/lib/android_build_client.py @@ -27,6 +27,7 @@ import stat import apiclient from acloud import errors +from acloud.internal import constants from acloud.internal.lib import base_cloud_client from acloud.internal.lib import utils @@ -65,7 +66,9 @@ class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): LATEST = "latest" # FETCH_CVD variables. FETCHER_NAME = "fetch_cvd" + FETCHER_BRANCH = "aosp-master" FETCHER_BUILD_TARGET = "aosp_cf_x86_64_phone-userdebug" + FETCHER_ARM_VERSION_BUILD_TARGET = "aosp_cf_arm64_phone-userdebug" MAX_RETRY = 3 RETRY_SLEEP_SECS = 3 @@ -111,21 +114,29 @@ class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): logger.error("Downloading artifact failed: %s", str(e)) raise errors.DriverError(str(e)) - def DownloadFetchcvd(self, local_dest, fetch_cvd_version): + def DownloadFetchcvd( + self, + local_dest, + fetch_cvd_version, + is_arm_version=False): """Get fetch_cvd from Android Build. Args: local_dest: A local path where the artifact should be stored. e.g. "/tmp/fetch_cvd" fetch_cvd_version: String of fetch_cvd version. + is_arm_version: is ARM version fetch_cvd. """ + if fetch_cvd_version == constants.LKGB: + fetch_cvd_version = self.GetFetcherVersion() utils.RetryExceptionType( - exception_types=ssl.SSLError, + exception_types=(ssl.SSLError, errors.DriverError), max_retries=self.MAX_RETRY, functor=self.DownloadArtifact, sleep_multiplier=self.RETRY_SLEEP_SECS, retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR, - build_target=self.FETCHER_BUILD_TARGET, + build_target=(self.FETCHER_ARM_VERSION_BUILD_TARGET + if is_arm_version else self.FETCHER_BUILD_TARGET), build_id=fetch_cvd_version, resource_id=self.FETCHER_NAME, local_dest=local_dest, @@ -134,17 +145,18 @@ class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): os.chmod(local_dest, fetch_cvd_stat.st_mode | stat.S_IEXEC) @staticmethod - def ProcessBuild(build_id=None, branch=None, build_target=None): + def ProcessBuild(build_info): """Create a Cuttlefish fetch_cvd build string. Args: - build_id: A specific build number to load from. Takes precedence over `branch`. - branch: A manifest-branch at which to get the latest build. - build_target: A particular device to load at the desired build. + build_info: The dictionary that contains build information. Returns: A string, used in the fetch_cvd cmd or None if all args are None. """ + build_id = build_info.get(constants.BUILD_ID) + build_target = build_info.get(constants.BUILD_TARGET) + branch = build_info.get(constants.BUILD_BRANCH) if not build_target: return build_id or branch @@ -152,62 +164,66 @@ class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): branch = _DEFAULT_BRANCH return (build_id or branch) + "/" + build_target - # pylint: disable=too-many-locals - def GetFetchBuildArgs(self, build_id, branch, build_target, system_build_id, - system_branch, system_build_target, kernel_build_id, - kernel_branch, kernel_build_target, bootloader_build_id, - bootloader_branch, bootloader_build_target, - ota_build_id, ota_branch, ota_build_target): + def GetFetchBuildArgs(self, default_build_info, system_build_info, + kernel_build_info, boot_build_info, + bootloader_build_info, ota_build_info): """Get args from build information for fetch_cvd. + Each build_info is a dictionary that contains 3 items, for example, + { + constants.BUILD_ID: "2263051", + constants.BUILD_TARGET: "aosp_cf_x86_64_phone-userdebug", + constants.BUILD_BRANCH: "aosp-master", + } + Args: - build_id: String of build id, e.g. "2263051", "P2804227" - branch: String of branch name, e.g. "aosp-master" - build_target: String of target name. - e.g. "aosp_cf_x86_64_phone-userdebug" - system_build_id: String of the system image build id. - system_branch: String of the system image branch name. - system_build_target: String of the system image target name, - e.g. "cf_x86_phone-userdebug" - kernel_build_id: String of the kernel image build id. - kernel_branch: String of the kernel image branch name. - kernel_build_target: String of the kernel image target name, - bootloader_build_id: String of the bootloader build id. - bootloader_branch: String of the bootloader branch name. - bootloader_build_target: String of the bootloader target name. - ota_build_id: String of the bootloader build id. - ota_branch: String of the bootloader branch name. - ota_build_target: String of the bootloader target name. + default_build_info: The build that provides full cuttlefish images. + system_build_info: The build that provides the system image. + kernel_build_info: The build that provides the kernel. + boot_build_info: The build that provides the boot image. This + dictionary may contain an additional key + constants.BUILD_ARTIFACT which is mapped to the + boot image name. + bootloader_build_info: The build that provides the bootloader. + ota_build_info: The build that provides the OTA tools. Returns: List of string args for fetch_cvd. """ fetch_cvd_args = [] - default_build = self.ProcessBuild(build_id, branch, build_target) + default_build = self.ProcessBuild(default_build_info) if default_build: - fetch_cvd_args.append("-default_build=%s" % default_build) - system_build = self.ProcessBuild( - system_build_id, system_branch, system_build_target) + fetch_cvd_args.append(f"-default_build={default_build}") + system_build = self.ProcessBuild(system_build_info) if system_build: - fetch_cvd_args.append("-system_build=%s" % system_build) - bootloader_build = self.ProcessBuild(bootloader_build_id, - bootloader_branch, - bootloader_build_target) + fetch_cvd_args.append(f"-system_build={system_build}") + bootloader_build = self.ProcessBuild(bootloader_build_info) if bootloader_build: - fetch_cvd_args.append("-bootloader_build=%s" % bootloader_build) - kernel_build = self.GetKernelBuild(kernel_build_id, - kernel_branch, - kernel_build_target) + fetch_cvd_args.append(f"-bootloader_build={bootloader_build}") + kernel_build = self.GetKernelBuild(kernel_build_info) if kernel_build: - fetch_cvd_args.append("-kernel_build=%s" % kernel_build) - ota_build = self.ProcessBuild( - ota_build_id, ota_branch, ota_build_target) + fetch_cvd_args.append(f"-kernel_build={kernel_build}") + boot_build = self.ProcessBuild(boot_build_info) + if boot_build: + fetch_cvd_args.append(f"-boot_build={boot_build}") + boot_artifact = boot_build_info.get(constants.BUILD_ARTIFACT) + if boot_artifact: + fetch_cvd_args.append(f"-boot_artifact={boot_artifact}") + ota_build = self.ProcessBuild(ota_build_info) if ota_build: - fetch_cvd_args.append("-otatools_build=%s" % ota_build) + fetch_cvd_args.append(f"-otatools_build={ota_build}") return fetch_cvd_args + def GetFetcherVersion(self): + """Get fetch_cvd build id from LKGB. + + Returns: + The build id of fetch_cvd. + """ + return self.GetLKGB(self.FETCHER_BUILD_TARGET, self.FETCHER_BRANCH) + @staticmethod # pylint: disable=broad-except def GetFetchCertArg(certification_file): @@ -237,26 +253,24 @@ class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): certification file, return empty string for aosp branch. """ cert_arg = "" - try: with open(certification_file) as cert_file: auth_token = json.load(cert_file).get("data")[-1].get( "credential").get("access_token") if auth_token: - cert_arg = "-credential_source=%s" % auth_token + cert_arg = f"-credential_source={auth_token}" except Exception as e: utils.PrintColorString( - "Fail to open the certification file(%s): %s" % - (certification_file, e), utils.TextColors.WARNING) + f"Fail to open the certification file " + f"({certification_file}): {e}", + utils.TextColors.WARNING) return cert_arg - def GetKernelBuild(self, kernel_build_id, kernel_branch, kernel_build_target): + def GetKernelBuild(self, kernel_build_info): """Get kernel build args for fetch_cvd. Args: - kernel_branch: Kernel branch name, e.g. "kernel-common-android-4.14" - kernel_build_id: Kernel build id, a string, e.g. "223051", "P280427" - kernel_build_target: String, Kernel build target name. + kernel_build_info: The dictionary that contains build information. Returns: String of kernel build args for fetch_cvd. @@ -264,8 +278,9 @@ class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): """ # kernel_target have default value "kernel". If user provide kernel_build_id # or kernel_branch, then start to process kernel image. - if kernel_build_id or kernel_branch: - return self.ProcessBuild(kernel_build_id, kernel_branch, kernel_build_target) + if (kernel_build_info.get(constants.BUILD_ID) or + kernel_build_info.get(constants.BUILD_BRANCH)): + return self.ProcessBuild(kernel_build_info) return None def CopyTo(self, diff --git a/internal/lib/android_build_client_test.py b/internal/lib/android_build_client_test.py index f589dc3b..9039f2da 100644 --- a/internal/lib/android_build_client_test.py +++ b/internal/lib/android_build_client_test.py @@ -26,6 +26,7 @@ from unittest import mock import apiclient from acloud import errors +from acloud.internal import constants from acloud.internal.lib import android_build_client from acloud.internal.lib import driver_test_lib @@ -166,26 +167,28 @@ class AndroidBuildClientTest(driver_test_lib.BaseDriverTest): def testGetFetchBuildArgs(self): """Test GetFetchBuildArgs.""" - build_id = "1234" - build_branch = "base_branch" - build_target = "base_target" - system_build_id = "2345" - system_build_branch = "system_branch" - system_build_target = "system_target" - kernel_build_id = "3456" - kernel_build_branch = "kernel_branch" - kernel_build_target = "kernel_target" - ota_build_id = "4567" - ota_build_branch = "ota_branch" - ota_build_target = "ota_target" + default_build = {constants.BUILD_ID: "1234", + constants.BUILD_BRANCH: "base_branch", + constants.BUILD_TARGET: "base_target"} + system_build = {constants.BUILD_ID: "2345", + constants.BUILD_BRANCH: "system_branch", + constants.BUILD_TARGET: "system_target"} + kernel_build = {constants.BUILD_ID: "3456", + constants.BUILD_BRANCH: "kernel_branch", + constants.BUILD_TARGET: "kernel_target"} + ota_build = {constants.BUILD_ID: "4567", + constants.BUILD_BRANCH: "ota_branch", + constants.BUILD_TARGET: "ota_target"} + boot_build = {constants.BUILD_ID: "5678", + constants.BUILD_BRANCH: "boot_branch", + constants.BUILD_TARGET: "boot_target", + constants.BUILD_ARTIFACT: "boot-5.10.img"} # Test base image. expected_args = ["-default_build=1234/base_target"] self.assertEqual( expected_args, - self.client.GetFetchBuildArgs( - build_id, build_branch, build_target, None, None, None, None, - None, None, None, None, None, None, None, None)) + self.client.GetFetchBuildArgs(default_build, {}, {}, {}, {}, {})) # Test base image with system image. expected_args = ["-default_build=1234/base_target", @@ -193,9 +196,7 @@ class AndroidBuildClientTest(driver_test_lib.BaseDriverTest): self.assertEqual( expected_args, self.client.GetFetchBuildArgs( - build_id, build_branch, build_target, system_build_id, - system_build_branch, system_build_target, None, None, None, - None, None, None, None, None, None)) + default_build, system_build, {}, {}, {}, {})) # Test base image with kernel image. expected_args = ["-default_build=1234/base_target", @@ -203,9 +204,16 @@ class AndroidBuildClientTest(driver_test_lib.BaseDriverTest): self.assertEqual( expected_args, self.client.GetFetchBuildArgs( - build_id, build_branch, build_target, None, None, None, - kernel_build_id, kernel_build_branch, kernel_build_target, - None, None, None, None, None, None)) + default_build, {}, kernel_build, {}, {}, {})) + + # Test base image with boot image. + expected_args = ["-default_build=1234/base_target", + "-boot_build=5678/boot_target", + "-boot_artifact=boot-5.10.img"] + self.assertEqual( + expected_args, + self.client.GetFetchBuildArgs( + default_build, {}, {}, boot_build, {}, {})) # Test base image with otatools. expected_args = ["-default_build=1234/base_target", @@ -213,9 +221,7 @@ class AndroidBuildClientTest(driver_test_lib.BaseDriverTest): self.assertEqual( expected_args, self.client.GetFetchBuildArgs( - build_id, build_branch, build_target, None, None, None, - None, None, None, None, None, None, ota_build_id, - ota_build_branch, ota_build_target)) + default_build, {}, {}, {}, {}, ota_build)) def testGetFetchCertArg(self): """Test GetFetchCertArg.""" @@ -239,27 +245,37 @@ class AndroidBuildClientTest(driver_test_lib.BaseDriverTest): def testProcessBuild(self): """Test creating "cuttlefish build" strings.""" + build_id = constants.BUILD_ID + branch = constants.BUILD_BRANCH + build_target = constants.BUILD_TARGET self.assertEqual( self.client.ProcessBuild( - build_id="123", branch="abc", build_target="def"), "123/def") + {build_id: "123", branch: "abc", build_target: "def"}), + "123/def") self.assertEqual( self.client.ProcessBuild( - build_id=None, branch="abc", build_target="def"), "abc/def") + {build_id: None, branch: "abc", build_target: "def"}), + "abc/def") self.assertEqual( self.client.ProcessBuild( - build_id="123", branch=None, build_target="def"), "123/def") + {build_id: "123", branch: None, build_target: "def"}), + "123/def") self.assertEqual( self.client.ProcessBuild( - build_id="123", branch="abc", build_target=None), "123") + {build_id: "123", branch: "abc", build_target: None}), + "123") self.assertEqual( self.client.ProcessBuild( - build_id=None, branch="abc", build_target=None), "abc") + {build_id: None, branch: "abc", build_target: None}), + "abc") self.assertEqual( self.client.ProcessBuild( - build_id="123", branch=None, build_target=None), "123") + {build_id: "123", branch: None, build_target: None}), + "123") self.assertEqual( self.client.ProcessBuild( - build_id=None, branch=None, build_target=None), None) + {build_id: None, branch: None, build_target: None}), + None) if __name__ == "__main__": diff --git a/internal/lib/android_compute_client.py b/internal/lib/android_compute_client.py index 8d7c7f48..4dd07eee 100755 --- a/internal/lib/android_compute_client.py +++ b/internal/lib/android_compute_client.py @@ -80,6 +80,7 @@ class AndroidComputeClient(gcompute_client.ComputeClient): self._ssh_public_key_path = acloud_config.ssh_public_key_path self._launch_args = acloud_config.launch_args self._instance_name_pattern = acloud_config.instance_name_pattern + self._gce_hostname = None self._AddPerInstanceSshkey() self._dict_report = {_ZONE: self._zone, _VERSION: config.GetVersion()} @@ -359,6 +360,7 @@ class AndroidComputeClient(gcompute_client.ComputeClient): if e.code == 400: logger.debug("CheckBoot: Instance is not ready yet %s", str(e)) return False + logger.error("Unexpected http status: %d, %s", e.code, e.message) raise def WaitForBoot(self, instance, boot_timeout_secs=None): @@ -426,3 +428,8 @@ class AndroidComputeClient(gcompute_client.ComputeClient): def dict_report(self): """Return dict_report""" return self._dict_report + + @property + def gce_hostname(self): + """Return gce_hostname""" + return self._gce_hostname diff --git a/internal/lib/auth.py b/internal/lib/auth.py index ad03d6b1..58a72f13 100644 --- a/internal/lib/auth.py +++ b/internal/lib/auth.py @@ -52,6 +52,7 @@ from acloud import errors logger = logging.getLogger(__name__) HOME_FOLDER = os.path.expanduser("~") +_WEB_SERVER_DEFAULT_PORT = 8080 # If there is no specific scope use case, we will always use this default full # scopes to run CreateCredentials func and user will only go oauth2 flow once # after login with this full scopes credentials. @@ -82,8 +83,8 @@ def _CreateOauthServiceAccountCreds(email, private_key_path, scopes): email, private_key_path, scopes=scopes) except EnvironmentError as e: raise errors.AuthenticationError( - "Could not authenticate using private key file (%s) " - " error message: %s" % (private_key_path, str(e))) + f"Could not authenticate using private key file ({private_key_path}) " + f" error message: {str(e)}") return credentials @@ -118,8 +119,8 @@ def _CreateOauthServiceAccountCredsWithJsonKey(json_private_key_path, scopes, credentials.set_store(storage) except EnvironmentError as e: raise errors.AuthenticationError( - "Could not authenticate using json private key file (%s) " - " error message: %s" % (json_private_key_path, str(e))) + f"Could not authenticate using json private key file ({json_private_key_path}) " + f"error message: {str(e)}") return credentials @@ -137,6 +138,8 @@ class RunFlowFlags(): def _RunAuthFlow(storage, client_id, client_secret, user_agent, scopes): """Get user oauth2 credentials. + Using the loopback IP address flow for desktop clients. + Args: client_id: String, client id from the cloud project. client_secret: String, client secret for the client_id. @@ -146,12 +149,13 @@ def _RunAuthFlow(storage, client_id, client_secret, user_agent, scopes): Returns: An oauth2client.OAuth2Credentials instance. """ - flags = RunFlowFlags(browser_auth=False) + flags = RunFlowFlags(browser_auth=True) flow = oauth2_client.OAuth2WebServerFlow( client_id=client_id, client_secret=client_secret, scope=scopes, - user_agent=user_agent) + user_agent=user_agent, + redirect_uri=f"http://localhost:{_WEB_SERVER_DEFAULT_PORT}") credentials = oauth2_tools.run_flow( flow=flow, storage=storage, flags=flags) return credentials diff --git a/internal/lib/cvd_compute_client.py b/internal/lib/cvd_compute_client.py deleted file mode 100644 index e4245e76..00000000 --- a/internal/lib/cvd_compute_client.py +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2018 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""A client that manages Cuttlefish Virtual Device on compute engine. - -** CvdComputeClient ** - -CvdComputeClient derives from AndroidComputeClient. It manges a google -compute engine project that is setup for running Cuttlefish Virtual Devices. -It knows how to create a host instance from Cuttlefish Stable Host Image, fetch -Android build, and start Android within the host instance. - -** Class hierarchy ** - - base_cloud_client.BaseCloudApiClient - ^ - | - gcompute_client.ComputeClient - ^ - | - android_compute_client.AndroidComputeClient - ^ - | - cvd_compute_client.CvdComputeClient - -""" - -import getpass -import logging - -from acloud.internal import constants -from acloud.internal.lib import android_compute_client -from acloud.internal.lib import gcompute_client -from acloud.internal.lib import utils - - -logger = logging.getLogger(__name__) - -_METADATA_TO_UNSET = ["cvd_01_launch", - "cvd_01_fetch_android_build_target", - "cvd_01_fetch_android_bid", - "cvd_01_fetch_system_bid", - "cvd_01_fetch_system_build_target", - "cvd_01_fetch_kernel_bid", - "cvd_01_fetch_kernel_build_target"] - -# TODO(228405515): Delete CvdComputeClient class. -class CvdComputeClient(android_compute_client.AndroidComputeClient): - """Client that manages Anadroid Virtual Device.""" - - DATA_POLICY_CREATE_IF_MISSING = "create_if_missing" - - # TODO: refactor CreateInstance to take in an object that contains these - # args, this method differs too and holds way too cf-specific args to put in - # the parent method. - # pylint: disable=arguments-differ,too-many-locals - @utils.TimeExecute(function_description="Creating GCE instance") - def CreateInstance(self, instance, image_name, image_project, - build_target=None, branch=None, build_id=None, - kernel_branch=None, kernel_build_id=None, - kernel_build_target=None, blank_data_disk_size_gb=None, - avd_spec=None, extra_scopes=None, - system_build_target=None, system_branch=None, - system_build_id=None): - """Create a cuttlefish instance given stable host image and build id. - - Args: - instance: instance name. - image_name: A string, the name of the GCE image. - image_project: A string, name of the project where the image lives. - Assume the default project if None. - build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" - branch: Branch name, e.g. "aosp-master" - build_id: Build id, a string, e.g. "2263051", "P2804227" - kernel_branch: Kernel branch name, e.g. "kernel-common-android-4.14" - kernel_build_id: Kernel build id, a string, e.g. "223051", "P280427" - kernel_build_target: String, Kernel build target name. - blank_data_disk_size_gb: Size of the blank data disk in GB. - avd_spec: An AVDSpec instance. - extra_scopes: A list of extra scopes to be passed to the instance. - system_build_target: Target name for the system image, - e.g. "cf_x86_phone-userdebug" - system_branch: A String, branch name for the system image. - system_build_id: A string, build id for the system image. - """ - self._CheckMachineSize() - - # A blank data disk would be created on the host. Make sure the size of - # the boot disk is large enough to hold it. - boot_disk_size_gb = ( - int(self.GetImage(image_name, image_project)["diskSizeGb"]) + - blank_data_disk_size_gb) - disk_args = self._GetDiskArgs( - instance, image_name, image_project, boot_disk_size_gb) - - # Transitional metadata variable as outlined in go/cuttlefish-deployment - # These metadata tell the host instance to fetch and launch one - # cuttlefish device (cvd-01). Ideally we should use a separate tool to - # manage CVD devices on the host instance and not through metadata. - # TODO(b/77626419): Remove these metadata once the - # cuttlefish-google.service is turned off on the host instance. - metadata = self._metadata.copy() - metadata["cvd_01_fetch_android_build_target"] = build_target - metadata["cvd_01_fetch_android_bid"] = "{branch}/{build_id}".format( - branch=branch, build_id=build_id) - if kernel_branch and kernel_build_id: - metadata["cvd_01_fetch_kernel_bid"] = "{branch}/{build_id}".format( - branch=kernel_branch, build_id=kernel_build_id) - if kernel_build_target: - metadata["cvd_01_fetch_kernel_build_target"] = kernel_build_target - if system_build_target: - metadata["cvd_01_fetch_system_build_target"] = system_build_target - if system_branch and system_build_id: - metadata["cvd_01_fetch_system_bid"] = "{branch}/{build_id}".format( - branch=system_branch, build_id=system_build_id) - metadata["cvd_01_launch"] = self._GetLaunchCvdArgs(avd_spec) - - # For the local image, we unset the _METADATA_TO_UNSET from - # metadata to tell server not to launch cvd and not to fetch image - # while instance is booted up. - if avd_spec and avd_spec.image_source == constants.IMAGE_SRC_LOCAL: - for meta in _METADATA_TO_UNSET: - metadata.pop(meta, None) - - if blank_data_disk_size_gb > 0: - # Policy 'create_if_missing' would create a blank userdata disk if - # missing. If already exist, reuse the disk. - metadata["cvd_01_data_policy"] = self.DATA_POLICY_CREATE_IF_MISSING - metadata["cvd_01_blank_data_disk_size"] = str( - blank_data_disk_size_gb * 1024) - metadata["user"] = getpass.getuser() - # Update metadata by avd_spec - # for legacy create_cf cmd, we will keep using resolution. - # And always use avd_spec for acloud create cmd. - # TODO(b/118406018): deprecate resolution config and use hw_proprty for - # all create cmds. - if avd_spec: - metadata[constants.INS_KEY_AVD_TYPE] = avd_spec.avd_type - metadata[constants.INS_KEY_AVD_FLAVOR] = avd_spec.flavor - metadata["cvd_01_x_res"] = avd_spec.hw_property[constants.HW_X_RES] - metadata["cvd_01_y_res"] = avd_spec.hw_property[constants.HW_Y_RES] - metadata["cvd_01_dpi"] = avd_spec.hw_property[constants.HW_ALIAS_DPI] - if constants.HW_ALIAS_DISK in avd_spec.hw_property: - metadata["cvd_01_blank_data_disk_size"] = avd_spec.hw_property[ - constants.HW_ALIAS_DISK] - # Use another METADATA_DISPLAY to record resolution which will be - # retrieved in acloud list cmd. We try not to use cvd_01_x_res - # since cvd_01_xxx metadata is going to deprecated by cuttlefish. - metadata[constants.INS_KEY_DISPLAY] = ("%sx%s (%s)" % ( - avd_spec.hw_property[constants.HW_X_RES], - avd_spec.hw_property[constants.HW_Y_RES], - avd_spec.hw_property[constants.HW_ALIAS_DPI])) - else: - resolution = self._resolution.split("x") - metadata["cvd_01_dpi"] = resolution[3] - metadata["cvd_01_x_res"] = resolution[0] - metadata["cvd_01_y_res"] = resolution[1] - - gcompute_client.ComputeClient.CreateInstance( - self, - instance=instance, - image_name=image_name, - image_project=image_project, - disk_args=disk_args, - metadata=metadata, - machine_type=self._machine_type, - network=self._network, - zone=self._zone, - disk_type=avd_spec.disk_type if avd_spec else None, - extra_scopes=extra_scopes) - - def _GetLaunchCvdArgs(self, avd_spec): - """Define the launch_cvd args. - - Set launch_cvd args with following priority. - -First: Set args from config. - -Second: Set args from cpu and memory settings. - -Third: Set args as "1" to don't pass any args. - - Args: - avd_spec: An AVDSpec instance. - - Returns: - String of launch_cvd args. - """ - if self._launch_args: - return self._launch_args - - if avd_spec: - cpu_arg = "" - mem_arg = "" - if constants.HW_ALIAS_CPUS in avd_spec.hw_property: - cpu_arg = ("-cpus=%s" % - avd_spec.hw_property[constants.HW_ALIAS_CPUS]) - if constants.HW_ALIAS_MEMORY in avd_spec.hw_property: - mem_arg = ("-memory_mb=%s" % - avd_spec.hw_property[constants.HW_ALIAS_MEMORY]) - if cpu_arg or mem_arg: - return cpu_arg + " " + mem_arg - - return "1" diff --git a/internal/lib/cvd_compute_client_multi_stage.py b/internal/lib/cvd_compute_client_multi_stage.py index 5464dee3..a490c1c9 100644 --- a/internal/lib/cvd_compute_client_multi_stage.py +++ b/internal/lib/cvd_compute_client_multi_stage.py @@ -55,26 +55,8 @@ from acloud.setup import mkcert logger = logging.getLogger(__name__) -_CONFIG_ARG = "-config" -_DECOMPRESS_KERNEL_ARG = "-decompress_kernel=true" -_AGREEMENT_PROMPT_ARG = "-report_anonymous_usage_stats=y" -_UNDEFOK_ARG = "-undefok=report_anonymous_usage_stats,config" -_NUM_AVDS_ARG = "-num_instances=%(num_AVD)s" -# Connect the OpenWrt device via console file. -_ENABLE_CONSOLE_ARG = "-console=true" -_DEFAULT_BRANCH = "aosp-master" -_FETCHER_BUILD_TARGET = "aosp_cf_x86_64_phone-userdebug" +_DEFAULT_WEBRTC_DEVICE_ID = "cvd-1" _FETCHER_NAME = "fetch_cvd" -# Time info to write in report. -_FETCH_ARTIFACT = "fetch_artifact_time" -_GCE_CREATE = "gce_create_time" -_LAUNCH_CVD = "launch_cvd_time" -# WebRTC args for launching AVD -_START_WEBRTC = "--start_webrtc" -_WEBRTC_ID = "--webrtc_device_id=%(instance)s" -_VM_MANAGER = "--vm_manager=crosvm" -_WEBRTC_ARGS = [_START_WEBRTC, _VM_MANAGER] -_VNC_ARGS = ["--start_vnc_server=true"] _NO_RETRY = 0 # Launch cvd command for acloud report _LAUNCH_CVD_COMMAND = "launch_cvd_command" @@ -83,11 +65,6 @@ _TRUST_REMOTE_INSTANCE_COMMAND = ( f"\"sudo cp -p ~/{constants.WEBRTC_CERTS_PATH}/{constants.SSL_CA_NAME}.pem " f"{constants.SSL_TRUST_CA_DIR}/{constants.SSL_CA_NAME}.crt;" "sudo update-ca-certificates;\"") -# Remote host instance name -_HOST_INSTANCE_NAME_FORMAT = (constants.INSTANCE_TYPE_HOST + - "-%(ip_addr)s-%(build_id)s-%(build_target)s") -_HOST_INSTANCE_NAME_PATTERN = re.compile(constants.INSTANCE_TYPE_HOST + - r"-(?P<ip_addr>[\d.]+)-.+") class CvdComputeClient(android_compute_client.AndroidComputeClient): @@ -100,7 +77,6 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): def __init__(self, acloud_config, oauth2_credentials, - boot_timeout_secs=None, ins_timeout_secs=None, report_internal_ip=None, gpu=None): @@ -109,8 +85,6 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): Args: acloud_config: An AcloudConfig object. oauth2_credentials: An oauth2client.OAuth2Credentials instance. - boot_timeout_secs: Integer, the maximum time to wait for the AVD - to boot up. ins_timeout_secs: Integer, the maximum time to wait for the instance ready. report_internal_ip: Boolean to report the internal ip instead of @@ -119,11 +93,9 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): """ super().__init__(acloud_config, oauth2_credentials) - self._fetch_cvd_version = acloud_config.fetch_cvd_version self._build_api = ( android_build_client.AndroidBuildClient(oauth2_credentials)) self._ssh_private_key_path = acloud_config.ssh_private_key_path - self._boot_timeout_secs = boot_timeout_secs self._ins_timeout_secs = ins_timeout_secs self._report_internal_ip = report_internal_ip self._gpu = gpu @@ -136,40 +108,11 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): self._user = constants.GCE_USER self._openwrt = None self._stage = constants.STAGE_INIT - self._execution_time = {_FETCH_ARTIFACT: 0, _GCE_CREATE: 0, _LAUNCH_CVD: 0} + self._execution_time = {constants.TIME_ARTIFACT: 0, + constants.TIME_GCE: 0, + constants.TIME_LAUNCH: 0} - @staticmethod - def FormatRemoteHostInstanceName(ip_addr, build_id, build_target): - """Convert an IP address and build info to an instance name. - - Args: - ip_addr: String, the IP address of the remote host. - build_id: String, the build id. - build_target: String, the build target, e.g., aosp_cf_x86_64_phone. - - Return: - String, the instance name. - """ - return _HOST_INSTANCE_NAME_FORMAT % { - "ip_addr": ip_addr, - "build_id": build_id, - "build_target": build_target} - - @staticmethod - def ParseRemoteHostAddress(instance_name): - """Parse IP address from a remote host instance name. - - Args: - instance_name: String, the instance name. - - Returns: - The IP address as a string. - None if the name does not represent a remote host instance. - """ - match = _HOST_INSTANCE_NAME_PATTERN.fullmatch(instance_name) - return match.group("ip_addr") if match else None - - def InitRemoteHost(self, ssh, ip, user): + def InitRemoteHost(self, ssh, ip, user, base_dir): """Init remote host. Check if we can ssh to the remote host, stop any cf instances running @@ -180,206 +123,101 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): ip: namedtuple (internal, external) that holds IP address of the remote host, e.g. "external:140.110.20.1, internal:10.0.0.1" user: String of user log in to the instance. + base_dir: The remote directory containing the images and tools. """ self.SetStage(constants.STAGE_SSH_CONNECT) self._ssh = ssh self._ip = ip self._user = user self._ssh.WaitForSsh(timeout=self._ins_timeout_secs) - cvd_utils.CleanUpRemoteCvd(self._ssh, raise_error=False) + cvd_utils.CleanUpRemoteCvd(self._ssh, base_dir, raise_error=False) - # TODO(171376263): Refactor CreateInstance() args with avd_spec. - # pylint: disable=arguments-differ,too-many-locals,broad-except + # pylint: disable=arguments-differ,broad-except def CreateInstance(self, instance, image_name, image_project, - build_target=None, branch=None, build_id=None, - kernel_branch=None, kernel_build_id=None, - kernel_build_target=None, blank_data_disk_size_gb=None, - avd_spec=None, extra_scopes=None, - system_build_target=None, system_branch=None, - system_build_id=None, bootloader_build_target=None, - bootloader_branch=None, bootloader_build_id=None, - ota_build_target=None, ota_branch=None, - ota_build_id=None): - - """Create/Reuse a single configured cuttlefish device. - 1. Prepare GCE instance. - Create a new instnace or get IP address for reusing the specific instance. - 2. Put fetch_cvd on the instance. - 3. Invoke fetch_cvd to fetch and run the instance. + avd_spec, extra_scopes=None): + """Create/Reuse a GCE instance. Args: instance: instance name. image_name: A string, the name of the GCE image. image_project: A string, name of the project where the image lives. Assume the default project if None. - build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" - branch: Branch name, e.g. "aosp-master" - build_id: Build id, a string, e.g. "2263051", "P2804227" - kernel_branch: Kernel branch name, e.g. "kernel-common-android-4.14" - kernel_build_id: Kernel build id, a string, e.g. "223051", "P280427" - kernel_build_target: String, Kernel build target name. - blank_data_disk_size_gb: Size of the blank data disk in GB. avd_spec: An AVDSpec instance. extra_scopes: A list of extra scopes to be passed to the instance. - system_build_target: String of the system image target name, - e.g. "cf_x86_phone-userdebug" - system_branch: String of the system image branch name. - system_build_id: String of the system image build id. - bootloader_build_target: String of the bootloader target name. - bootloader_branch: String of the bootloader branch name. - bootloader_build_id: String of the bootloader build id. - ota_build_target: String of the otatools target name. - ota_branch: String of the otatools branch name. - ota_build_id: String of the otatools build id. Returns: A string, representing instance name. """ - # A blank data disk would be created on the host. Make sure the size of # the boot disk is large enough to hold it. boot_disk_size_gb = ( int(self.GetImage(image_name, image_project)["diskSizeGb"]) + - blank_data_disk_size_gb) + avd_spec.cfg.extra_data_disk_size_gb) - if avd_spec and avd_spec.instance_name_to_reuse: + if avd_spec.instance_name_to_reuse: self._ip = self._ReusingGceInstance(avd_spec) else: self._VerifyZoneByQuota() self._ip = self._CreateGceInstance(instance, image_name, image_project, extra_scopes, boot_disk_size_gb, avd_spec) + if avd_spec.connect_hostname: + self._gce_hostname = gcompute_client.GetGCEHostName( + self._project, instance, self._zone) self._ssh = Ssh(ip=self._ip, user=constants.GCE_USER, ssh_private_key_path=self._ssh_private_key_path, extra_args_ssh_tunnel=self._extra_args_ssh_tunnel, - report_internal_ip=self._report_internal_ip) + report_internal_ip=self._report_internal_ip, + gce_hostname=self._gce_hostname) try: self.SetStage(constants.STAGE_SSH_CONNECT) self._ssh.WaitForSsh(timeout=self._ins_timeout_secs) - if avd_spec: - if avd_spec.instance_name_to_reuse: - cvd_utils.CleanUpRemoteCvd(self._ssh, raise_error=False) - return instance - - # TODO: Remove following code after create_cf deprecated. - self.UpdateFetchCvd() - - self.FetchBuild(build_id, branch, build_target, system_build_id, - system_branch, system_build_target, kernel_build_id, - kernel_branch, kernel_build_target, bootloader_build_id, - bootloader_branch, bootloader_build_target, - ota_build_id, ota_branch, ota_build_target) - failures = self.LaunchCvd( - instance, - blank_data_disk_size_gb=blank_data_disk_size_gb, - boot_timeout_secs=self._boot_timeout_secs, - extra_args=[]) - self._all_failures.update(failures) - return instance + if avd_spec.instance_name_to_reuse: + cvd_utils.CleanUpRemoteCvd(self._ssh, cvd_utils.GCE_BASE_DIR, + raise_error=False) except Exception as e: self._all_failures[instance] = e - return instance + return instance - def _GetConfigFromAndroidInfo(self): + def _GetGCEHostName(self, instance): + """Get the GCE host name with specific rule. + + Args: + instance: Sting, instance name. + + Returns: + One host name coverted by instance name, project name, and zone. + """ + if ":" in self._project: + domain = self._project.split(":")[0] + project_no_domain = self._project.split(":")[1] + project = f"{project_no_domain}.{domain}" + return f"nic0.{instance}.{self._zone}.c.{project}.internal.gcpnode.com" + return f"nic0.{instance}.{self._zone}.c.{self._project}.internal.gcpnode.com" + + def _GetConfigFromAndroidInfo(self, base_dir): """Get config value from android-info.txt. The config in android-info.txt would like "config=phone". + Args: + base_dir: The remote directory containing the images. + Returns: Strings of config value. """ android_info = self._ssh.GetCmdOutput( - "cat %s" % constants.ANDROID_INFO_FILE) + f"cat {base_dir}/{constants.ANDROID_INFO_FILE}") logger.debug("Android info: %s", android_info) config_match = _CONFIG_RE.match(android_info) if config_match: return config_match.group("config") return None - # pylint: disable=too-many-branches - def _GetLaunchCvdArgs(self, avd_spec=None, blank_data_disk_size_gb=None, - decompress_kernel=None, instance=None): - """Get launch_cvd args. - - Args: - avd_spec: An AVDSpec instance. - blank_data_disk_size_gb: Size of the blank data disk in GB. - decompress_kernel: Boolean, if true decompress the kernel. - instance: String, instance name. - - Returns: - String, args of launch_cvd. - """ - launch_cvd_args = [] - if blank_data_disk_size_gb and blank_data_disk_size_gb > 0: - # Policy 'create_if_missing' would create a blank userdata disk if - # missing. If already exist, reuse the disk. - launch_cvd_args.append( - "-data_policy=" + self.DATA_POLICY_CREATE_IF_MISSING) - launch_cvd_args.append( - "-blank_data_image_mb=%d" % (blank_data_disk_size_gb * 1024)) - if avd_spec: - config = self._GetConfigFromAndroidInfo() - if config: - launch_cvd_args.append("-config=%s" % config) - if avd_spec.hw_customize or not config: - launch_cvd_args.append( - "-x_res=" + avd_spec.hw_property[constants.HW_X_RES]) - launch_cvd_args.append( - "-y_res=" + avd_spec.hw_property[constants.HW_Y_RES]) - launch_cvd_args.append( - "-dpi=" + avd_spec.hw_property[constants.HW_ALIAS_DPI]) - if constants.HW_ALIAS_DISK in avd_spec.hw_property: - launch_cvd_args.append( - "-data_policy=" + self.DATA_POLICY_ALWAYS_CREATE) - launch_cvd_args.append( - "-blank_data_image_mb=" - + avd_spec.hw_property[constants.HW_ALIAS_DISK]) - if constants.HW_ALIAS_CPUS in avd_spec.hw_property: - launch_cvd_args.append( - "-cpus=%s" % avd_spec.hw_property[constants.HW_ALIAS_CPUS]) - if constants.HW_ALIAS_MEMORY in avd_spec.hw_property: - launch_cvd_args.append( - "-memory_mb=%s" % avd_spec.hw_property[constants.HW_ALIAS_MEMORY]) - if avd_spec.connect_webrtc: - launch_cvd_args.extend(_WEBRTC_ARGS) - launch_cvd_args.append(_WEBRTC_ID % {"instance": instance}) - if avd_spec.connect_vnc: - launch_cvd_args.extend(_VNC_ARGS) - if avd_spec.openwrt: - launch_cvd_args.append(_ENABLE_CONSOLE_ARG) - if avd_spec.num_avds_per_instance > 1: - launch_cvd_args.append( - _NUM_AVDS_ARG % {"num_AVD": avd_spec.num_avds_per_instance}) - if avd_spec.base_instance_num: - launch_cvd_args.append( - "--base-instance-num=%s" % avd_spec.base_instance_num) - if avd_spec.launch_args: - launch_cvd_args.append(avd_spec.launch_args) - else: - resolution = self._resolution.split("x") - launch_cvd_args.append("-x_res=" + resolution[0]) - launch_cvd_args.append("-y_res=" + resolution[1]) - launch_cvd_args.append("-dpi=" + resolution[3]) - - if not avd_spec and self._launch_args: - launch_cvd_args.append(self._launch_args) - - if decompress_kernel: - launch_cvd_args.append(_DECOMPRESS_KERNEL_ARG) - - launch_cvd_args.append(_UNDEFOK_ARG) - launch_cvd_args.append(_AGREEMENT_PROMPT_ARG) - return launch_cvd_args - @utils.TimeExecute(function_description="Launching AVD(s) and waiting for boot up", result_evaluator=utils.BootEvaluator) - def LaunchCvd(self, instance, avd_spec=None, - blank_data_disk_size_gb=None, - decompress_kernel=None, - boot_timeout_secs=None, - extra_args=()): + def LaunchCvd(self, instance, avd_spec, base_dir, extra_args): """Launch CVD. Launch AVD with launch_cvd. If the process is failed, acloud would show @@ -388,10 +226,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): Args: instance: String, instance name. avd_spec: An AVDSpec instance. - blank_data_disk_size_gb: Size of the blank data disk in GB. - decompress_kernel: Boolean, if true decompress the kernel. - boot_timeout_secs: Integer, the maximum time to wait for the - command to respond. + base_dir: The remote directory containing the images and tools. extra_args: Collection of strings, the extra arguments generated by acloud. e.g., remote image paths. @@ -403,22 +238,22 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): timestart = time.time() error_msg = "" launch_cvd_args = list(extra_args) - launch_cvd_args.extend( - self._GetLaunchCvdArgs(avd_spec, blank_data_disk_size_gb, - decompress_kernel, instance)) + config = self._GetConfigFromAndroidInfo(base_dir) + launch_cvd_args.extend(cvd_utils.GetLaunchCvdArgs(avd_spec, config)) + boot_timeout_secs = self._GetBootTimeout( - boot_timeout_secs or constants.DEFAULT_CF_BOOT_TIMEOUT) - ssh_command = "./bin/launch_cvd -daemon " + " ".join(launch_cvd_args) + avd_spec.boot_timeout_secs or constants.DEFAULT_CF_BOOT_TIMEOUT) + ssh_command = (f"'HOME=$HOME/{base_dir} " + f"{base_dir}/bin/launch_cvd -daemon " + f"{' '.join(launch_cvd_args)}'") try: - if avd_spec and avd_spec.base_instance_num: - self.ExtendReportData(constants.BASE_INSTANCE_NUM, avd_spec.base_instance_num) self.ExtendReportData(_LAUNCH_CVD_COMMAND, ssh_command) self._ssh.Run(ssh_command, boot_timeout_secs, retry=_NO_RETRY) self._UpdateOpenWrtStatus(avd_spec) except (subprocess.CalledProcessError, errors.DeviceConnectionError, errors.LaunchCVDFail) as e: - error_msg = ("Device %s did not finish on boot within timeout (%s secs)" - % (instance, boot_timeout_secs)) + error_msg = (f"Device {instance} did not finish on boot within " + f"timeout ({boot_timeout_secs} secs)") if constants.ERROR_MSG_VNC_NOT_SUPPORT in str(e): error_msg = ( "VNC is not supported in the current build. Please try WebRTC such " @@ -429,7 +264,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): "as '$acloud create --autoconnect vnc'") utils.PrintColorString(str(e), utils.TextColors.FAIL) - self._execution_time[_LAUNCH_CVD] = round(time.time() - timestart, 2) + self._execution_time[constants.TIME_LAUNCH] = time.time() - timestart return {instance: error_msg} if error_msg else {} def _GetBootTimeout(self, timeout_secs): @@ -443,7 +278,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): Returns: The timeout values for device boots up. """ - boot_timeout_secs = timeout_secs - self._execution_time[_FETCH_ARTIFACT] + boot_timeout_secs = timeout_secs - self._execution_time[constants.TIME_ARTIFACT] logger.debug("Timeout for boot: %s secs", boot_timeout_secs) return boot_timeout_secs @@ -489,6 +324,8 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): if avd_spec: metadata[constants.INS_KEY_AVD_TYPE] = avd_spec.avd_type metadata[constants.INS_KEY_AVD_FLAVOR] = avd_spec.flavor + metadata[constants.INS_KEY_WEBRTC_DEVICE_ID] = ( + avd_spec.webrtc_device_id or _DEFAULT_WEBRTC_DEVICE_ID) metadata[constants.INS_KEY_DISPLAY] = ("%sx%s (%s)" % ( avd_spec.hw_property[constants.HW_X_RES], avd_spec.hw_property[constants.HW_Y_RES], @@ -522,51 +359,41 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): logger.debug("'instance_ip': %s", ip.internal if self._report_internal_ip else ip.external) - self._execution_time[_GCE_CREATE] = round(time.time() - timestart, 2) + self._execution_time[constants.TIME_GCE] = time.time() - timestart return ip @utils.TimeExecute(function_description="Uploading build fetcher to instance") - def UpdateFetchCvd(self): + def UpdateFetchCvd(self, fetch_cvd_version): """Download fetch_cvd from the Build API, and upload it to a remote instance. The version of fetch_cvd to use is retrieved from the configuration file. Once fetch_cvd is on the instance, future commands can use it to download relevant Cuttlefish files from the Build API on the instance itself. + + Args: + fetch_cvd_version: String. The build id of fetch_cvd. """ self.SetStage(constants.STAGE_ARTIFACT) download_dir = tempfile.mkdtemp() download_target = os.path.join(download_dir, _FETCHER_NAME) - self._build_api.DownloadFetchcvd(download_target, self._fetch_cvd_version) + self._build_api.DownloadFetchcvd(download_target, fetch_cvd_version) self._ssh.ScpPushFile(src_file=download_target, dst_file=_FETCHER_NAME) os.remove(download_target) os.rmdir(download_dir) @utils.TimeExecute(function_description="Downloading build on instance") - def FetchBuild(self, build_id, branch, build_target, system_build_id, - system_branch, system_build_target, kernel_build_id, - kernel_branch, kernel_build_target, bootloader_build_id, - bootloader_branch, bootloader_build_target, ota_build_id, - ota_branch, ota_build_target): + def FetchBuild(self, default_build_info, system_build_info, + kernel_build_info, boot_build_info, bootloader_build_info, + ota_build_info): """Execute fetch_cvd on the remote instance to get Cuttlefish runtime files. Args: - build_id: String of build id, e.g. "2263051", "P2804227" - branch: String of branch name, e.g. "aosp-master" - build_target: String of target name. - e.g. "aosp_cf_x86_64_phone-userdebug" - system_build_id: String of the system image build id. - system_branch: String of the system image branch name. - system_build_target: String of the system image target name, - e.g. "cf_x86_phone-userdebug" - kernel_build_id: String of the kernel image build id. - kernel_branch: String of the kernel image branch name. - kernel_build_target: String of the kernel image target name, - bootloader_build_id: String of the bootloader build id. - bootloader_branch: String of the bootloader branch name. - bootloader_build_target: String of the bootloader target name. - ota_build_id: String of the otatools build id. - ota_branch: String of the otatools branch name. - ota_build_target: String of the otatools target name. + default_build_info: The build that provides full cuttlefish images. + system_build_info: The build that provides the system image. + kernel_build_info: The build that provides the kernel. + boot_build_info: The build that provides the boot image. + bootloader_build_info: The build that provides the bootloader. + ota_build_info: The build that provides the OTA tools. Returns: List of string args for fetch_cvd. @@ -574,15 +401,13 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): timestart = time.time() fetch_cvd_args = ["-credential_source=gce"] fetch_cvd_build_args = self._build_api.GetFetchBuildArgs( - build_id, branch, build_target, system_build_id, system_branch, - system_build_target, kernel_build_id, kernel_branch, - kernel_build_target, bootloader_build_id, bootloader_branch, - bootloader_build_target, ota_build_id, ota_branch, ota_build_target) + default_build_info, system_build_info, kernel_build_info, + boot_build_info, bootloader_build_info, ota_build_info) fetch_cvd_args.extend(fetch_cvd_build_args) self._ssh.Run("./fetch_cvd " + " ".join(fetch_cvd_args), timeout=constants.DEFAULT_SSH_TIMEOUT) - self._execution_time[_FETCH_ARTIFACT] = round(time.time() - timestart, 2) + self._execution_time[constants.TIME_ARTIFACT] = time.time() - timestart @utils.TimeExecute(function_description="Update instance's certificates") def UpdateCertificate(self): @@ -622,7 +447,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): for extra_file in extra_files: if not os.path.exists(extra_file.source): raise errors.CheckPathError( - "The path doesn't exist: %s" % extra_file.source) + f"The path doesn't exist: {extra_file.source}") self._ssh.ScpPushFile(extra_file.source, extra_file.target) def GetSshConnectCmd(self): diff --git a/internal/lib/cvd_compute_client_multi_stage_test.py b/internal/lib/cvd_compute_client_multi_stage_test.py index d9646714..b85a8ed0 100644 --- a/internal/lib/cvd_compute_client_multi_stage_test.py +++ b/internal/lib/cvd_compute_client_multi_stage_test.py @@ -19,7 +19,6 @@ import collections import glob import os -import subprocess import unittest from unittest import mock @@ -51,6 +50,7 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): MACHINE_TYPE = "fake-machine-type" NETWORK = "fake-network" ZONE = "fake-zone" + PROJECT = "fake-project" BRANCH = "fake-branch" TARGET = "aosp_cf_x86_64_phone-userdebug" BUILD_ID = "2263051" @@ -68,8 +68,6 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): GPU = "fake-gpu" DISK_TYPE = "fake-disk-type" FAKE_IP = IP(external="1.1.1.1", internal="10.1.1.1") - REMOTE_HOST_IP = "192.0.2.1" - REMOTE_HOST_INSTANCE_NAME = "host-192.0.2.1-2263051-aosp_cf_x86_64_phone" def _GetFakeConfig(self): """Create a fake configuration object. @@ -82,6 +80,7 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): fake_cfg.machine_type = self.MACHINE_TYPE fake_cfg.network = self.NETWORK fake_cfg.zone = self.ZONE + fake_cfg.project = self.PROJECT fake_cfg.resolution = "{x}x{y}x32x{dpi}".format( x=self.X_RES, y=self.Y_RES, dpi=self.DPI) fake_cfg.metadata_variable = self.METADATA @@ -117,6 +116,7 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): self.args.avd_type = constants.TYPE_CF self.args.flavor = "phone" self.args.adb_port = None + self.args.fastboot_port = None self.args.base_instance_num = None self.args.hw_property = "cpu:2,resolution:1080x1920,dpi:240,memory:4g,disk:10g" self.args.num_avds_per_instance = 2 @@ -126,44 +126,7 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): self.args.autoconnect = False self.args.disk_type = self.DISK_TYPE self.args.openwrt = False - - # pylint: disable=protected-access - @mock.patch.object(utils, "GetBuildEnvironmentVariable", return_value="fake_env_cf_x86") - @mock.patch.object(glob, "glob", return_value=["fake.img"]) - def testGetLaunchCvdArgs(self, _mock_check_img, _mock_env): - """test GetLaunchCvdArgs.""" - # test GetLaunchCvdArgs with avd_spec - self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, - "_GetConfigFromAndroidInfo", return_value="phone") - fake_avd_spec = avd_spec.AVDSpec(self.args) - expected_args = ["-config=phone", "-x_res=1080", "-y_res=1920", "-dpi=240", - "-data_policy=always_create", "-blank_data_image_mb=10240", - "-cpus=2", "-memory_mb=4096", "-num_instances=2", - "--setupwizard_mode=REQUIRED", - "-undefok=report_anonymous_usage_stats,config", - "-report_anonymous_usage_stats=y"] - launch_cvd_args = self.cvd_compute_client_multi_stage._GetLaunchCvdArgs(fake_avd_spec) - self.assertEqual(launch_cvd_args, expected_args) - - self.args.openwrt = True - fake_avd_spec = avd_spec.AVDSpec(self.args) - expected_args = ["-config=phone", "-x_res=1080", "-y_res=1920", "-dpi=240", - "-data_policy=always_create", "-blank_data_image_mb=10240", - "-cpus=2", "-memory_mb=4096", "-console=true", - "-num_instances=2", "--setupwizard_mode=REQUIRED", - "-undefok=report_anonymous_usage_stats,config", - "-report_anonymous_usage_stats=y"] - launch_cvd_args = self.cvd_compute_client_multi_stage._GetLaunchCvdArgs(fake_avd_spec) - self.assertEqual(launch_cvd_args, expected_args) - - # test GetLaunchCvdArgs without avd_spec - expected_args = ["-x_res=720", "-y_res=1280", "-dpi=160", - "--setupwizard_mode=REQUIRED", - "-undefok=report_anonymous_usage_stats,config", - "-report_anonymous_usage_stats=y"] - launch_cvd_args = self.cvd_compute_client_multi_stage._GetLaunchCvdArgs( - avd_spec=None) - self.assertEqual(launch_cvd_args, expected_args) + self.args.webrtc_device_id = "cvd-1" @mock.patch.object(utils, "GetBuildEnvironmentVariable", return_value="fake_env_cf_x86") @mock.patch.object(glob, "glob", return_value=["fake.img"]) @@ -174,71 +137,29 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): @mock.patch.object(gcompute_client.ComputeClient, "CreateInstance") @mock.patch.object(cvd_compute_client_multi_stage.CvdComputeClient, "_GetDiskArgs", return_value=[{"fake_arg": "fake_value"}]) - @mock.patch("getpass.getuser", return_value="fake_user") - def testCreateInstance(self, _get_user, _get_disk_args, mock_create, - _get_image, _compare_machine_size, mock_check_img, - _mock_env): + def testCreateInstance(self, _get_disk_args, mock_create, _get_image, + _compare_machine_size, _mock_check_img, _mock_env): """Test CreateInstance.""" - expected_metadata = dict() - expected_metadata_local_image = dict() - expected_metadata.update(self.METADATA) - expected_metadata_local_image.update(self.METADATA) - remote_image_metadata = dict(expected_metadata) expected_disk_args = [{"fake_arg": "fake_value"}] fake_avd_spec = avd_spec.AVDSpec(self.args) fake_avd_spec._instance_name_to_reuse = None - - created_subprocess = mock.MagicMock() - created_subprocess.stdout = mock.MagicMock() - created_subprocess.stdout.readline = mock.MagicMock(return_value=b"") - created_subprocess.poll = mock.MagicMock(return_value=0) - created_subprocess.returncode = 0 - created_subprocess.communicate = mock.MagicMock(return_value=('', '')) - self.Patch(subprocess, "Popen", return_value=created_subprocess) - self.Patch(subprocess, "check_call") - self.Patch(os, "chmod") - self.Patch(os, "stat") - self.Patch(os, "remove") - self.Patch(os, "rmdir") - self.cvd_compute_client_multi_stage.CreateInstance( - self.INSTANCE, self.IMAGE, self.IMAGE_PROJECT, self.TARGET, - self.BRANCH, self.BUILD_ID, self.KERNEL_BRANCH, - self.KERNEL_BUILD_ID, self.KERNEL_BUILD_TARGET, - self.EXTRA_DATA_DISK_SIZE_GB, extra_scopes=self.EXTRA_SCOPES) - mock_create.assert_called_with( - self.cvd_compute_client_multi_stage, - instance=self.INSTANCE, - image_name=self.IMAGE, - image_project=self.IMAGE_PROJECT, - disk_args=expected_disk_args, - metadata=remote_image_metadata, - machine_type=self.MACHINE_TYPE, - network=self.NETWORK, - zone=self.ZONE, - extra_scopes=self.EXTRA_SCOPES, - gpu=self.GPU, - disk_type=None, - disable_external_ip=False) - - mock_check_img.return_value = True - #test use local image in the remote instance. - local_image_metadata = dict(expected_metadata_local_image) fake_avd_spec.hw_property[constants.HW_X_RES] = str(self.X_RES) fake_avd_spec.hw_property[constants.HW_Y_RES] = str(self.Y_RES) fake_avd_spec.hw_property[constants.HW_ALIAS_DPI] = str(self.DPI) fake_avd_spec.hw_property[constants.HW_ALIAS_DISK] = str( self.EXTRA_DATA_DISK_SIZE_GB * 1024) + + local_image_metadata = dict(self.METADATA) local_image_metadata["avd_type"] = constants.TYPE_CF local_image_metadata["flavor"] = "phone" + local_image_metadata[constants.INS_KEY_WEBRTC_DEVICE_ID] = "cvd-1" local_image_metadata[constants.INS_KEY_DISPLAY] = ("%sx%s (%s)" % ( fake_avd_spec.hw_property[constants.HW_X_RES], fake_avd_spec.hw_property[constants.HW_Y_RES], fake_avd_spec.hw_property[constants.HW_ALIAS_DPI])) self.cvd_compute_client_multi_stage.CreateInstance( - self.INSTANCE, self.IMAGE, self.IMAGE_PROJECT, self.TARGET, self.BRANCH, - self.BUILD_ID, self.KERNEL_BRANCH, self.KERNEL_BUILD_ID, - self.KERNEL_BUILD_TARGET, self.EXTRA_DATA_DISK_SIZE_GB, - fake_avd_spec, extra_scopes=self.EXTRA_SCOPES) + self.INSTANCE, self.IMAGE, self.IMAGE_PROJECT, + fake_avd_spec, self.EXTRA_SCOPES) mock_create.assert_called_with( self.cvd_compute_client_multi_stage, @@ -255,22 +176,6 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): disk_type=self.DISK_TYPE, disable_external_ip=False) - def testFormatRemoteHostInstanceName(self): - """Test FormatRemoteHostInstanceName.""" - name = self.cvd_compute_client_multi_stage.FormatRemoteHostInstanceName( - self.REMOTE_HOST_IP, self.BUILD_ID, self.TARGET.split("-")[0]) - self.assertEqual(name, self.REMOTE_HOST_INSTANCE_NAME) - - def testParseRemoteHostAddress(self): - """Test ParseRemoteHostAddress.""" - ip_addr = self.cvd_compute_client_multi_stage.ParseRemoteHostAddress( - self.REMOTE_HOST_INSTANCE_NAME) - self.assertEqual(ip_addr, self.REMOTE_HOST_IP) - - ip_addr = self.cvd_compute_client_multi_stage.ParseRemoteHostAddress( - "host-goldfish-192.0.2.1-5554-123456-sdk_x86_64-sdk") - self.assertIsNone(ip_addr) - def testSetStage(self): """Test SetStage""" device_stage = "fake_stage" @@ -283,7 +188,7 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): self.Patch(Ssh, "GetCmdOutput", return_value="config=phone") expected = "phone" self.assertEqual( - self.cvd_compute_client_multi_stage._GetConfigFromAndroidInfo(), + self.cvd_compute_client_multi_stage._GetConfigFromAndroidInfo("dir"), expected) @mock.patch.object(Ssh, "Run") diff --git a/internal/lib/cvd_compute_client_test.py b/internal/lib/cvd_compute_client_test.py deleted file mode 100644 index 235530d1..00000000 --- a/internal/lib/cvd_compute_client_test.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2018 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for acloud.internal.lib.cvd_compute_client.""" - -import glob -import unittest - -from unittest import mock - -from acloud.create import avd_spec -from acloud.internal import constants -from acloud.internal.lib import cvd_compute_client -from acloud.internal.lib import driver_test_lib -from acloud.internal.lib import gcompute_client -from acloud.internal.lib import utils -from acloud.list import list as list_instances - - -class CvdComputeClientTest(driver_test_lib.BaseDriverTest): - """Test CvdComputeClient.""" - - SSH_PUBLIC_KEY_PATH = "" - INSTANCE = "fake-instance" - IMAGE = "fake-image" - IMAGE_PROJECT = "fake-iamge-project" - MACHINE_TYPE = "fake-machine-type" - NETWORK = "fake-network" - ZONE = "fake-zone" - BRANCH = "fake-branch" - TARGET = "aosp_cf_x86_64_phone-userdebug" - BUILD_ID = "2263051" - KERNEL_BRANCH = "fake-kernel-branch" - KERNEL_BUILD_ID = "1234567" - KERNEL_BUILD_TARGET = "kernel" - DPI = 160 - X_RES = 720 - Y_RES = 1280 - METADATA = {"metadata_key": "metadata_value"} - EXTRA_DATA_DISK_SIZE_GB = 4 - BOOT_DISK_SIZE_GB = 10 - LAUNCH_ARGS = "--setupwizard_mode=REQUIRED" - EXTRA_SCOPES = ["scope1"] - DISK_TYPE = "fake-disk-type" - - def _GetFakeConfig(self): - """Create a fake configuration object. - - Returns: - A fake configuration mock object. - """ - fake_cfg = mock.MagicMock() - fake_cfg.ssh_public_key_path = self.SSH_PUBLIC_KEY_PATH - fake_cfg.machine_type = self.MACHINE_TYPE - fake_cfg.network = self.NETWORK - fake_cfg.zone = self.ZONE - fake_cfg.resolution = "{x}x{y}x32x{dpi}".format( - x=self.X_RES, y=self.Y_RES, dpi=self.DPI) - fake_cfg.metadata_variable = self.METADATA - fake_cfg.extra_data_disk_size_gb = self.EXTRA_DATA_DISK_SIZE_GB - fake_cfg.launch_args = self.LAUNCH_ARGS - fake_cfg.extra_scopes = self.EXTRA_SCOPES - return fake_cfg - - def setUp(self): - """Set up the test.""" - super().setUp() - self.Patch(cvd_compute_client.CvdComputeClient, "InitResourceHandle") - self.Patch(list_instances, "ChooseOneRemoteInstance", return_value=mock.MagicMock()) - self.Patch(list_instances, "GetInstancesFromInstanceNames", return_value=mock.MagicMock()) - self.cvd_compute_client = cvd_compute_client.CvdComputeClient( - self._GetFakeConfig(), mock.MagicMock()) - - @mock.patch.object(utils, "GetBuildEnvironmentVariable", return_value="fake_cf_x86") - @mock.patch.object(glob, "glob", return_value=["fake.img"]) - @mock.patch.object(gcompute_client.ComputeClient, "CompareMachineSize", - return_value=1) - @mock.patch.object(gcompute_client.ComputeClient, "GetImage", - return_value={"diskSizeGb": 10}) - @mock.patch.object(gcompute_client.ComputeClient, "CreateInstance") - @mock.patch.object(cvd_compute_client.CvdComputeClient, "_GetDiskArgs", - return_value=[{"fake_arg": "fake_value"}]) - @mock.patch("getpass.getuser", return_value="fake_user") - def testCreateInstance(self, _get_user, _get_disk_args, mock_create, - _get_image, _compare_machine_size, mock_check_img, - _mock_env): - """Test CreateInstance.""" - expected_metadata = { - "cvd_01_dpi": str(self.DPI), - "cvd_01_fetch_android_build_target": self.TARGET, - "cvd_01_fetch_android_bid": "{branch}/{build_id}".format( - branch=self.BRANCH, build_id=self.BUILD_ID), - "cvd_01_fetch_kernel_bid": "{branch}/{build_id}".format( - branch=self.KERNEL_BRANCH, build_id=self.KERNEL_BUILD_ID), - "cvd_01_fetch_kernel_build_target": self.KERNEL_BUILD_TARGET, - "cvd_01_x_res": str(self.X_RES), - "cvd_01_y_res": str(self.Y_RES), - "user": "fake_user", - "cvd_01_data_policy": - self.cvd_compute_client.DATA_POLICY_CREATE_IF_MISSING, - "cvd_01_blank_data_disk_size": str(self.EXTRA_DATA_DISK_SIZE_GB * 1024), - } - expected_metadata_local_image = { - "cvd_01_dpi": str(self.DPI), - "cvd_01_x_res": str(self.X_RES), - "cvd_01_y_res": str(self.Y_RES), - "user": "fake_user", - "cvd_01_data_policy": - self.cvd_compute_client.DATA_POLICY_CREATE_IF_MISSING, - "cvd_01_blank_data_disk_size": str(self.EXTRA_DATA_DISK_SIZE_GB * 1024), - } - expected_metadata.update(self.METADATA) - expected_metadata_local_image.update(self.METADATA) - remote_image_metadata = dict(expected_metadata) - remote_image_metadata["cvd_01_launch"] = self.LAUNCH_ARGS - expected_disk_args = [{"fake_arg": "fake_value"}] - - self.cvd_compute_client.CreateInstance( - self.INSTANCE, self.IMAGE, self.IMAGE_PROJECT, self.TARGET, - self.BRANCH, self.BUILD_ID, self.KERNEL_BRANCH, - self.KERNEL_BUILD_ID, self.KERNEL_BUILD_TARGET, - self.EXTRA_DATA_DISK_SIZE_GB, extra_scopes=self.EXTRA_SCOPES) - mock_create.assert_called_with( - self.cvd_compute_client, - instance=self.INSTANCE, - image_name=self.IMAGE, - image_project=self.IMAGE_PROJECT, - disk_args=expected_disk_args, - metadata=remote_image_metadata, - machine_type=self.MACHINE_TYPE, - network=self.NETWORK, - zone=self.ZONE, - disk_type=None, - extra_scopes=self.EXTRA_SCOPES) - - #test use local image in the remote instance. - local_image_metadata = dict(expected_metadata_local_image) - args = mock.MagicMock() - mock_check_img.return_value = True - args.local_image = constants.FIND_IN_BUILD_ENV - args.local_system_image = None - args.config_file = "" - args.avd_type = constants.TYPE_CF - args.flavor = "phone" - args.adb_port = None - args.remote_host = False - args.launch_args = None - args.disk_type = self.DISK_TYPE - fake_avd_spec = avd_spec.AVDSpec(args) - fake_avd_spec.hw_property[constants.HW_X_RES] = str(self.X_RES) - fake_avd_spec.hw_property[constants.HW_Y_RES] = str(self.Y_RES) - fake_avd_spec.hw_property[constants.HW_ALIAS_DPI] = str(self.DPI) - fake_avd_spec.hw_property[constants.HW_ALIAS_DISK] = str( - self.EXTRA_DATA_DISK_SIZE_GB * 1024) - local_image_metadata["avd_type"] = constants.TYPE_CF - local_image_metadata["flavor"] = "phone" - local_image_metadata[constants.INS_KEY_DISPLAY] = ("%sx%s (%s)" % ( - fake_avd_spec.hw_property[constants.HW_X_RES], - fake_avd_spec.hw_property[constants.HW_Y_RES], - fake_avd_spec.hw_property[constants.HW_ALIAS_DPI])) - self.cvd_compute_client.CreateInstance( - self.INSTANCE, self.IMAGE, self.IMAGE_PROJECT, self.TARGET, self.BRANCH, - self.BUILD_ID, self.KERNEL_BRANCH, self.KERNEL_BUILD_ID, - self.KERNEL_BUILD_TARGET, self.EXTRA_DATA_DISK_SIZE_GB, - fake_avd_spec, extra_scopes=self.EXTRA_SCOPES) - - mock_create.assert_called_with( - self.cvd_compute_client, - instance=self.INSTANCE, - image_name=self.IMAGE, - image_project=self.IMAGE_PROJECT, - disk_args=expected_disk_args, - metadata=local_image_metadata, - machine_type=self.MACHINE_TYPE, - network=self.NETWORK, - zone=self.ZONE, - disk_type=self.DISK_TYPE, - extra_scopes=self.EXTRA_SCOPES) - - # pylint: disable=protected-access - def testGetLaunchCvdArgs(self): - """Test GetLaunchCvdArgs""" - fake_avd_spec = mock.MagicMock() - fake_avd_spec.hw_property = {} - fake_avd_spec.hw_property[constants.HW_ALIAS_CPUS] = "2" - fake_avd_spec.hw_property[constants.HW_ALIAS_MEMORY] = "4096" - - # Test get launch_args exist from config - self.assertEqual(self.cvd_compute_client._GetLaunchCvdArgs(fake_avd_spec), - self.LAUNCH_ARGS) - - # Test get launch_args from cpu and memory - expected_args = "-cpus=2 -memory_mb=4096" - self.cvd_compute_client._launch_args = None - self.assertEqual(self.cvd_compute_client._GetLaunchCvdArgs(fake_avd_spec), - expected_args) - - # Test to set launch_args as "1" for no customized args - expected_args = "1" - fake_avd_spec.hw_property = {} - self.assertEqual(self.cvd_compute_client._GetLaunchCvdArgs(fake_avd_spec), - expected_args) - - self.cvd_compute_client._launch_args = self.LAUNCH_ARGS - - -if __name__ == "__main__": - unittest.main() diff --git a/internal/lib/cvd_runtime_config.py b/internal/lib/cvd_runtime_config.py index bcfcc1e6..8f4cba04 100644 --- a/internal/lib/cvd_runtime_config.py +++ b/internal/lib/cvd_runtime_config.py @@ -20,17 +20,17 @@ import re from acloud import errors _CFG_KEY_CROSVM_BINARY = "crosvm_binary" -_CFG_KEY_X_RES = "x_res" -_CFG_KEY_Y_RES = "y_res" -_CFG_KEY_DPI = "dpi" +_CFG_KEY_DISPLAY_CONFIGS = "display_configs" _CFG_KEY_VIRTUAL_DISK_PATHS = "virtual_disk_paths" _CFG_KEY_INSTANCES = "instances" _CFG_KEY_ADB_IP_PORT = "adb_ip_and_port" _CFG_KEY_INSTANCE_DIR = "instance_dir" +_CFG_KEY_ROOT_DIR = "root_dir" _CFG_KEY_VNC_PORT = "vnc_server_port" # The adb port field name changes from "host_port" to "adb_host_port". _CFG_KEY_ADB_PORT = "host_port" _CFG_KEY_ADB_HOST_PORT = "adb_host_port" +_CFG_KEY_FASTBOOT_HOST_PORT = "fastboot_host_port" _CFG_KEY_ENABLE_WEBRTC = "enable_webrtc" # TODO(148648620): Check instance_home_[id] for backward compatible. _RE_LOCAL_INSTANCE_ID = re.compile(r".+(?:local-instance-|instance_home_)" @@ -64,6 +64,14 @@ class CvdRuntimeConfig(): { "memory_mb" : 4096, "cpus" : 2, + "display_configs" : + [ + { + "dpi" : 160, + "x_res" : 1280, + "y_res" : 700 + } + ], "dpi" : 320, "virtual_disk_paths" : [ @@ -84,6 +92,7 @@ class CvdRuntimeConfig(): { "adb_ip_and_port" : "0.0.0.0:6520", "instance_dir" : "/path-to-instance-dir", + "webrtc_device_id" : "cvd-1", "virtual_disk_paths" : [ "/path-to-image" @@ -99,7 +108,6 @@ class CvdRuntimeConfig(): "webrtc_assets_dir" : "/home/vsoc-01/usr/share/webrtc/assets", "webrtc_binary" : "/home/vsoc-01/bin/webRTC", "webrtc_certs_dir" : "/home/vsoc-01/usr/share/webrtc/certs", - "webrtc_enable_adb_websocket" : false, "webrtc_public_ip" : "0.0.0.0", } @@ -113,9 +121,11 @@ class CvdRuntimeConfig(): config_path) self._config_dict = self._GetCuttlefishRuntimeConfig(config_path, raw_data) - self._x_res = self._config_dict.get(_CFG_KEY_X_RES) - self._y_res = self._config_dict.get(_CFG_KEY_Y_RES) - self._dpi = self._config_dict.get(_CFG_KEY_DPI) + self._instances = self._config_dict.get(_CFG_KEY_INSTANCES) + # Old runtime config doesn't have "instances" information. + self._instance_ids = list(self._instances.keys()) if self._instances else ["1"] + self._display_configs = self._config_dict.get(_CFG_KEY_DISPLAY_CONFIGS, {}) + self._root_dir = self._config_dict.get(_CFG_KEY_ROOT_DIR) crosvm_bin = self._config_dict.get(_CFG_KEY_CROSVM_BINARY) self._cvd_tools_path = (os.path.dirname(crosvm_bin) if crosvm_bin else None) @@ -131,8 +141,7 @@ class CvdRuntimeConfig(): _CFG_KEY_VIRTUAL_DISK_PATHS) self._enable_webrtc = self._config_dict.get(_CFG_KEY_ENABLE_WEBRTC) if not self._instance_dir: - ins_cfg = self._config_dict.get(_CFG_KEY_INSTANCES) - ins_dict = ins_cfg.get(self._instance_id) + ins_dict = self._instances.get(self._instance_id) if not ins_dict: raise errors.ConfigError("instances[%s] property does not exist" " in: %s" % @@ -142,7 +151,11 @@ class CvdRuntimeConfig(): self._adb_port = (ins_dict.get(_CFG_KEY_ADB_PORT) or ins_dict.get(_CFG_KEY_ADB_HOST_PORT)) self._adb_ip_port = ins_dict.get(_CFG_KEY_ADB_IP_PORT) + self._fastboot_port = ins_dict.get(_CFG_KEY_FASTBOOT_HOST_PORT) self._virtual_disk_paths = ins_dict.get(_CFG_KEY_VIRTUAL_DISK_PATHS) + if not self._cvd_tools_path: + self._cvd_tools_path = os.path.dirname( + ins_dict.get(_CFG_KEY_CROSVM_BINARY)) @staticmethod def _GetCuttlefishRuntimeConfig(runtime_cf_config_path, raw_data=None): @@ -180,19 +193,9 @@ class CvdRuntimeConfig(): return self._cvd_tools_path @property - def x_res(self): - """Return x_res.""" - return self._x_res - - @property - def y_res(self): - """Return y_res.""" - return self._y_res - - @property - def dpi(self): - """Return dpi.""" - return self._dpi + def display_configs(self): + """Return display_configs.""" + return self._display_configs @property def adb_ip_port(self): @@ -205,6 +208,11 @@ class CvdRuntimeConfig(): return self._instance_dir @property + def root_dir(self): + """Return root_dir.""" + return self._root_dir + + @property def vnc_port(self): """Return vnc_port.""" return self._vnc_port @@ -215,6 +223,11 @@ class CvdRuntimeConfig(): return self._adb_port @property + def fastboot_port(self): + """Return fastboot_port""" + return self._fastboot_port + + @property def config_path(self): """Return config_path.""" return self._config_path @@ -230,6 +243,16 @@ class CvdRuntimeConfig(): return self._instance_id @property + def instance_ids(self): + """Return _instance_ids""" + return self._instance_ids + + @property + def instances(self): + """Return _instances""" + return self._instances + + @property def enable_webrtc(self): """Return _enable_webrtc""" return self._enable_webrtc diff --git a/internal/lib/cvd_runtime_config_test.py b/internal/lib/cvd_runtime_config_test.py index 42bd48b1..8845a2e4 100644 --- a/internal/lib/cvd_runtime_config_test.py +++ b/internal/lib/cvd_runtime_config_test.py @@ -30,13 +30,21 @@ class CvdRuntimeconfigTest(driver_test_lib.BaseDriverTest): CF_RUNTIME_CONFIG = """ {"x_display" : ":20", - "x_res" : 720, - "y_res" : 1280, + "display_configs" : + [ + { + "dpi" : 320, + "x_res" : 720, + "y_res" : 1280 + } + ], "instances": { "2":{ "adb_ip_and_port": "127.0.0.1:6520", "adb_host_port": 6520, + "fastboot_host_port": 7520, "instance_dir": "/path-to-instance-dir", + "crosvm_binary" : "/home/vsoc-01/bin/crosvm", "vnc_server_port": 6444 } } @@ -45,13 +53,19 @@ class CvdRuntimeconfigTest(driver_test_lib.BaseDriverTest): CF_RUNTIME_CONFIG_WEBRTC = """ {"x_display" : ":20", - "x_res" : 720, - "y_res" : 1280, - "dpi" : 320, + "display_configs" : + [ + { + "dpi" : 320, + "x_res" : 720, + "y_res" : 1280 + } + ], "instances" : { "1":{ "adb_ip_and_port": "127.0.0.1:6520", "adb_host_port": 6520, + "fastboot_host_port": 7520, "instance_dir": "/path-to-instance-dir", "vnc_server_port": 6444, "virtual_disk_paths": ["/path-to-image"] @@ -63,11 +77,25 @@ class CvdRuntimeconfigTest(driver_test_lib.BaseDriverTest): "webrtc_assets_dir" : "/home/vsoc-01/usr/share/webrtc/assets", "webrtc_binary" : "/home/vsoc-01/bin/webRTC", "webrtc_certs_dir" : "/home/vsoc-01/usr/share/webrtc/certs", - "webrtc_enable_adb_websocket" : false, "webrtc_public_ip" : "127.0.0.1" } """ + CF_RUNTIME_CONFIG_NO_INSTANCES = """ +{"x_display" : ":20", + "display_configs" : + [ + { + "dpi" : 320, + "x_res" : 720, + "y_res" : 1280 + } + ], + "instance_dir" : "fake_instance_dir", + "instances": {} +} +""" + # pylint: disable=protected-access, no-member def testGetCuttlefishRuntimeConfig(self): @@ -76,15 +104,17 @@ class CvdRuntimeconfigTest(driver_test_lib.BaseDriverTest): self.Patch(os.path, "exists", return_value=False) # Verify return data. self.Patch(os.path, "exists", return_value=True) - expected_dict = {u'y_res': 1280, - u'x_res': 720, - u'x_display': u':20', - u'instances': - {u'2': - {u'adb_ip_and_port': u'127.0.0.1:6520', - u'adb_host_port': 6520, - u'instance_dir': u'/path-to-instance-dir', - u'vnc_server_port': 6444} + expected_dict = { + 'display_configs': [{'dpi': 320, 'x_res': 720, 'y_res': 1280}], + 'x_display': ':20', + 'instances': + {'2': + {'adb_ip_and_port': '127.0.0.1:6520', + 'crosvm_binary': '/home/vsoc-01/bin/crosvm', + 'adb_host_port': 6520, + 'fastboot_host_port': 7520, + 'instance_dir': '/path-to-instance-dir', + 'vnc_server_port': 6444} }, } mock_open = mock.mock_open(read_data=self.CF_RUNTIME_CONFIG) @@ -104,17 +134,23 @@ class CvdRuntimeconfigTest(driver_test_lib.BaseDriverTest): cf_cfg._GetIdFromInstanceDirStr.assert_not_called() self.assertEqual(fake_cvd_runtime_config_webrtc.config_path, None) self.assertEqual(fake_cvd_runtime_config_webrtc.instance_id, "1") + self.assertEqual(fake_cvd_runtime_config_webrtc.instance_ids, ["1"]) self.assertEqual(fake_cvd_runtime_config_webrtc.enable_webrtc, True) - self.assertEqual(fake_cvd_runtime_config_webrtc.x_res, 720) - self.assertEqual(fake_cvd_runtime_config_webrtc.y_res, 1280) - self.assertEqual(fake_cvd_runtime_config_webrtc.dpi, 320) + self.assertEqual(fake_cvd_runtime_config_webrtc.display_configs, + [{'dpi': 320, 'x_res': 720, 'y_res': 1280}]) self.assertEqual(fake_cvd_runtime_config_webrtc.adb_ip_port, "127.0.0.1:6520") self.assertEqual(fake_cvd_runtime_config_webrtc.instance_dir, "/path-to-instance-dir") self.assertEqual(fake_cvd_runtime_config_webrtc.vnc_port, 6444) - self.assertEqual(fake_cvd_runtime_config_webrtc.adb_port, 6520) + self.assertEqual(fake_cvd_runtime_config_webrtc.fastboot_port, 7520) self.assertEqual(fake_cvd_runtime_config_webrtc.virtual_disk_paths, ['/path-to-image']) self.assertEqual(fake_cvd_runtime_config_webrtc.cvd_tools_path, "/home/vsoc-01/bin") + # Test read runtime config with no instances data. + fake_cvd_runtime_config_no_instances = cf_cfg.CvdRuntimeConfig( + raw_data=self.CF_RUNTIME_CONFIG_NO_INSTANCES) + self.assertEqual(fake_cvd_runtime_config_no_instances.instance_id, "1") + self.assertEqual(fake_cvd_runtime_config_no_instances.instance_ids, ["1"]) + # Test exception with no config file and no raw_data. self.assertRaises(errors.ConfigError, cf_cfg.CvdRuntimeConfig, diff --git a/internal/lib/cvd_utils.py b/internal/lib/cvd_utils.py index 5194806e..d7079b46 100644 --- a/internal/lib/cvd_utils.py +++ b/internal/lib/cvd_utils.py @@ -14,15 +14,19 @@ """Utility functions that process cuttlefish images.""" +import collections import glob import logging import os import posixpath as remote_path +import re import subprocess +import tempfile from acloud import errors from acloud.create import create_common from acloud.internal import constants +from acloud.internal.lib import ota_tools from acloud.internal.lib import ssh from acloud.internal.lib import utils from acloud.public import report @@ -30,9 +34,8 @@ from acloud.public import report logger = logging.getLogger(__name__) -# bootloader and kernel are files required to launch AVD. +# Local build artifacts to be uploaded. _ARTIFACT_FILES = ["*.img", "bootloader", "kernel"] -_REMOTE_IMAGE_DIR = "acloud_cf" # The boot image name pattern corresponds to the use cases: # - In a cuttlefish build environment, ANDROID_PRODUCT_OUT conatins boot.img # and boot-debug.img. The former is the default boot image. The latter is not @@ -43,44 +46,147 @@ _BOOT_IMAGE_NAME_PATTERN = r"boot(-[\d.]+)?\.img" _VENDOR_BOOT_IMAGE_NAME = "vendor_boot.img" _KERNEL_IMAGE_NAMES = ("kernel", "bzImage", "Image") _INITRAMFS_IMAGE_NAME = "initramfs.img" +_VENDOR_IMAGE_NAMES = ("vendor.img", "vendor_dlkm.img", "odm.img", + "odm_dlkm.img") +VendorImagePaths = collections.namedtuple( + "VendorImagePaths", + ["vendor", "vendor_dlkm", "odm", "odm_dlkm"]) + +# The relative path to the base directory containing cuttelfish images, tools, +# and runtime files. On a GCE instance, the directory is the SSH user's HOME. +GCE_BASE_DIR = "." +_REMOTE_HOST_BASE_DIR_FORMAT = "acloud_cf_%(num)d" +# Relative paths in a base directory. +_REMOTE_IMAGE_DIR = "acloud_image" _REMOTE_BOOT_IMAGE_PATH = remote_path.join(_REMOTE_IMAGE_DIR, "boot.img") _REMOTE_VENDOR_BOOT_IMAGE_PATH = remote_path.join( _REMOTE_IMAGE_DIR, _VENDOR_BOOT_IMAGE_NAME) +_REMOTE_VBMETA_IMAGE_PATH = remote_path.join(_REMOTE_IMAGE_DIR, "vbmeta.img") _REMOTE_KERNEL_IMAGE_PATH = remote_path.join( _REMOTE_IMAGE_DIR, _KERNEL_IMAGE_NAMES[0]) _REMOTE_INITRAMFS_IMAGE_PATH = remote_path.join( _REMOTE_IMAGE_DIR, _INITRAMFS_IMAGE_NAME) - -_ANDROID_BOOT_IMAGE_MAGIC = b"ANDROID!" - +_REMOTE_SUPER_IMAGE_DIR = remote_path.join(_REMOTE_IMAGE_DIR, + "super_image_dir") + +# Remote host instance name +_REMOTE_HOST_INSTANCE_NAME_FORMAT = ( + constants.INSTANCE_TYPE_HOST + + "-%(ip_addr)s-%(num)d-%(build_id)s-%(build_target)s") +_REMOTE_HOST_INSTANCE_NAME_PATTERN = re.compile( + constants.INSTANCE_TYPE_HOST + r"-(?P<ip_addr>[\d.]+)-(?P<num>\d+)-.+") +# launch_cvd arguments. +_DATA_POLICY_CREATE_IF_MISSING = "create_if_missing" +_DATA_POLICY_ALWAYS_CREATE = "always_create" +_NUM_AVDS_ARG = "-num_instances=%(num_AVD)s" +AGREEMENT_PROMPT_ARG = "-report_anonymous_usage_stats=y" +UNDEFOK_ARG = "-undefok=report_anonymous_usage_stats,config" +# Connect the OpenWrt device via console file. +_ENABLE_CONSOLE_ARG = "-console=true" +# WebRTC args +_WEBRTC_ID = "--webrtc_device_id=%(instance)s" +_WEBRTC_ARGS = ["--start_webrtc", "--vm_manager=crosvm"] +_VNC_ARGS = ["--start_vnc_server=true"] + +# Cuttlefish runtime directory is specified by `-instance_dir <runtime_dir>`. +# Cuttlefish tools may create a symbolic link at the specified path. +# The actual location of the runtime directory depends on the version: +# +# In Android 10, the directory is `<runtime_dir>`. +# +# In Android 11 and 12, the directory is `<runtime_dir>.<num>`. +# `<runtime_dir>` is a symbolic link to the first device's directory. +# +# In the latest version, if `--instance-dir <runtime_dir>` is specified, the +# directory is `<runtime_dir>/instances/cvd-<num>`. +# `<runtime_dir>_runtime` and `<runtime_dir>.<num>` are symbolic links. +# +# If `--instance-dir <runtime_dir>` is not specified, the directory is +# `~/cuttlefish/instances/cvd-<num>`. +# `~/cuttlefish_runtime` and `~/cuttelfish_runtime.<num>` are symbolic links. +_LOCAL_LOG_DIR_FORMAT = os.path.join( + "%(runtime_dir)s", "instances", "cvd-%(num)d", "logs") +# Relative paths in a base directory. +_REMOTE_RUNTIME_DIR_FORMAT = remote_path.join( + "cuttlefish", "instances", "cvd-%(num)d") +_REMOTE_LEGACY_RUNTIME_DIR_FORMAT = "cuttlefish_runtime.%(num)d" HOST_KERNEL_LOG = report.LogFile( "/var/log/kern.log", constants.LOG_TYPE_KERNEL_LOG, "host_kernel.log") -TOMBSTONES = report.LogFile( - constants.REMOTE_LOG_FOLDER + "/tombstones", constants.LOG_TYPE_DIR, - "tombstones-zip") -FETCHER_CONFIG_JSON = report.LogFile( - "fetcher_config.json", constants.LOG_TYPE_TEXT) + +# Contents of the target_files archive. +_DOWNLOAD_MIX_IMAGE_NAME = "{build_target}-target_files-{build_id}.zip" +_TARGET_FILES_META_DIR_NAME = "META" +_TARGET_FILES_IMAGES_DIR_NAME = "IMAGES" +_MISC_INFO_FILE_NAME = "misc_info.txt" + +# ARM flavor build target pattern. +_ARM_TARGET_PATTERN = "arm" + + +def GetAdbPorts(base_instance_num, num_avds_per_instance): + """Get ADB ports of cuttlefish. + + Args: + base_instance_num: An integer or None, the instance number of the first + device. + num_avds_per_instance: An integer or None, the number of devices. + + Returns: + The port numbers as a list of integers. + """ + return [constants.CF_ADB_PORT + (base_instance_num or 1) - 1 + index + for index in range(num_avds_per_instance or 1)] + +def GetFastbootPorts(base_instance_num, num_avds_per_instance): + """Get Fastboot ports of cuttlefish. + + Args: + base_instance_num: An integer or None, the instance number of the first + device. + num_avds_per_instance: An integer or None, the number of devices. + + Returns: + The port numbers as a list of integers. + """ + return [constants.CF_FASTBOOT_PORT + (base_instance_num or 1) - 1 + index + for index in range(num_avds_per_instance or 1)] + +def GetVncPorts(base_instance_num, num_avds_per_instance): + """Get VNC ports of cuttlefish. + + Args: + base_instance_num: An integer or None, the instance number of the first + device. + num_avds_per_instance: An integer or None, the number of devices. + + Returns: + The port numbers as a list of integers. + """ + return [constants.CF_VNC_PORT + (base_instance_num or 1) - 1 + index + for index in range(num_avds_per_instance or 1)] -def _UploadImageZip(ssh_obj, image_zip): +def _UploadImageZip(ssh_obj, remote_dir, image_zip): """Upload an image zip to a remote host and a GCE instance. Args: ssh_obj: An Ssh object. + remote_dir: The remote base directory. image_zip: The path to the image zip. """ - remote_cmd = f"/usr/bin/install_zip.sh . < {image_zip}" + remote_cmd = f"/usr/bin/install_zip.sh {remote_dir} < {image_zip}" logger.debug("remote_cmd:\n %s", remote_cmd) ssh_obj.Run(remote_cmd) -def _UploadImageDir(ssh_obj, image_dir): +def _UploadImageDir(ssh_obj, remote_dir, image_dir): """Upload an image directory to a remote host or a GCE instance. The images are compressed for faster upload. Args: ssh_obj: An Ssh object. + remote_dir: The remote base directory. image_dir: The directory containing the files to be uploaded. """ try: @@ -98,53 +204,48 @@ def _UploadImageDir(ssh_obj, image_dir): # Upload android-info.txt to parse config value. artifact_files.append(constants.ANDROID_INFO_FILE) cmd = (f"tar -cf - --lzop -S -C {image_dir} {' '.join(artifact_files)} | " - f"{ssh_obj.GetBaseCmd(constants.SSH_BIN)} -- tar -xf - --lzop -S") + f"{ssh_obj.GetBaseCmd(constants.SSH_BIN)} -- " + f"tar -xf - --lzop -S -C {remote_dir}") logger.debug("cmd:\n %s", cmd) ssh.ShellCmdWithRetry(cmd) -def _UploadCvdHostPackage(ssh_obj, cvd_host_package): +def _UploadCvdHostPackage(ssh_obj, remote_dir, cvd_host_package): """Upload a CVD host package to a remote host or a GCE instance. Args: ssh_obj: An Ssh object. + remote_dir: The remote base directory. cvd_host_package: The path to the CVD host package. """ - remote_cmd = f"tar -x -z -f - < {cvd_host_package}" - logger.debug("remote_cmd:\n %s", remote_cmd) - ssh_obj.Run(remote_cmd) + if cvd_host_package.endswith(".tar.gz"): + remote_cmd = f"tar -xzf - -C {remote_dir} < {cvd_host_package}" + logger.debug("remote_cmd:\n %s", remote_cmd) + ssh_obj.Run(remote_cmd) + else: + cmd = (f"tar -cf - --lzop -S -C {cvd_host_package} . | " + f"{ssh_obj.GetBaseCmd(constants.SSH_BIN)} -- " + f"tar -xf - --lzop -S -C {remote_dir}") + logger.debug("cmd:\n %s", cmd) + ssh.ShellCmdWithRetry(cmd) @utils.TimeExecute(function_description="Processing and uploading local images") -def UploadArtifacts(ssh_obj, image_path, cvd_host_package): +def UploadArtifacts(ssh_obj, remote_dir, image_path, cvd_host_package): """Upload images and a CVD host package to a remote host or a GCE instance. Args: ssh_obj: An Ssh object. + remote_dir: The remote base directory. image_path: A string, the path to the image zip built by `m dist` or the directory containing the images built by `m`. cvd_host_package: A string, the path to the CVD host package in gzip. """ if os.path.isdir(image_path): - _UploadImageDir(ssh_obj, image_path) + _UploadImageDir(ssh_obj, remote_dir, image_path) else: - _UploadImageZip(ssh_obj, image_path) - _UploadCvdHostPackage(ssh_obj, cvd_host_package) - - -def _IsBootImage(image_path): - """Check if a file is an Android boot image by reading the magic bytes. - - Args: - image_path: The file path. - - Returns: - A boolean, whether the file is a boot image. - """ - if not os.path.isfile(image_path): - return False - with open(image_path, "rb") as image_file: - return image_file.read(8) == _ANDROID_BOOT_IMAGE_MAGIC + _UploadImageZip(ssh_obj, remote_dir, image_path) + _UploadCvdHostPackage(ssh_obj, remote_dir, cvd_host_package) def FindBootImages(search_path): @@ -161,12 +262,8 @@ def FindBootImages(search_path): errors.GetLocalImageError if search_path contains more than one boot image or the file format is not correct. """ - boot_image_path = create_common.FindLocalImage( - search_path, _BOOT_IMAGE_NAME_PATTERN, raise_error=False) - if boot_image_path and not _IsBootImage(boot_image_path): - raise errors.GetLocalImageError( - f"{boot_image_path} is not a boot image.") - + boot_image_path = create_common.FindBootImage(search_path, + raise_error=False) vendor_boot_image_path = os.path.join(search_path, _VENDOR_BOOT_IMAGE_NAME) if not os.path.isfile(vendor_boot_image_path): vendor_boot_image_path = None @@ -174,7 +271,7 @@ def FindBootImages(search_path): return boot_image_path, vendor_boot_image_path -def _FindKernelImages(search_path): +def FindKernelImages(search_path): """Find kernel and initramfs images in a path. Args: @@ -196,11 +293,13 @@ def _FindKernelImages(search_path): @utils.TimeExecute(function_description="Uploading local kernel images.") -def _UploadKernelImages(ssh_obj, search_path): - """Find and upload kernel images to a remote host or a GCE instance. +def _UploadKernelImages(ssh_obj, remote_dir, search_path): + """Find and upload kernel or boot images to a remote host or a GCE + instance. Args: ssh_obj: An Ssh object. + remote_dir: The remote base directory. search_path: A path to an image file or an image directory. Returns: @@ -211,35 +310,76 @@ def _UploadKernelImages(ssh_obj, search_path): images. """ # Assume that the caller cleaned up the remote home directory. - ssh_obj.Run("mkdir -p " + _REMOTE_IMAGE_DIR) + ssh_obj.Run("mkdir -p " + remote_path.join(remote_dir, _REMOTE_IMAGE_DIR)) + + kernel_image_path, initramfs_image_path = FindKernelImages(search_path) + if kernel_image_path and initramfs_image_path: + remote_kernel_image_path = remote_path.join( + remote_dir, _REMOTE_KERNEL_IMAGE_PATH) + remote_initramfs_image_path = remote_path.join( + remote_dir, _REMOTE_INITRAMFS_IMAGE_PATH) + ssh_obj.ScpPushFile(kernel_image_path, remote_kernel_image_path) + ssh_obj.ScpPushFile(initramfs_image_path, remote_initramfs_image_path) + return ["-kernel_path", remote_kernel_image_path, + "-initramfs_path", remote_initramfs_image_path] boot_image_path, vendor_boot_image_path = FindBootImages(search_path) if boot_image_path: - ssh_obj.ScpPushFile(boot_image_path, _REMOTE_BOOT_IMAGE_PATH) - launch_cvd_args = ["-boot_image", _REMOTE_BOOT_IMAGE_PATH] + remote_boot_image_path = remote_path.join( + remote_dir, _REMOTE_BOOT_IMAGE_PATH) + ssh_obj.ScpPushFile(boot_image_path, remote_boot_image_path) + launch_cvd_args = ["-boot_image", remote_boot_image_path] if vendor_boot_image_path: + remote_vendor_boot_image_path = remote_path.join( + remote_dir, _REMOTE_VENDOR_BOOT_IMAGE_PATH) ssh_obj.ScpPushFile(vendor_boot_image_path, - _REMOTE_VENDOR_BOOT_IMAGE_PATH) + remote_vendor_boot_image_path) launch_cvd_args.extend(["-vendor_boot_image", - _REMOTE_VENDOR_BOOT_IMAGE_PATH]) + remote_vendor_boot_image_path]) return launch_cvd_args - kernel_image_path, initramfs_image_path = _FindKernelImages(search_path) - if kernel_image_path and initramfs_image_path: - ssh_obj.ScpPushFile(kernel_image_path, _REMOTE_KERNEL_IMAGE_PATH) - ssh_obj.ScpPushFile(initramfs_image_path, _REMOTE_INITRAMFS_IMAGE_PATH) - return ["-kernel_path", _REMOTE_KERNEL_IMAGE_PATH, - "-initramfs_path", _REMOTE_INITRAMFS_IMAGE_PATH] - raise errors.GetLocalImageError( f"{search_path} is not a boot image or a directory containing images.") -def UploadExtraImages(ssh_obj, avd_spec): +@utils.TimeExecute(function_description="Uploading disabled vbmeta image.") +def _UploadDisabledVbmetaImage(ssh_obj, remote_dir, local_tool_dirs): + """Upload disabled vbmeta image to a remote host or a GCE instance. + + Args: + ssh_obj: An Ssh object. + remote_dir: The remote base directory. + local_tool_dirs: A list of local directories containing tools. + + Returns: + A list of strings, the launch_cvd arguments including the remote paths. + + Raises: + CheckPathError if local_tool_dirs do not contain OTA tools. + """ + # Assume that the caller cleaned up the remote home directory. + ssh_obj.Run("mkdir -p " + remote_path.join(remote_dir, _REMOTE_IMAGE_DIR)) + + remote_vbmeta_image_path = remote_path.join(remote_dir, + _REMOTE_VBMETA_IMAGE_PATH) + with tempfile.NamedTemporaryFile(prefix="vbmeta", + suffix=".img") as temp_file: + tool_dirs = local_tool_dirs + create_common.GetNonEmptyEnvVars( + constants.ENV_ANDROID_SOONG_HOST_OUT, + constants.ENV_ANDROID_HOST_OUT) + ota = ota_tools.FindOtaTools(tool_dirs) + ota.MakeDisabledVbmetaImage(temp_file.name) + ssh_obj.ScpPushFile(temp_file.name, remote_vbmeta_image_path) + + return ["-vbmeta_image", remote_vbmeta_image_path] + + +def UploadExtraImages(ssh_obj, remote_dir, avd_spec): """Find and upload the images specified in avd_spec. Args: ssh_obj: An Ssh object. + remote_dir: The remote base directory. avd_spec: An AvdSpec object containing extra image paths. Returns: @@ -248,58 +388,334 @@ def UploadExtraImages(ssh_obj, avd_spec): Raises: errors.GetLocalImageError if any specified image path does not exist. """ + extra_img_args = [] if avd_spec.local_kernel_image: - return _UploadKernelImages(ssh_obj, avd_spec.local_kernel_image) - return [] + extra_img_args += _UploadKernelImages(ssh_obj, remote_dir, + avd_spec.local_kernel_image) + if avd_spec.local_vendor_image: + extra_img_args += _UploadDisabledVbmetaImage(ssh_obj, remote_dir, + avd_spec.local_tool_dirs) + return extra_img_args + + +@utils.TimeExecute(function_description="Uploading local super image") +def UploadSuperImage(ssh_obj, remote_dir, super_image_path): + """Upload a super image to a remote host or a GCE instance. + + Args: + ssh_obj: An Ssh object. + remote_dir: The remote base directory. + super_image_path: Path to the super image file. + Returns: + A list of strings, the launch_cvd arguments including the remote paths. + """ + # Assume that the caller cleaned up the remote home directory. + super_image_stem = os.path.basename(super_image_path) + remote_super_image_dir = remote_path.join( + remote_dir, _REMOTE_SUPER_IMAGE_DIR) + remote_super_image_path = remote_path.join( + remote_super_image_dir, super_image_stem) + ssh_obj.Run(f"mkdir -p {remote_super_image_dir}") + cmd = (f"tar -cf - --lzop -S -C {os.path.dirname(super_image_path)} " + f"{super_image_stem} | " + f"{ssh_obj.GetBaseCmd(constants.SSH_BIN)} -- " + f"tar -xf - --lzop -S -C {remote_super_image_dir}") + ssh.ShellCmdWithRetry(cmd) + launch_cvd_args = ["-super_image", remote_super_image_path] + return launch_cvd_args -def CleanUpRemoteCvd(ssh_obj, raise_error): + +def CleanUpRemoteCvd(ssh_obj, remote_dir, raise_error): """Call stop_cvd and delete the files on a remote host or a GCE instance. Args: ssh_obj: An Ssh object. + remote_dir: The remote base directory. raise_error: Whether to raise an error if the remote instance is not running. Raises: subprocess.CalledProcessError if any command fails. """ - stop_cvd_cmd = "./bin/stop_cvd" + home = remote_path.join("$HOME", remote_dir) + stop_cvd_path = remote_path.join(remote_dir, "bin", "stop_cvd") + stop_cvd_cmd = f"'HOME={home} {stop_cvd_path}'" if raise_error: ssh_obj.Run(stop_cvd_cmd) else: try: ssh_obj.Run(stop_cvd_cmd, retry=0) - except subprocess.CalledProcessError as e: + except Exception as e: logger.debug( "Failed to stop_cvd (possibly no running device): %s", e) # This command deletes all files except hidden files under HOME. # It does not raise an error if no files can be deleted. - ssh_obj.Run("'rm -rf ./*'") + ssh_obj.Run(f"'rm -rf {remote_path.join(remote_dir, '*')}'") + + +def GetRemoteHostBaseDir(base_instance_num): + """Get remote base directory by instance number. + + Args: + base_instance_num: Integer or None, the instance number of the device. + + Returns: + The remote base directory. + """ + return _REMOTE_HOST_BASE_DIR_FORMAT % {"num": base_instance_num or 1} + + +def FormatRemoteHostInstanceName(ip_addr, base_instance_num, build_id, + build_target): + """Convert an IP address and build info to an instance name. + + Args: + ip_addr: String, the IP address of the remote host. + base_instance_num: Integer or None, the instance number of the device. + build_id: String, the build id. + build_target: String, the build target, e.g., aosp_cf_x86_64_phone. + + Return: + String, the instance name. + """ + return _REMOTE_HOST_INSTANCE_NAME_FORMAT % { + "ip_addr": ip_addr, + "num": base_instance_num or 1, + "build_id": build_id, + "build_target": build_target} + + +def ParseRemoteHostAddress(instance_name): + """Parse IP address from a remote host instance name. + + Args: + instance_name: String, the instance name. + + Returns: + The IP address and the base directory as strings. + None if the name does not represent a remote host instance. + """ + match = _REMOTE_HOST_INSTANCE_NAME_PATTERN.fullmatch(instance_name) + if match: + return (match.group("ip_addr"), + GetRemoteHostBaseDir(int(match.group("num")))) + return None -def ConvertRemoteLogs(log_paths): - """Convert paths on a remote host or a GCE instance to log objects. +# pylint:disable=too-many-branches +def GetLaunchCvdArgs(avd_spec, config=None): + """Get launch_cvd arguments for remote instances. Args: - log_paths: A collection of strings, the remote paths to the logs. + avd_spec: An AVDSpec instance. + config: A string, the name of the predefined hardware config. + e.g., "auto", "phone", and "tv". + + Returns: + A list of strings, arguments of launch_cvd. + """ + launch_cvd_args = [] + + blank_data_disk_size_gb = avd_spec.cfg.extra_data_disk_size_gb + if blank_data_disk_size_gb and blank_data_disk_size_gb > 0: + launch_cvd_args.append( + "-data_policy=" + _DATA_POLICY_CREATE_IF_MISSING) + launch_cvd_args.append( + "-blank_data_image_mb=" + str(blank_data_disk_size_gb * 1024)) + + if config: + launch_cvd_args.append("-config=" + config) + if avd_spec.hw_customize or not config: + launch_cvd_args.append( + "-x_res=" + avd_spec.hw_property[constants.HW_X_RES]) + launch_cvd_args.append( + "-y_res=" + avd_spec.hw_property[constants.HW_Y_RES]) + launch_cvd_args.append( + "-dpi=" + avd_spec.hw_property[constants.HW_ALIAS_DPI]) + if constants.HW_ALIAS_DISK in avd_spec.hw_property: + launch_cvd_args.append( + "-data_policy=" + _DATA_POLICY_ALWAYS_CREATE) + launch_cvd_args.append( + "-blank_data_image_mb=" + + avd_spec.hw_property[constants.HW_ALIAS_DISK]) + if constants.HW_ALIAS_CPUS in avd_spec.hw_property: + launch_cvd_args.append( + "-cpus=" + str(avd_spec.hw_property[constants.HW_ALIAS_CPUS])) + if constants.HW_ALIAS_MEMORY in avd_spec.hw_property: + launch_cvd_args.append( + "-memory_mb=" + + str(avd_spec.hw_property[constants.HW_ALIAS_MEMORY])) + + if avd_spec.connect_webrtc: + launch_cvd_args.extend(_WEBRTC_ARGS) + if avd_spec.webrtc_device_id: + launch_cvd_args.append( + _WEBRTC_ID % {"instance": avd_spec.webrtc_device_id}) + if avd_spec.connect_vnc: + launch_cvd_args.extend(_VNC_ARGS) + if avd_spec.openwrt: + launch_cvd_args.append(_ENABLE_CONSOLE_ARG) + if avd_spec.num_avds_per_instance > 1: + launch_cvd_args.append( + _NUM_AVDS_ARG % {"num_AVD": avd_spec.num_avds_per_instance}) + if avd_spec.base_instance_num: + launch_cvd_args.append( + "--base-instance-num=" + str(avd_spec.base_instance_num)) + if avd_spec.launch_args: + launch_cvd_args.append(avd_spec.launch_args) + + launch_cvd_args.append(UNDEFOK_ARG) + launch_cvd_args.append(AGREEMENT_PROMPT_ARG) + return launch_cvd_args + + +def _GetRemoteRuntimeDirs(ssh_obj, remote_dir, base_instance_num, + num_avds_per_instance): + """Get cuttlefish runtime directories on a remote host or a GCE instance. + + Args: + ssh_obj: An Ssh object. + remote_dir: The remote base directory. + base_instance_num: An integer, the instance number of the first device. + num_avds_per_instance: An integer, the number of devices. + + Returns: + A list of strings, the paths to the runtime directories. + """ + runtime_dir = remote_path.join( + remote_dir, _REMOTE_RUNTIME_DIR_FORMAT % {"num": base_instance_num}) + try: + ssh_obj.Run(f"test -d {runtime_dir}", retry=0) + return [remote_path.join(remote_dir, + _REMOTE_RUNTIME_DIR_FORMAT % + {"num": base_instance_num + num}) + for num in range(num_avds_per_instance)] + except subprocess.CalledProcessError: + logger.debug("%s is not the runtime directory.", runtime_dir) + + legacy_runtime_dirs = [ + remote_path.join(remote_dir, constants.REMOTE_LOG_FOLDER)] + legacy_runtime_dirs.extend( + remote_path.join(remote_dir, + _REMOTE_LEGACY_RUNTIME_DIR_FORMAT % + {"num": base_instance_num + num}) + for num in range(1, num_avds_per_instance)) + return legacy_runtime_dirs + + +def GetRemoteFetcherConfigJson(remote_dir): + """Get the config created by fetch_cvd on a remote host or a GCE instance. + + Args: + remote_dir: The remote base directory. + + Returns: + An object of report.LogFile. + """ + return report.LogFile(remote_path.join(remote_dir, "fetcher_config.json"), + constants.LOG_TYPE_CUTTLEFISH_LOG) + + +def _GetRemoteTombstone(runtime_dir, name_suffix): + """Get log object for tombstones in a remote cuttlefish runtime directory. + + Args: + runtime_dir: The path to the remote cuttlefish runtime directory. + name_suffix: The string appended to the log name. It is used to + distinguish log files found in different runtime_dirs. + + Returns: + A report.LogFile object. + """ + return report.LogFile(remote_path.join(runtime_dir, "tombstones"), + constants.LOG_TYPE_DIR, + "tombstones-zip" + name_suffix) + + +def _GetLogType(file_name): + """Determine log type by file name. + + Args: + file_name: A file name. + + Returns: + A string, one of the log types defined in constants. + None if the file is not a log file. + """ + if file_name == "kernel.log": + return constants.LOG_TYPE_KERNEL_LOG + if file_name == "logcat": + return constants.LOG_TYPE_LOGCAT + if file_name.endswith(".log") or file_name == "cuttlefish_config.json": + return constants.LOG_TYPE_CUTTLEFISH_LOG + return None + + +def FindRemoteLogs(ssh_obj, remote_dir, base_instance_num, + num_avds_per_instance): + """Find log objects on a remote host or a GCE instance. + + Args: + ssh_obj: An Ssh object. + remote_dir: The remote base directory. + base_instance_num: An integer or None, the instance number of the first + device. + num_avds_per_instance: An integer or None, the number of devices. Returns: A list of report.LogFile objects. """ + runtime_dirs = _GetRemoteRuntimeDirs( + ssh_obj, remote_dir, + (base_instance_num or 1), (num_avds_per_instance or 1)) logs = [] - for log_path in log_paths: - log = report.LogFile(log_path, constants.LOG_TYPE_TEXT) - if log_path.endswith("kernel.log"): - log = report.LogFile(log_path, constants.LOG_TYPE_KERNEL_LOG) - elif log_path.endswith("logcat"): - log = report.LogFile(log_path, constants.LOG_TYPE_LOGCAT, - "full_gce_logcat") - elif not (log_path.endswith(".log") or - log_path.endswith("cuttlefish_config.json")): + for log_path in utils.FindRemoteFiles(ssh_obj, runtime_dirs): + file_name = remote_path.basename(log_path) + log_type = _GetLogType(file_name) + if not log_type: continue - logs.append(log) + base, ext = remote_path.splitext(file_name) + # The index of the runtime_dir containing log_path. + index_str = "" + for index, runtime_dir in enumerate(runtime_dirs): + if log_path.startswith(runtime_dir + remote_path.sep): + index_str = "." + str(index) if index else "" + log_name = ("full_gce_logcat" + index_str if file_name == "logcat" else + base + index_str + ext) + + logs.append(report.LogFile(log_path, log_type, log_name)) + + logs.extend(_GetRemoteTombstone(runtime_dir, + ("." + str(index) if index else "")) + for index, runtime_dir in enumerate(runtime_dirs)) + return logs + + +def FindLocalLogs(runtime_dir, instance_num): + """Find log objects in a local runtime directory. + + Args: + runtime_dir: A string, the runtime directory path. + instance_num: An integer, the instance number. + + Returns: + A list of report.LogFile. + """ + log_dir = _LOCAL_LOG_DIR_FORMAT % {"runtime_dir": runtime_dir, + "num": instance_num} + if not os.path.isdir(log_dir): + log_dir = runtime_dir + + logs = [] + for parent_dir, _, file_names in os.walk(log_dir, followlinks=False): + for file_name in file_names: + log_path = os.path.join(parent_dir, file_name) + log_type = _GetLogType(file_name) + if os.path.islink(log_path) or not log_type: + continue + logs.append(report.LogFile(log_path, log_type)) return logs @@ -332,3 +748,104 @@ def GetRemoteBuildInfoDict(avd_spec): for key, val in avd_spec.bootloader_build_info.items() if val} ) return build_info_dict + + +def GetMixBuildTargetFilename(build_target, build_id): + """Get the mix build target filename. + + Args: + build_id: String, Build id, e.g. "2263051", "P2804227" + build_target: String, the build target, e.g. cf_x86_phone-userdebug + + Returns: + String, a file name, e.g. "cf_x86_phone-target_files-2263051.zip" + """ + return _DOWNLOAD_MIX_IMAGE_NAME.format( + build_target=build_target.split('-')[0], + build_id=build_id) + + +def FindMiscInfo(image_dir): + """Find misc info in build output dir or extracted target files. + + Args: + image_dir: The directory to search for misc info. + + Returns: + image_dir if the directory structure looks like an output directory + in build environment. + image_dir/META if it looks like extracted target files. + + Raises: + errors.CheckPathError if this function cannot find misc info. + """ + misc_info_path = os.path.join(image_dir, _MISC_INFO_FILE_NAME) + if os.path.isfile(misc_info_path): + return misc_info_path + misc_info_path = os.path.join(image_dir, _TARGET_FILES_META_DIR_NAME, + _MISC_INFO_FILE_NAME) + if os.path.isfile(misc_info_path): + return misc_info_path + raise errors.CheckPathError( + f"Cannot find {_MISC_INFO_FILE_NAME} in {image_dir}. The " + f"directory is expected to be an extracted target files zip or " + f"{constants.ENV_ANDROID_PRODUCT_OUT}.") + + +def FindImageDir(image_dir): + """Find images in build output dir or extracted target files. + + Args: + image_dir: The directory to search for images. + + Returns: + image_dir if the directory structure looks like an output directory + in build environment. + image_dir/IMAGES if it looks like extracted target files. + + Raises: + errors.GetLocalImageError if this function cannot find any image. + """ + if glob.glob(os.path.join(image_dir, "*.img")): + return image_dir + subdir = os.path.join(image_dir, _TARGET_FILES_IMAGES_DIR_NAME) + if glob.glob(os.path.join(subdir, "*.img")): + return subdir + raise errors.GetLocalImageError( + "Cannot find images in %s." % image_dir) + + +def IsArmImage(image): + """Check if the image is built for ARM. + + Args: + image: Image meta info. + + Returns: + A boolean, whether the image is for ARM. + """ + return _ARM_TARGET_PATTERN in image.get("build_target", "") + + +def FindVendorImages(image_dir): + """Find vendor, vendor_dlkm, odm, and odm_dlkm image in build output dir. + + Args: + image_dir: The directory to search for images. + + Returns: + An object of VendorImagePaths. + + Raises: + errors.GetLocalImageError if this function cannot find images. + """ + + image_paths = [] + for image_name in _VENDOR_IMAGE_NAMES: + image_path = os.path.join(image_dir, image_name) + if not os.path.isfile(image_path): + raise errors.GetLocalImageError( + f"Cannot find {image_path} in {image_dir}.") + image_paths.append(image_path) + + return VendorImagePaths(*image_paths) diff --git a/internal/lib/cvd_utils_test.py b/internal/lib/cvd_utils_test.py index 28dc4410..707b19a9 100644 --- a/internal/lib/cvd_utils_test.py +++ b/internal/lib/cvd_utils_test.py @@ -21,18 +21,42 @@ import unittest from unittest import mock from acloud import errors +from acloud.create import create_common from acloud.internal import constants from acloud.internal.lib import cvd_utils +from acloud.internal.lib import driver_test_lib -class CvdUtilsTest(unittest.TestCase): +# pylint: disable=too-many-public-methods +class CvdUtilsTest(driver_test_lib.BaseDriverTest): """Test the functions in cvd_utils.""" - @staticmethod - def _CreateFile(path, data=b""): - """Create and write binary data to a file.""" - with open(path, "wb") as file_obj: - file_obj.write(data) + # Remote host instance name. + _PRODUCT_NAME = "aosp_cf_x86_64_phone" + _BUILD_ID = "2263051" + _REMOTE_HOST_IP = "192.0.2.1" + _REMOTE_HOST_INSTANCE_NAME_1 = ( + "host-192.0.2.1-1-2263051-aosp_cf_x86_64_phone") + _REMOTE_HOST_INSTANCE_NAME_2 = ( + "host-192.0.2.1-2-2263051-aosp_cf_x86_64_phone") + + def testGetAdbPorts(self): + """Test GetAdbPorts.""" + self.assertEqual([6520], cvd_utils.GetAdbPorts(None, None)) + self.assertEqual([6520], cvd_utils.GetAdbPorts(1, 1)) + self.assertEqual([6521, 6522], cvd_utils.GetAdbPorts(2, 2)) + + def testGetFastbootPorts(self): + """Test GetFastbootPorts.""" + self.assertEqual([7520], cvd_utils.GetFastbootPorts(None, None)) + self.assertEqual([7520], cvd_utils.GetFastbootPorts(1, 1)) + self.assertEqual([7521, 7522], cvd_utils.GetFastbootPorts(2, 2)) + + def testGetVncPorts(self): + """Test GetVncPorts.""" + self.assertEqual([6444], cvd_utils.GetVncPorts(None, None)) + self.assertEqual([6444], cvd_utils.GetVncPorts(1, 1)) + self.assertEqual([6445, 6446], cvd_utils.GetVncPorts(2, 2)) @staticmethod @mock.patch("acloud.internal.lib.cvd_utils.os.path.isdir", @@ -40,10 +64,11 @@ class CvdUtilsTest(unittest.TestCase): def testUploadImageZip(_mock_isdir): """Test UploadArtifacts with image zip.""" mock_ssh = mock.Mock() - cvd_utils.UploadArtifacts(mock_ssh, "/mock/img.zip", "/mock/cvd.tgz") - mock_ssh.Run.assert_any_call("/usr/bin/install_zip.sh . < " + cvd_utils.UploadArtifacts(mock_ssh, "dir", "/mock/img.zip", + "/mock/cvd.tar.gz") + mock_ssh.Run.assert_any_call("/usr/bin/install_zip.sh dir < " "/mock/img.zip") - mock_ssh.Run.assert_any_call("tar -x -z -f - < /mock/cvd.tgz") + mock_ssh.Run.assert_any_call("tar -xzf - -C dir < /mock/cvd.tar.gz") @staticmethod @mock.patch("acloud.internal.lib.cvd_utils.glob") @@ -54,20 +79,38 @@ class CvdUtilsTest(unittest.TestCase): """Test UploadArtifacts with image directory.""" mock_ssh = mock.Mock() mock_ssh.GetBaseCmd.return_value = "/mock/ssh" - expected_shell_cmd = ("tar -cf - --lzop -S -C /mock/dir " - "super.img bootloader kernel android-info.txt | " - "/mock/ssh -- tar -xf - --lzop -S") - expected_ssh_cmd = "tar -x -z -f - < /mock/cvd.tgz" + expected_image_shell_cmd = ("tar -cf - --lzop -S -C local/dir " + "super.img bootloader kernel android-info.txt | " + "/mock/ssh -- " + "tar -xf - --lzop -S -C remote/dir") + expected_cvd_tar_ssh_cmd = "tar -xzf - -C remote/dir < /mock/cvd.tar.gz" + expected_cvd_dir_shell_cmd = ("tar -cf - --lzop -S -C /mock/cvd . | " + "/mock/ssh -- " + "tar -xf - --lzop -S -C remote/dir") + + # Test with cvd directory. + mock_open = mock.mock_open(read_data="super.img\nbootloader\nkernel") + with mock.patch("acloud.internal.lib.cvd_utils.open", mock_open): + cvd_utils.UploadArtifacts(mock_ssh, "remote/dir","local/dir", + "/mock/cvd") + mock_open.assert_called_with("local/dir/required_images", "r", + encoding="utf-8") + mock_glob.glob.assert_not_called() + mock_shell.assert_has_calls([mock.call(expected_image_shell_cmd), + mock.call(expected_cvd_dir_shell_cmd)]) # Test with required_images file. + mock_ssh.reset_mock() + mock_shell.reset_mock() mock_open = mock.mock_open(read_data="super.img\nbootloader\nkernel") with mock.patch("acloud.internal.lib.cvd_utils.open", mock_open): - cvd_utils.UploadArtifacts(mock_ssh, "/mock/dir", "/mock/cvd.tgz") - mock_open.assert_called_with("/mock/dir/required_images", "r", + cvd_utils.UploadArtifacts(mock_ssh, "remote/dir","local/dir", + "/mock/cvd.tar.gz") + mock_open.assert_called_with("local/dir/required_images", "r", encoding="utf-8") mock_glob.glob.assert_not_called() - mock_shell.assert_called_with(expected_shell_cmd) - mock_ssh.Run.assert_called_with(expected_ssh_cmd) + mock_shell.assert_called_with(expected_image_shell_cmd) + mock_ssh.Run.assert_called_with(expected_cvd_tar_ssh_cmd) # Test with glob. mock_ssh.reset_mock() @@ -76,94 +119,358 @@ class CvdUtilsTest(unittest.TestCase): lambda path: [path.replace("*", "super")]) with mock.patch("acloud.internal.lib.cvd_utils.open", side_effect=IOError("file does not exist")): - cvd_utils.UploadArtifacts(mock_ssh, "/mock/dir", "/mock/cvd.tgz") + cvd_utils.UploadArtifacts(mock_ssh, "remote/dir", "local/dir", + "/mock/cvd.tar.gz") mock_glob.glob.assert_called() - mock_shell.assert_called_with(expected_shell_cmd) - mock_ssh.Run.assert_called_with(expected_ssh_cmd) + mock_shell.assert_called_with(expected_image_shell_cmd) + mock_ssh.Run.assert_called_with(expected_cvd_tar_ssh_cmd) - def testUploadBootImages(self): + @mock.patch("acloud.internal.lib.cvd_utils.create_common") + def testUploadBootImages(self, mock_create_common): """Test FindBootImages and UploadExtraImages.""" mock_ssh = mock.Mock() with tempfile.TemporaryDirectory(prefix="cvd_utils") as image_dir: - boot_image_path = os.path.join(image_dir, "boot.img") - self._CreateFile(boot_image_path, b"ANDROID!test") - self._CreateFile(os.path.join(image_dir, "vendor_boot.img")) + mock_create_common.FindBootImage.return_value = "boot.img" + self.CreateFile(os.path.join(image_dir, "vendor_boot.img")) - mock_avd_spec = mock.Mock(local_kernel_image=boot_image_path) - args = cvd_utils.UploadExtraImages(mock_ssh, mock_avd_spec) - self.assertEqual(["-boot_image", "acloud_cf/boot.img"], args) - mock_ssh.Run.assert_called_once_with("mkdir -p acloud_cf") - mock_ssh.ScpPushFile.assert_called_once() + mock_avd_spec = mock.Mock(local_kernel_image="boot.img", + local_vendor_image=None) + args = cvd_utils.UploadExtraImages(mock_ssh, "dir", mock_avd_spec) + self.assertEqual(["-boot_image", "dir/acloud_image/boot.img"], + args) + mock_ssh.Run.assert_called_once_with("mkdir -p dir/acloud_image") + mock_ssh.ScpPushFile.assert_called_once_with( + "boot.img", "dir/acloud_image/boot.img") mock_ssh.reset_mock() mock_avd_spec.local_kernel_image = image_dir - args = cvd_utils.UploadExtraImages(mock_ssh, mock_avd_spec) + mock_avd_spec.local_vendor_image = None + args = cvd_utils.UploadExtraImages(mock_ssh, "dir", mock_avd_spec) self.assertEqual( - ["-boot_image", "acloud_cf/boot.img", - "-vendor_boot_image", "acloud_cf/vendor_boot.img"], + ["-boot_image", "dir/acloud_image/boot.img", + "-vendor_boot_image", "dir/acloud_image/vendor_boot.img"], args) mock_ssh.Run.assert_called_once() self.assertEqual(2, mock_ssh.ScpPushFile.call_count) def testUploadKernelImages(self): - """Test UploadExtraImages with kernel images.""" + """Test FindKernelImages and UploadExtraImages.""" mock_ssh = mock.Mock() with tempfile.TemporaryDirectory(prefix="cvd_utils") as image_dir: kernel_image_path = os.path.join(image_dir, "Image") - self._CreateFile(kernel_image_path) - self._CreateFile(os.path.join(image_dir, "initramfs.img")) + self.CreateFile(kernel_image_path) + self.CreateFile(os.path.join(image_dir, "initramfs.img")) + self.CreateFile(os.path.join(image_dir, "boot.img")) - mock_avd_spec = mock.Mock(local_kernel_image=kernel_image_path) + mock_avd_spec = mock.Mock(local_kernel_image=kernel_image_path, + local_vendor_image=None) with self.assertRaises(errors.GetLocalImageError): - cvd_utils.UploadExtraImages(mock_ssh, mock_avd_spec) + cvd_utils.UploadExtraImages(mock_ssh, "dir", mock_avd_spec) mock_ssh.reset_mock() mock_avd_spec.local_kernel_image = image_dir - args = cvd_utils.UploadExtraImages(mock_ssh, mock_avd_spec) + mock_avd_spec.local_vendor_image = None + args = cvd_utils.UploadExtraImages(mock_ssh, "dir", mock_avd_spec) self.assertEqual( - ["-kernel_path", "acloud_cf/kernel", - "-initramfs_path", "acloud_cf/initramfs.img"], + ["-kernel_path", "dir/acloud_image/kernel", + "-initramfs_path", "dir/acloud_image/initramfs.img"], args) mock_ssh.Run.assert_called_once() self.assertEqual(2, mock_ssh.ScpPushFile.call_count) + @mock.patch("acloud.internal.lib.ota_tools.FindOtaTools") + def testUploadVbmetaImages(self, mock_find_ota_tools): + """Test UploadExtraImages.""" + self.Patch(create_common, "GetNonEmptyEnvVars", return_value=[]) + mock_ssh = mock.Mock() + mock_ota_tools_object = mock.Mock() + mock_find_ota_tools.return_value = mock_ota_tools_object + mock_avd_spec = mock.Mock( + local_kernel_image=None, + local_vendor_image="vendor.img", + local_tool_dirs=[]) + + args = cvd_utils.UploadExtraImages(mock_ssh, "dir", mock_avd_spec) + self.assertEqual( + ["-vbmeta_image", "dir/acloud_image/vbmeta.img"], + args) + mock_ssh.Run.assert_called_once() + mock_ssh.ScpPushFile.assert_called_once() + mock_find_ota_tools.assert_called_once_with([]) + mock_ota_tools_object.MakeDisabledVbmetaImage.assert_called_once() + + @mock.patch("acloud.internal.lib.cvd_utils.ssh.ShellCmdWithRetry") + def testUploadSuperImage(self, mock_shell_cmd_with_retry): + """Test UploadSuperImage.""" + mock_ssh = mock.Mock() + self.assertEqual( + ["-super_image", + "/remote/cvd/dir/acloud_image/super_image_dir/super.img"], + cvd_utils.UploadSuperImage(mock_ssh, "/remote/cvd/dir", + "/local/path/to/super.img")) + mock_shell_cmd_with_retry.assert_called_once() + args = mock_shell_cmd_with_retry.call_args[0] + self.assertEqual(1, len(args)) + self.assertIn("/local/path/to", args[0]) + self.assertIn("super.img", args[0]) + self.assertIn("/remote/cvd/dir/acloud_image/super_image_dir", args[0]) def testCleanUpRemoteCvd(self): """Test CleanUpRemoteCvd.""" mock_ssh = mock.Mock() - cvd_utils.CleanUpRemoteCvd(mock_ssh, raise_error=True) - mock_ssh.Run.assert_any_call("./bin/stop_cvd") - mock_ssh.Run.assert_any_call("'rm -rf ./*'") + cvd_utils.CleanUpRemoteCvd(mock_ssh, "dir", raise_error=True) + mock_ssh.Run.assert_any_call("'HOME=$HOME/dir dir/bin/stop_cvd'") + mock_ssh.Run.assert_any_call("'rm -rf dir/*'") mock_ssh.reset_mock() mock_ssh.Run.side_effect = [ subprocess.CalledProcessError(cmd="should raise", returncode=1)] with self.assertRaises(subprocess.CalledProcessError): - cvd_utils.CleanUpRemoteCvd(mock_ssh, raise_error=True) + cvd_utils.CleanUpRemoteCvd(mock_ssh, "dir", raise_error=True) mock_ssh.reset_mock() mock_ssh.Run.side_effect = [ subprocess.CalledProcessError(cmd="should ignore", returncode=1), None] - cvd_utils.CleanUpRemoteCvd(mock_ssh, raise_error=False) - mock_ssh.Run.assert_any_call("./bin/stop_cvd", retry=0) - mock_ssh.Run.assert_any_call("'rm -rf ./*'") - - def testConvertRemoteLogs(self): - """Test ConvertRemoteLogs.""" - logs = cvd_utils.ConvertRemoteLogs( - ["/kernel.log", "/logcat", "/launcher.log", "/access-kregistry"]) + cvd_utils.CleanUpRemoteCvd(mock_ssh, "dir", raise_error=False) + mock_ssh.Run.assert_any_call("'HOME=$HOME/dir dir/bin/stop_cvd'", + retry=0) + mock_ssh.Run.assert_any_call("'rm -rf dir/*'") + + def testGetRemoteHostBaseDir(self): + """Test GetRemoteHostBaseDir.""" + self.assertEqual("acloud_cf_1", cvd_utils.GetRemoteHostBaseDir(None)) + self.assertEqual("acloud_cf_2", cvd_utils.GetRemoteHostBaseDir(2)) + + def testFormatRemoteHostInstanceName(self): + """Test FormatRemoteHostInstanceName.""" + name = cvd_utils.FormatRemoteHostInstanceName( + self._REMOTE_HOST_IP, None, self._BUILD_ID, self._PRODUCT_NAME) + self.assertEqual(name, self._REMOTE_HOST_INSTANCE_NAME_1) + + name = cvd_utils.FormatRemoteHostInstanceName( + self._REMOTE_HOST_IP, 2, self._BUILD_ID, self._PRODUCT_NAME) + self.assertEqual(name, self._REMOTE_HOST_INSTANCE_NAME_2) + + def testParseRemoteHostAddress(self): + """Test ParseRemoteHostAddress.""" + result = cvd_utils.ParseRemoteHostAddress( + self._REMOTE_HOST_INSTANCE_NAME_1) + self.assertEqual(result, (self._REMOTE_HOST_IP, "acloud_cf_1")) + + result = cvd_utils.ParseRemoteHostAddress( + self._REMOTE_HOST_INSTANCE_NAME_2) + self.assertEqual(result, (self._REMOTE_HOST_IP, "acloud_cf_2")) + + result = cvd_utils.ParseRemoteHostAddress( + "host-goldfish-192.0.2.1-5554-123456-sdk_x86_64-sdk") + self.assertIsNone(result) + + def testGetLaunchCvdArgs(self): + """Test GetLaunchCvdArgs.""" + # Minimum arguments + mock_cfg = mock.Mock(extra_data_disk_size_gb=0) + hw_property = { + constants.HW_X_RES: "1080", + constants.HW_Y_RES: "1920", + constants.HW_ALIAS_DPI: "240"} + mock_avd_spec = mock.Mock( + spec=[], + cfg=mock_cfg, + hw_customize=False, + hw_property=hw_property, + connect_webrtc=False, + connect_vnc=False, + openwrt=False, + num_avds_per_instance=1, + base_instance_num=0, + launch_args="") + expected_args = [ + "-x_res=1080", "-y_res=1920", "-dpi=240", + "-undefok=report_anonymous_usage_stats,config", + "-report_anonymous_usage_stats=y"] + launch_cvd_args = cvd_utils.GetLaunchCvdArgs(mock_avd_spec) + self.assertEqual(launch_cvd_args, expected_args) + + # All arguments. + mock_cfg = mock.Mock(extra_data_disk_size_gb=20) + hw_property = { + constants.HW_X_RES: "1080", + constants.HW_Y_RES: "1920", + constants.HW_ALIAS_DPI: "240", + constants.HW_ALIAS_DISK: "10240", + constants.HW_ALIAS_CPUS: "2", + constants.HW_ALIAS_MEMORY: "4096"} + mock_avd_spec = mock.Mock( + spec=[], + cfg=mock_cfg, + hw_customize=True, + hw_property=hw_property, + connect_webrtc=True, + webrtc_device_id="pet-name", + connect_vnc=True, + openwrt=True, + num_avds_per_instance=2, + base_instance_num=3, + launch_args="--setupwizard_mode=REQUIRED") + expected_args = [ + "-data_policy=create_if_missing", "-blank_data_image_mb=20480", + "-config=phone", "-x_res=1080", "-y_res=1920", "-dpi=240", + "-data_policy=always_create", "-blank_data_image_mb=10240", + "-cpus=2", "-memory_mb=4096", + "--start_webrtc", "--vm_manager=crosvm", + "--webrtc_device_id=pet-name", + "--start_vnc_server=true", + "-console=true", + "-num_instances=2", "--base-instance-num=3", + "--setupwizard_mode=REQUIRED", + "-undefok=report_anonymous_usage_stats,config", + "-report_anonymous_usage_stats=y"] + launch_cvd_args = cvd_utils.GetLaunchCvdArgs( + mock_avd_spec, config="phone") + self.assertEqual(launch_cvd_args, expected_args) + + def testGetRemoteFetcherConfigJson(self): + """Test GetRemoteFetcherConfigJson.""" + expected_log = {"path": "dir/fetcher_config.json", + "type": constants.LOG_TYPE_CUTTLEFISH_LOG} + self.assertEqual(expected_log, + cvd_utils.GetRemoteFetcherConfigJson("dir")) + + @mock.patch("acloud.internal.lib.cvd_utils.utils") + def testFindRemoteLogs(self, mock_utils): + """Test FindRemoteLogs with the runtime directories in Android 13.""" + mock_ssh = mock.Mock() + mock_utils.FindRemoteFiles.return_value = [ + "/kernel.log", "/logcat", "/launcher.log", "/access-kregistry", + "/cuttlefish_config.json"] + + logs = cvd_utils.FindRemoteLogs(mock_ssh, "dir", None, None) + mock_ssh.Run.assert_called_with( + "test -d dir/cuttlefish/instances/cvd-1", retry=0) + mock_utils.FindRemoteFiles.assert_called_with( + mock_ssh, ["dir/cuttlefish/instances/cvd-1"]) expected_logs = [ - {"path": "/kernel.log", "type": constants.LOG_TYPE_KERNEL_LOG}, + { + "path": "/kernel.log", + "type": constants.LOG_TYPE_KERNEL_LOG, + "name": "kernel.log" + }, { "path": "/logcat", "type": constants.LOG_TYPE_LOGCAT, "name": "full_gce_logcat" }, - {"path": "/launcher.log", "type": constants.LOG_TYPE_TEXT} + { + "path": "/launcher.log", + "type": constants.LOG_TYPE_CUTTLEFISH_LOG, + "name": "launcher.log" + }, + { + "path": "/cuttlefish_config.json", + "type": constants.LOG_TYPE_CUTTLEFISH_LOG, + "name": "cuttlefish_config.json" + }, + { + "path": "dir/cuttlefish/instances/cvd-1/tombstones", + "type": constants.LOG_TYPE_DIR, + "name": "tombstones-zip" + }, ] self.assertEqual(expected_logs, logs) + @mock.patch("acloud.internal.lib.cvd_utils.utils") + def testFindRemoteLogsWithLegacyDirs(self, mock_utils): + """Test FindRemoteLogs with the runtime directories in Android 11.""" + mock_ssh = mock.Mock() + mock_ssh.Run.side_effect = subprocess.CalledProcessError( + cmd="test", returncode=1) + mock_utils.FindRemoteFiles.return_value = [ + "dir/cuttlefish_runtime/kernel.log", + "dir/cuttlefish_runtime.4/kernel.log", + ] + + logs = cvd_utils.FindRemoteLogs(mock_ssh, "dir", 3, 2) + mock_ssh.Run.assert_called_with( + "test -d dir/cuttlefish/instances/cvd-3", retry=0) + mock_utils.FindRemoteFiles.assert_called_with( + mock_ssh, ["dir/cuttlefish_runtime", "dir/cuttlefish_runtime.4"]) + expected_logs = [ + { + "path": "dir/cuttlefish_runtime/kernel.log", + "type": constants.LOG_TYPE_KERNEL_LOG, + "name": "kernel.log" + }, + { + "path": "dir/cuttlefish_runtime.4/kernel.log", + "type": constants.LOG_TYPE_KERNEL_LOG, + "name": "kernel.1.log" + }, + { + "path": "dir/cuttlefish_runtime/tombstones", + "type": constants.LOG_TYPE_DIR, + "name": "tombstones-zip" + }, + { + "path": "dir/cuttlefish_runtime.4/tombstones", + "type": constants.LOG_TYPE_DIR, + "name": "tombstones-zip.1" + }, + ] + self.assertEqual(expected_logs, logs) + + def testFindLocalLogs(self): + """Test FindLocalLogs with the runtime directory in Android 13.""" + with tempfile.TemporaryDirectory() as temp_dir: + log_dir = os.path.join(temp_dir, "instances", "cvd-2", "logs") + kernel_log = os.path.join(os.path.join(log_dir, "kernel.log")) + launcher_log = os.path.join(os.path.join(log_dir, "launcher.log")) + logcat = os.path.join(os.path.join(log_dir, "logcat")) + self.CreateFile(kernel_log) + self.CreateFile(launcher_log) + self.CreateFile(logcat) + self.CreateFile(os.path.join(temp_dir, "legacy.log")) + self.CreateFile(os.path.join(log_dir, "log.txt")) + os.symlink(os.path.join(log_dir, "launcher.log"), + os.path.join(log_dir, "link.log")) + + logs = cvd_utils.FindLocalLogs(temp_dir, 2) + expected_logs = [ + { + "path": kernel_log, + "type": constants.LOG_TYPE_KERNEL_LOG, + }, + { + "path": launcher_log, + "type": constants.LOG_TYPE_CUTTLEFISH_LOG, + }, + { + "path": logcat, + "type": constants.LOG_TYPE_LOGCAT, + }, + ] + self.assertEqual(expected_logs, + sorted(logs, key=lambda log: log["path"])) + + def testFindLocalLogsWithLegacyDir(self): + """Test FindLocalLogs with the runtime directory in Android 11.""" + with tempfile.TemporaryDirectory() as temp_dir: + log_dir = os.path.join(temp_dir, "cuttlefish_runtime.2") + log_dir_link = os.path.join(temp_dir, "cuttlefish_runtime") + os.mkdir(log_dir) + os.symlink(log_dir, log_dir_link, target_is_directory=True) + launcher_log = os.path.join(log_dir_link, "launcher.log") + self.CreateFile(launcher_log) + + logs = cvd_utils.FindLocalLogs(log_dir_link, 2) + expected_logs = [ + { + "path": launcher_log, + "type": constants.LOG_TYPE_CUTTLEFISH_LOG, + }, + ] + self.assertEqual(expected_logs, logs) + def testGetRemoteBuildInfoDict(self): """Test GetRemoteBuildInfoDict.""" remote_image = { @@ -211,6 +518,24 @@ class CvdUtilsTest(unittest.TestCase): self.assertEqual(all_build_info, cvd_utils.GetRemoteBuildInfoDict(mock_avd_spec)) + def testFindMiscInfo(self): + """Test FindMiscInfo.""" + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(errors.CheckPathError): + cvd_utils.FindMiscInfo(temp_dir) + misc_info_path = os.path.join(temp_dir, "META", "misc_info.txt") + self.CreateFile(misc_info_path, b"key=value") + self.assertEqual(misc_info_path, cvd_utils.FindMiscInfo(temp_dir)) + + def testFindImageDir(self): + """Test FindImageDir.""" + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaises(errors.GetLocalImageError): + cvd_utils.FindImageDir(temp_dir) + image_dir = os.path.join(temp_dir, "IMAGES") + self.CreateFile(os.path.join(image_dir, "super.img")) + self.assertEqual(image_dir, cvd_utils.FindImageDir(temp_dir)) + if __name__ == "__main__": unittest.main() diff --git a/internal/lib/driver_test_lib.py b/internal/lib/driver_test_lib.py index 339a8fcd..91ce438c 100644 --- a/internal/lib/driver_test_lib.py +++ b/internal/lib/driver_test_lib.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """Driver test library.""" + +import os import unittest from unittest import mock @@ -47,3 +49,10 @@ class BaseDriverTest(unittest.TestCase): patcher = mock.patch.object(*args, **kwargs) self._patchers.append(patcher) return patcher.start() + + @staticmethod + def CreateFile(path, data=b""): + """Create and write binary data to a file.""" + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as file_obj: + file_obj.write(data) diff --git a/internal/lib/gcompute_client.py b/internal/lib/gcompute_client.py index b30b6868..d8460afd 100755 --- a/internal/lib/gcompute_client.py +++ b/internal/lib/gcompute_client.py @@ -1786,3 +1786,21 @@ def GetInstanceIP(instance): external_ip = access_configs.get("natIP", "") internal_ip = network_interface.get("networkIP", "") return IP(internal=internal_ip, external=external_ip) + +def GetGCEHostName(gce_project, instance, zone): + """Get the GCE host name with specific rule. + + Args: + gce_project: String, GCE project name. + instance: String, GCE instance name. + zone: String, Instance zone name. + + Returns: + One host name coverted by instance name, project name, and zone. + """ + if ":" in gce_project: + domain = gce_project.split(":")[0] + project_no_domain = gce_project.split(":")[1] + project = f"{project_no_domain}.{domain}" + return f"nic0.{instance}.{zone}.c.{project}.internal.gcpnode.com" + return f"nic0.{instance}.{zone}.c.{gce_project}.internal.gcpnode.com" diff --git a/internal/lib/gcompute_client_test.py b/internal/lib/gcompute_client_test.py index 27eff81d..a37faa33 100644 --- a/internal/lib/gcompute_client_test.py +++ b/internal/lib/gcompute_client_test.py @@ -1526,6 +1526,20 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): disk=self.DISK) self.assertTrue(self.compute_client.CheckDiskExists(self.DISK, self.ZONE)) + def testGetGCEHostName(self): + """Test GetGCEHostName.""" + instance_name = "instance_name" + project = "fake-project" + zone = "fake-zone" + expected = "nic0.instance_name.fake-zone.c.fake-project.internal.gcpnode.com" + self.assertEqual(expected, gcompute_client.GetGCEHostName( + project, instance_name, zone)) + + project = "test.com:project" + expected = "nic0.instance_name.fake-zone.c.project.test.com.internal.gcpnode.com" + self.assertEqual(expected, gcompute_client.GetGCEHostName( + project, instance_name, zone)) + if __name__ == "__main__": unittest.main() diff --git a/internal/lib/goldfish_compute_client.py b/internal/lib/goldfish_compute_client.py index 33de884b..9ae7922b 100644 --- a/internal/lib/goldfish_compute_client.py +++ b/internal/lib/goldfish_compute_client.py @@ -159,6 +159,7 @@ class GoldfishComputeClient(android_compute_client.AndroidComputeClient): kernel_build_target=None, emulator_branch=None, emulator_build_id=None, + emulator_build_target=None, blank_data_disk_size_gb=None, gpu=None, avd_spec=None, @@ -180,6 +181,7 @@ class GoldfishComputeClient(android_compute_client.AndroidComputeClient): kernel_build_target: kernel target, e.g. "kernel_x86_64" emulator_branch: String, emulator branch name, e.g."aosp-emu-master-dev" emulator_build_id: String, emulator build id, a string, e.g. "2263051", "P2804227" + emulator_build_target: String, emulator build target. blank_data_disk_size_gb: Integer, size of the blank data disk in GB. gpu: String, GPU that should be attached to the instance, or None of no acceleration is needed. e.g. "nvidia-tesla-k80" @@ -220,6 +222,8 @@ class GoldfishComputeClient(android_compute_client.AndroidComputeClient): metadata[ "cvd_01_fetch_emulator_bid"] = "{branch}/{build_id}".format( branch=emulator_branch, build_id=emulator_build_id) + if emulator_build_target: + metadata["cvd_01_fetch_emulator_build_target"] = emulator_build_target if launch_args: metadata["launch_args"] = launch_args metadata["cvd_01_launch"] = "1" diff --git a/internal/lib/goldfish_compute_client_test.py b/internal/lib/goldfish_compute_client_test.py index e0ab45d6..02f649ad 100644 --- a/internal/lib/goldfish_compute_client_test.py +++ b/internal/lib/goldfish_compute_client_test.py @@ -60,6 +60,7 @@ Jan 12 12:00:00 ins-abcdefgh-5000000-sdk-x86-64-sdk launch_emulator[123]: VIRTUA KERNEL_BUILD_ARTIFACT = "bzImage" EMULATOR_BRANCH = "aosp-emu-master-dev" EMULATOR_BUILD_ID = "1234567" + EMULATOR_BUILD_TARGET = "emulator-linux_x64_nolocationui" DPI = 160 X_RES = 720 Y_RES = 1280 @@ -162,6 +163,7 @@ Jan 12 12:00:00 ins-abcdefgh-5000000-sdk-x86-64-sdk launch_emulator[123]: VIRTUA "{branch}/{build_id}".format( branch=self.EMULATOR_BRANCH, build_id=self.EMULATOR_BUILD_ID), + "cvd_01_fetch_emulator_build_target": self.EMULATOR_BUILD_TARGET, "cvd_01_launch": "1", "cvd_01_dpi": str(self.DPI), "cvd_01_x_res": str(self.X_RES), @@ -178,7 +180,9 @@ Jan 12 12:00:00 ins-abcdefgh-5000000-sdk-x86-64-sdk launch_emulator[123]: VIRTUA self.KERNEL_BUILD_ID, self.KERNEL_BUILD_TARGET, self.EMULATOR_BRANCH, - self.EMULATOR_BUILD_ID, self.EXTRA_DATA_DISK_SIZE_GB, self.GPU, + self.EMULATOR_BUILD_ID, + self.EMULATOR_BUILD_TARGET, + self.EXTRA_DATA_DISK_SIZE_GB, self.GPU, extra_scopes=self.EXTRA_SCOPES, tags=self.TAGS, launch_args=self.LAUNCH_ARGS) @@ -218,6 +222,7 @@ Jan 12 12:00:00 ins-abcdefgh-5000000-sdk-x86-64-sdk launch_emulator[123]: VIRTUA "{branch}/{build_id}".format( branch=self.EMULATOR_BRANCH, build_id=self.EMULATOR_BUILD_ID), + "cvd_01_fetch_emulator_build_target": self.EMULATOR_BUILD_TARGET, "cvd_01_launch": "1", "display": "{x}x{y} ({dpi})".format( @@ -246,16 +251,12 @@ Jan 12 12:00:00 ins-abcdefgh-5000000-sdk-x86-64-sdk launch_emulator[123]: VIRTUA self.goldfish_compute_client.CreateInstance( self.INSTANCE, self.IMAGE, self.IMAGE_PROJECT, self.TARGET, - self.BRANCH, self.BUILD_ID, - self.KERNEL_BRANCH, - self.KERNEL_BUILD_ID, - self.KERNEL_BUILD_TARGET, - self.EMULATOR_BRANCH, - self.EMULATOR_BUILD_ID, self.EXTRA_DATA_DISK_SIZE_GB, self.GPU, - avd_spec=mock_avd_spec, - extra_scopes=self.EXTRA_SCOPES, - tags=self.TAGS, - launch_args=self.LAUNCH_ARGS) + self.BRANCH, self.BUILD_ID, self.KERNEL_BRANCH, + self.KERNEL_BUILD_ID, self.KERNEL_BUILD_TARGET, + self.EMULATOR_BRANCH, self.EMULATOR_BUILD_ID, + self.EMULATOR_BUILD_TARGET, self.EXTRA_DATA_DISK_SIZE_GB, self.GPU, + avd_spec=mock_avd_spec, extra_scopes=self.EXTRA_SCOPES, + tags=self.TAGS, launch_args=self.LAUNCH_ARGS) self._mock_create_instance.assert_called_with( self.goldfish_compute_client, diff --git a/internal/lib/goldfish_remote_host_client.py b/internal/lib/goldfish_remote_host_client.py deleted file mode 100644 index 08e2f367..00000000 --- a/internal/lib/goldfish_remote_host_client.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2021 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This module implements the classes and functions needed for the common -creation flow.""" - -import re - -from acloud.internal import constants -from acloud.internal.lib import ssh -from acloud.public import config - - -_INSTANCE_NAME_FORMAT = ("host-goldfish-%(ip_addr)s-%(console_port)s-" - "%(build_id)s-%(build_target)s") -_INSTANCE_NAME_PATTERN = re.compile( - r"host-goldfish-(?P<ip_addr>[\d.]+)-(?P<console_port>\d+)-.+") -# Report keys -_VERSION = "version" - - -def FormatInstanceName(ip_addr, console_port, build_info): - """Convert address and build info to an instance name. - - Args: - ip_addr: A string, the IP address of the host. - console_port: An integer, the emulator console port. - build_info: A dict containing the build ID and target. - - Returns: - A string, the instance name. - """ - return _INSTANCE_NAME_FORMAT % { - "ip_addr": ip_addr, - "console_port": console_port, - "build_id": build_info.get(constants.BUILD_ID), - "build_target": build_info.get(constants.BUILD_TARGET)} - - -def ParseEmulatorConsoleAddress(instance_name): - """Parse emulator console address from an instance name. - - Args: - instance_name: A string, the instance name. - - Returns: - The IP address as a string and the console port as an integer. - None if the name does not represent a goldfish instance on remote host. - """ - match = _INSTANCE_NAME_PATTERN.fullmatch(instance_name) - return ((match.group("ip_addr"), int(match.group("console_port"))) - if match else None) - - -class GoldfishRemoteHostClient: - """A client that manages goldfish instance on a remote host.""" - - @staticmethod - def GetInstanceIP(instance_name): - """Parse the IP address from an instance name.""" - ip_and_port = ParseEmulatorConsoleAddress(instance_name) - if not ip_and_port: - raise ValueError("Cannot parse instance name: %s" % instance_name) - return ssh.IP(ip=ip_and_port[0]) - - @staticmethod - def WaitForBoot(_instance_name, _boot_timeout_secs): - """Should not be called in the common creation flow.""" - raise NotImplementedError("The common creation flow should call " - "GetFailures instead of this method.") - - @staticmethod - def GetSerialPortOutput(): - """Remote hosts do not support serial log.""" - return "" - - @property - def dict_report(self): - """Return the key-value pairs to be written to the report.""" - return {_VERSION: config.GetVersion()} diff --git a/internal/lib/goldfish_remote_host_client_test.py b/internal/lib/goldfish_remote_host_client_test.py deleted file mode 100644 index 3b3c72a5..00000000 --- a/internal/lib/goldfish_remote_host_client_test.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2021 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for GoldfishRemoteHostClient.""" - -import unittest - -from acloud.internal.lib import driver_test_lib -from acloud.internal.lib import goldfish_remote_host_client - - -class GoldfishRemoteHostClientTest(driver_test_lib.BaseDriverTest): - """Unit tests for GoldfishRemoteHostClient.""" - - _IP_ADDRESS = "192.0.2.1" - _CONSOLE_PORT = 5554 - _BUILD_INFO = {"build_id": "123456", - "build_target": "sdk_phone_x86_64-userdebug"} - _INSTANCE_NAME = ("host-goldfish-192.0.2.1-5554-" - "123456-sdk_phone_x86_64-userdebug") - _INVALID_NAME = "host-192.0.2.1-123456-aosp_cf_x86_phone-userdebug" - - def testParseEmulatorConsoleAddress(self): - """Test ParseEmulatorConsoleAddress.""" - console_addr = goldfish_remote_host_client.ParseEmulatorConsoleAddress( - self._INSTANCE_NAME) - self.assertEqual((self._IP_ADDRESS, self._CONSOLE_PORT), console_addr) - - console_addr = goldfish_remote_host_client.ParseEmulatorConsoleAddress( - self._INVALID_NAME) - self.assertIsNone(console_addr) - - def testFormatInstanceName(self): - """Test FormatInstanceName.""" - instance_name = goldfish_remote_host_client.FormatInstanceName( - self._IP_ADDRESS, self._CONSOLE_PORT, self._BUILD_INFO) - self.assertEqual(self._INSTANCE_NAME, instance_name) - - def testGetInstanceIP(self): - """Test GetInstanceIP.""" - client = goldfish_remote_host_client.GoldfishRemoteHostClient() - ip_addr = client.GetInstanceIP(self._INSTANCE_NAME) - self.assertEqual(ip_addr.external, self._IP_ADDRESS) - self.assertEqual(ip_addr.internal, self._IP_ADDRESS) - -if __name__ == "__main__": - unittest.main() diff --git a/internal/lib/goldfish_utils.py b/internal/lib/goldfish_utils.py index 5e10cb84..41a1fe96 100644 --- a/internal/lib/goldfish_utils.py +++ b/internal/lib/goldfish_utils.py @@ -15,6 +15,7 @@ """Utility functions that process goldfish images and arguments.""" import os +import re import shutil from acloud import errors @@ -37,6 +38,11 @@ _SYSTEM_QEMU_CONFIG_FILE_NAME = "system-qemu-config.txt" _DISK_IMAGE_NAMES = (SYSTEM_QEMU_IMAGE_NAME, _SDK_REPO_SYSTEM_IMAGE_NAME) _KERNEL_IMAGE_NAMES = ("kernel-ranchu", "kernel-ranchu-64", "kernel") _RAMDISK_IMAGE_NAMES = ("ramdisk-qemu.img", "ramdisk.img") +# Remote host instance name. +_REMOTE_HOST_INSTANCE_NAME_FORMAT = ( + "host-goldfish-%(ip_addr)s-%(console_port)s-%(build_info)s") +_REMOTE_HOST_INSTANCE_NAME_PATTERN = re.compile( + r"host-goldfish-(?P<ip_addr>[\d.]+)-(?P<console_port>\d+)-.+") def _FindFileByNames(parent_dir, names): @@ -209,6 +215,44 @@ def MixWithSystemImage(output_dir, image_dir, system_image_path, ota): return disk_image +def FormatRemoteHostInstanceName(ip_addr, console_port, build_info): + """Convert address and build info to a remote host instance name. + + Args: + ip_addr: A string, the IP address of the host. + console_port: An integer, the emulator console port. + build_info: A dict containing the build ID and target. + + Returns: + A string, the instance name. + """ + build_id = build_info.get(constants.BUILD_ID) + build_target = build_info.get(constants.BUILD_TARGET) + build_info_str = (f"{build_id}-{build_target}" if + build_id and build_target else + "userbuild") + return _REMOTE_HOST_INSTANCE_NAME_FORMAT % { + "ip_addr": ip_addr, + "console_port": console_port, + "build_info": build_info_str, + } + + +def ParseRemoteHostConsoleAddress(instance_name): + """Parse emulator console address from a remote host instance name. + + Args: + instance_name: A string, the instance name. + + Returns: + The IP address as a string and the console port as an integer. + None if the name does not represent a goldfish instance on remote host. + """ + match = _REMOTE_HOST_INSTANCE_NAME_PATTERN.fullmatch(instance_name) + return ((match.group("ip_addr"), int(match.group("console_port"))) + if match else None) + + def ConvertAvdSpecToArgs(avd_spec): """Convert hardware specification to emulator arguments. diff --git a/internal/lib/goldfish_utils_test.py b/internal/lib/goldfish_utils_test.py index 85a10a36..fe787931 100644 --- a/internal/lib/goldfish_utils_test.py +++ b/internal/lib/goldfish_utils_test.py @@ -30,6 +30,16 @@ from acloud.internal.lib import goldfish_utils class GoldfishUtilsTest(unittest.TestCase): """Test functions in goldfish_utils.""" + # Remote host instance name. + _IP_ADDRESS = "192.0.2.1" + _CONSOLE_PORT = 5554 + _BUILD_INFO = {"build_id": "123456", + "build_target": "sdk_phone_x86_64-userdebug"} + _INSTANCE_NAME = ("host-goldfish-192.0.2.1-5554-" + "123456-sdk_phone_x86_64-userdebug") + _INSTANCE_NAME_WITHOUT_INFO = "host-goldfish-192.0.2.1-5554-userbuild" + _INVALID_NAME = "host-192.0.2.1-123456-aosp_cf_x86_phone-userdebug" + @staticmethod def _CreateEmptyFile(path): os.makedirs(os.path.dirname(path), exist_ok=True) @@ -146,6 +156,26 @@ class GoldfishUtilsTest(unittest.TestCase): self.assertEqual(vbmeta_image_path, get_image("vbmeta")) self.assertEqual(super_image_path, get_image("super")) + def testParseRemoteHostConsoleAddress(self): + """Test ParseRemoteHostConsoleAddress.""" + console_addr = goldfish_utils.ParseRemoteHostConsoleAddress( + self._INSTANCE_NAME) + self.assertEqual((self._IP_ADDRESS, self._CONSOLE_PORT), console_addr) + + console_addr = goldfish_utils.ParseRemoteHostConsoleAddress( + self._INVALID_NAME) + self.assertIsNone(console_addr) + + def testFormatInstanceName(self): + """Test FormatRemoteHostInstanceName.""" + instance_name = goldfish_utils.FormatRemoteHostInstanceName( + self._IP_ADDRESS, self._CONSOLE_PORT, self._BUILD_INFO) + self.assertEqual(self._INSTANCE_NAME, instance_name) + + instance_name = goldfish_utils.FormatRemoteHostInstanceName( + self._IP_ADDRESS, self._CONSOLE_PORT, {}) + self.assertEqual(self._INSTANCE_NAME_WITHOUT_INFO, instance_name) + def testConvertAvdSpecToArgs(self): """Test ConvertAvdSpecToArgs.""" hw_property = { diff --git a/internal/lib/ota_tools.py b/internal/lib/ota_tools.py index edfe23fa..f2ea9580 100644 --- a/internal/lib/ota_tools.py +++ b/internal/lib/ota_tools.py @@ -298,3 +298,30 @@ class OtaTools: utils.Popen(unpack_bootimg, "--out", out_dir, "--boot_img", boot_img) + + def MixSuperImage(self, super_image, misc_info, image_dir, + system_image=None, vendor_image=None, + vendor_dlkm_image=None, odm_image=None, + odm_dlkm_image=None): + """Create mixed super image from device images and given partition + images. + + Args: + super_image: Path to the output super image. + misc_info: Path to the misc_info.txt. + image_dir: Path to image files excluding system image. + system_image: Path to the system image. + vendor_image: Path to the vendor image. + vendor_dlkm_image: Path to the vendor_dlkm image. + odm_image: Path to the odm image. + odm_dlkm_image: Path to the odm_dlkm image. + """ + self.BuildSuperImage( + super_image, misc_info, + lambda partition: GetImageForPartition( + partition, image_dir, + system=system_image, + vendor=vendor_image, + vendor_dlkm=vendor_dlkm_image, + odm=odm_image, + odm_dlkm=odm_dlkm_image)) diff --git a/internal/lib/remote_host_client.py b/internal/lib/remote_host_client.py new file mode 100644 index 00000000..37c9069e --- /dev/null +++ b/internal/lib/remote_host_client.py @@ -0,0 +1,93 @@ +# Copyright 2022 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module implements the classes and functions needed for the common +creation flow.""" + +import time + +from acloud.internal import constants +from acloud.internal.lib import ssh +from acloud.public import config + + +# Report keys +_VERSION = "version" + + +class RemoteHostClient: + """A client that manages an instance on a remote host. + + Attributes: + ip_addr: A string, the IP address of the host. + execution_time: A dictionary that records the execution time. The + possible keys are defined as TIME_* in constants.py. + stage: An integer. The possible values are defined as STAGE_* in + constants.py. + """ + + def __init__(self, ip_addr): + """Initialize the attribtues.""" + self._ip_addr = ip_addr + self._execution_time = {} + self._stage = constants.STAGE_INIT + + def RecordTime(self, key, start_time): + """Record the interval between the start time and the current time. + + Args: + key: A string, the stage name. + start_time: A float, the timestamp when the stage starts. + + Returns: + A float, the current time. + """ + current = time.time() + self._execution_time[key] = current - start_time + return current + + def SetStage(self, stage): + """Set device creation progress.""" + self._stage = stage + + # The following methods are called by common_operations.py. + def GetInstanceIP(self, _instance_name): + """Return the IP address of the host.""" + return ssh.IP(ip=self._ip_addr) + + @staticmethod + def WaitForBoot(_instance_name, _boot_timeout_secs): + """Should not be called in the common creation flow.""" + raise NotImplementedError("The common creation flow should call " + "GetFailures instead of this method.") + + @staticmethod + def GetSerialPortOutput(): + """Remote hosts do not support serial log.""" + return "" + + @property + def execution_time(self): + """Return execution_time.""" + return self._execution_time + + @property + def stage(self): + """Return stage.""" + return self._stage + + @property + def dict_report(self): + """Return the key-value pairs to be written to the report.""" + return {_VERSION: config.GetVersion()} diff --git a/internal/lib/remote_host_client_test.py b/internal/lib/remote_host_client_test.py new file mode 100644 index 00000000..cdafac8f --- /dev/null +++ b/internal/lib/remote_host_client_test.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for RemoteHostClient.""" + +import unittest +from unittest import mock + +from acloud.internal.lib import driver_test_lib +from acloud.internal.lib import remote_host_client + + +class RemoteHostClientTest(driver_test_lib.BaseDriverTest): + """Unit tests for RemoteHostClient.""" + + _IP_ADDRESS = "192.0.2.1" + + def testGetInstanceIP(self): + """Test GetInstanceIP.""" + client = remote_host_client.RemoteHostClient(self._IP_ADDRESS) + ip_addr = client.GetInstanceIP("name") + self.assertEqual(ip_addr.external, self._IP_ADDRESS) + self.assertEqual(ip_addr.internal, self._IP_ADDRESS) + + def testRecordTime(self): + """Test RecordTime and execution_time.""" + client = remote_host_client.RemoteHostClient(self._IP_ADDRESS) + self.assertFalse(client.execution_time) + with mock.patch( + "acloud.internal.lib.remote_host_client.time") as mock_time: + mock_time.time.return_value = 1.0 + self.assertEqual(1.0, client.RecordTime("TIME", 0.25)) + self.assertDictEqual({"TIME": 0.75}, client.execution_time) + + +if __name__ == "__main__": + unittest.main() diff --git a/internal/lib/ssh.py b/internal/lib/ssh.py index 778bcb6e..53916b88 100755 --- a/internal/lib/ssh.py +++ b/internal/lib/ssh.py @@ -26,7 +26,7 @@ from acloud.internal.lib import utils logger = logging.getLogger(__name__) -_SSH_CMD = ("-i %(rsa_key_file)s -o LogLevel=ERROR " +_SSH_CMD = ("-i %(rsa_key_file)s -o LogLevel=ERROR -o ControlPath=none " "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no") _SSH_IDENTITY = "-l %(login_user)s %(ip_addr)s" _SSH_CMD_MAX_RETRY = 5 @@ -94,7 +94,7 @@ def _SshCall(cmd, timeout=None): return process.returncode -def _SshLogOutput(cmd, timeout=None, show_output=False): +def _SshLogOutput(cmd, timeout=None, show_output=False, hide_error_msg=False): """Runs a single SSH command while logging its output and processes its return code. Output is streamed to the log at the debug level for more interactive debugging. @@ -106,6 +106,7 @@ def _SshLogOutput(cmd, timeout=None, show_output=False): cmd: String of the full SSH command to run, including the SSH binary and its arguments. timeout: Optional integer, number of seconds to give. show_output: Boolean, True to show command output in screen. + hide_error_msg: Boolean, True to hide error message. Raises: errors.DeviceConnectionError: Failed to connect to the GCE instance. @@ -125,7 +126,7 @@ def _SshLogOutput(cmd, timeout=None, show_output=False): timer.start() stdout, _ = process.communicate() if stdout: - if show_output or process.returncode != 0: + if (show_output or process.returncode != 0) and not hide_error_msg: print(stdout.strip(), file=sys.stderr) else: # fetch_cvd and launch_cvd can be noisy, so left at debug @@ -133,12 +134,11 @@ def _SshLogOutput(cmd, timeout=None, show_output=False): if timeout: timer.cancel() if process.returncode == 255: - raise errors.DeviceConnectionError( - "Failed to send command to instance (%(ssh_cmd)s)\n" - "Error message: %(error_message)s" % { - "ssh_cmd": cmd, - "error_message": _GetErrorMessage(stdout)} - ) + error_msg = (f"Failed to send command to instance {cmd}\n" + f"Error message: {_GetErrorMessage(stdout)}") + if constants.ERROR_MSG_SSO_INVALID in stdout: + raise errors.SshConnectFail(error_msg) + raise errors.DeviceConnectionError(error_msg) if process.returncode != 0: if constants.ERROR_MSG_VNC_NOT_SUPPORT in stdout: raise errors.LaunchCVDFail(constants.ERROR_MSG_VNC_NOT_SUPPORT) @@ -239,13 +239,22 @@ class Ssh(): _user: String of user login into the instance. _ssh_private_key_path: Path to the private key file. _extra_args_ssh_tunnel: String, extra args for ssh or scp. + _report_internal_ip: Boolean, True to use internal ip. + _gce_hostname: String, the hostname for ssh connect. """ def __init__(self, ip, user, ssh_private_key_path, - extra_args_ssh_tunnel=None, report_internal_ip=False): + extra_args_ssh_tunnel=None, report_internal_ip=False, + gce_hostname=None): self._ip = ip.internal if report_internal_ip else ip.external self._user = user self._ssh_private_key_path = ssh_private_key_path self._extra_args_ssh_tunnel = extra_args_ssh_tunnel + if gce_hostname: + self._ip = gce_hostname + self._extra_args_ssh_tunnel = None + logger.debug( + "To connect with hostname, erase the extra_args_ssh_tunnel: %s", + extra_args_ssh_tunnel) def Run(self, target_command, timeout=None, show_output=False, retry=_SSH_CMD_MAX_RETRY): @@ -330,11 +339,11 @@ class Ssh(): """ remote_cmd = [self.GetBaseCmd(constants.SSH_BIN)] remote_cmd.append("uptime") - - if _SshCallWait(" ".join(remote_cmd), timeout) == 0: - return - raise errors.DeviceConnectionError( - "Ssh isn't ready in the remote instance.") + try: + _SshLogOutput(" ".join(remote_cmd), timeout, hide_error_msg=True) + except subprocess.CalledProcessError as e: + raise errors.DeviceConnectionError( + "Ssh isn't ready in the remote instance.") from e @utils.TimeExecute(function_description="Waiting for SSH server") def WaitForSsh(self, timeout=None, max_retry=_SSH_CMD_MAX_RETRY): diff --git a/internal/lib/ssh_test.py b/internal/lib/ssh_test.py index 1c188b40..d35a6086 100644 --- a/internal/lib/ssh_test.py +++ b/internal/lib/ssh_test.py @@ -28,6 +28,7 @@ from acloud import errors from acloud.internal import constants from acloud.internal.lib import driver_test_lib from acloud.internal.lib import ssh +from acloud.internal.lib import utils class SshTest(driver_test_lib.BaseDriverTest): @@ -42,6 +43,8 @@ class SshTest(driver_test_lib.BaseDriverTest): def setUp(self): """Set up the test.""" super().setUp() + self.Patch(utils, "FindExecutable", + side_effect=lambda name: f"/usr/bin/{name}") self.created_subprocess = mock.MagicMock() self.created_subprocess.stdout = mock.MagicMock() self.created_subprocess.stdout.readline = mock.MagicMock(return_value=b"") @@ -67,21 +70,23 @@ class SshTest(driver_test_lib.BaseDriverTest): ssh_private_key_path=self.FAKE_SSH_PRIVATE_KEY_PATH, report_internal_ip=self.FAKE_REPORT_INTERNAL_IP) expected_ssh_cmd = ( - "/usr/bin/ssh -i /fake/acloud_rea -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null " - "-o StrictHostKeyChecking=no -l fake_user 10.1.1.1") + "/usr/bin/ssh -i /fake/acloud_rea -o LogLevel=ERROR -o ControlPath=none " + "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " + "-l fake_user 10.1.1.1") self.assertEqual(ssh_object.GetBaseCmd(constants.SSH_BIN), expected_ssh_cmd) def testGetBaseCmd(self): """Test get base command.""" ssh_object = ssh.Ssh(self.FAKE_IP, self.FAKE_SSH_USER, self.FAKE_SSH_PRIVATE_KEY_PATH) expected_ssh_cmd = ( - "/usr/bin/ssh -i /fake/acloud_rea -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null " - "-o StrictHostKeyChecking=no -l fake_user 1.1.1.1") + "/usr/bin/ssh -i /fake/acloud_rea -o LogLevel=ERROR -o ControlPath=none " + "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " + "-l fake_user 1.1.1.1") self.assertEqual(ssh_object.GetBaseCmd(constants.SSH_BIN), expected_ssh_cmd) expected_scp_cmd = ( - "/usr/bin/scp -i /fake/acloud_rea -o LogLevel=ERROR -o UserKnownHostsFile=/dev/null " - "-o StrictHostKeyChecking=no") + "/usr/bin/scp -i /fake/acloud_rea -o LogLevel=ERROR -o ControlPath=none " + "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no") self.assertEqual(ssh_object.GetBaseCmd(constants.SCP_BIN), expected_scp_cmd) # pylint: disable=no-member @@ -91,7 +96,7 @@ class SshTest(driver_test_lib.BaseDriverTest): ssh_object = ssh.Ssh(self.FAKE_IP, self.FAKE_SSH_USER, self.FAKE_SSH_PRIVATE_KEY_PATH) ssh_object.Run("command") expected_cmd = ( - "exec /usr/bin/ssh -i /fake/acloud_rea -o LogLevel=ERROR " + "exec /usr/bin/ssh -i /fake/acloud_rea -o LogLevel=ERROR -o ControlPath=none " "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " "-l fake_user 1.1.1.1 command") subprocess.Popen.assert_called_with(expected_cmd, @@ -110,7 +115,7 @@ class SshTest(driver_test_lib.BaseDriverTest): self.FAKE_EXTRA_ARGS_SSH) ssh_object.Run("command") expected_cmd = ( - "exec /usr/bin/ssh -i /fake/acloud_rea -o LogLevel=ERROR " + "exec /usr/bin/ssh -i /fake/acloud_rea -o LogLevel=ERROR -o ControlPath=none " "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " "-o ProxyCommand='ssh fake_user@2.2.2.2 Server 22' " "-l fake_user 1.1.1.1 command") @@ -127,7 +132,7 @@ class SshTest(driver_test_lib.BaseDriverTest): ssh_object = ssh.Ssh(self.FAKE_IP, self.FAKE_SSH_USER, self.FAKE_SSH_PRIVATE_KEY_PATH) ssh_object.ScpPullFile("/tmp/test", "/tmp/test_1.log") expected_cmd = ( - "exec /usr/bin/scp -i /fake/acloud_rea -o LogLevel=ERROR " + "exec /usr/bin/scp -i /fake/acloud_rea -o LogLevel=ERROR -o ControlPath=none " "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " "fake_user@1.1.1.1:/tmp/test /tmp/test_1.log") subprocess.Popen.assert_called_with(expected_cmd, @@ -146,8 +151,8 @@ class SshTest(driver_test_lib.BaseDriverTest): self.FAKE_EXTRA_ARGS_SSH) ssh_object.ScpPullFile("/tmp/test", "/tmp/test_1.log") expected_cmd = ( - "exec /usr/bin/scp -i /fake/acloud_rea -o LogLevel=ERROR -o " - "UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " + "exec /usr/bin/scp -i /fake/acloud_rea -o LogLevel=ERROR -o ControlPath=none " + "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " "-o ProxyCommand='ssh fake_user@2.2.2.2 Server 22' " "fake_user@1.1.1.1:/tmp/test /tmp/test_1.log") subprocess.Popen.assert_called_with(expected_cmd, @@ -163,7 +168,7 @@ class SshTest(driver_test_lib.BaseDriverTest): ssh_object = ssh.Ssh(self.FAKE_IP, self.FAKE_SSH_USER, self.FAKE_SSH_PRIVATE_KEY_PATH) ssh_object.ScpPushFile("/tmp/test", "/tmp/test_1.log") expected_cmd = ( - "exec /usr/bin/scp -i /fake/acloud_rea -o LogLevel=ERROR " + "exec /usr/bin/scp -i /fake/acloud_rea -o LogLevel=ERROR -o ControlPath=none " "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " "/tmp/test fake_user@1.1.1.1:/tmp/test_1.log") subprocess.Popen.assert_called_with(expected_cmd, @@ -182,7 +187,7 @@ class SshTest(driver_test_lib.BaseDriverTest): self.FAKE_EXTRA_ARGS_SSH) ssh_object.ScpPushFile("/tmp/test", "/tmp/test_1.log") expected_cmd = ( - "exec /usr/bin/scp -i /fake/acloud_rea -o LogLevel=ERROR " + "exec /usr/bin/scp -i /fake/acloud_rea -o LogLevel=ERROR -o ControlPath=none " "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no " "-o ProxyCommand='ssh fake_user@2.2.2.2 Server 22' " "/tmp/test fake_user@1.1.1.1:/tmp/test_1.log") @@ -224,9 +229,9 @@ class SshTest(driver_test_lib.BaseDriverTest): user=self.FAKE_SSH_USER, ssh_private_key_path=self.FAKE_SSH_PRIVATE_KEY_PATH, report_internal_ip=self.FAKE_REPORT_INTERNAL_IP) - self.Patch(ssh, "_SshCallWait", return_value=-1) - self.Patch(ssh, "_SshLogOutput") - self.assertRaises(errors.DeviceConnectionError, + self.created_subprocess.returncode = -1 + self.Patch(subprocess, "Popen", return_value=self.created_subprocess) + self.assertRaises(subprocess.CalledProcessError, ssh_object.WaitForSsh, timeout=1, max_retry=1) diff --git a/internal/lib/utils.py b/internal/lib/utils.py index 0295b292..a2319a19 100755 --- a/internal/lib/utils.py +++ b/internal/lib/utils.py @@ -71,30 +71,40 @@ _WEBRTC_OPERATOR_PATTERN = re.compile(r"(.+)(webrtc_operator )(.+)") _PORT_8443 = 8443 _PORT_1443 = 1443 PortMapping = collections.namedtuple("PortMapping", ["local", "target"]) -WEBRTC_PORTS_MAPPING = [PortMapping(15550, 15550), - PortMapping(15551, 15551), - PortMapping(15552, 15552)] +# Acloud uses only part of default webrtc port range to support both local and remote. +# The default webrtc port range is [15550, 15599]. +WEBRTC_PORT_START = 15555 +WEBRTC_PORT_END = 15579 +WEBRTC_PORTS_MAPPING = [PortMapping(port, port) for port in range(WEBRTC_PORT_START, WEBRTC_PORT_END + 1)] _RE_GROUP_WEBRTC = "local_webrtc_port" _RE_WEBRTC_SSH_TUNNEL_PATTERN = ( r"((.*-L\s)(?P<local_webrtc_port>\d+):127.0.0.1:%s)(.+%s)") _ADB_CONNECT_ARGS = "connect 127.0.0.1:%(adb_port)d" # Store the ports that vnc/adb are forwarded to, both are integers. ForwardedPorts = collections.namedtuple("ForwardedPorts", [constants.VNC_PORT, - constants.ADB_PORT]) + constants.ADB_PORT, + constants.FASTBOOT_PORT]) AVD_PORT_DICT = { constants.TYPE_GCE: ForwardedPorts(constants.GCE_VNC_PORT, - constants.GCE_ADB_PORT), + constants.GCE_ADB_PORT, + None), constants.TYPE_CF: ForwardedPorts(constants.CF_VNC_PORT, - constants.CF_ADB_PORT), + constants.CF_ADB_PORT, + constants.CF_FASTBOOT_PORT), constants.TYPE_GF: ForwardedPorts(constants.GF_VNC_PORT, - constants.GF_ADB_PORT), + constants.GF_ADB_PORT, + None), constants.TYPE_CHEEPS: ForwardedPorts(constants.CHEEPS_VNC_PORT, - constants.CHEEPS_ADB_PORT), - constants.TYPE_FVP: ForwardedPorts(None, constants.FVP_ADB_PORT), + constants.CHEEPS_ADB_PORT, + None), + constants.TYPE_FVP: ForwardedPorts(None, constants.FVP_ADB_PORT, None), } _VNC_BIN = "ssvnc" +# search_dirs and the files can be symbolic links. The -H flag makes the +# command skip the links except search_dirs. The returned files are unique. +_CMD_FIND_FILES = "find -H %(search_dirs)s -type f" _CMD_KILL = ["pkill", "-9", "-f"] _CMD_SG = "sg " _CMD_START_VNC = "%(bin)s vnc://127.0.0.1:%(port)d" @@ -839,7 +849,7 @@ def EstablishSshTunnel(ip_addr, rsa_key_file, ssh_user, """Create an ssh tunnel. Args: - ip_addr: String, use to build the adb & vnc tunnel between local + ip_addr: String, use to build the adb, fastboot & vnc tunnel between local and remote instance. rsa_key_file: String, Private key file path to use when creating the ssh tunnels. @@ -873,7 +883,7 @@ def EstablishWebRTCSshTunnel(ip_addr, webrtc_local_port, rsa_key_file, ssh_user, the port of the webrtc operator of the remote instance. Args: - ip_addr: String, use to build the adb & vnc tunnel between local + ip_addr: String, use to build the adb, fastboot & vnc tunnel between local and remote instance. webrtc_local_port: Integer, pick a free port as webrtc local port. rsa_key_file: String, Private key file path to use when creating @@ -909,7 +919,7 @@ def GetWebRTCServerPort(ip_addr, rsa_key_file, ssh_user, determine the WebRTC server port is 8443 or 1443. Args: - ip_addr: String, use to build the adb & vnc tunnel between local + ip_addr: String, use to build the adb, fastboot & vnc tunnel between local and remote instance. rsa_key_file: String, Private key file path to use when creating the ssh tunnels. @@ -965,29 +975,35 @@ def GetWebrtcPortFromSSHTunnel(ip): return None -# TODO(147337696): create ssh tunnels tear down as adb and vnc. +# TODO(147337696): create ssh tunnels tear down as adb, fastboot and vnc. # pylint: disable=too-many-locals -def AutoConnect(ip_addr, rsa_key_file, target_vnc_port, target_adb_port, - ssh_user, client_adb_port=None, extra_args_ssh_tunnel=None): +def AutoConnect(ip_addr, rsa_key_file, target_vnc_port, target_adb_port, target_fastboot_port, + ssh_user, client_adb_port=None, client_fastboot_port=None, + extra_args_ssh_tunnel=None): """Autoconnect to an AVD instance. Args: - ip_addr: String, use to build the adb & vnc tunnel between local + ip_addr: String, use to build the adb, fastboot & vnc tunnel between local and remote instance. rsa_key_file: String, Private key file path to use when creating the ssh tunnels. target_vnc_port: Integer of target vnc port number. target_adb_port: Integer of target adb port number. + target_fastboot_port: Integer of target fastboot port number. ssh_user: String of user login into the instance. client_adb_port: Integer, Specified adb port to establish connection. + client_fastboot_port: Integer, Specified fastboot port to establish connection. extra_args_ssh_tunnel: String, extra args for ssh tunnel connection. Returns: - NamedTuple of (vnc_port, adb_port) SSHTUNNEL of the connect, both are + NamedTuple of (vnc_port, adb_port, fastboot_port) SSHTUNNEL of the connect, both are integers. """ local_adb_port = client_adb_port or PickFreePort() port_mapping = [(local_adb_port, target_adb_port)] + local_fastboot_port = client_fastboot_port or PickFreePort() + if target_fastboot_port: + port_mapping.append((local_fastboot_port, target_fastboot_port)) local_free_vnc_port = None if target_vnc_port: local_free_vnc_port = PickFreePort() @@ -998,7 +1014,7 @@ def AutoConnect(ip_addr, rsa_key_file, target_vnc_port, target_adb_port, except subprocess.CalledProcessError as e: PrintColorString("\n%s\nFailed to create ssh tunnels, retry with '#acloud " "reconnect'." % e, TextColors.FAIL) - return ForwardedPorts(vnc_port=None, adb_port=None) + return ForwardedPorts(vnc_port=None, adb_port=None, fastboot_port=None) try: adb_connect_args = _ADB_CONNECT_ARGS % {"adb_port": local_adb_port} @@ -1008,7 +1024,31 @@ def AutoConnect(ip_addr, rsa_key_file, target_vnc_port, target_adb_port, "'#acloud reconnect'", TextColors.FAIL) return ForwardedPorts(vnc_port=local_free_vnc_port, - adb_port=local_adb_port) + adb_port=local_adb_port, + fastboot_port=local_fastboot_port) + + +def FindRemoteFiles(ssh_obj, search_dirs): + """Get all files, except symbolic links, under remote directories. + + Args: + ssh_obj: An Ssh object. + search_dirs: A list of strings, the remote directories. + + Returns: + A list of strings, the file paths. + """ + if not search_dirs: + return [] + ssh_cmd = (ssh_obj.GetBaseCmd(constants.SSH_BIN) + " " + + _CMD_FIND_FILES % {"search_dirs": " ".join(search_dirs)}) + proc = subprocess.run(ssh_cmd, shell=True, capture_output=True, + check=False) + if proc.stderr: + logger.debug("`%s` stderr: %s", ssh_cmd, proc.stderr.decode()) + if proc.stdout: + return proc.stdout.decode().splitlines() + return [] def GetAnswerFromList(answer_list, enable_choose_all=False): @@ -1574,7 +1614,7 @@ def GetCvdPorts(): Returns: - ForwardedPorts: vnc port and adb port. + ForwardedPorts: vnc, adb and fastboot ports. """ return AVD_PORT_DICT[constants.TYPE_CF] @@ -1587,6 +1627,8 @@ def SetCvdPorts(base_instance_num): """ offset = (base_instance_num or 1) - 1 AVD_PORT_DICT[constants.TYPE_CF] = ForwardedPorts( - constants.CF_VNC_PORT + offset, constants.CF_ADB_PORT + offset) + constants.CF_VNC_PORT + offset, + constants.CF_ADB_PORT + offset, + constants.CF_FASTBOOT_PORT + offset) # TODO: adjust WebRTC ports diff --git a/internal/lib/utils_test.py b/internal/lib/utils_test.py index ccbca330..7f467357 100644 --- a/internal/lib/utils_test.py +++ b/internal/lib/utils_test.py @@ -390,15 +390,17 @@ class UtilsTest(driver_test_lib.BaseDriverTest): fake_rsa_key_file = "/tmp/rsa_file" fake_target_vnc_port = 8888 target_adb_port = 9999 + target_fastboot_port = 7777 ssh_user = "fake_user" call_side_effect = subprocess.CalledProcessError(123, "fake", "fake error") - result = utils.ForwardedPorts(vnc_port=None, adb_port=None) + result = utils.ForwardedPorts(vnc_port=None, adb_port=None, fastboot_port=None) self.Patch(utils, "EstablishSshTunnel", side_effect=call_side_effect) self.assertEqual(result, utils.AutoConnect(fake_ip_addr, fake_rsa_key_file, fake_target_vnc_port, target_adb_port, + target_fastboot_port, ssh_user)) def testAutoConnectWithExtraArgs(self): @@ -407,6 +409,7 @@ class UtilsTest(driver_test_lib.BaseDriverTest): fake_rsa_key_file = "/tmp/rsa_file" fake_target_vnc_port = 8888 target_adb_port = 9999 + target_fastboot_port = 7777 ssh_user = "fake_user" fake_port = 12345 self.Patch(utils, "PickFreePort", return_value=fake_port) @@ -417,14 +420,17 @@ class UtilsTest(driver_test_lib.BaseDriverTest): rsa_key_file=fake_rsa_key_file, target_vnc_port=fake_target_vnc_port, target_adb_port=target_adb_port, + target_fastboot_port=target_fastboot_port, ssh_user=ssh_user, client_adb_port=fake_port, + client_fastboot_port=fake_port, extra_args_ssh_tunnel=extra_args_ssh_tunnel) mock_establish_ssh_tunnel.assert_called_with( fake_ip_addr, fake_rsa_key_file, ssh_user, [utils.PortMapping(fake_port, target_adb_port), + utils.PortMapping(fake_port, target_fastboot_port), utils.PortMapping(fake_port, fake_target_vnc_port)], extra_args_ssh_tunnel) mock_execute_command.assert_called_with( @@ -438,10 +444,7 @@ class UtilsTest(driver_test_lib.BaseDriverTest): fake_webrtc_local_port = 12345 self.Patch(utils, "GetWebRTCServerPort", return_value=8443) mock_establish_ssh_tunnel = self.Patch(utils, "EstablishSshTunnel") - fake_port_mapping = [utils.PortMapping(15550, 15550), - utils.PortMapping(15551, 15551), - utils.PortMapping(15552, 15552), - utils.PortMapping(12345, 8443)] + fake_port_mapping = [utils.PortMapping(port, port) for port in range(15555, 15579 + 1)] + [utils.PortMapping(12345, 8443)] utils.EstablishWebRTCSshTunnel( ip_addr=fake_ip_addr, rsa_key_file=fake_rsa_key_file, @@ -499,6 +502,28 @@ class UtilsTest(driver_test_lib.BaseDriverTest): webrtc_ports = utils.GetWebrtcPortFromSSHTunnel("1.1.1.1") self.assertEqual(12345, webrtc_ports) + @mock.patch("acloud.internal.lib.utils.subprocess") + def testFindRemoteFiles(self, mock_subprocess): + """Test FindRemoteFiles.""" + mock_ssh = mock.Mock() + + paths = utils.FindRemoteFiles(mock_ssh, []) + mock_subprocess.run.assert_not_called() + self.assertEqual([], paths) + + mock_ssh.GetBaseCmd.return_value = "mock_ssh" + mock_subprocess.run.return_value = mock.Mock( + stderr=b'stderr', stdout=b'file1\nfile2\n') + paths = utils.FindRemoteFiles(mock_ssh, ["dir1", "dir2"]) + self.assertEqual(["file1", "file2"], paths) + mock_subprocess.run.assert_called_with( + 'mock_ssh find -H dir1 dir2 -type f', + shell=True, capture_output=True, check=False) + + mock_subprocess.run.return_value = mock.Mock(stderr=None, stdout=b'') + paths = utils.FindRemoteFiles(mock_ssh, ["dir1", "dir2"]) + self.assertEqual([], paths) + # pylint: disable=protected-access, no-member def testCleanupSSVncviwer(self): """test cleanup ssvnc viewer.""" @@ -564,6 +589,7 @@ class UtilsTest(driver_test_lib.BaseDriverTest): """test base_instance_num.""" utils.SetCvdPorts(2) self.assertEqual(utils.GetCvdPorts().adb_port, 6521) + self.assertEqual(utils.GetCvdPorts().fastboot_port, 7521) self.assertEqual(utils.GetCvdPorts().vnc_port, 6445) utils.SetCvdPorts(None) diff --git a/internal/proto/internal_config.proto b/internal/proto/internal_config.proto index 0bfd80a2..1646c8ae 100755 --- a/internal/proto/internal_config.proto +++ b/internal/proto/internal_config.proto @@ -102,7 +102,7 @@ message InternalConfig { // [CVD only] The kernel build target: "kernel". This is unlikely to change. optional string kernel_build_target = 16; - // [GOLDFISH only] The emulator build target: "sdk_tools_linux". + // [GOLDFISH only] The emulator build target: "emulator-linux_x64_nolocationui". // It's very unlikely that this will ever change. optional string emulator_build_target = 17; diff --git a/internal/proto/user_config.proto b/internal/proto/user_config.proto index 2d6ffdcb..33904b6e 100755 --- a/internal/proto/user_config.proto +++ b/internal/proto/user_config.proto @@ -127,4 +127,7 @@ message UserConfig { // Storage options of created GCP instance, e.g. pd-standard, pd-ssd. optional string disk_type = 36; + + // [CVD only] Ssh connect with hostname. + optional bool connect_hostname = 37; } diff --git a/list/instance.py b/list/instance.py index 5b238357..5fdf776d 100644 --- a/list/instance.py +++ b/list/instance.py @@ -35,7 +35,7 @@ import re import subprocess import tempfile -# pylint: disable=import-error +# pylint: disable=import-error,too-many-lines import dateutil.parser import dateutil.tz @@ -46,21 +46,26 @@ from acloud.internal.lib import utils from acloud.internal.lib.adb_tools import AdbTools from acloud.internal.lib.local_instance_lock import LocalInstanceLock from acloud.internal.lib.gcompute_client import GetInstanceIP +from acloud.internal.lib.gcompute_client import GetGCEHostName logger = logging.getLogger(__name__) _ACLOUD_CVD_TEMP = os.path.join(tempfile.gettempdir(), "acloud_cvd_temp") -_CVD_CONFIG_FOLDER = "%(cvd_runtime)s/instances/cvd-%(id)d" -_CVD_LOG_FOLDER = _CVD_CONFIG_FOLDER + "/logs" _CVD_RUNTIME_FOLDER_NAME = "cuttlefish_runtime" _CVD_BIN = "cvd" _CVD_BIN_FOLDER = "host_bins/bin" _CVD_STATUS_BIN = "cvd_status" -_CVD_SERVER = "cvd_server" _CVD_STOP_ERROR_KEYWORDS = "cvd_internal_stop E" # Default timeout 30 secs for cvd commands. _CVD_TIMEOUT = 30 +# Keywords read from runtime config. +_ADB_HOST_PORT = "adb_host_port" +_FASTBOOT_HOST_PORT = "fastboot_host_port" +# Keywords read from the output of "cvd status". +_DISPLAYS = "displays" +_WEBRTC_PORT = "webrtc_port" +_ADB_SERIAL = "adb_serial" _INSTANCE_ASSEMBLY_DIR = "cuttlefish_assembly" _LOCAL_INSTANCE_NAME_FORMAT = "local-instance-%(id)d" _LOCAL_INSTANCE_NAME_PATTERN = re.compile(r"^local-instance-(?P<id>\d+)$") @@ -68,20 +73,24 @@ _ACLOUDWEB_INSTANCE_START_STRING = "cf-" _MSG_UNABLE_TO_CALCULATE = "Unable to calculate" _NO_ANDROID_ENV = "android source not available" _RE_GROUP_ADB = "local_adb_port" +_RE_GROUP_FASTBOOT = "local_fastboot_port" _RE_GROUP_VNC = "local_vnc_port" _RE_SSH_TUNNEL_PATTERN = (r"((.*\s*-L\s)(?P<%s>\d+):127.0.0.1:%s)" r"((.*\s*-L\s)(?P<%s>\d+):127.0.0.1:%s)" - r"(.+%s)") + r"((.*\s*-L\s)(?P<%s>\d+):127.0.0.1:%s)" + r"(.+(%s|%s))") _RE_TIMEZONE = re.compile(r"^(?P<time>[0-9\-\.:T]*)(?P<timezone>[+-]\d+:\d+)$") _RE_DEVICE_INFO = re.compile(r"(?s).*(?P<device_info>[{][\s\w\W]+})") _COMMAND_PS_LAUNCH_CVD = ["ps", "-wweo", "lstart,cmd"] _RE_RUN_CVD = re.compile(r"(?P<date_str>^[^/]+)(.*run_cvd)") +_X_RES = "x_res" +_Y_RES = "y_res" +_DPI = "dpi" _DISPLAY_STRING = "%(x_res)sx%(y_res)s (%(dpi)s)" _RE_ZONE = re.compile(r".+/zones/(?P<zone>.+)$") +_RE_PROJECT = re.compile(r".+/projects/(?P<project>.+)/zones/.+$") _LOCAL_ZONE = "local" -_FULL_NAME_STRING = ("device serial: %(device_serial)s (%(instance_name)s) " - "elapsed time: %(elapsed_time)s") _INDENT = " " * 3 LocalPorts = collections.namedtuple("LocalPorts", [constants.VNC_PORT, constants.ADB_PORT]) @@ -221,40 +230,25 @@ def GetLocalInstanceRuntimeDir(local_instance_id): _CVD_RUNTIME_FOLDER_NAME) -def GetLocalInstanceLogDir(local_instance_id): - """Get local instance log directory. - - Cuttlefish log directories are different between versions: - - In Android 10, the logs are in `<runtime_dir>`. - - In Android 11, the logs are in `<runtime_dir>.<id>`. - `<runtime_dir>` is a symbolic link to `<runtime_dir>.<id>`. - - In the latest version, the logs are in - `<runtime_dir>/instances/cvd-<id>/logs`. - `<runtime_dir>_runtime` and `<runtime_dir>.<id>` are symbolic links to - `<runtime_dir>/instances/cvd-<id>`. - - This method looks for `<runtime_dir>/instances/cvd-<id>/logs` which is the - latest known location. If it doesn't exist, this method returns - `<runtime_dir>` which is compatible with the old versions. +def GetCuttleFishLocalInstances(cf_config_path): + """Get all instances information from cf runtime config. Args: - local_instance_id: Integer of instance id. + cf_config_path: String, path to the cf runtime config. Returns: - The path to the log directory. + List of LocalInstance object. """ - runtime_dir = GetLocalInstanceRuntimeDir(local_instance_id) - log_dir = _CVD_LOG_FOLDER % {"cvd_runtime": runtime_dir, - "id": local_instance_id} - return log_dir if os.path.isdir(log_dir) else runtime_dir + cf_runtime_cfg = cvd_runtime_config.CvdRuntimeConfig(cf_config_path) + local_instances = [] + for ins_id in cf_runtime_cfg.instance_ids: + local_instances.append(LocalInstance(cf_config_path, ins_id)) + return local_instances def _GetCurrentLocalTime(): """Return a datetime object for current time in local time zone.""" - return datetime.datetime.now(dateutil.tz.tzlocal()) + return datetime.datetime.now(dateutil.tz.tzlocal()).replace(microsecond=0) def _GetElapsedTime(start_time): @@ -272,14 +266,39 @@ def _GetElapsedTime(start_time): # Check start_time has timezone or not. If timezone can't be found, # use local timezone to get elapsed time. if match: - return _GetCurrentLocalTime() - dateutil.parser.parse(start_time) + return _GetCurrentLocalTime() - dateutil.parser.parse( + start_time).replace(microsecond=0) return _GetCurrentLocalTime() - dateutil.parser.parse( - start_time).replace(tzinfo=dateutil.tz.tzlocal()) + start_time).replace(tzinfo=dateutil.tz.tzlocal(), microsecond=0) except ValueError: logger.debug(("Can't parse datetime string(%s)."), start_time) return _MSG_UNABLE_TO_CALCULATE +def _GetDeviceFullName(device_serial, instance_name, elapsed_time, + webrtc_device_id=None): + """Get the full name of device. + + The full name is composed with device serial, webrtc device id, instance + name, and elapsed_time. + + Args: + device_serial: String of device serial. e.g. 127.0.0.1:6520 + instance_name: String of instance name. + elapsed time: String of elapsed time. + webrtc_device_id: String of webrtc device id. + + Returns: + String of device full name. + """ + if webrtc_device_id: + return (f"device serial: {device_serial} {webrtc_device_id} " + f"({instance_name}) elapsed time: {elapsed_time}") + + return (f"device serial: {device_serial} ({instance_name}) " + f"elapsed time: {elapsed_time}") + + def _IsProcessRunning(process): """Check if this process is running. @@ -301,8 +320,8 @@ class Instance(object): # pylint: disable=too-many-locals def __init__(self, name, fullname, display, ip, status=None, adb_port=None, - vnc_port=None, ssh_tunnel_is_connected=None, createtime=None, - elapsed_time=None, avd_type=None, avd_flavor=None, + fastboot_port=None, vnc_port=None, ssh_tunnel_is_connected=None, + createtime=None, elapsed_time=None, avd_type=None, avd_flavor=None, is_local=False, device_information=None, zone=None, webrtc_port=None, webrtc_forward_port=None): self._name = name @@ -310,8 +329,9 @@ class Instance(object): self._status = status self._display = display # Resolution and dpi self._ip = ip - self._adb_port = adb_port # adb port which is forwarding to remote - self._vnc_port = vnc_port # vnc port which is forwarding to remote + self._adb_port = adb_port # adb port which is forwarding to remote + self._fastboot_port = fastboot_port # fastboot port which is forwarding to remote + self._vnc_port = vnc_port # vnc port which is forwarding to remote self._webrtc_port = webrtc_port self._webrtc_forward_port = webrtc_forward_port # True if ssh tunnel is still connected @@ -385,6 +405,8 @@ class Instance(object): return constants.INS_KEY_VNC if self._adb_port: return constants.INS_KEY_ADB + if self._fastboot_port: + return constants.INS_KEY_FASTBOOT return None @property @@ -443,6 +465,11 @@ class Instance(object): return self._adb_port @property + def fastboot_port(self): + """Return fastboot_port.""" + return self._fastboot_port + + @property def vnc_port(self): """Return vnc_port.""" return self._vnc_port @@ -470,33 +497,45 @@ class Instance(object): class LocalInstance(Instance): """Class to store data of local cuttlefish instance.""" - def __init__(self, cf_config_path): + def __init__(self, cf_config_path, ins_id=None): """Initialize a localInstance object. Args: cf_config_path: String, path to the cf runtime config. + ins_id: Integer, the id to specify the instance information. """ self._cf_runtime_cfg = cvd_runtime_config.CvdRuntimeConfig(cf_config_path) self._instance_dir = self._cf_runtime_cfg.instance_dir self._virtual_disk_paths = self._cf_runtime_cfg.virtual_disk_paths - self._local_instance_id = int(self._cf_runtime_cfg.instance_id) - display = _DISPLAY_STRING % {"x_res": self._cf_runtime_cfg.x_res, - "y_res": self._cf_runtime_cfg.y_res, - "dpi": self._cf_runtime_cfg.dpi} + self._local_instance_id = int(ins_id or self._cf_runtime_cfg.instance_id) + self._instance_home = GetLocalInstanceHomeDir(self._local_instance_id) + if self._cf_runtime_cfg.root_dir: + self._instance_home = os.path.dirname(self._cf_runtime_cfg.root_dir) + + ins_info = self._cf_runtime_cfg.instances.get(ins_id, {}) + adb_port = ins_info.get(_ADB_HOST_PORT) or self._cf_runtime_cfg.adb_port + fastboot_port = ins_info.get(_FASTBOOT_HOST_PORT) or self._cf_runtime_cfg.fastboot_port + webrtc_device_id = (ins_info.get(constants.INS_KEY_WEBRTC_DEVICE_ID) + or f"cvd-{self._local_instance_id}") + adb_serial = f"0.0.0.0:{adb_port}" + display = [] + for display_config in self._cf_runtime_cfg.display_configs: + display.append(_DISPLAY_STRING % {"x_res": display_config.get(_X_RES), + "y_res": display_config.get(_Y_RES), + "dpi": display_config.get(_DPI)}) # TODO(143063678), there's no createtime info in # cuttlefish_config.json so far. - name = GetLocalInstanceName(self._local_instance_id) - fullname = (_FULL_NAME_STRING % - {"device_serial": "0.0.0.0:%s" % self._cf_runtime_cfg.adb_port, - "instance_name": name, - "elapsed_time": None}) - adb_device = AdbTools(device_serial="0.0.0.0:%s" % self._cf_runtime_cfg.adb_port) webrtc_port = local_image_local_instance.LocalImageLocalInstance.GetWebrtcSigServerPort( self._local_instance_id) - cvd_fleet_info = self.GetDevidInfoFromCvdFleet() - if cvd_fleet_info: - display = cvd_fleet_info.get("displays") + cvd_status_info = self._GetDevidInfoFromCvdStatus() + if cvd_status_info: + display = cvd_status_info.get(_DISPLAYS) + webrtc_port = int(cvd_status_info.get(_WEBRTC_PORT)) + adb_serial = cvd_status_info.get(_ADB_SERIAL) + name = GetLocalInstanceName(self._local_instance_id) + fullname = _GetDeviceFullName(adb_serial, name, None, webrtc_device_id) + adb_device = AdbTools(device_serial=adb_serial) device_information = None if adb_device.IsAdbConnected(): device_information = adb_device.device_information @@ -504,7 +543,8 @@ class LocalInstance(Instance): super().__init__( name=name, fullname=fullname, display=display, ip="0.0.0.0", status=constants.INS_STATUS_RUNNING, - adb_port=self._cf_runtime_cfg.adb_port, + adb_port=adb_port, + fastboot_port=fastboot_port, vnc_port=self._cf_runtime_cfg.vnc_port, createtime=None, elapsed_time=None, avd_type=constants.TYPE_CF, is_local=True, device_information=device_information, @@ -522,43 +562,43 @@ class LocalInstance(Instance): os.environ with cuttlefish variables updated. """ cvd_env = os.environ.copy() + cvd_env[constants.ENV_ANDROID_HOST_OUT] = os.path.dirname( + self._cf_runtime_cfg.cvd_tools_path) cvd_env[constants.ENV_ANDROID_SOONG_HOST_OUT] = os.path.dirname( self._cf_runtime_cfg.cvd_tools_path) cvd_env[constants.ENV_CUTTLEFISH_CONFIG_FILE] = self._cf_runtime_cfg.config_path - cvd_env[constants.ENV_CVD_HOME] = GetLocalInstanceHomeDir(self._local_instance_id) + cvd_env[constants.ENV_CVD_HOME] = self._instance_home cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str(self._local_instance_id) return cvd_env - def GetDevidInfoFromCvdFleet(self): - """Get device information from 'cvd fleet'. + def _GetDevidInfoFromCvdStatus(self): + """Get device information from 'cvd status'. - Execute 'cvd fleet' cmd to get device information. + Execute 'cvd status --print -instance_name=name' cmd to get devices + information. Returns - Output of 'cvd fleet'. None for fail to run 'cvd fleet'. + Output of 'cvd status'. None for fail to run 'cvd status'. """ - ins_home_dir = GetLocalInstanceHomeDir(self._local_instance_id) try: - cvd_tool = os.path.join(ins_home_dir, _CVD_BIN_FOLDER, _CVD_BIN) - cvd_fleet_cmd = f"{cvd_tool} fleet" + cvd_tool = os.path.join(self._instance_home, _CVD_BIN_FOLDER, _CVD_BIN) + ins_name = f"cvd-{self._local_instance_id}" + cvd_status_cmd = f"{cvd_tool} status -print -instance_name={ins_name}" if not os.path.exists(cvd_tool): logger.warning("Cvd tools path doesn't exist:%s", cvd_tool) return None - if not _IsProcessRunning(_CVD_SERVER): - logger.warning("The %s is not active.", _CVD_SERVER) - return None - logger.debug("Running cmd [%s] to get device info.", cvd_fleet_cmd) - process = subprocess.Popen(cvd_fleet_cmd, shell=True, text=True, + logger.debug("Running cmd [%s] to get device info.", cvd_status_cmd) + process = subprocess.Popen(cvd_status_cmd, shell=True, text=True, env=self._GetCvdEnv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, _ = process.communicate(timeout=_CVD_TIMEOUT) - logger.debug("Output of cvd fleet: %s", stdout) + logger.debug("Output of cvd status: %s", stdout) return json.loads(self._ParsingCvdFleetOutput(stdout)) except (subprocess.CalledProcessError, subprocess.TimeoutExpired, json.JSONDecodeError) as error: - logger.error("Failed to run 'cvd fleet': %s", str(error)) - return None + logger.error("Failed to run 'cvd status': %s", str(error)) + return None @staticmethod def _ParsingCvdFleetOutput(output): @@ -743,9 +783,7 @@ class LocalGoldfishInstance(Instance): elapsed_time = _GetElapsedTime(create_time) if create_time else None - fullname = _FULL_NAME_STRING % {"device_serial": self.device_serial, - "instance_name": name, - "elapsed_time": elapsed_time} + fullname = _GetDeviceFullName(self.device_serial, name, elapsed_time) if x_res and y_res and dpi: display = _DISPLAY_STRING % {"x_res": x_res, "y_res": y_res, @@ -880,12 +918,15 @@ class RemoteInstance(Instance): instance_ip = GetInstanceIP(gce_instance) ip = instance_ip.external or instance_ip.internal + project = self._GetProjectName(gce_instance.get(constants.INS_KEY_ZONE)) + hostname = GetGCEHostName(project, name, zone) # Get metadata, webrtc_port will be removed if "cvd fleet" show it. display = None avd_type = None avd_flavor = None webrtc_port = None + webrtc_device_id = None for metadata in gce_instance.get("metadata", {}).get("items", []): key = metadata["key"] value = metadata["value"] @@ -897,18 +938,22 @@ class RemoteInstance(Instance): avd_flavor = value elif key == constants.INS_KEY_WEBRTC_PORT: webrtc_port = value + elif key == constants.INS_KEY_WEBRTC_DEVICE_ID: + webrtc_device_id = value # TODO(176884236): Insert avd information into metadata of instance. if not avd_type and name.startswith(_ACLOUDWEB_INSTANCE_START_STRING): avd_type = constants.TYPE_CF # Find ssl tunnel info. adb_port = None + fastboot_port = None vnc_port = None webrtc_forward_port = None device_information = None if ip: - forwarded_ports = self.GetAdbVncPortFromSSHTunnel(ip, avd_type) + forwarded_ports = self.GetForwardedPortsFromSSHTunnel(ip, hostname, avd_type) adb_port = forwarded_ports.adb_port + fastboot_port = forwarded_ports.fastboot_port vnc_port = forwarded_ports.vnc_port ssh_tunnel_is_connected = adb_port is not None webrtc_forward_port = utils.GetWebrtcPortFromSSHTunnel(ip) @@ -916,26 +961,20 @@ class RemoteInstance(Instance): adb_device = AdbTools(adb_port) if adb_device.IsAdbConnected(): device_information = adb_device.device_information - fullname = (_FULL_NAME_STRING % - {"device_serial": "127.0.0.1:%d" % adb_port, - "instance_name": name, - "elapsed_time": elapsed_time}) + fullname = _GetDeviceFullName("127.0.0.1:%d" % adb_port, name, + elapsed_time, webrtc_device_id) else: - fullname = (_FULL_NAME_STRING % - {"device_serial": "not connected", - "instance_name": name, - "elapsed_time": elapsed_time}) + fullname = _GetDeviceFullName("not connected", name, + elapsed_time, webrtc_device_id) # If instance is terminated, its ip is None. else: ssh_tunnel_is_connected = False - fullname = (_FULL_NAME_STRING % - {"device_serial": "terminated", - "instance_name": name, - "elapsed_time": elapsed_time}) + fullname = _GetDeviceFullName("terminated", name, elapsed_time, + webrtc_device_id) super().__init__( name=name, fullname=fullname, display=display, ip=ip, status=status, - adb_port=adb_port, vnc_port=vnc_port, + adb_port=adb_port, fastboot_port=fastboot_port, vnc_port=vnc_port, ssh_tunnel_is_connected=ssh_tunnel_is_connected, createtime=create_time, elapsed_time=elapsed_time, avd_type=avd_type, avd_flavor=avd_flavor, is_local=False, @@ -965,34 +1004,60 @@ class RemoteInstance(Instance): return None @staticmethod - def GetAdbVncPortFromSSHTunnel(ip, avd_type): + def _GetProjectName(zone_info): + """Get the project name from the zone information of gce instance. + + Zone information is like: + "https://www.googleapis.com/compute/v1/projects/project/zones/us-central1-c" + We want to get "project" as project name. + + Args: + zone_info: String, zone information of gce instance. + + Returns: + Project name of gce instance. None if project name can't find. + """ + project_match = _RE_PROJECT.match(zone_info) + if project_match: + return project_match.group("project") + + logger.debug("Can't get project name from %s.", zone_info) + return None + + @staticmethod + def GetForwardedPortsFromSSHTunnel(ip, hostname, avd_type): """Get forwarding adb and vnc port from ssh tunnel. Args: ip: String, ip address. + hostname: String, hostname of GCE instance. avd_type: String, the AVD type. Returns: - NamedTuple ForwardedPorts(vnc_port, adb_port) holding the ports + NamedTuple ForwardedPorts(vnc_port, adb_port, fastboot_port) holding the ports used in the ssh forwarded call. Both fields are integers. """ if avd_type not in utils.AVD_PORT_DICT: - return utils.ForwardedPorts(vnc_port=None, adb_port=None) + return utils.ForwardedPorts(vnc_port=None, adb_port=None, fastboot_port=None) default_vnc_port = utils.AVD_PORT_DICT[avd_type].vnc_port default_adb_port = utils.AVD_PORT_DICT[avd_type].adb_port + default_fastboot_port = utils.AVD_PORT_DICT[avd_type].fastboot_port # TODO(165888525): Align the SSH tunnel for the order of adb port and # vnc port. re_pattern = re.compile(_RE_SSH_TUNNEL_PATTERN % (_RE_GROUP_ADB, default_adb_port, - _RE_GROUP_VNC, default_vnc_port, ip)) + _RE_GROUP_FASTBOOT, default_fastboot_port, + _RE_GROUP_VNC, default_vnc_port, ip, hostname)) adb_port = None + fastboot_port = None vnc_port = None process_output = utils.CheckOutput(constants.COMMAND_PS) for line in process_output.splitlines(): match = re_pattern.match(line) if match: adb_port = int(match.group(_RE_GROUP_ADB)) + fastboot_port = int(match.group(_RE_GROUP_FASTBOOT)) vnc_port = int(match.group(_RE_GROUP_VNC)) break @@ -1000,4 +1065,6 @@ class RemoteInstance(Instance): "IP:%s, forwarding (adb:%s, vnc:%s)"), ip, adb_port, vnc_port) - return utils.ForwardedPorts(vnc_port=vnc_port, adb_port=adb_port) + return utils.ForwardedPorts(vnc_port=vnc_port, + adb_port=adb_port, + fastboot_port=fastboot_port) diff --git a/list/instance_test.py b/list/instance_test.py index 8475653d..078fddab 100644 --- a/list/instance_test.py +++ b/list/instance_test.py @@ -30,18 +30,24 @@ import dateutil.tz from acloud.internal import constants from acloud.internal.lib import cvd_runtime_config from acloud.internal.lib import driver_test_lib +from acloud.internal.lib import gcompute_client from acloud.internal.lib import utils from acloud.internal.lib.adb_tools import AdbTools from acloud.list import instance +ForwardedPorts = collections.namedtuple("ForwardedPorts", + [constants.VNC_PORT, + constants.ADB_PORT, + constants.FASTBOOT_PORT]) + class InstanceTest(driver_test_lib.BaseDriverTest): """Test instance.""" PS_SSH_TUNNEL = ("/fake_ps_1 --fake arg \n" "/fake_ps_2 --fake arg \n" "/usr/bin/ssh -i ~/.ssh/acloud_rsa " "-o UserKnownHostsFile=/dev/null " - "-o StrictHostKeyChecking=no -L 54321:127.0.0.1:6520 " + "-o StrictHostKeyChecking=no -L 54321:127.0.0.1:6520 -L 6789:127.0.0.1:7520" "-L 12345:127.0.0.1:6444 -N -f -l user 1.1.1.1").encode() GCE_INSTANCE = { constants.INS_KEY_NAME: "fake_ins_name", @@ -65,15 +71,15 @@ class InstanceTest(driver_test_lib.BaseDriverTest): """Create a mock CvdRuntimeConfig.""" return mock.MagicMock( instance_id=2, - x_res=1080, - y_res=1920, - dpi=480, + display_configs=[{'dpi': 480, 'x_res': 1080, 'y_res': 1920}], instance_dir="fake_instance_dir", adb_port=6521, vnc_port=6445, adb_ip_port="127.0.0.1:6521", cvd_tools_path="fake_cvd_tools_path", config_path="fake_config_path", + instances={}, + root_dir="/tmp/acloud_cvd_temp/local-instance-2/cuttlefish_runtime" ) @mock.patch("acloud.list.instance.AdbTools") @@ -84,15 +90,16 @@ class InstanceTest(driver_test_lib.BaseDriverTest): mock_adb_tools.return_value = mock_adb_tools_object self.Patch(cvd_runtime_config, "CvdRuntimeConfig", return_value=self._MockCvdRuntimeConfig()) - self.Patch(instance.LocalInstance, "GetDevidInfoFromCvdFleet", + self.Patch(instance.LocalInstance, "_GetDevidInfoFromCvdStatus", return_value=None) local_instance = instance.LocalInstance("fake_config_path") self.assertEqual("local-instance-2", local_instance.name) self.assertEqual(True, local_instance.islocal) - self.assertEqual("1080x1920 (480)", local_instance.display) - expected_full_name = ("device serial: 0.0.0.0:%s (%s) elapsed time: %s" + self.assertEqual(["1080x1920 (480)"], local_instance.display) + expected_full_name = ("device serial: 0.0.0.0:%s %s (%s) elapsed time: %s" % ("6521", + "cvd-2", "local-instance-2", "None")) self.assertEqual(expected_full_name, local_instance.fullname) @@ -151,7 +158,7 @@ class InstanceTest(driver_test_lib.BaseDriverTest): mock_adb_tools.return_value = mock_adb_tools_object self.Patch(utils, "AddUserGroupsToCmd", side_effect=lambda cmd, groups: cmd) - self.Patch(instance.LocalInstance, "GetDevidInfoFromCvdFleet", + self.Patch(instance.LocalInstance, "_GetDevidInfoFromCvdStatus", return_value=None) mock_check_call = self.Patch(subprocess, "check_call") mock_check_output = self.Patch( @@ -166,6 +173,7 @@ class InstanceTest(driver_test_lib.BaseDriverTest): "CUTTLEFISH_INSTANCE": "2", "HOME": "/tmp/acloud_cvd_temp/local-instance-2", "CUTTLEFISH_CONFIG_FILE": "fake_config_path", + "ANDROID_HOST_OUT": "", "ANDROID_SOONG_HOST_OUT": "", } mock_check_output.assert_called_with( @@ -248,36 +256,46 @@ class InstanceTest(driver_test_lib.BaseDriverTest): datetime.timedelta(hours=2), instance._GetElapsedTime(start_time)) # pylint: disable=protected-access - def testGetAdbVncPortFromSSHTunnel(self): + def testGetForwardedPortsFromSSHTunnel(self): """"Test Get forwarding adb and vnc port from ssh tunnel.""" self.Patch(subprocess, "check_output", return_value=self.PS_SSH_TUNNEL) self.Patch(instance, "_GetElapsedTime", return_value="fake_time") self.Patch(instance.RemoteInstance, "_GetZoneName", return_value="fake_zone") + self.Patch(instance.RemoteInstance, + "_GetProjectName", + return_value="fake_project") + self.Patch(gcompute_client, "GetGCEHostName", return_value="fake_hostname") forwarded_ports = instance.RemoteInstance( - mock.MagicMock()).GetAdbVncPortFromSSHTunnel( - "1.1.1.1", constants.TYPE_CF) + mock.MagicMock()).GetForwardedPortsFromSSHTunnel( + "1.1.1.1", "fake_hostname", constants.TYPE_CF) self.assertEqual(54321, forwarded_ports.adb_port) + self.assertEqual(6789, forwarded_ports.fastboot_port) self.assertEqual(12345, forwarded_ports.vnc_port) # If avd_type is undefined in utils.AVD_PORT_DICT. forwarded_ports = instance.RemoteInstance( - mock.MagicMock()).GetAdbVncPortFromSSHTunnel( - "1.1.1.1", "undefined_avd_type") + mock.MagicMock()).GetForwardedPortsFromSSHTunnel( + "1.1.1.1", "fake_hostname", "undefined_avd_type") self.assertEqual(None, forwarded_ports.adb_port) + self.assertEqual(None, forwarded_ports.fastboot_port) self.assertEqual(None, forwarded_ports.vnc_port) # pylint: disable=protected-access def testProcessGceInstance(self): """"Test process instance detail.""" fake_adb = 123456 + fake_fastboot = 654321 fake_vnc = 654321 - forwarded_ports = collections.namedtuple("ForwardedPorts", - [constants.VNC_PORT, - constants.ADB_PORT]) + + self.Patch(instance.RemoteInstance, + "_GetProjectName", + return_value="fake_project") self.Patch( instance.RemoteInstance, - "GetAdbVncPortFromSSHTunnel", - return_value=forwarded_ports(vnc_port=fake_vnc, adb_port=fake_adb)) + "GetForwardedPortsFromSSHTunnel", + return_value=ForwardedPorts(vnc_port=fake_vnc, + adb_port=fake_adb, + fastboot_port=fake_fastboot)) self.Patch(utils, "GetWebrtcPortFromSSHTunnel", return_value="fake_webrtc_port") self.Patch(instance, "_GetElapsedTime", return_value="fake_time") @@ -307,8 +325,8 @@ class InstanceTest(driver_test_lib.BaseDriverTest): # test ssh_tunnel_is_connected will be false if ssh tunnel connection is not found self.Patch( instance.RemoteInstance, - "GetAdbVncPortFromSSHTunnel", - return_value=forwarded_ports(vnc_port=None, adb_port=None)) + "GetForwardedPortsFromSSHTunnel", + return_value=ForwardedPorts(vnc_port=None, adb_port=None, fastboot_port=None)) instance_info = instance.RemoteInstance(self.GCE_INSTANCE) self.assertFalse(instance_info.ssh_tunnel_is_connected) expected_full_name = "device serial: not connected (%s) elapsed time: %s" % ( @@ -318,14 +336,17 @@ class InstanceTest(driver_test_lib.BaseDriverTest): def testInstanceSummary(self): """Test instance summary.""" fake_adb = 123456 + fake_fastboot = 654321 fake_vnc = 654321 - forwarded_ports = collections.namedtuple("ForwardedPorts", - [constants.VNC_PORT, - constants.ADB_PORT]) + self.Patch(instance.RemoteInstance, + "_GetProjectName", + return_value="fake_project") self.Patch( instance.RemoteInstance, - "GetAdbVncPortFromSSHTunnel", - return_value=forwarded_ports(vnc_port=fake_vnc, adb_port=fake_adb)) + "GetForwardedPortsFromSSHTunnel", + return_value=ForwardedPorts(vnc_port=fake_vnc, + adb_port=fake_adb, + fastboot_port=fake_fastboot)) self.Patch(utils, "GetWebrtcPortFromSSHTunnel", return_value=8443) self.Patch(instance, "_GetElapsedTime", return_value="fake_time") self.Patch(AdbTools, "IsAdbConnected", return_value=True) @@ -351,8 +372,8 @@ class InstanceTest(driver_test_lib.BaseDriverTest): self.Patch( instance.RemoteInstance, - "GetAdbVncPortFromSSHTunnel", - return_value=forwarded_ports(vnc_port=None, adb_port=None)) + "GetForwardedPortsFromSSHTunnel", + return_value=ForwardedPorts(vnc_port=None, adb_port=None, fastboot_port=None)) self.Patch(instance, "_GetElapsedTime", return_value="fake_time") self.Patch(AdbTools, "IsAdbConnected", return_value=False) remote_instance = instance.RemoteInstance(self.GCE_INSTANCE) @@ -381,6 +402,13 @@ class InstanceTest(driver_test_lib.BaseDriverTest): zone_info = "v1/projects/project/us-central1-c" self.assertEqual(instance.RemoteInstance._GetZoneName(zone_info), None) + def testGetProjectName(self): + """Test GetProjectName.""" + zone_info = "v1/projects/fake_project/zones/us-central1-c" + expected_result = "fake_project" + self.assertEqual(instance.RemoteInstance._GetProjectName(zone_info), + expected_result) + def testGetLocalInstanceConfig(self): """Test GetLocalInstanceConfig.""" self.Patch(instance, "GetLocalInstanceHomeDir", @@ -395,18 +423,6 @@ class InstanceTest(driver_test_lib.BaseDriverTest): self.assertEqual( instance.GetLocalInstanceConfig(instance_id), expected_result) - def testGetLocalInstanceLogDir(self): - """Test GetLocalInstanceLogDir.""" - self.Patch(instance, "GetLocalInstanceRuntimeDir", - return_value="ins_runtime_dir") - self.Patch(os.path, "isdir", return_value=False) - self.assertEqual(instance.GetLocalInstanceLogDir(1), "ins_runtime_dir") - - expected_path = "ins_runtime_dir/instances/cvd-1/logs" - self.Patch(os.path, "isdir", - side_effect=lambda path: path == expected_path) - self.assertEqual(instance.GetLocalInstanceLogDir(1), expected_path) - def testGetAutoConnect(self): """Test GetAutoConnect.""" name = "ins_name" @@ -425,9 +441,40 @@ class InstanceTest(driver_test_lib.BaseDriverTest): name, fullname, display, ip, adb_port=6666) self.assertEqual(ins_webrtc._GetAutoConnect(), constants.INS_KEY_ADB) + ins_webrtc = instance.Instance( + name, fullname, display, ip, fastboot_port=6666) + self.assertEqual(ins_webrtc._GetAutoConnect(), constants.INS_KEY_FASTBOOT) + ins_webrtc = instance.Instance(name, fullname, display, ip) self.assertEqual(ins_webrtc._GetAutoConnect(), None) + @mock.patch("acloud.list.instance.LocalInstance") + def testGetCuttleFishLocalInstances(self, mock_local_instance): + """Test GetCuttleFishLocalInstances.""" + self.Patch(cvd_runtime_config, "CvdRuntimeConfig", + return_value=mock.MagicMock(instance_ids=["2", "3"])) + instance.GetCuttleFishLocalInstances("fake_config_path") + self.assertEqual(mock_local_instance.call_count, 2) + + def testGetDeviceFullName(self): + """Test GetDeviceFullName.""" + device_serial = "0.0.0.0:6520" + webrtc_device_id = "codelab" + instance_name = "local-instance-1" + elapsed_time = "10:10:24" + + expected_result = ("device serial: 0.0.0.0:6520 codelab " + "(local-instance-1) elapsed time: 10:10:24") + self.assertEqual(expected_result, instance._GetDeviceFullName( + device_serial, instance_name, elapsed_time, webrtc_device_id)) + + # Test with no webrtc_device_id + webrtc_device_id = None + expected_result = ("device serial: 0.0.0.0:6520 (local-instance-1) " + "elapsed time: 10:10:24") + self.assertEqual(expected_result, instance._GetDeviceFullName( + device_serial, instance_name, elapsed_time, webrtc_device_id)) + if __name__ == "__main__": unittest.main() diff --git a/list/list.py b/list/list.py index 6ccbac1a..a008bb90 100644 --- a/list/list.py +++ b/list/list.py @@ -34,6 +34,9 @@ from acloud.public import config logger = logging.getLogger(__name__) _COMMAND_PS_LAUNCH_CVD = ["ps", "-wweo", "lstart,cmd"] +_NOT_CONNECTED_DEVICE_HINT = ( + "\nFor not connected device, you can try \"$ acloud reconnect\" or " + "\"$ acloud restart\" to get the device back.") def _ProcessInstances(instance_list): @@ -91,11 +94,12 @@ def PrintInstancesDetails(instance_list, verbose=False): verbose: Boolean, True to print all details and only full name if False. instance_list: List of instances. """ + not_any_connected_device = False if not instance_list: print("No remote or local instances found") for num, instance_info in enumerate(instance_list, 1): - idx_str = "[%d]" % num + idx_str = f"[{num}]" utils.PrintColorString(idx_str, end="") if verbose: print(instance_info.Summary()) @@ -104,6 +108,11 @@ def PrintInstancesDetails(instance_list, verbose=False): else: print(instance_info) + if not instance_info.AdbConnected(): + not_any_connected_device = True + if not_any_connected_device: + utils.PrintColorString(_NOT_CONNECTED_DEVICE_HINT) + def GetRemoteInstances(cfg): """Look for remote instances. @@ -118,7 +127,7 @@ def GetRemoteInstances(cfg): """ credentials = auth.CreateCredentials(cfg) compute_client = gcompute_client.ComputeClient(cfg, credentials) - filter_item = "labels.%s=%s" % (constants.LABEL_CREATE_BY, getpass.getuser()) + filter_item = f"labels.{constants.LABEL_CREATE_BY}={getpass.getuser()}" all_instances = compute_client.ListInstances(instance_filter=filter_item) logger.debug("Instance list from: (filter: %s\n%s):", @@ -149,12 +158,13 @@ def _GetLocalCuttlefishInstances(id_cfg_pairs): try: if not os.path.isfile(cfg_path): continue - ins = instance.LocalInstance(cfg_path) - if ins.CvdStatus(): - local_instance_list.append(ins) - else: - logger.info("Cvd runtime config is found at %s but instance " - "%d is not active.", cfg_path, ins_id) + instances = instance.GetCuttleFishLocalInstances(cfg_path) + for ins in instances: + if ins.CvdStatus(): + local_instance_list.append(ins) + else: + logger.info("Cvd runtime config is found at %s but instance " + "%d is not active.", cfg_path, ins_id) finally: ins_lock.Unlock() return local_instance_list diff --git a/list/list_test.py b/list/list_test.py index 6e5d031c..a5296577 100644 --- a/list/list_test.py +++ b/list/list_test.py @@ -45,8 +45,9 @@ class ListTest(driver_test_lib.BaseDriverTest): super().setUp() self.Patch(instance, "_GetElapsedTime", return_value=0) self.Patch(instance.RemoteInstance, "_GetZoneName") + self.Patch(instance.RemoteInstance, "_GetProjectName") self.Patch(instance, "GetInstanceIP", return_value=ssh.IP()) - self.Patch(instance.RemoteInstance, "GetAdbVncPortFromSSHTunnel") + self.Patch(instance.RemoteInstance, "GetForwardedPortsFromSSHTunnel") self.Patch(adb_tools, "AdbTools") self.Patch(adb_tools.AdbTools, "IsAdbConnected", return_value=False) self.Patch(auth, "CreateCredentials") @@ -161,7 +162,8 @@ class ListTest(driver_test_lib.BaseDriverTest): local_ins = mock.MagicMock() local_ins.CvdStatus.return_value = True - self.Patch(instance, "LocalInstance", return_value=local_ins) + self.Patch(instance, "GetCuttleFishLocalInstances", + return_value=[local_ins]) ins_list = list_instance._GetLocalCuttlefishInstances(id_cfg_pairs) self.assertEqual(2, len(ins_list)) @@ -198,11 +200,12 @@ class ListTest(driver_test_lib.BaseDriverTest): y_res=728, dpi=240, instance_dir="fake_dir", - adb_ip_port="127.0.0.1:6520" + adb_ip_port="127.0.0.1:6520", + root_dir="root/cuttlefish_runtime" ) self.Patch(cvd_runtime_config, "CvdRuntimeConfig", return_value=cf_config) - self.Patch(instance.LocalInstance, "GetDevidInfoFromCvdFleet", + self.Patch(instance.LocalInstance, "_GetDevidInfoFromCvdStatus", return_value=None) ins = instance.LocalInstance("fake_cf_path") diff --git a/metrics/metrics.py b/metrics/metrics.py index eba0411e..e9cc39c0 100644 --- a/metrics/metrics.py +++ b/metrics/metrics.py @@ -17,6 +17,7 @@ import logging from acloud.internal import constants _NO_METRICS = "--no-metrics" +_NO_METRICS_COMMANDS = ["delete"] logger = logging.getLogger(__name__) @@ -44,11 +45,12 @@ def LogUsage(argv): """ if _NO_METRICS in argv: return False + if len(argv) > 0 and argv[0] in _NO_METRICS_COMMANDS: + return False try: - from asuite import atest_utils from asuite.metrics import metrics_utils - atest_utils.print_data_collection_notice() + metrics_utils.print_data_collection_notice() metrics_utils.send_start_event(tool_name=constants.TOOL_NAME, command_line=' '.join(argv), test_references=[argv[0]]) diff --git a/metrics/metrics_test.py b/metrics/metrics_test.py index 77e8670a..39c88abf 100644 --- a/metrics/metrics_test.py +++ b/metrics/metrics_test.py @@ -21,7 +21,6 @@ from unittest import mock # pylint: disable=import-error, no-name-in-module, wrong-import-position sys.modules["asuite"] = mock.MagicMock() sys.modules["asuite.metrics"] = mock.MagicMock() -from asuite import atest_utils from asuite.metrics import metrics_utils from acloud.internal.lib import driver_test_lib from acloud.metrics import metrics @@ -31,13 +30,17 @@ class MetricsTest(driver_test_lib.BaseDriverTest): """Test metrics methods.""" def testLogUsage(self): """Test LogUsage.""" - self.Patch(atest_utils, "print_data_collection_notice") + self.Patch(metrics_utils, "print_data_collection_notice") self.Patch(metrics_utils, "send_start_event") - argv = ["acloud", "create"] + argv = ["create", "--local-instance"] self.assertTrue(metrics.LogUsage(argv)) # Test arguments with "--no-metrics" - argv = ["acloud", "create", "--no-metrics"] + argv = ["create", "--no-metrics"] + self.assertFalse(metrics.LogUsage(argv)) + + # Don't collect metrics for "delete" command. + argv = ["delete", "--all"] self.assertFalse(metrics.LogUsage(argv)) def testLogExitEvent(self): diff --git a/public/acloud_common.py b/public/acloud_common.py index a3c07201..e1a71438 100755 --- a/public/acloud_common.py +++ b/public/acloud_common.py @@ -28,7 +28,7 @@ def AddCommonArguments(parser): parser.add_argument("--email", type=str, dest="email", - help="Email account to use for authentcation.") + help="Email account to use for authentication.") parser.add_argument("--config-file", type=str, dest="config_file", diff --git a/public/acloud_main.py b/public/acloud_main.py index 96648d6e..1eef9992 100644 --- a/public/acloud_main.py +++ b/public/acloud_main.py @@ -80,14 +80,6 @@ if sys.version_info.major == 2: sys.version_info.micro)) sys.exit(1) -# (b/219847353) Move googleapiclient to the last position of sys.path when -# existed. -for lib in sys.path: - if 'googleapiclient' in lib: - sys.path.remove(lib) - sys.path.append(lib) - break - # By Default silence root logger's stream handler since 3p lib may initial # root logger no matter what level we're using. The acloud logger behavior will # be defined in _SetupLogging(). This also could workaround to get rid of below @@ -196,6 +188,12 @@ def _ParseArgs(args): help="Emulator build branch name, e.g. aosp-emu-master-dev. If specified" " without emulator_build_id, the last green build will be used.") create_gf_parser.add_argument( + "--emulator-build-target", + dest="emulator_build_target", + required=False, + help="Emulator build target used to run the images. e.g. " + "emulator-linux_x64_nolocationui.") + create_gf_parser.add_argument( "--base_image", type=str, dest="base_image", @@ -405,10 +403,11 @@ def main(argv=None): reporter = create_goldfish_action.CreateDevices( cfg=cfg, build_target=args.build_target, + branch=args.branch, build_id=args.build_id, emulator_build_id=args.emulator_build_id, - branch=args.branch, emulator_branch=args.emulator_branch, + emulator_build_target=args.emulator_build_target, kernel_build_id=args.kernel_build_id, kernel_branch=args.kernel_branch, kernel_build_target=args.kernel_build_target, diff --git a/public/actions/base_device_factory.py b/public/actions/base_device_factory.py index 847f19c3..8ac52720 100644 --- a/public/actions/base_device_factory.py +++ b/public/actions/base_device_factory.py @@ -50,6 +50,42 @@ class BaseDeviceFactory(): return # pylint: disable=no-self-use + def GetAdbPorts(self): + """Get ADB ports of the created devices. + + Subclasses should define this function if their ADB ports are not + constant. + + Returns: + The port numbers as a list of integers. + """ + return [None] + + # pylint: disable=no-self-use + def GetFastbootPorts(self): + """Get Fastboot ports of the created devices. + + Subclasses should define this function if their ADB ports are not + constant. + + Returns: + The port numbers as a list of integers. + """ + return [None] + + # pylint: disable=no-self-use + def GetVncPorts(self): + """Get VNC ports of the created devices. + + Subclasses should define this function if they support VNC and their + VNC ports are not constant. + + Returns: + The port numbers as a list of integers. + """ + return [None] + + # pylint: disable=no-self-use def GetBuildInfoDict(self): """Get build info dictionary. diff --git a/public/actions/common_operations.py b/public/actions/common_operations.py index 2b64ec54..f7823121 100644 --- a/public/actions/common_operations.py +++ b/public/actions/common_operations.py @@ -13,12 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Common operations between managing GCE and Cuttlefish devices. - -This module provides the common operations between managing GCE (device_driver) -and Cuttlefish (create_cuttlefish_action) devices. Should not be called -directly. -""" +"""Common operations to create remote devices.""" import logging import os @@ -109,16 +104,19 @@ class DevicePool: for _ in range(num): instance = self._device_factory.CreateInstance() ip = self._compute_client.GetInstanceIP(instance) - time_info = self._compute_client.execution_time if hasattr( - self._compute_client, "execution_time") else {} + time_info = { + stage: round(exec_time, 2) for stage, exec_time in + getattr(self._compute_client, "execution_time", {}).items()} stage = self._compute_client.stage if hasattr( self._compute_client, "stage") else 0 openwrt = self._compute_client.openwrt if hasattr( self._compute_client, "openwrt") else False + gce_hostname = self._compute_client.gce_hostname if hasattr( + self._compute_client, "gce_hostname") else None self.devices.append( avd.AndroidVirtualDevice(ip=ip, instance_name=instance, time_info=time_info, stage=stage, - openwrt=openwrt)) + openwrt=openwrt, gce_hostname=gce_hostname)) @utils.TimeExecute(function_description="Waiting for AVD(s) to boot up", result_evaluator=utils.BootEvaluator) @@ -208,24 +206,10 @@ def _GetErrorType(error): return constants.GCE_QUOTA_ERROR return constants.ACLOUD_UNKNOWN_ERROR -def _GetAdbPort(avd_type, base_instance_num): - """Get Adb port according to avd_type and device offset. - - Args: - avd_type: String, the AVD type(cuttlefish, goldfish...). - base_instance_num: int, device offset. - - Returns: - int, adb port. - """ - if avd_type in utils.AVD_PORT_DICT: - return utils.AVD_PORT_DICT[avd_type].adb_port + base_instance_num - 1 - return None - -# pylint: disable=too-many-locals,unused-argument,too-many-branches +# pylint: disable=too-many-locals,unused-argument,too-many-branches,too-many-statements def CreateDevices(command, cfg, device_factory, num, avd_type, - report_internal_ip=False, autoconnect=False, - serial_log_file=None, client_adb_port=None, + report_internal_ip=False, autoconnect=False, serial_log_file=None, + client_adb_port=None, client_fastboot_port=None, boot_timeout_secs=None, unlock_screen=False, wait_for_boot=True, connect_webrtc=False, ssh_private_key_path=None, @@ -247,6 +231,7 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, serial_log_file: String, the file path to tar the serial logs. autoconnect: Boolean, whether to auto connect to device. client_adb_port: Integer, Specify port for adb forwarding. + client_fastboot_port: Integer, Specify port for fastboot forwarding. boot_timeout_secs: Integer, boot timeout secs. unlock_screen: Boolean, whether to unlock screen after invoke vnc client. wait_for_boot: Boolean, True to check serial log include boot up @@ -288,15 +273,16 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, for device in device_pool.devices: ip = (device.ip.internal if report_internal_ip else device.ip.external) - base_instance_num = 1 - if constants.BASE_INSTANCE_NUM in device_pool._compute_client.dict_report: - base_instance_num = device_pool._compute_client.dict_report[constants.BASE_INSTANCE_NUM] - adb_port = _GetAdbPort( - avd_type, - base_instance_num - ) + extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel + # TODO(b/154175542): Report multiple devices. + vnc_ports = device_factory.GetVncPorts() + adb_ports = device_factory.GetAdbPorts() + fastboot_ports = device_factory.GetFastbootPorts() + if not vnc_ports[0] and not adb_ports[0] and not fastboot_ports[0]: + vnc_ports[0], adb_ports[0], fastboot_ports[0] = utils.AVD_PORT_DICT[avd_type] + device_dict = { - "ip": ip + (":" + str(adb_port) if adb_port else ""), + "ip": ip + (":" + str(adb_ports[0]) if adb_ports[0] else ""), "instance_name": device.instance_name } if device.build_info: @@ -305,35 +291,45 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, device_dict.update(device.time_info) if device.openwrt: device_dict.update(device_factory.GetOpenWrtInfoDict()) + if device.gce_hostname: + device_dict[constants.GCE_HOSTNAME] = device.gce_hostname + logger.debug( + "To connect with hostname, erase the extra_args_ssh_tunnel: %s", + extra_args_ssh_tunnel) + extra_args_ssh_tunnel="" if autoconnect and reporter.status == report.Status.SUCCESS: - forwarded_ports = utils.AutoConnect( - ip_addr=ip, - rsa_key_file=(ssh_private_key_path or - cfg.ssh_private_key_path), - target_vnc_port=utils.AVD_PORT_DICT[avd_type].vnc_port, - target_adb_port=adb_port, - ssh_user=ssh_user, - client_adb_port=client_adb_port, - extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) - device_dict[constants.VNC_PORT] = forwarded_ports.vnc_port - device_dict[constants.ADB_PORT] = forwarded_ports.adb_port - device_dict[constants.DEVICE_SERIAL] = ( - constants.REMOTE_INSTANCE_ADB_SERIAL % - forwarded_ports.adb_port) - if unlock_screen: - AdbTools(forwarded_ports.adb_port).AutoUnlockScreen() + forwarded_ports = _EstablishDeviceConnections( + device.gce_hostname or ip, + vnc_ports, adb_ports, fastboot_ports, + client_adb_port, client_fastboot_port, ssh_user, + ssh_private_key_path=(ssh_private_key_path or + cfg.ssh_private_key_path), + extra_args_ssh_tunnel=extra_args_ssh_tunnel, + unlock_screen=unlock_screen) + if forwarded_ports: + forwarded_port = forwarded_ports[0] + device_dict[constants.VNC_PORT] = forwarded_port.vnc_port + device_dict[constants.ADB_PORT] = forwarded_port.adb_port + device_dict[constants.FASTBOOT_PORT] = forwarded_port.fastboot_port + device_dict[constants.DEVICE_SERIAL] = ( + constants.REMOTE_INSTANCE_ADB_SERIAL % + forwarded_port.adb_port) if connect_webrtc and reporter.status == report.Status.SUCCESS: webrtc_local_port = utils.PickFreePort() device_dict[constants.WEBRTC_PORT] = webrtc_local_port utils.EstablishWebRTCSshTunnel( - ip_addr=ip, + ip_addr=device.gce_hostname or ip, webrtc_local_port=webrtc_local_port, rsa_key_file=(ssh_private_key_path or cfg.ssh_private_key_path), ssh_user=ssh_user, - extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) + extra_args_ssh_tunnel=extra_args_ssh_tunnel) if device.instance_name in logs: device_dict[constants.LOGS] = logs[device.instance_name] + if hasattr(device_factory, 'GetFetchCvdWrapperLogIfExist'): + fetch_cvd_wrapper_log = device_factory.GetFetchCvdWrapperLogIfExist() + if fetch_cvd_wrapper_log: + device_dict["fetch_cvd_wrapper_log"] = fetch_cvd_wrapper_log if device.instance_name in failures: reporter.SetErrorType(constants.ACLOUD_BOOT_UP_ERROR) if device.stage: @@ -347,3 +343,45 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, reporter.AddError(str(e)) reporter.SetStatus(report.Status.FAIL) return reporter + + +def _EstablishDeviceConnections(ip, vnc_ports, adb_ports, fastboot_ports, + client_adb_port, client_fastboot_port, + ssh_user, ssh_private_key_path, + extra_args_ssh_tunnel, unlock_screen): + """Establish the adb and vnc connections. + + Create the ssh tunnels with adb ports and vnc ports. Then unlock the device + screen via the adb port. + + Args: + ip: String, the IPv4 address. + vnc_ports: List of integer, the vnc ports. + adb_ports: List of integer, the adb ports. + fastboot_ports: List of integer, the fastboot ports. + client_adb_port: Integer, Specify port for adb forwarding. + client_fastboot_port: Integer, Specify port for fastboot forwarding. + ssh_user: String, the user name for SSH tunneling. + ssh_private_key_path: String, the private key for SSH tunneling. + extra_args_ssh_tunnel: String, extra args for ssh tunnel connection. + unlock_screen: Boolean, whether to unlock screen after invoking vnc client. + + Returns: + A list of namedtuple of (vnc_port, adb_port) + """ + forwarded_ports = [] + for vnc_port, adb_port, fastboot_port in zip(vnc_ports, adb_ports, fastboot_ports): + forwarded_port = utils.AutoConnect( + ip_addr=ip, + rsa_key_file=ssh_private_key_path, + target_vnc_port=vnc_port, + target_adb_port=adb_port, + target_fastboot_port=fastboot_port, + ssh_user=ssh_user, + client_adb_port=client_adb_port, + client_fastboot_port=client_fastboot_port, + extra_args_ssh_tunnel=extra_args_ssh_tunnel) + forwarded_ports.append(forwarded_port) + if unlock_screen: + AdbTools(forwarded_port.adb_port).AutoUnlockScreen() + return forwarded_ports diff --git a/public/actions/common_operations_test.py b/public/actions/common_operations_test.py index b06a60cd..45a3d314 100644 --- a/public/actions/common_operations_test.py +++ b/public/actions/common_operations_test.py @@ -36,6 +36,7 @@ from acloud.public.actions import common_operations class CommonOperationsTest(driver_test_lib.BaseDriverTest): """Test Common Operations.""" + maxDiff = None IP = ssh.IP(external="127.0.0.1", internal="10.0.0.1") INSTANCE = "fake-instance" CMD = "test-cmd" @@ -56,6 +57,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "AndroidBuildClient", return_value=self.build_client) self.compute_client = mock.MagicMock() + self.compute_client.gce_hostname = None self.Patch( android_compute_client, "AndroidComputeClient", @@ -68,11 +70,9 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): self.device_factory, "GetComputeClient", return_value=self.compute_client) - self.Patch(self.device_factory, "GetBuildInfoDict", - return_value={"branch": self.BRANCH, - "build_id": self.BUILD_ID, - "build_target": self.BUILD_TARGET, - "gcs_bucket_build_id": self.BUILD_ID}) + self.Patch(self.device_factory, "GetVncPorts", return_value=[6444]) + self.Patch(self.device_factory, "GetAdbPorts", return_value=[6520]) + self.Patch(self.device_factory, "GetFastbootPorts", return_value=[7520]) self.Patch(self.device_factory, "GetBuildInfoDict", return_value={"branch": self.BRANCH, "build_id": self.BUILD_ID, @@ -80,6 +80,9 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "gcs_bucket_build_id": self.BUILD_ID}) self.Patch(self.device_factory, "GetLogs", return_value={self.INSTANCE: self.LOGS}) + self.Patch( + self.device_factory, + "GetFetchCvdWrapperLogIfExist", return_value={}) @staticmethod def _CreateCfg(): @@ -113,7 +116,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): self.assertEqual( _report.data, {"devices": [{ - "ip": self.IP.external, + "ip": self.IP.external + ":6520", "instance_name": self.INSTANCE, "branch": self.BRANCH, "build_id": self.BUILD_ID, @@ -122,9 +125,9 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "logs": self.LOGS }]}) - def testCreateDevicesWithAdbPort(self): + def testCreateDevicesWithAdbAndFastbootPorts(self): """Test Create Devices with adb port for cuttlefish avd type.""" - forwarded_ports = mock.Mock(adb_port=12345, vnc_port=56789) + forwarded_ports = mock.Mock(adb_port=12345, fastboot_port=54321, vnc_port=56789) mock_auto_connect = self.Patch(utils, "AutoConnect", return_value=forwarded_ports) cfg = self._CreateCfg() @@ -132,12 +135,13 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): self.device_factory, 1, "cuttlefish", autoconnect=True, - client_adb_port=12345) + client_adb_port=12345, + client_fastboot_port=54321) mock_auto_connect.assert_called_with( ip_addr="127.0.0.1", rsa_key_file="cfg/private/key", - target_vnc_port=6444, target_adb_port=6520, - ssh_user=constants.GCE_USER, client_adb_port=12345, + target_vnc_port=6444, target_adb_port=6520, target_fastboot_port=7520, + ssh_user=constants.GCE_USER, client_adb_port=12345, client_fastboot_port=54321, extra_args_ssh_tunnel="extra args") self.assertEqual(_report.command, self.CMD) self.assertEqual(_report.status, report.Status.SUCCESS) @@ -149,6 +153,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "branch": self.BRANCH, "build_id": self.BUILD_ID, "adb_port": 12345, + "fastboot_port": 54321, "device_serial": "127.0.0.1:12345", "vnc_port": 56789, "build_target": self.BUILD_TARGET, @@ -156,6 +161,37 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "logs": self.LOGS }]}) + def testCreateDevicesMultipleDevices(self): + """Test Create Devices with multiple cuttlefish devices.""" + forwarded_ports_1 = mock.Mock(adb_port=12345, vnc_port=56789) + forwarded_ports_2 = mock.Mock(adb_port=23456, vnc_port=67890) + self.Patch(self.device_factory, "GetVncPorts", return_value=[6444, 6445]) + self.Patch(self.device_factory, "GetAdbPorts", return_value=[6520, 6521]) + self.Patch(self.device_factory, "GetFastbootPorts", return_value=[7520, 7521]) + self.Patch(utils, "PickFreePort", return_value=12345) + mock_auto_connect = self.Patch( + utils, "AutoConnect", side_effects=[forwarded_ports_1, + forwarded_ports_2]) + cfg = self._CreateCfg() + _report = common_operations.CreateDevices(self.CMD, cfg, + self.device_factory, 1, + "cuttlefish", + autoconnect=True, + client_adb_port=None) + self.assertEqual(2, mock_auto_connect.call_count) + mock_auto_connect.assert_any_call( + ip_addr="127.0.0.1", rsa_key_file="cfg/private/key", + target_vnc_port=6444, target_adb_port=6520, target_fastboot_port=7520, + ssh_user=constants.GCE_USER, client_adb_port=None, client_fastboot_port=None, + extra_args_ssh_tunnel="extra args") + mock_auto_connect.assert_any_call( + ip_addr="127.0.0.1", rsa_key_file="cfg/private/key", + target_vnc_port=6444, target_adb_port=6520, target_fastboot_port=7520, + ssh_user=constants.GCE_USER, client_adb_port=None, client_fastboot_port=None, + extra_args_ssh_tunnel="extra args") + self.assertEqual(_report.command, self.CMD) + self.assertEqual(_report.status, report.Status.SUCCESS) + def testCreateDevicesInternalIP(self): """Test Create Devices and report internal IP.""" cfg = self._CreateCfg() @@ -168,7 +204,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): self.assertEqual( _report.data, {"devices": [{ - "ip": self.IP.internal, + "ip": self.IP.internal + ":6520", "instance_name": self.INSTANCE, "branch": self.BRANCH, "build_id": self.BUILD_ID, @@ -179,7 +215,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): def testCreateDevicesWithSshParameters(self): """Test Create Devices with ssh user and key.""" - forwarded_ports = mock.Mock(adb_port=12345, vnc_port=56789) + forwarded_ports = mock.Mock(adb_port=12345, fastboot_port=54321, vnc_port=56789) mock_auto_connect = self.Patch(utils, "AutoConnect", return_value=forwarded_ports) mock_establish_webrtc = self.Patch(utils, "EstablishWebRTCSshTunnel") @@ -192,8 +228,9 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): mock_auto_connect.assert_called_with( ip_addr="127.0.0.1", rsa_key_file="private/key", - target_vnc_port=6444, target_adb_port=6520, ssh_user="user", - client_adb_port=None, extra_args_ssh_tunnel="extra args") + target_vnc_port=6444, target_adb_port=6520, target_fastboot_port=7520, + ssh_user="user", client_adb_port=None, client_fastboot_port=None, + extra_args_ssh_tunnel="extra args") mock_establish_webrtc.assert_called_with( ip_addr="127.0.0.1", rsa_key_file="private/key", ssh_user="user", extra_args_ssh_tunnel="extra args", @@ -231,6 +268,32 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): expected_result = constants.GCE_QUOTA_ERROR self.assertEqual(common_operations._GetErrorType(error), expected_result) + def testCreateDevicesWithFetchCvdWrapper(self): + """Test Create Devices with FetchCvdWrapper.""" + self.Patch( + self.device_factory, + "GetFetchCvdWrapperLogIfExist", return_value={"fetch_log": "abc"}) + cfg = self._CreateCfg() + _report = common_operations.CreateDevices(self.CMD, cfg, + self.device_factory, 1, + constants.TYPE_CF) + self.assertEqual(_report.command, self.CMD) + self.assertEqual(_report.status, report.Status.SUCCESS) + self.assertEqual( + _report.data, + {"devices": [{ + "ip": self.IP.external + ":6520", + "instance_name": self.INSTANCE, + "branch": self.BRANCH, + "build_id": self.BUILD_ID, + "build_target": self.BUILD_TARGET, + "gcs_bucket_build_id": self.BUILD_ID, + "logs": self.LOGS, + "fetch_cvd_wrapper_log": { + "fetch_log": "abc" + }, + }]}) + if __name__ == "__main__": unittest.main() diff --git a/public/actions/create_cuttlefish_action.py b/public/actions/create_cuttlefish_action.py deleted file mode 100644 index 33545550..00000000 --- a/public/actions/create_cuttlefish_action.py +++ /dev/null @@ -1,299 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2018 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Create cuttlefish instances. - -TODO: This module now just contains the skeleton but not the actual logic. - Need to fill in the actuall logic. -""" - -import logging - -from acloud.public.actions import common_operations -from acloud.public.actions import base_device_factory -from acloud.internal import constants -from acloud.internal.lib import android_build_client -from acloud.internal.lib import auth -from acloud.internal.lib import cvd_compute_client -from acloud.internal.lib import cvd_compute_client_multi_stage -from acloud.internal.lib import utils - - -logger = logging.getLogger(__name__) - - -class CuttlefishDeviceFactory(base_device_factory.BaseDeviceFactory): - """A class that can produce a cuttlefish device. - - Attributes: - cfg: An AcloudConfig instance. - build_target: String,Target name. - build_id: String, Build id, e.g. "2263051", "P2804227" - kernel_build_id: String, Kernel build id. - gpu: String, GPU to attach to the device or None. e.g. "nvidia-tesla-k80" - """ - - LOG_FILES = ["/home/vsoc-01/cuttlefish_runtime/kernel.log", - "/home/vsoc-01/cuttlefish_runtime/logcat", - "/home/vsoc-01/cuttlefish_runtime/cuttlefish_config.json"] - - #pylint: disable=too-many-locals - def __init__(self, cfg, build_target, build_id, branch=None, - kernel_build_id=None, kernel_branch=None, - kernel_build_target=None, system_branch=None, - system_build_id=None, system_build_target=None, - bootloader_branch=None, bootloader_build_id=None, - bootloader_build_target=None, boot_timeout_secs=None, - ins_timeout_secs=None, report_internal_ip=None, gpu=None): - - self.credentials = auth.CreateCredentials(cfg) - - if cfg.enable_multi_stage: - compute_client = cvd_compute_client_multi_stage.CvdComputeClient( - cfg, self.credentials, boot_timeout_secs, ins_timeout_secs, - report_internal_ip, gpu) - else: - compute_client = cvd_compute_client.CvdComputeClient( - cfg, self.credentials) - super().__init__(compute_client) - - # Private creation parameters - self._cfg = cfg - self._build_target = build_target - self._build_id = build_id - self._branch = branch - self._kernel_build_id = kernel_build_id - self._blank_data_disk_size_gb = cfg.extra_data_disk_size_gb - self._extra_scopes = cfg.extra_scopes - - # Configure clients for interaction with GCE/Build servers - self._build_client = android_build_client.AndroidBuildClient( - self.credentials) - - # Get build_info namedtuple for platform, kernel, system build - self.build_info = self._build_client.GetBuildInfo( - build_target, build_id, branch) - self.kernel_build_info = self._build_client.GetBuildInfo( - kernel_build_target or cfg.kernel_build_target, kernel_build_id, - kernel_branch) - self.system_build_info = self._build_client.GetBuildInfo( - system_build_target or build_target, system_build_id, system_branch) - self.bootloader_build_info = self._build_client.GetBuildInfo( - bootloader_build_target, bootloader_build_id, bootloader_branch) - - def GetBuildInfoDict(self): - """Get build info dictionary. - - Returns: - A build info dictionary. - """ - build_info_dict = { - key: val for key, val in utils.GetDictItems(self.build_info) if val} - - build_info_dict.update( - {"kernel_%s" % key: val - for key, val in utils.GetDictItems(self.kernel_build_info) if val} - ) - build_info_dict.update( - {"system_%s" % key: val - for key, val in utils.GetDictItems(self.system_build_info) if val} - ) - build_info_dict.update( - {"bootloader_%s" % key: val - for key, val in utils.GetDictItems(self.bootloader_build_info) if val} - ) - return build_info_dict - - def GetFailures(self): - """Get failures from all devices. - - Returns: - A dictionary that contains all the failures. - The key is the name of the instance that fails to boot, - and the value is an errors.DeviceBootError object. - """ - return self._compute_client.all_failures - - @staticmethod - def _GetGcsBucketBuildId(build_id, release_id): - """Get GCS Bucket Build Id. - - Args: - build_id: The incremental build id. For example 5325535. - release_id: The release build id, None if not a release build. - For example AAAA.190220.001. - - Returns: - GCS bucket build id. For example: AAAA.190220.001-5325535 - """ - return "-".join([release_id, build_id]) if release_id else build_id - - def CreateInstance(self): - """Creates singe configured cuttlefish device. - - Override method from parent class. - - Returns: - A string, representing instance name. - """ - - # Create host instances for cuttlefish device. Currently one host instance - # has one cuttlefish device. In the future, these logics should be modified - # to support multiple cuttlefish devices per host instance. - instance = self._compute_client.GenerateInstanceName( - build_id=self.build_info.build_id, build_target=self._build_target) - - if self._cfg.enable_multi_stage: - remote_build_id = self.build_info.build_id - else: - remote_build_id = self._GetGcsBucketBuildId( - self.build_info.build_id, self.build_info.release_build_id) - - if self._cfg.enable_multi_stage: - remote_system_build_id = self.system_build_info.build_id - else: - remote_system_build_id = self._GetGcsBucketBuildId( - self.system_build_info.build_id, self.system_build_info.release_build_id) - - host_image_name = self._compute_client.GetHostImageName( - self._cfg.stable_host_image_name, - self._cfg.stable_host_image_family, - self._cfg.stable_host_image_project) - # Create an instance from Stable Host Image - self._compute_client.CreateInstance( - instance=instance, - image_name=host_image_name, - image_project=self._cfg.stable_host_image_project, - build_target=self.build_info.build_target, - branch=self.build_info.branch, - build_id=remote_build_id, - kernel_branch=self.kernel_build_info.branch, - kernel_build_id=self.kernel_build_info.build_id, - kernel_build_target=self.kernel_build_info.build_target, - blank_data_disk_size_gb=self._blank_data_disk_size_gb, - extra_scopes=self._extra_scopes, - system_build_target=self.system_build_info.build_target, - system_branch=self.system_build_info.branch, - system_build_id=remote_system_build_id, - bootloader_build_target=self.bootloader_build_info.build_target, - bootloader_branch=self.bootloader_build_info.branch, - bootloader_build_id=self.bootloader_build_info.build_id) - - return instance - - -#pylint: disable=too-many-locals -def CreateDevices(cfg, - build_target=None, - build_id=None, - branch=None, - kernel_build_id=None, - kernel_branch=None, - kernel_build_target=None, - system_branch=None, - system_build_id=None, - system_build_target=None, - bootloader_branch=None, - bootloader_build_id=None, - bootloader_build_target=None, - gpu=None, - num=1, - serial_log_file=None, - autoconnect=False, - report_internal_ip=False, - boot_timeout_secs=None, - ins_timeout_secs=None): - """Create one or multiple Cuttlefish devices. - - Args: - cfg: An AcloudConfig instance. - build_target: String, Target name. - build_id: String, Build id, e.g. "2263051", "P2804227" - branch: Branch name, a string, e.g. aosp_master - kernel_build_id: String, Kernel build id. - kernel_branch: String, Kernel branch name. - kernel_build_target: String, Kernel build target name. - system_branch: Branch name to consume the system.img from, a string. - system_build_id: System branch build id, a string. - system_build_target: System image build target, a string. - bootloader_branch: String of the bootloader branch name. - bootloader_build_id: String of the bootloader build id. - bootloader_build_target: String of the bootloader target name. - gpu: String, GPU to attach to the device or None. e.g. "nvidia-tesla-k80" - num: Integer, Number of devices to create. - serial_log_file: String, A path to a tar file where serial output should - be saved to. - autoconnect: Boolean, Create ssh tunnel(s) and adb connect after device - creation. - report_internal_ip: Boolean to report the internal ip instead of - external ip. - boot_timeout_secs: Integer, the maximum time in seconds used to wait - for the AVD to boot. - ins_timeout_secs: Integer, the maximum time in seconds used to wait for - the instance ready. - - Returns: - A Report instance. - """ - client_adb_port = None - unlock_screen = False - wait_for_boot = True - logger.info( - "Creating a cuttlefish device in project %s, " - "build_target: %s, " - "build_id: %s, " - "branch: %s, " - "kernel_build_id: %s, " - "kernel_branch: %s, " - "kernel_build_target: %s, " - "system_branch: %s, " - "system_build_id: %s, " - "system_build_target: %s, " - "bootloader_branch: %s, " - "bootloader_build_id: %s, " - "bootloader_build_target: %s, " - "gpu: %s" - "num: %s, " - "serial_log_file: %s, " - "autoconnect: %s, " - "report_internal_ip: %s", cfg.project, build_target, - build_id, branch, kernel_build_id, kernel_branch, kernel_build_target, - system_branch, system_build_id, system_build_target, bootloader_branch, - bootloader_build_id, bootloader_build_target, gpu, num, serial_log_file, - autoconnect, report_internal_ip) - # If multi_stage enable, launch_cvd don't write serial log to instance. So - # it doesn't go WaitForBoot function. - if cfg.enable_multi_stage: - wait_for_boot = False - device_factory = CuttlefishDeviceFactory( - cfg, build_target, build_id, branch=branch, - kernel_build_id=kernel_build_id, kernel_branch=kernel_branch, - kernel_build_target=kernel_build_target, system_branch=system_branch, - system_build_id=system_build_id, - system_build_target=system_build_target, - bootloader_branch=bootloader_branch, - bootloader_build_id=bootloader_build_id, - bootloader_build_target=bootloader_build_target, - boot_timeout_secs=boot_timeout_secs, - ins_timeout_secs=ins_timeout_secs, - report_internal_ip=report_internal_ip, - gpu=gpu) - return common_operations.CreateDevices("create_cf", cfg, device_factory, - num, constants.TYPE_CF, - report_internal_ip, autoconnect, - serial_log_file, client_adb_port, - boot_timeout_secs, unlock_screen, - wait_for_boot) diff --git a/public/actions/create_cuttlefish_action_test.py b/public/actions/create_cuttlefish_action_test.py deleted file mode 100644 index 0ddef21e..00000000 --- a/public/actions/create_cuttlefish_action_test.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2018 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for create_cuttlefish_action. - -Tests for acloud.public.actions.create_cuttlefish_action. -""" - -import uuid -import unittest - -from unittest import mock - -from acloud.internal.lib import android_build_client -from acloud.internal.lib import android_compute_client -from acloud.internal.lib import auth -from acloud.internal.lib import cvd_compute_client -from acloud.internal.lib import cvd_compute_client_multi_stage -from acloud.internal.lib import driver_test_lib -from acloud.internal.lib import ssh -from acloud.public.actions import create_cuttlefish_action - - -class CreateCuttlefishActionTest(driver_test_lib.BaseDriverTest): - """Test create_cuttlefish_action.""" - - IP = ssh.IP(external="127.0.0.1", internal="10.0.0.1") - INSTANCE = "fake-instance" - IMAGE = "fake-image" - BRANCH = "fake-branch" - BUILD_ID = "12345" - BUILD_TARGET = "fake-build-target" - KERNEL_BRANCH = "fake-kernel-branch" - KERNEL_BUILD_ID = "54321" - KERNEL_BUILD_TARGET = "kernel" - SYSTEM_BRANCH = "fake-system-branch" - SYSTEM_BUILD_ID = "23456" - SYSTEM_BUILD_TARGET = "fake-system-build-target" - BOOTLOADER_BRANCH = "fake-bootloader-branch" - BOOTLOADER_BUILD_ID = "34567" - BOOTLOADER_BUILD_TARGET = "fake-bootloader-build-target" - STABLE_HOST_IMAGE_NAME = "fake-stable-host-image-name" - STABLE_HOST_IMAGE_PROJECT = "fake-stable-host-image-project" - EXTRA_DATA_DISK_GB = 4 - EXTRA_SCOPES = ["scope1", "scope2"] - DEFAULT_ADB_PORT = 6520 - - def setUp(self): - """Set up the test.""" - super().setUp() - self.build_client = mock.MagicMock() - self.Patch( - android_build_client, - "AndroidBuildClient", - return_value=self.build_client) - self.compute_client = mock.MagicMock() - self.compute_client.openwrt = False - self.Patch( - cvd_compute_client, - "CvdComputeClient", - return_value=self.compute_client) - self.Patch( - cvd_compute_client_multi_stage, - "CvdComputeClient", - return_value=self.compute_client) - self.Patch( - android_compute_client, - "AndroidComputeClient", - return_value=self.compute_client) - self.Patch(auth, "CreateCredentials", return_value=mock.MagicMock()) - - def _CreateCfg(self): - """A helper method that creates a mock configuration object.""" - cfg = mock.MagicMock() - cfg.service_account_name = "fake@service.com" - cfg.service_account_private_key_path = "/fake/path/to/key" - cfg.zone = "fake_zone" - cfg.disk_image_name = "fake_image.tar.gz" - cfg.disk_image_mime_type = "fake/type" - cfg.ssh_private_key_path = "" - cfg.ssh_public_key_path = "" - cfg.stable_host_image_name = self.STABLE_HOST_IMAGE_NAME - cfg.stable_host_image_project = self.STABLE_HOST_IMAGE_PROJECT - cfg.extra_data_disk_size_gb = self.EXTRA_DATA_DISK_GB - cfg.kernel_build_target = self.KERNEL_BUILD_TARGET - cfg.extra_scopes = self.EXTRA_SCOPES - cfg.enable_multi_stage = False - return cfg - - def testCreateDevices(self): - """Test CreateDevices.""" - cfg = self._CreateCfg() - - # Mock uuid - fake_uuid = mock.MagicMock(hex="1234") - self.Patch(uuid, "uuid4", return_value=fake_uuid) - - # Mock compute client methods - self.compute_client.GetInstanceIP.return_value = self.IP - self.compute_client.GenerateImageName.return_value = self.IMAGE - self.compute_client.GenerateInstanceName.return_value = self.INSTANCE - self.compute_client.GetHostImageName.return_value = self.STABLE_HOST_IMAGE_NAME - - # Mock build client method - self.build_client.GetBuildInfo.side_effect = [ - android_build_client.BuildInfo( - self.BRANCH, self.BUILD_ID, self.BUILD_TARGET, None), - android_build_client.BuildInfo( - self.KERNEL_BRANCH, self.KERNEL_BUILD_ID, - self.KERNEL_BUILD_TARGET, None), - android_build_client.BuildInfo( - self.SYSTEM_BRANCH, self.SYSTEM_BUILD_ID, - self.SYSTEM_BUILD_TARGET, None), - android_build_client.BuildInfo( - self.BOOTLOADER_BRANCH, self.BOOTLOADER_BUILD_ID, - self.BOOTLOADER_BUILD_TARGET, None)] - - # Call CreateDevices - report = create_cuttlefish_action.CreateDevices( - cfg, self.BUILD_TARGET, self.BUILD_ID, branch=self.BRANCH, - kernel_build_id=self.KERNEL_BUILD_ID, - system_build_target=self.SYSTEM_BUILD_TARGET, - system_branch=self.SYSTEM_BRANCH, - system_build_id=self.SYSTEM_BUILD_ID, - bootloader_build_target=self.BOOTLOADER_BUILD_TARGET, - bootloader_branch=self.BOOTLOADER_BRANCH, - bootloader_build_id=self.BOOTLOADER_BUILD_ID) - - # Verify - self.compute_client.CreateInstance.assert_called_with( - instance=self.INSTANCE, - image_name=self.STABLE_HOST_IMAGE_NAME, - image_project=self.STABLE_HOST_IMAGE_PROJECT, - build_target=self.BUILD_TARGET, - branch=self.BRANCH, - build_id=self.BUILD_ID, - kernel_branch=self.KERNEL_BRANCH, - kernel_build_id=self.KERNEL_BUILD_ID, - kernel_build_target=self.KERNEL_BUILD_TARGET, - system_branch=self.SYSTEM_BRANCH, - system_build_id=self.SYSTEM_BUILD_ID, - system_build_target=self.SYSTEM_BUILD_TARGET, - bootloader_branch=self.BOOTLOADER_BRANCH, - bootloader_build_id=self.BOOTLOADER_BUILD_ID, - bootloader_build_target=self.BOOTLOADER_BUILD_TARGET, - blank_data_disk_size_gb=self.EXTRA_DATA_DISK_GB, - extra_scopes=self.EXTRA_SCOPES) - - self.assertEqual(report.data, { - "devices": [ - { - "branch": self.BRANCH, - "build_id": self.BUILD_ID, - "build_target": self.BUILD_TARGET, - "kernel_branch": self.KERNEL_BRANCH, - "kernel_build_id": self.KERNEL_BUILD_ID, - "kernel_build_target": self.KERNEL_BUILD_TARGET, - "system_branch": self.SYSTEM_BRANCH, - "system_build_id": self.SYSTEM_BUILD_ID, - "system_build_target": self.SYSTEM_BUILD_TARGET, - "bootloader_branch": self.BOOTLOADER_BRANCH, - "bootloader_build_id": self.BOOTLOADER_BUILD_ID, - "bootloader_build_target": self.BOOTLOADER_BUILD_TARGET, - "instance_name": self.INSTANCE, - "ip": self.IP.external + ":" + str(self.DEFAULT_ADB_PORT), - }, - ], - }) - self.assertEqual(report.command, "create_cf") - self.assertEqual(report.status, "SUCCESS") - - -if __name__ == "__main__": - unittest.main() diff --git a/public/actions/create_goldfish_action.py b/public/actions/create_goldfish_action.py index 069b9d93..d4617433 100644 --- a/public/actions/create_goldfish_action.py +++ b/public/actions/create_goldfish_action.py @@ -159,6 +159,7 @@ class GoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): build_id=self.build_info.build_id, emulator_branch=self.emulator_build_info.branch, emulator_build_id=self.emulator_build_info.build_id, + emulator_build_target=self.emulator_build_info.build_target, kernel_branch=self.kernel_build_info.branch, kernel_build_id=self.kernel_build_info.build_id, kernel_build_target=self.kernel_build_info.build_target, @@ -234,6 +235,7 @@ def CreateDevices(avd_spec=None, build_id=None, emulator_build_id=None, emulator_branch=None, + emulator_build_target=None, kernel_build_id=None, kernel_branch=None, kernel_build_target=None, @@ -254,7 +256,8 @@ def CreateDevices(avd_spec=None, build_id: String, Build id, e.g. "2263051", "P2804227" branch: String, Branch name for system image. emulator_build_id: String, emulator build id. - emulator_branch: String, Emulator branch name. + emulator_branch: String, emulator branch name. + emulator_build_target: String, emulator build target. gpu: String, GPU to attach to the device or None. e.g. "nvidia-k80" kernel_build_id: Kernel build id, a string. kernel_branch: Kernel branch name, a string. @@ -283,6 +286,7 @@ def CreateDevices(avd_spec=None, branch = avd_spec.remote_image[constants.BUILD_BRANCH] num = avd_spec.num emulator_build_id = avd_spec.emulator_build_id + emulator_build_target = avd_spec.emulator_build_target gpu = avd_spec.gpu serial_log_file = avd_spec.serial_log_file autoconnect = avd_spec.autoconnect @@ -290,12 +294,15 @@ def CreateDevices(avd_spec=None, client_adb_port = avd_spec.client_adb_port boot_timeout_secs = avd_spec.boot_timeout_secs + if not emulator_build_target: + emulator_build_target = cfg.emulator_build_target + # If emulator_build_id and emulator_branch is None, retrieve emulator # build id from platform build emulator-info.txt artifact # Example: require version-emulator=5292001 if not emulator_build_id and not emulator_branch: logger.info("emulator_build_id not provided. " - "Try to get %s from build %s/%s.", _EMULATOR_INFO_FILENAME, + "Attempting to get %s from build %s/%s.", _EMULATOR_INFO_FILENAME, build_id, build_target) emulator_build_id = _FetchBuildIdFromFile(cfg, build_target, @@ -311,7 +318,7 @@ def CreateDevices(avd_spec=None, # Example: version-sysimage-git_pi-dev-sdk_gphone_x86_64-userdebug=4833817 if not build_id and not branch: build_id = _FetchBuildIdFromFile(cfg, - cfg.emulator_build_target, + emulator_build_target, emulator_build_id, _SYSIMAGE_INFO_FILENAME) @@ -320,16 +327,16 @@ def CreateDevices(avd_spec=None, "in %s" % _SYSIMAGE_INFO_FILENAME) logger.info( "Creating a goldfish device in project %s, build_target: %s, " - "build_id: %s, emulator_bid: %s, kernel_build_id: %s, " + "build_id: %s, emulator_bid: %s, emulator_branch: %s, kernel_build_id: %s, " "kernel_branch: %s, kernel_build_target: %s, GPU: %s, num: %s, " "serial_log_file: %s, " "autoconnect: %s", cfg.project, build_target, build_id, - emulator_build_id, kernel_build_id, kernel_branch, kernel_build_target, - gpu, num, serial_log_file, autoconnect) + emulator_build_id, emulator_branch, kernel_build_id, kernel_branch, + kernel_build_target, gpu, num, serial_log_file, autoconnect) device_factory = GoldfishDeviceFactory( cfg, build_target, build_id, - cfg.emulator_build_target, + emulator_build_target, emulator_build_id, gpu=gpu, avd_spec=avd_spec, tags=tags, branch=branch, diff --git a/public/actions/create_goldfish_action_test.py b/public/actions/create_goldfish_action_test.py index dbc47b5b..b784be44 100644 --- a/public/actions/create_goldfish_action_test.py +++ b/public/actions/create_goldfish_action_test.py @@ -62,6 +62,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): return_value=self.build_client) self.compute_client = mock.MagicMock() self.compute_client.openwrt = False + self.compute_client.gce_hostname = None self.Patch( goldfish_compute_client, "GoldfishComputeClient", @@ -144,6 +145,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, @@ -201,6 +203,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, @@ -266,6 +269,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, @@ -321,6 +325,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, @@ -379,6 +384,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, @@ -434,6 +440,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): build_id=self.BUILD_ID, emulator_branch=self.EMULATOR_BRANCH, emulator_build_id=self.EMULATOR_BUILD_ID, + emulator_build_target=self.EMULATOR_BUILD_TARGET, kernel_branch=self.KERNEL_BRANCH, kernel_build_id=self.KERNEL_BUILD_ID, kernel_build_target=self.KERNEL_BUILD_TARGET, diff --git a/public/actions/gce_device_factory.py b/public/actions/gce_device_factory.py index 5502f296..42970178 100644 --- a/public/actions/gce_device_factory.py +++ b/public/actions/gce_device_factory.py @@ -51,7 +51,7 @@ class GCEDeviceFactory(base_device_factory.BaseDeviceFactory): super().__init__(compute_client) self._ssh = None - def _CreateGceInstance(self): + def CreateGceInstance(self): """Create a single configured GCE instance. build_target: The format is like "aosp_cf_x86_phone". We only get info @@ -87,14 +87,15 @@ class GCEDeviceFactory(base_device_factory.BaseDeviceFactory): instance=instance, image_name=host_image_name, image_project=self._cfg.stable_host_image_project, - blank_data_disk_size_gb=self._cfg.extra_data_disk_size_gb, avd_spec=self._avd_spec) ip = self._compute_client.GetInstanceIP(instance) + self._all_failures = self._compute_client.all_failures self._ssh = ssh.Ssh(ip=ip, user=constants.GCE_USER, ssh_private_key_path=self._cfg.ssh_private_key_path, extra_args_ssh_tunnel=self._cfg.extra_args_ssh_tunnel, - report_internal_ip=self._report_internal_ip) + report_internal_ip=self._report_internal_ip, + gce_hostname=self._compute_client.gce_hostname) return instance def GetFailures(self): diff --git a/public/actions/remote_host_cf_device_factory.py b/public/actions/remote_host_cf_device_factory.py index 4b4181e7..1ba46174 100644 --- a/public/actions/remote_host_cf_device_factory.py +++ b/public/actions/remote_host_cf_device_factory.py @@ -16,11 +16,14 @@ cuttlefish instances on a remote host.""" import glob +import json import logging import os +import posixpath as remote_path import shutil import subprocess import tempfile +import time from acloud import errors from acloud.internal import constants @@ -79,19 +82,42 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: A string, representing instance name. """ + init_remote_host_timestart = time.time() instance = self._InitRemotehost() + self._compute_client.execution_time[constants.TIME_GCE] = ( + time.time() - init_remote_host_timestart) + + process_artifacts_timestart = time.time() image_args = self._ProcessRemoteHostArtifacts() + self._compute_client.execution_time[constants.TIME_ARTIFACT] = ( + time.time() - process_artifacts_timestart) + + launch_cvd_timestart = time.time() failures = self._compute_client.LaunchCvd( - instance, - self._avd_spec, - self._avd_spec.cfg.extra_data_disk_size_gb, - boot_timeout_secs=self._avd_spec.boot_timeout_secs, - extra_args=image_args) + instance, self._avd_spec, self._GetInstancePath(), image_args) + self._compute_client.execution_time[constants.TIME_LAUNCH] = ( + time.time() - launch_cvd_timestart) + self._all_failures.update(failures) self._FindLogFiles( instance, instance in failures and not self._avd_spec.no_pull_log) return instance + def _GetInstancePath(self, relative_path=""): + """Append a relative path to the remote base directory. + + Args: + relative_path: The remote relative path. + + Returns: + The remote base directory if relative_path is empty. + The remote path under the base directory otherwise. + """ + base_dir = cvd_utils.GetRemoteHostBaseDir( + self._avd_spec.base_instance_num) + return (remote_path.join(base_dir, relative_path) if relative_path else + base_dir) + def _InitRemotehost(self): """Initialize remote host. @@ -116,8 +142,9 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: build_id = self._avd_spec.remote_image[constants.BUILD_ID] - instance = self._compute_client.FormatRemoteHostInstanceName( - self._avd_spec.remote_host, build_id, build_target) + instance = cvd_utils.FormatRemoteHostInstanceName( + self._avd_spec.remote_host, self._avd_spec.base_instance_num, + build_id, build_target) ip = ssh.IP(ip=self._avd_spec.remote_host) self._ssh = ssh.Ssh( ip=ip, @@ -127,7 +154,7 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): extra_args_ssh_tunnel=self._avd_spec.cfg.extra_args_ssh_tunnel, report_internal_ip=self._avd_spec.report_internal_ip) self._compute_client.InitRemoteHost( - self._ssh, ip, self._avd_spec.host_user) + self._ssh, ip, self._avd_spec.host_user, self._GetInstancePath()) return instance def _ProcessRemoteHostArtifacts(self): @@ -143,21 +170,120 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): A list of strings, the launch_cvd arguments. """ self._compute_client.SetStage(constants.STAGE_ARTIFACT) + self._ssh.Run(f"mkdir -p {self._GetInstancePath()}") if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: cvd_utils.UploadArtifacts( - self._ssh, + self._ssh, self._GetInstancePath(), self._local_image_artifact or self._avd_spec.local_image_dir, self._cvd_host_package_artifact) else: try: artifacts_path = tempfile.mkdtemp() logger.debug("Extracted path of artifacts: %s", artifacts_path) - self._DownloadArtifacts(artifacts_path) - self._UploadRemoteImageArtifacts(artifacts_path) + if self._avd_spec.remote_fetch: + # TODO: Check fetch cvd wrapper file is valid. + if self._avd_spec.fetch_cvd_wrapper: + self._UploadFetchCvd(artifacts_path) + self._DownloadArtifactsByFetchWrapper() + else: + self._UploadFetchCvd(artifacts_path) + self._DownloadArtifactsRemotehost() + else: + self._DownloadArtifacts(artifacts_path) + self._UploadRemoteImageArtifacts(artifacts_path) finally: shutil.rmtree(artifacts_path) - return cvd_utils.UploadExtraImages(self._ssh, self._avd_spec) + return cvd_utils.UploadExtraImages(self._ssh, self._GetInstancePath(), + self._avd_spec) + + def _GetRemoteFetchCredentialArg(self): + """Get the credential source argument for remote fetch_cvd. + + Remote fetch_cvd uses the service account key uploaded by + _UploadFetchCvd if it is available. Otherwise, fetch_cvd uses the + token extracted from the local credential file. + + Returns: + A string, the credential source argument. + """ + cfg = self._avd_spec.cfg + if cfg.service_account_json_private_key_path: + return "-credential_source=" + self._GetInstancePath( + constants.FETCH_CVD_CREDENTIAL_SOURCE) + + return self._compute_client.build_api.GetFetchCertArg( + os.path.join(_HOME_FOLDER, cfg.creds_cache_file)) + + @utils.TimeExecute( + function_description="Downloading artifacts on remote host by fetch cvd wrapper.") + def _DownloadArtifactsByFetchWrapper(self): + """Generate fetch_cvd args and run fetch cvd wrapper on remote host to download artifacts. + + Fetch cvd wrapper will fetch from cluster cached artifacts, and fallback to fetch_cvd if + the artifacts not exist. + """ + fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( + self._avd_spec.remote_image, + self._avd_spec.system_build_info, + self._avd_spec.kernel_build_info, + self._avd_spec.boot_build_info, + self._avd_spec.bootloader_build_info, + self._avd_spec.ota_build_info) + + fetch_cvd_args = self._avd_spec.fetch_cvd_wrapper.split(',') + [ + f"-directory={self._GetInstancePath()}", + f"-fetch_cvd_path={self._GetInstancePath(constants.FETCH_CVD)}", + self._GetRemoteFetchCredentialArg()] + fetch_cvd_args.extend(fetch_cvd_build_args) + + ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) + cmd = (f"{ssh_cmd} -- " + " ".join(fetch_cvd_args)) + logger.debug("cmd:\n %s", cmd) + ssh.ShellCmdWithRetry(cmd) + + @utils.TimeExecute(function_description="Downloading artifacts on remote host") + def _DownloadArtifactsRemotehost(self): + """Generate fetch_cvd args and run fetch_cvd on remote host to download artifacts.""" + fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( + self._avd_spec.remote_image, + self._avd_spec.system_build_info, + self._avd_spec.kernel_build_info, + self._avd_spec.boot_build_info, + self._avd_spec.bootloader_build_info, + self._avd_spec.ota_build_info) + + fetch_cvd_args = [self._GetInstancePath(constants.FETCH_CVD), + f"-directory={self._GetInstancePath()}", + self._GetRemoteFetchCredentialArg()] + fetch_cvd_args.extend(fetch_cvd_build_args) + + ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) + cmd = (f"{ssh_cmd} -- " + " ".join(fetch_cvd_args)) + logger.debug("cmd:\n %s", cmd) + ssh.ShellCmdWithRetry(cmd) + + @utils.TimeExecute(function_description="Download and upload fetch_cvd") + def _UploadFetchCvd(self, extract_path): + """Download fetch_cvd, duplicate service account json private key when available and upload + to remote host. + + Args: + extract_path: String, a path include extracted files. + """ + cfg = self._avd_spec.cfg + is_arm_img = (cvd_utils.IsArmImage(self._avd_spec.remote_image) + and self._avd_spec.remote_fetch) + fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) + self._compute_client.build_api.DownloadFetchcvd( + fetch_cvd, self._avd_spec.fetch_cvd_version, is_arm_img) + # Duplicate fetch_cvd API key when available + if cfg.service_account_json_private_key_path: + shutil.copyfile( + cfg.service_account_json_private_key_path, + os.path.join(extract_path, constants.FETCH_CVD_CREDENTIAL_SOURCE)) + + self._UploadRemoteImageArtifacts(extract_path) @utils.TimeExecute(function_description="Downloading Android Build artifact") def _DownloadArtifacts(self, extract_path): @@ -173,28 +299,18 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): errors.GetRemoteImageError: Fails to download rom images. """ cfg = self._avd_spec.cfg - build_id = self._avd_spec.remote_image[constants.BUILD_ID] - build_branch = self._avd_spec.remote_image[constants.BUILD_BRANCH] - build_target = self._avd_spec.remote_image[constants.BUILD_TARGET] # Download images with fetch_cvd fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) - self._compute_client.build_api.DownloadFetchcvd(fetch_cvd, - cfg.fetch_cvd_version) + self._compute_client.build_api.DownloadFetchcvd( + fetch_cvd, self._avd_spec.fetch_cvd_version) fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( - build_id, build_branch, build_target, - self._avd_spec.system_build_info.get(constants.BUILD_ID), - self._avd_spec.system_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.system_build_info.get(constants.BUILD_TARGET), - self._avd_spec.kernel_build_info.get(constants.BUILD_ID), - self._avd_spec.kernel_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.kernel_build_info.get(constants.BUILD_TARGET), - self._avd_spec.bootloader_build_info.get(constants.BUILD_ID), - self._avd_spec.bootloader_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.bootloader_build_info.get(constants.BUILD_TARGET), - self._avd_spec.ota_build_info.get(constants.BUILD_ID), - self._avd_spec.ota_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.ota_build_info.get(constants.BUILD_TARGET)) + self._avd_spec.remote_image, + self._avd_spec.system_build_info, + self._avd_spec.kernel_build_info, + self._avd_spec.boot_build_info, + self._avd_spec.bootloader_build_info, + self._avd_spec.ota_build_info) creds_cache_file = os.path.join(_HOME_FOLDER, cfg.creds_cache_file) fetch_cvd_cert_arg = self._compute_client.build_api.GetFetchCertArg( creds_cache_file) @@ -223,7 +339,8 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): # TODO(b/182259589): Refactor upload image command into a function. cmd = (f"tar -cf - --lzop -S -C {images_dir} " f"{' '.join(artifact_files)} | " - f"{ssh_cmd} -- tar -xf - --lzop -S") + f"{ssh_cmd} -- " + f"tar -xf - --lzop -S -C {self._GetInstancePath()}") logger.debug("cmd:\n %s", cmd) ssh.ShellCmdWithRetry(cmd) @@ -235,11 +352,22 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): download: Whether to download the files to a temporary directory and show messages to the user. """ - self._all_logs[instance] = [cvd_utils.TOMBSTONES] - log_files = pull.GetAllLogFilePaths(self._ssh) - self._all_logs[instance].extend(cvd_utils.ConvertRemoteLogs(log_files)) + logs = [] + if (self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE and + self._avd_spec.remote_fetch): + logs.append( + cvd_utils.GetRemoteFetcherConfigJson(self._GetInstancePath())) + logs.extend(cvd_utils.FindRemoteLogs( + self._ssh, + self._GetInstancePath(), + self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance)) + self._all_logs[instance] = logs if download: + # To avoid long download time, fetch from the first device only. + log_files = pull.GetAllLogFilePaths( + self._ssh, self._GetInstancePath(constants.REMOTE_LOG_FOLDER)) error_log_folder = pull.PullLogs(self._ssh, log_files, instance) self._compute_client.ExtendReportData(constants.ERROR_LOG_FOLDER, error_log_folder) @@ -265,6 +393,33 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): return None return cvd_utils.GetRemoteBuildInfoDict(self._avd_spec) + def GetAdbPorts(self): + """Get ADB ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetAdbPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + + def GetFastbootPorts(self): + """Get Fastboot ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetFastbootPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + + def GetVncPorts(self): + """Get VNC ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetVncPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + def GetFailures(self): """Get failures from all devices. @@ -282,3 +437,24 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): A dictionary that maps instance names to lists of report.LogFile. """ return self._all_logs + + def GetFetchCvdWrapperLogIfExist(self): + """Get FetchCvdWrapper log if exist. + + Returns: + A dictionary that includes FetchCvdWrapper logs. + """ + if not self._avd_spec.fetch_cvd_wrapper: + return {} + path = os.path.join(self._GetInstancePath(), "fetch_cvd_wrapper_log.json") + ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) + " cat " + path + proc = subprocess.run(ssh_cmd, shell=True, capture_output=True, + check=False) + if proc.stderr: + logger.debug("`%s` stderr: %s", ssh_cmd, proc.stderr.decode()) + if proc.stdout: + try: + return json.loads(proc.stdout) + except ValueError as e: + return {"status": "FETCH_WRAPPER_REPORT_PARSE_ERROR"} + return {} diff --git a/public/actions/remote_host_cf_device_factory_test.py b/public/actions/remote_host_cf_device_factory_test.py index 4b9c3e68..76670888 100644 --- a/public/actions/remote_host_cf_device_factory_test.py +++ b/public/actions/remote_host_cf_device_factory_test.py @@ -36,11 +36,11 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): def _CreateMockAvdSpec(): """Create a mock AvdSpec with necessary attributes.""" mock_cfg = mock.Mock(spec=[], - extra_data_disk_size_gb=10, ssh_private_key_path="/mock/id_rsa", extra_args_ssh_tunnel="extra args", fetch_cvd_version="123456", - creds_cache_file="credential") + creds_cache_file="credential", + service_account_json_private_key_path="/mock/key") return mock.Mock(spec=[], remote_image={ "branch": "aosp-android12-gsi", @@ -48,6 +48,7 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): "build_target": "aosp_cf_x86_64_phone-userdebug"}, system_build_info={}, kernel_build_info={}, + boot_build_info={}, bootloader_build_info={}, ota_build_info={}, remote_host="192.0.2.100", @@ -60,6 +61,11 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): boot_timeout_secs=100, gpu="auto", no_pull_log=False, + remote_fetch=False, + fetch_cvd_wrapper=None, + base_instance_num=None, + num_avds_per_instance=None, + fetch_cvd_version="123456", cfg=mock_cfg) @mock.patch("acloud.public.actions.remote_host_cf_device_factory." @@ -74,34 +80,42 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_avd_spec = self._CreateMockAvdSpec() mock_avd_spec.image_source = constants.IMAGE_SRC_LOCAL mock_avd_spec.local_image_dir = "/mock/img" + mock_avd_spec.base_instance_num = 2 + mock_avd_spec.num_avds_per_instance = 3 + mock_ssh_obj = mock.Mock() + mock_ssh.Ssh.return_value = mock_ssh_obj factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( mock_avd_spec, cvd_host_package_artifact="/mock/cvd.tar.gz") mock_client_obj = factory.GetComputeClient() - mock_client_obj.FormatRemoteHostInstanceName.return_value = "inst" mock_client_obj.LaunchCvd.return_value = {"inst": "failure"} log = {"path": "/log.txt"} - tombstones = {"path": "/tombstones"} - mock_cvd_utils.TOMBSTONES = tombstones + mock_cvd_utils.GetRemoteHostBaseDir.return_value = "acloud_cf_2" + mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" mock_cvd_utils.UploadExtraImages.return_value = ["extra"] - mock_cvd_utils.ConvertRemoteLogs.return_value = [log] + mock_cvd_utils.FindRemoteLogs.return_value = [log] self.assertEqual("inst", factory.CreateInstance()) - mock_ssh.Ssh.assert_called_once() mock_client_obj.InitRemoteHost.assert_called_once() + mock_cvd_utils.GetRemoteHostBaseDir.assert_called_with(2) + mock_ssh_obj.Run.assert_called_with("mkdir -p acloud_cf_2") mock_cvd_utils.UploadArtifacts.assert_called_with( - mock.ANY, "/mock/img", "/mock/cvd.tar.gz") + mock.ANY, "acloud_cf_2", "/mock/img", "/mock/cvd.tar.gz") + mock_cvd_utils.FindRemoteLogs.assert_called_with( + mock.ANY, "acloud_cf_2", 2, 3) mock_client_obj.LaunchCvd.assert_called_with( - "inst", - mock_avd_spec, - mock_avd_spec.cfg.extra_data_disk_size_gb, - boot_timeout_secs=mock_avd_spec.boot_timeout_secs, - extra_args=["extra"]) + "inst", mock_avd_spec, "acloud_cf_2", ["extra"]) mock_pull.GetAllLogFilePaths.assert_called_once() mock_pull.PullLogs.assert_called_once() + factory.GetAdbPorts() + mock_cvd_utils.GetAdbPorts.assert_called_with(2, 3) + factory.GetFastbootPorts() + mock_cvd_utils.GetFastbootPorts.assert_called_with(2, 3) + factory.GetVncPorts() + mock_cvd_utils.GetVncPorts.assert_called_with(2, 3) self.assertEqual({"inst": "failure"}, factory.GetFailures()) - self.assertEqual({"inst": [tombstones, log]}, factory.GetLogs()) + self.assertDictEqual({"inst": [log]}, factory.GetLogs()) @mock.patch("acloud.public.actions.remote_host_cf_device_factory." "cvd_compute_client_multi_stage") @@ -114,24 +128,38 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): """Test CreateInstance with local image zip.""" mock_avd_spec = self._CreateMockAvdSpec() mock_avd_spec.image_source = constants.IMAGE_SRC_LOCAL + mock_ssh_obj = mock.Mock() + mock_ssh.Ssh.return_value = mock_ssh_obj factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( mock_avd_spec, local_image_artifact="/mock/img.zip", cvd_host_package_artifact="/mock/cvd.tar.gz") mock_client_obj = factory.GetComputeClient() - mock_client_obj.FormatRemoteHostInstanceName.return_value = "inst" mock_client_obj.LaunchCvd.return_value = {} + mock_cvd_utils.GetRemoteHostBaseDir.return_value = "acloud_cf_1" + mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" + mock_cvd_utils.FindRemoteLogs.return_value = [] + self.assertEqual("inst", factory.CreateInstance()) - mock_ssh.Ssh.assert_called_once() + mock_cvd_utils.GetRemoteHostBaseDir.assert_called_with(None) mock_client_obj.InitRemoteHost.assert_called_once() + mock_ssh_obj.Run.assert_called_with("mkdir -p acloud_cf_1") mock_cvd_utils.UploadArtifacts.assert_called_with( - mock.ANY, "/mock/img.zip", "/mock/cvd.tar.gz") + mock.ANY, "acloud_cf_1", "/mock/img.zip", "/mock/cvd.tar.gz") + mock_cvd_utils.FindRemoteLogs.assert_called_with( + mock.ANY, "acloud_cf_1", None, None) mock_client_obj.LaunchCvd.assert_called() - mock_pull.GetAllLogFilePaths.assert_called_once() + mock_pull.GetAllLogFilePaths.assert_not_called() mock_pull.PullLogs.assert_not_called() + factory.GetAdbPorts() + mock_cvd_utils.GetAdbPorts.assert_called_with(None, None) + factory.GetFastbootPorts() + mock_cvd_utils.GetFastbootPorts.assert_called_with(None, None) + factory.GetVncPorts() + mock_cvd_utils.GetVncPorts.assert_called_with(None, None) self.assertFalse(factory.GetFailures()) - self.assertEqual(1, len(factory.GetLogs()["inst"])) + self.assertDictEqual({"inst": []}, factory.GetLogs()) @mock.patch("acloud.public.actions.remote_host_cf_device_factory." "cvd_compute_client_multi_stage") @@ -143,7 +171,7 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): @mock.patch("acloud.public.actions.remote_host_cf_device_factory.glob") @mock.patch("acloud.public.actions.remote_host_cf_device_factory.pull") def testCreateInstanceWithRemoteImages(self, mock_pull, mock_glob, - mock_check_call, _mock_cvd_utils, + mock_check_call, mock_cvd_utils, mock_ssh, _mock_client): """Test CreateInstance with remote images.""" mock_avd_spec = self._CreateMockAvdSpec() @@ -155,24 +183,135 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( mock_avd_spec) + mock_cvd_utils.GetRemoteHostBaseDir.return_value = "acloud_cf_1" + mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" + mock_cvd_utils.FindRemoteLogs.return_value = [] + mock_client_obj = factory.GetComputeClient() - mock_client_obj.FormatRemoteHostInstanceName.return_value = "inst" mock_client_obj.LaunchCvd.return_value = {} self.assertEqual("inst", factory.CreateInstance()) - mock_ssh.Ssh.assert_called_once() mock_client_obj.InitRemoteHost.assert_called_once() + mock_ssh_obj.Run.assert_called_with("mkdir -p acloud_cf_1") mock_check_call.assert_called_once() mock_ssh.ShellCmdWithRetry.assert_called_once() self.assertRegex(mock_ssh.ShellCmdWithRetry.call_args[0][0], r"^tar -cf - --lzop -S -C \S+ super\.img \| " - r"/mock/ssh -- tar -xf - --lzop -S$") + r"/mock/ssh -- tar -xf - --lzop -S -C acloud_cf_1$") mock_client_obj.LaunchCvd.assert_called() - mock_pull.GetAllLogFilePaths.assert_called_once() + mock_pull.GetAllLogFilePaths.assert_not_called() mock_pull.PullLogs.assert_not_called() self.assertFalse(factory.GetFailures()) - self.assertEqual(1, len(factory.GetLogs()["inst"])) + self.assertDictEqual({"inst": []}, factory.GetLogs()) + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_compute_client_multi_stage") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.ssh") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_utils") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.glob") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.shutil") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.pull") + def testCreateInstanceWithRemoteFetch(self, mock_pull, mock_shutil, + mock_glob, mock_cvd_utils, mock_ssh, + _mock_client): + """Test CreateInstance with remotely fetched images.""" + mock_avd_spec = self._CreateMockAvdSpec() + mock_avd_spec.remote_fetch = True + mock_ssh_obj = mock.Mock() + mock_ssh.Ssh.return_value = mock_ssh_obj + mock_ssh_obj.GetBaseCmd.return_value = "/mock/ssh" + mock_glob.glob.return_value = ["/mock/fetch_cvd"] + factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( + mock_avd_spec) + + log = {"path": "/log.txt"} + mock_cvd_utils.GetRemoteHostBaseDir.return_value = "acloud_cf_1" + mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" + mock_cvd_utils.FindRemoteLogs.return_value = [] + mock_cvd_utils.GetRemoteFetcherConfigJson.return_value = log + + mock_client_obj = factory.GetComputeClient() + mock_client_obj.LaunchCvd.return_value = {} + mock_client_obj.build_api.GetFetchBuildArgs.return_value = ["-test"] + + self.assertEqual("inst", factory.CreateInstance()) + mock_client_obj.InitRemoteHost.assert_called_once() + mock_ssh_obj.Run.assert_called_with("mkdir -p acloud_cf_1") + mock_client_obj.build_api.DownloadFetchcvd.assert_called_once() + mock_shutil.copyfile.assert_called_with("/mock/key", mock.ANY) + self.assertRegex(mock_ssh.ShellCmdWithRetry.call_args_list[0][0][0], + r"^tar -cf - --lzop -S -C \S+ fetch_cvd \| " + r"/mock/ssh -- tar -xf - --lzop -S -C acloud_cf_1$") + self.assertRegex(mock_ssh.ShellCmdWithRetry.call_args_list[1][0][0], + r"^/mock/ssh -- acloud_cf_1/fetch_cvd " + r"-directory=acloud_cf_1 " + r"-credential_source=acloud_cf_1/credential_key.json " + r"-test$") + mock_client_obj.LaunchCvd.assert_called() + mock_pull.GetAllLogFilePaths.assert_not_called() + mock_pull.PullLogs.assert_not_called() + self.assertFalse(factory.GetFailures()) + self.assertDictEqual({"inst": [log]}, factory.GetLogs()) + + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_compute_client_multi_stage") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.ssh") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_utils") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.glob") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.shutil") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.pull") + def testCreateInstanceWithFetchCvdWrapper(self, mock_pull, mock_shutil, + mock_glob, mock_cvd_utils, mock_ssh, + _mock_client): + """Test CreateInstance with remotely fetched images.""" + mock_avd_spec = self._CreateMockAvdSpec() + mock_avd_spec.remote_fetch = True + mock_avd_spec.fetch_cvd_wrapper = ( + r"GOOGLE_APPLICATION_CREDENTIALS=/fake_key.json," + r"CACHE_CONFIG=/home/shared/cache.properties," + r"java,-jar,/home/shared/FetchCvdWrapper.jar" + ) + mock_ssh_obj = mock.Mock() + mock_ssh.Ssh.return_value = mock_ssh_obj + mock_ssh_obj.GetBaseCmd.return_value = "/mock/ssh" + mock_glob.glob.return_value = ["/mock/fetch_cvd"] + factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( + mock_avd_spec) + + log = {"path": "/log.txt"} + mock_cvd_utils.GetRemoteHostBaseDir.return_value = "acloud_cf_1" + mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" + mock_cvd_utils.FindRemoteLogs.return_value = [] + mock_cvd_utils.GetRemoteFetcherConfigJson.return_value = log + + mock_client_obj = factory.GetComputeClient() + mock_client_obj.LaunchCvd.return_value = {} + mock_client_obj.build_api.GetFetchBuildArgs.return_value = ["-test"] + + self.assertEqual("inst", factory.CreateInstance()) + mock_client_obj.InitRemoteHost.assert_called_once() + mock_ssh_obj.Run.assert_called_with("mkdir -p acloud_cf_1") + mock_client_obj.build_api.DownloadFetchcvd.assert_called_once() + mock_shutil.copyfile.assert_called_with("/mock/key", mock.ANY) + self.assertRegex(mock_ssh.ShellCmdWithRetry.call_args_list[0][0][0], + r"^tar -cf - --lzop -S -C \S+ fetch_cvd \| " + r"/mock/ssh -- tar -xf - --lzop -S -C acloud_cf_1$") + self.assertRegex(mock_ssh.ShellCmdWithRetry.call_args_list[1][0][0], + r"^/mock/ssh -- " + r"GOOGLE_APPLICATION_CREDENTIALS=/fake_key.json " + r"CACHE_CONFIG=/home/shared/cache.properties " + r"java -jar /home/shared/FetchCvdWrapper.jar " + r"-directory=acloud_cf_1 " + r"-fetch_cvd_path=acloud_cf_1/fetch_cvd " + r"-credential_source=acloud_cf_1/credential_key.json " + r"-test$") + mock_client_obj.LaunchCvd.assert_called() + mock_pull.GetAllLogFilePaths.assert_not_called() + mock_pull.PullLogs.assert_not_called() + self.assertFalse(factory.GetFailures()) + self.assertDictEqual({"inst": [log]}, factory.GetLogs()) if __name__ == "__main__": unittest.main() diff --git a/public/actions/remote_host_gf_device_factory.py b/public/actions/remote_host_gf_device_factory.py index 5cf82503..c4b44c4d 100644 --- a/public/actions/remote_host_gf_device_factory.py +++ b/public/actions/remote_host_gf_device_factory.py @@ -21,17 +21,20 @@ import os import posixpath as remote_path import re import shutil +import subprocess import tempfile +import time import zipfile from acloud import errors +from acloud.create import create_common from acloud.internal import constants from acloud.internal.lib import android_build_client from acloud.internal.lib import auth -from acloud.internal.lib import goldfish_remote_host_client from acloud.internal.lib import goldfish_utils from acloud.internal.lib import emulator_console from acloud.internal.lib import ota_tools +from acloud.internal.lib import remote_host_client from acloud.internal.lib import utils from acloud.internal.lib import ssh from acloud.public import report @@ -45,32 +48,41 @@ _SDK_REPO_IMAGE_ZIP_NAME_FORMAT = ("sdk-repo-linux-system-images-" _EXTRA_IMAGE_ZIP_NAME_FORMAT = "emu-extra-linux-system-images-%(build_id)s.zip" _IMAGE_ZIP_NAME_FORMAT = "%(build_target)s-img-%(build_id)s.zip" _OTA_TOOLS_ZIP_NAME = "otatools.zip" -_SYSTEM_IMAGE_NAME = "system.img" - _EMULATOR_INFO_NAME = "emulator-info.txt" _EMULATOR_VERSION_PATTERN = re.compile(r"require\s+version-emulator=" r"(?P<build_id>\w+)") _EMULATOR_ZIP_NAME_FORMAT = "sdk-repo-%(os)s-emulator-%(build_id)s.zip" _EMULATOR_BIN_DIR_NAMES = ("bin64", "qemu") _EMULATOR_BIN_NAME = "emulator" -# Remote paths -_REMOTE_WORKING_DIR = "acloud_gf" -_REMOTE_ARTIFACT_DIR = remote_path.join(_REMOTE_WORKING_DIR, "artifact") -_REMOTE_IMAGE_DIR = remote_path.join(_REMOTE_WORKING_DIR, "image") -_REMOTE_KERNEL_PATH = remote_path.join(_REMOTE_WORKING_DIR, "kernel") -_REMOTE_RAMDISK_PATH = remote_path.join(_REMOTE_WORKING_DIR, "mixed_ramdisk") -_REMOTE_EMULATOR_DIR = remote_path.join(_REMOTE_WORKING_DIR, "emulator") -_REMOTE_INSTANCE_DIR = remote_path.join(_REMOTE_WORKING_DIR, "instance") -_REMOTE_LOGCAT_PATH = os.path.join(_REMOTE_INSTANCE_DIR, "logcat.txt") -_REMOTE_STDOUTERR_PATH = os.path.join(_REMOTE_INSTANCE_DIR, "kernel.log") +_SDK_REPO_EMULATOR_DIR_NAME = "emulator" +# Files in temporary artifact directory. +_DOWNLOAD_DIR_NAME = "download" +_OTA_TOOLS_DIR_NAME = "ota_tools" +_SYSTEM_IMAGE_NAME = "system.img" +# Base directory of an instance. +_REMOTE_INSTANCE_DIR_FORMAT = "acloud_gf_%d" +# Relative paths in a base directory. +_REMOTE_IMAGE_ZIP_PATH = "image.zip" +_REMOTE_EMULATOR_ZIP_PATH = "emulator.zip" +_REMOTE_IMAGE_DIR = "image" +_REMOTE_KERNEL_PATH = "kernel" +_REMOTE_RAMDISK_PATH = "mixed_ramdisk" +_REMOTE_EMULATOR_DIR = "emulator" +_REMOTE_RUNTIME_DIR = "instance" +_REMOTE_LOGCAT_PATH = remote_path.join(_REMOTE_RUNTIME_DIR, "logcat.txt") +_REMOTE_STDOUT_PATH = remote_path.join(_REMOTE_RUNTIME_DIR, "kernel.log") +_REMOTE_STDERR_PATH = remote_path.join(_REMOTE_RUNTIME_DIR, "emu_stderr.txt") # Runtime parameters _EMULATOR_DEFAULT_CONSOLE_PORT = 5554 _DEFAULT_BOOT_TIMEOUT_SECS = 150 +# Error messages +_MISSING_EMULATOR_MSG = ("No emulator zip. Specify " + "--emulator-build-id, or --emulator-zip.") ArtifactPaths = collections.namedtuple( "ArtifactPaths", - ["image_zip", "emulator_zip", "ota_tools_zip", - "system_image_zip", "boot_image"]) + ["image_zip", "emulator_zip", "ota_tools_dir", + "system_image", "boot_image"]) RemotePaths = collections.namedtuple( "RemotePaths", @@ -82,6 +94,9 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Attributes: avd_spec: AVDSpec object that tells us what we're going to create. + android_build_client: An AndroidBuildClient that is lazily initialized. + temp_artifact_dir: The temporary artifact directory that is lazily + initialized during PrepareArtifacts. ssh: Ssh object that executes commands on the remote host. failures: A dictionary the maps instance names to error.DeviceBootError objects. @@ -90,6 +105,8 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): def __init__(self, avd_spec): """Initialize the attributes and the compute client.""" self._avd_spec = avd_spec + self._android_build_client = None + self._temp_artifact_dir = None self._ssh = ssh.Ssh( ip=ssh.IP(ip=self._avd_spec.remote_host), user=self._ssh_user, @@ -99,7 +116,32 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): self._failures = {} self._logs = {} super().__init__(compute_client=( - goldfish_remote_host_client.GoldfishRemoteHostClient())) + remote_host_client.RemoteHostClient(avd_spec.remote_host))) + + @property + def _build_api(self): + """Initialize android_build_client.""" + if not self._android_build_client: + credentials = auth.CreateCredentials(self._avd_spec.cfg) + self._android_build_client = android_build_client.AndroidBuildClient( + credentials) + return self._android_build_client + + @property + def _artifact_dir(self): + """Initialize temp_artifact_dir.""" + if not self._temp_artifact_dir: + self._temp_artifact_dir = tempfile.mkdtemp("host_gf") + logger.info("Create temporary artifact directory: %s", + self._temp_artifact_dir) + return self._temp_artifact_dir + + @property + def _download_dir(self): + """Get the directory where the artifacts are downloaded.""" + if self._avd_spec.image_download_dir: + return self._avd_spec.image_download_dir + return os.path.join(self._artifact_dir, _DOWNLOAD_DIR_NAME) @property def _ssh_user(self): @@ -114,38 +156,69 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): def _ssh_extra_args(self): return self._avd_spec.cfg.extra_args_ssh_tunnel + def _GetConsolePort(self): + """Calculate the console port from the instance number. + + By convention, the console port is an even number, and the adb port is + the console port + 1. The first instance uses port 5554 and 5555. The + second instance uses 5556 and 5557, and so on. + """ + return (_EMULATOR_DEFAULT_CONSOLE_PORT + + ((self._avd_spec.base_instance_num or 1) - 1) * 2) + + def _GetInstancePath(self, relative_path): + """Append a relative path to the instance directory.""" + return remote_path.join( + _REMOTE_INSTANCE_DIR_FORMAT % + (self._avd_spec.base_instance_num or 1), + relative_path) + def CreateInstance(self): """Create a goldfish instance on the remote host. Returns: The instance name. """ - self._InitRemoteHost() - remote_paths = self._PrepareArtifacts() - - instance_name = goldfish_remote_host_client.FormatInstanceName( + instance_name = goldfish_utils.FormatRemoteHostInstanceName( self._avd_spec.remote_host, - _EMULATOR_DEFAULT_CONSOLE_PORT, + self._GetConsolePort(), self._avd_spec.remote_image) - self._logs[instance_name] = [ - report.LogFile(_REMOTE_STDOUTERR_PATH, - constants.LOG_TYPE_KERNEL_LOG), - report.LogFile(_REMOTE_LOGCAT_PATH, constants.LOG_TYPE_LOGCAT)] + + client = self.GetComputeClient() + timed_stage = constants.TIME_GCE + start_time = time.time() try: + client.SetStage(constants.STAGE_SSH_CONNECT) + self._InitRemoteHost() + + start_time = client.RecordTime(timed_stage, start_time) + timed_stage = constants.TIME_ARTIFACT + client.SetStage(constants.STAGE_ARTIFACT) + remote_paths = self._PrepareArtifacts() + + start_time = client.RecordTime(timed_stage, start_time) + timed_stage = constants.TIME_LAUNCH + client.SetStage(constants.STAGE_BOOT_UP) + self._logs[instance_name] = self._GetEmulatorLogs() self._StartEmulator(remote_paths) self._WaitForEmulator() - except errors.DeviceBootError as e: + except (errors.DriverError, subprocess.CalledProcessError) as e: + # Catch the generic runtime error and CalledProcessError which is + # raised by the ssh module. self._failures[instance_name] = e + finally: + client.RecordTime(timed_stage, start_time) + return instance_name def _InitRemoteHost(self): - """Remove existing instance and working directory.""" + """Remove the existing instance and the instance directory.""" # Disable authentication for emulator console. self._ssh.Run("""'echo -n "" > .emulator_console_auth_token'""") try: with emulator_console.RemoteEmulatorConsole( self._avd_spec.remote_host, - _EMULATOR_DEFAULT_CONSOLE_PORT, + self._GetConsolePort(), self._ssh_user, self._ssh_private_key_path, self._ssh_extra_args) as console: @@ -154,7 +227,7 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): except errors.DeviceConnectionError as e: logger.info("Did not kill existing emulator: %s", str(e)) # Delete instance files. - self._ssh.Run("rm -rf %s" % _REMOTE_WORKING_DIR) + self._ssh.Run(f"rm -rf {self._GetInstancePath('')}") def _PrepareArtifacts(self): """Prepare artifacts on remote host. @@ -165,21 +238,13 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: An object of RemotePaths. """ - if self._avd_spec.image_download_dir: - temp_download_dir = None - download_dir = self._avd_spec.image_download_dir - else: - temp_download_dir = tempfile.mkdtemp() - download_dir = temp_download_dir - logger.info("--image-download-dir is not specified. Create " - "temporary download directory: %s", download_dir) - try: - artifact_paths = self._RetrieveArtifacts(download_dir) + artifact_paths = self._RetrieveArtifacts() return self._UploadArtifacts(artifact_paths) finally: - if temp_download_dir: - shutil.rmtree(temp_download_dir, ignore_errors=True) + if self._temp_artifact_dir: + shutil.rmtree(self._temp_artifact_dir, ignore_errors=True) + self._temp_artifact_dir = None @staticmethod def _InferEmulatorZipName(build_target, build_id): @@ -191,7 +256,7 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Args: build_target: The emulator build target name, e.g., - "sdk_tools_linux", "aarch64_sdk_tools_mac". + "emulator-linux_x64_nolocationui", "aarch64_sdk_tools_mac". build_id: A string, the emulator build ID. Returns: @@ -210,14 +275,11 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): return _EMULATOR_ZIP_NAME_FORMAT % {"os": os_name, "build_id": build_id} - @staticmethod - def _RetrieveArtifact(download_dir, build_api, build_target, build_id, + def _RetrieveArtifact(self, build_target, build_id, resource_id): """Retrieve an artifact from cache or Android Build API. Args: - download_dir: The cache directory. - build_api: An AndroidBuildClient object. build_target: A string, the build target of the artifact. e.g., "sdk_phone_x86_64-userdebug". build_id: A string, the build ID of the artifact. @@ -227,7 +289,7 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: The path to the artifact in download_dir. """ - local_path = os.path.join(download_dir, build_id, build_target, + local_path = os.path.join(self._download_dir, build_id, build_target, resource_id) if os.path.isfile(local_path): logger.info("Skip downloading existing artifact: %s", local_path) @@ -236,129 +298,187 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): complete = False try: os.makedirs(os.path.dirname(local_path), exist_ok=True) - build_api.DownloadArtifact(build_target, build_id, resource_id, - local_path, build_api.LATEST) + self._build_api.DownloadArtifact( + build_target, build_id, resource_id, local_path, + self._build_api.LATEST) complete = True finally: if not complete and os.path.isfile(local_path): os.remove(local_path) return local_path - def _RetrieveEmulatorBuildID(self, download_dir, build_api, build_target, - build_id): - """Retrieve required emulator build from a goldfish image build.""" - emulator_info_path = self._RetrieveArtifact(download_dir, build_api, - build_target, build_id, - _EMULATOR_INFO_NAME) - with open(emulator_info_path, 'r') as emulator_info: - for line in emulator_info: - match = _EMULATOR_VERSION_PATTERN.fullmatch(line.strip()) - if match: - logger.info("Found emulator build ID: %s", line) - return match.group("build_id") - return None - @utils.TimeExecute(function_description="Download Android Build artifacts") - def _RetrieveArtifacts(self, download_dir): + def _RetrieveArtifacts(self): """Retrieve goldfish images and tools from cache or Android Build API. - Args: - download_dir: The cache directory. - Returns: An object of ArtifactPaths. Raises: errors.GetRemoteImageError: Fails to download rom images. + errors.GetLocalImageError: Fails to validate local image zip. + errors.GetSdkRepoPackageError: Fails to retrieve emulator zip. """ - credentials = auth.CreateCredentials(self._avd_spec.cfg) - build_api = android_build_client.AndroidBuildClient(credentials) # Device images. + if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: + image_zip_path = self._RetrieveDeviceImageZip() + elif self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: + image_zip_path = self._avd_spec.local_image_artifact + if not image_zip_path or not zipfile.is_zipfile(image_zip_path): + raise errors.GetLocalImageError( + f"{image_zip_path or self._avd_spec.local_image_dir} is " + "not an SDK repository zip.") + else: + raise errors.CreateError( + f"Unknown image source: {self._avd_spec.image_source}") + + # Emulator tools. + emu_zip_path = (self._avd_spec.emulator_zip or + self._RetrieveEmulatorZip()) + if not emu_zip_path: + raise errors.GetSdkRepoPackageError(_MISSING_EMULATOR_MSG) + + # System image. + if self._avd_spec.local_system_image: + system_image_path = create_common.FindSystemImage( + self._avd_spec.local_system_image) + else: + system_image_path = self._RetrieveSystemImage() + + # Boot image. + if self._avd_spec.local_kernel_image: + boot_image_path = create_common.FindBootImage( + self._avd_spec.local_kernel_image) + else: + boot_image_path = self._RetrieveBootImage() + + # OTA tools. + ota_tools_dir = None + if system_image_path or boot_image_path: + if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: + ota_tools_dir = self._RetrieveOtaTools() + else: + ota_tools_dir = ota_tools.FindOtaToolsDir( + self._avd_spec.local_tool_dirs + + create_common.GetNonEmptyEnvVars( + constants.ENV_ANDROID_SOONG_HOST_OUT, + constants.ENV_ANDROID_HOST_OUT)) + + return ArtifactPaths(image_zip_path, emu_zip_path, ota_tools_dir, + system_image_path, boot_image_path) + + def _RetrieveDeviceImageZip(self): + """Retrieve device image zip from cache or Android Build API. + + Returns: + The path to the device image zip in download_dir. + """ build_id = self._avd_spec.remote_image.get(constants.BUILD_ID) build_target = self._avd_spec.remote_image.get(constants.BUILD_TARGET) image_zip_name_format = (_EXTRA_IMAGE_ZIP_NAME_FORMAT if self._ShouldMixDiskImage() else _SDK_REPO_IMAGE_ZIP_NAME_FORMAT) - image_zip_path = self._RetrieveArtifact( - download_dir, build_api, build_target, build_id, + return self._RetrieveArtifact( + build_target, build_id, image_zip_name_format % {"build_id": build_id}) - # Emulator tools. - emu_build_id = self._avd_spec.emulator_build_id - if not emu_build_id: - emu_build_id = self._RetrieveEmulatorBuildID( - download_dir, build_api, build_target, build_id) - if not emu_build_id: - raise errors.GetRemoteImageError( - "No emulator build ID in command line or " - "emulator-info.txt.") + def _RetrieveEmulatorBuildID(self): + """Retrieve required emulator build from a goldfish image build. + + Returns: + A string, the emulator build ID. + None if the build info is empty. + """ + build_id = self._avd_spec.remote_image.get(constants.BUILD_ID) + build_target = self._avd_spec.remote_image.get(constants.BUILD_TARGET) + if build_id and build_target: + emu_info_path = self._RetrieveArtifact(build_target, build_id, + _EMULATOR_INFO_NAME) + with open(emu_info_path, "r", encoding="utf-8") as emu_info: + for line in emu_info: + match = _EMULATOR_VERSION_PATTERN.fullmatch(line.strip()) + if match: + logger.info("Found emulator build ID: %s", line) + return match.group("build_id") + return None + def _RetrieveEmulatorZip(self): + """Retrieve emulator zip from cache or Android Build API. + + Returns: + The path to the emulator zip in download_dir. + None if this method cannot determine the emulator build ID. + """ + emu_build_id = (self._avd_spec.emulator_build_id or + self._RetrieveEmulatorBuildID()) + if not emu_build_id: + return None emu_build_target = (self._avd_spec.emulator_build_target or self._avd_spec.cfg.emulator_build_target) emu_zip_name = self._InferEmulatorZipName(emu_build_target, emu_build_id) - emu_zip_path = self._RetrieveArtifact(download_dir, build_api, - emu_build_target, emu_build_id, - emu_zip_name) - - system_image_zip_path = self._RetrieveSystemImageZip( - download_dir, build_api) - boot_image_path = self._RetrieveBootImage(download_dir, build_api) - # Retrieve OTA tools from the goldfish build which contains - # mk_combined_img. - ota_tools_zip_path = ( - self._RetrieveArtifact(download_dir, build_api, build_target, - build_id, _OTA_TOOLS_ZIP_NAME) - if system_image_zip_path or boot_image_path else None) - - return ArtifactPaths(image_zip_path, emu_zip_path, - ota_tools_zip_path, system_image_zip_path, - boot_image_path) - - def _RetrieveSystemImageZip(self, download_dir, build_api): - """Retrieve system image zip if system build info is not empty. + return self._RetrieveArtifact(emu_build_target, emu_build_id, + emu_zip_name) - Args: - download_dir: The download cache directory. - build_api: An AndroidBuildClient object. + def _RetrieveSystemImage(self): + """Retrieve and unzip system image if system build info is not empty. Returns: - The path to the system image zip in download_dir. + The path to the temporary system image. None if the system build info is empty. """ build_id = self._avd_spec.system_build_info.get(constants.BUILD_ID) build_target = self._avd_spec.system_build_info.get( constants.BUILD_TARGET) - if build_id and build_target: - image_zip_name = _IMAGE_ZIP_NAME_FORMAT % { - "build_target": build_target.split("-", 1)[0], - "build_id": build_id} - return self._RetrieveArtifact( - download_dir, build_api, build_target, build_id, - image_zip_name) - return None - - def _RetrieveBootImage(self, download_dir, build_api): - """Retrieve boot image if kernel build info is not empty. - - Args: - download_dir: The download cache directory. - build_api: An AndroidBuildClient object. + if not build_id or not build_target: + return None + image_zip_name = _IMAGE_ZIP_NAME_FORMAT % { + "build_target": build_target.split("-", 1)[0], + "build_id": build_id} + image_zip_path = self._RetrieveArtifact(build_target, build_id, + image_zip_name) + logger.debug("Unzip %s from %s to %s.", + _SYSTEM_IMAGE_NAME, image_zip_path, self._artifact_dir) + with zipfile.ZipFile(image_zip_path, "r") as zip_file: + zip_file.extract(_SYSTEM_IMAGE_NAME, self._artifact_dir) + return os.path.join(self._artifact_dir, _SYSTEM_IMAGE_NAME) + + def _RetrieveBootImage(self): + """Retrieve boot image if boot build info is not empty. Returns: The path to the boot image in download_dir. - None if the kernel build info is empty. + None if the boot build info is empty. """ - build_id = self._avd_spec.kernel_build_info.get(constants.BUILD_ID) - build_target = self._avd_spec.kernel_build_info.get( + build_id = self._avd_spec.boot_build_info.get(constants.BUILD_ID) + build_target = self._avd_spec.boot_build_info.get( constants.BUILD_TARGET) - image_name = self._avd_spec.kernel_build_info.get( + image_name = self._avd_spec.boot_build_info.get( constants.BUILD_ARTIFACT) if build_id and build_target and image_name: - return self._RetrieveArtifact( - download_dir, build_api, build_target, build_id, image_name) + return self._RetrieveArtifact(build_target, build_id, image_name) return None + def _RetrieveOtaTools(self): + """Retrieve and unzip OTA tools. + + This method retrieves OTA tools from the goldfish build which contains + mk_combined_img. + + Returns: + The path to the temporary OTA tools directory. + """ + build_id = self._avd_spec.remote_image.get(constants.BUILD_ID) + build_target = self._avd_spec.remote_image.get(constants.BUILD_TARGET) + zip_path = self._RetrieveArtifact(build_target, build_id, + _OTA_TOOLS_ZIP_NAME) + ota_tools_dir = os.path.join(self._artifact_dir, _OTA_TOOLS_DIR_NAME) + logger.debug("Unzip %s to %s.", zip_path, ota_tools_dir) + os.mkdir(ota_tools_dir) + with zipfile.ZipFile(zip_path, "r") as zip_file: + zip_file.extractall(ota_tools_dir) + return ota_tools_dir + @staticmethod def _GetSubdirNameInZip(zip_path): """Get the name of the only subdirectory in a zip. @@ -367,12 +487,12 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): subdirectory. This class needs to find out the subdirectory name in order to construct the remote commands. - For example, in sdk-repo-*-emulator-*.zip, all files are in - "emulator/". The zip entries are: + For example, in a sdk-repo-linux-system-images-*.zip for arm64, all + files are in "arm64-v8a/". The zip entries are: - emulator/NOTICE.txt - emulator/emulator - emulator/lib64/libc++.so + arm64-v8a/NOTICE.txt + arm64-v8a/system.img + arm64-v8a/data/local.prop ... This method scans the entries and returns the common subdirectory name. @@ -388,7 +508,7 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): zip_path, " ".join(entries)) return "" - def _UploadArtifacts(self, artifacts_paths): + def _UploadArtifacts(self, artifact_paths): """Process and upload all images and tools to the remote host. Args: @@ -398,38 +518,33 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): An object of RemotePaths. """ remote_emulator_dir, remote_image_dir = self._UploadDeviceImages( - artifacts_paths.emulator_zip, artifacts_paths.image_zip) + artifact_paths.emulator_zip, artifact_paths.image_zip) remote_kernel_path = None remote_ramdisk_path = None - if artifacts_paths.boot_image or artifacts_paths.system_image_zip: + if artifact_paths.boot_image or artifact_paths.system_image: with tempfile.TemporaryDirectory("host_gf") as temp_dir: - ota_tools_dir = os.path.join(temp_dir, "ota_tools") - logger.debug("Unzip %s.", artifacts_paths.ota_tools_zip) - with zipfile.ZipFile(artifacts_paths.ota_tools_zip, - "r") as zip_file: - zip_file.extractall(ota_tools_dir) - ota = ota_tools.OtaTools(ota_tools_dir) + ota = ota_tools.OtaTools(artifact_paths.ota_tools_dir) image_dir = os.path.join(temp_dir, "images") - logger.debug("Unzip %s.", artifacts_paths.image_zip) - with zipfile.ZipFile(artifacts_paths.image_zip, + logger.debug("Unzip %s.", artifact_paths.image_zip) + with zipfile.ZipFile(artifact_paths.image_zip, "r") as zip_file: zip_file.extractall(image_dir) image_dir = os.path.join( image_dir, - self._GetSubdirNameInZip(artifacts_paths.image_zip)) + self._GetSubdirNameInZip(artifact_paths.image_zip)) - if artifacts_paths.system_image_zip: + if artifact_paths.system_image: self._MixAndUploadDiskImage( remote_image_dir, image_dir, - artifacts_paths.system_image_zip, ota) + artifact_paths.system_image, ota) - if artifacts_paths.boot_image: + if artifact_paths.boot_image: remote_kernel_path, remote_ramdisk_path = ( self._MixAndUploadKernelImages( - image_dir, artifacts_paths.boot_image, ota)) + image_dir, artifact_paths.boot_image, ota)) return RemotePaths(remote_image_dir, remote_emulator_dir, remote_kernel_path, remote_ramdisk_path) @@ -444,8 +559,9 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: Boolean, whether a mixed disk image is required. """ - return (self._avd_spec.system_build_info.get(constants.BUILD_ID) and - self._avd_spec.system_build_info.get(constants.BUILD_TARGET)) + return self._avd_spec.local_system_image or ( + self._avd_spec.system_build_info.get(constants.BUILD_ID) and + self._avd_spec.system_build_info.get(constants.BUILD_TARGET)) @utils.TimeExecute( function_description="Processing and uploading tools and images") @@ -459,24 +575,22 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: The remote paths to the extracted emulator tools and images. """ - self._ssh.Run("mkdir -p " + - " ".join([_REMOTE_INSTANCE_DIR, _REMOTE_ARTIFACT_DIR, - _REMOTE_EMULATOR_DIR, _REMOTE_IMAGE_DIR])) - self._ssh.ScpPushFile(emulator_zip_path, _REMOTE_ARTIFACT_DIR) - self._ssh.ScpPushFile(image_zip_path, _REMOTE_ARTIFACT_DIR) - - self._ssh.Run("unzip -d %s %s" % ( - _REMOTE_EMULATOR_DIR, - remote_path.join(_REMOTE_ARTIFACT_DIR, - os.path.basename(emulator_zip_path)))) - self._ssh.Run("unzip -d %s %s" % ( - _REMOTE_IMAGE_DIR, - remote_path.join(_REMOTE_ARTIFACT_DIR, - os.path.basename(image_zip_path)))) + remote_emulator_dir = self._GetInstancePath(_REMOTE_EMULATOR_DIR) + remote_image_dir = self._GetInstancePath(_REMOTE_IMAGE_DIR) + remote_emulator_zip_path = self._GetInstancePath( + _REMOTE_EMULATOR_ZIP_PATH) + remote_image_zip_path = self._GetInstancePath(_REMOTE_IMAGE_ZIP_PATH) + self._ssh.Run(f"mkdir -p {remote_emulator_dir} {remote_image_dir}") + self._ssh.ScpPushFile(emulator_zip_path, remote_emulator_zip_path) + self._ssh.ScpPushFile(image_zip_path, remote_image_zip_path) + + self._ssh.Run(f"unzip -d {remote_emulator_dir} " + f"{remote_emulator_zip_path}") + self._ssh.Run(f"unzip -d {remote_image_dir} {remote_image_zip_path}") remote_emulator_subdir = remote_path.join( - _REMOTE_EMULATOR_DIR, self._GetSubdirNameInZip(emulator_zip_path)) + remote_emulator_dir, _SDK_REPO_EMULATOR_DIR_NAME) remote_image_subdir = remote_path.join( - _REMOTE_IMAGE_DIR, self._GetSubdirNameInZip(image_zip_path)) + remote_image_dir, self._GetSubdirNameInZip(image_zip_path)) # TODO(b/141898893): In Android build environment, emulator gets build # information from $ANDROID_PRODUCT_OUT/system/build.prop. # If image_dir is an extacted SDK repository, the file is at @@ -493,30 +607,22 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): return remote_emulator_subdir, remote_image_subdir def _MixAndUploadDiskImage(self, remote_image_dir, image_dir, - system_image_zip_path, ota): + system_image_path, ota): """Mix emulator images with a system image and upload them. Args: remote_image_dir: The remote directory where the mixed disk image is uploaded. image_dir: The directory containing emulator images. - system_image_zip_path: The path to the zip containing the system - image. + system_image_path: The path to the system image. ota: An instance of ota_tools.OtaTools. Returns: The remote path to the mixed disk image. """ with tempfile.TemporaryDirectory("host_gf_disk") as temp_dir: - logger.debug("Unzip %s.", system_image_zip_path) - with zipfile.ZipFile(system_image_zip_path, "r") as zip_file: - zip_file.extract(_SYSTEM_IMAGE_NAME, temp_dir) - mixed_image = goldfish_utils.MixWithSystemImage( - os.path.join(temp_dir, "mix_disk"), - image_dir, - os.path.join(temp_dir, _SYSTEM_IMAGE_NAME), - ota) + temp_dir, image_dir, system_image_path, ota) # TODO(b/142228085): Use -system instead of overwriting the file. remote_disk_image_path = os.path.join( @@ -536,14 +642,25 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: The remote paths to the kernel image and the ramdisk image. """ + remote_kernel_path = self._GetInstancePath(_REMOTE_KERNEL_PATH) + remote_ramdisk_path = self._GetInstancePath(_REMOTE_RAMDISK_PATH) with tempfile.TemporaryDirectory("host_gf_kernel") as temp_dir: kernel_path, ramdisk_path = goldfish_utils.MixWithBootImage( temp_dir, image_dir, boot_image_path, ota) - self._ssh.ScpPushFile(kernel_path, _REMOTE_KERNEL_PATH) - self._ssh.ScpPushFile(ramdisk_path, _REMOTE_RAMDISK_PATH) + self._ssh.ScpPushFile(kernel_path, remote_kernel_path) + self._ssh.ScpPushFile(ramdisk_path, remote_ramdisk_path) - return _REMOTE_KERNEL_PATH, _REMOTE_RAMDISK_PATH + return remote_kernel_path, remote_ramdisk_path + + def _GetEmulatorLogs(self): + """Return the logs created by the remote emulator command.""" + return [report.LogFile(self._GetInstancePath(_REMOTE_STDOUT_PATH), + constants.LOG_TYPE_KERNEL_LOG), + report.LogFile(self._GetInstancePath(_REMOTE_STDERR_PATH), + constants.LOG_TYPE_TEXT), + report.LogFile(self._GetInstancePath(_REMOTE_LOGCAT_PATH), + constants.LOG_TYPE_LOGCAT)] @utils.TimeExecute(function_description="Start emulator") def _StartEmulator(self, remote_paths): @@ -561,16 +678,16 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): remote_bin_paths.append(remote_emulator_bin_path) self._ssh.Run("chmod -R +x %s" % " ".join(remote_bin_paths)) + remote_runtime_dir = self._GetInstancePath(_REMOTE_RUNTIME_DIR) + self._ssh.Run(f"mkdir -p {remote_runtime_dir}") env = {constants.ENV_ANDROID_PRODUCT_OUT: remote_paths.image_dir, - constants.ENV_ANDROID_TMP: _REMOTE_INSTANCE_DIR, - constants.ENV_ANDROID_BUILD_TOP: _REMOTE_INSTANCE_DIR} - adb_port = _EMULATOR_DEFAULT_CONSOLE_PORT + 1 + constants.ENV_ANDROID_TMP: remote_runtime_dir, + constants.ENV_ANDROID_BUILD_TOP: remote_runtime_dir} cmd = ["nohup", remote_emulator_bin_path, "-verbose", "-show-kernel", "-read-only", "-ports", - str(_EMULATOR_DEFAULT_CONSOLE_PORT) + "," + str(adb_port), + str(self._GetConsolePort()) + "," + str(self.GetAdbPorts()[0]), "-no-window", - "-logcat-output", _REMOTE_LOGCAT_PATH, - "-stdouterr-file", _REMOTE_STDOUTERR_PATH] + "-logcat-output", self._GetInstancePath(_REMOTE_LOGCAT_PATH)] if remote_paths.kernel: cmd.extend(("-kernel", remote_paths.kernel)) @@ -586,12 +703,13 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): cmd.extend(("-qemu", "-append", "androidboot.verifiedbootstate=orange")) - # Emulator doesn't create -stdouterr-file automatically. + # Emulator does not support -stdouterr-file on macOS. self._ssh.Run( - "'export {env} ; touch {stdouterr} ; {cmd} &'".format( + "'export {env} ; {cmd} 1> {stdout} 2> {stderr} &'".format( env=" ".join(k + "=~/" + v for k, v in env.items()), - stdouterr=_REMOTE_STDOUTERR_PATH, - cmd=" ".join(cmd))) + cmd=" ".join(cmd), + stdout=self._GetInstancePath(_REMOTE_STDOUT_PATH), + stderr=self._GetInstancePath(_REMOTE_STDERR_PATH))) @utils.TimeExecute(function_description="Wait for emulator") def _WaitForEmulator(self): @@ -602,7 +720,7 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): errors.DeviceBootTimeoutError if boot times out. """ ip_addr = self._avd_spec.remote_host - console_port = _EMULATOR_DEFAULT_CONSOLE_PORT + console_port = self._GetConsolePort() poll_timeout_secs = (self._avd_spec.boot_timeout_secs or _DEFAULT_BOOT_TIMEOUT_SECS) try: @@ -633,6 +751,16 @@ class RemoteHostGoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): self._avd_spec.remote_image.items() if val} return build_info_dict + def GetAdbPorts(self): + """Get ADB ports of the created devices. + + This class does not support --num-avds-per-instance. + + Returns: + The port numbers as a list of integers. + """ + return [self._GetConsolePort() + 1] + def GetFailures(self): """Get Failures from all devices. diff --git a/public/actions/remote_host_gf_device_factory_test.py b/public/actions/remote_host_gf_device_factory_test.py index 8321eb32..f0d5a9cd 100644 --- a/public/actions/remote_host_gf_device_factory_test.py +++ b/public/actions/remote_host_gf_device_factory_test.py @@ -44,14 +44,15 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): constants.BUILD_TARGET: "sdk_arm64-sdk", } _ARM64_INSTANCE_NAME = ( - "host-goldfish-192.0.2.1-5554-123456-sdk_arm64-sdk") + "host-goldfish-192.0.2.1-5556-123456-sdk_arm64-sdk") _CFG_ATTRS = { "ssh_private_key_path": "cfg_key_path", "extra_args_ssh_tunnel": "extra args", - "emulator_build_target": "sdk_tools_linux", + "emulator_build_target": "emulator-linux_x64_nolocationui", } _AVD_SPEC_ATTRS = { "cfg": None, + "image_source": constants.IMAGE_SRC_REMOTE, "remote_image": _X86_64_BUILD_INFO, "image_download_dir": None, "host_user": "user", @@ -59,33 +60,42 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): "host_ssh_private_key_path": None, "emulator_build_id": None, "emulator_build_target": None, + "emulator_zip": None, "system_build_info": {}, - "kernel_build_info": {}, + "boot_build_info": {}, + "local_image_artifact": None, + "local_kernel_image": None, + "local_system_image": None, + "local_tool_dirs": [], + "base_instance_num": None, "boot_timeout_secs": None, "hw_customize": False, "hw_property": {}, "gpu": "auto", } - _LOGS = [{"path": "acloud_gf/instance/kernel.log", "type": "KERNEL_LOG"}, - {"path": "acloud_gf/instance/logcat.txt", "type": "LOGCAT"}] + _LOGS = [{"path": "acloud_gf_1/instance/kernel.log", "type": "KERNEL_LOG"}, + {"path": "acloud_gf_1/instance/emu_stderr.txt", "type": "TEXT"}, + {"path": "acloud_gf_1/instance/logcat.txt", "type": "LOGCAT"}] _SSH_COMMAND = ( - "'export ANDROID_PRODUCT_OUT=~/acloud_gf/image/x86_64 " - "ANDROID_TMP=~/acloud_gf/instance " - "ANDROID_BUILD_TOP=~/acloud_gf/instance ; " - "touch acloud_gf/instance/kernel.log ; " - "nohup acloud_gf/emulator/x86_64/emulator -verbose " + "'export ANDROID_PRODUCT_OUT=~/acloud_gf_1/image/x86_64 " + "ANDROID_TMP=~/acloud_gf_1/instance " + "ANDROID_BUILD_TOP=~/acloud_gf_1/instance ; " + "nohup acloud_gf_1/emulator/emulator/emulator -verbose " "-show-kernel -read-only -ports 5554,5555 -no-window " - "-logcat-output acloud_gf/instance/logcat.txt " - "-stdouterr-file acloud_gf/instance/kernel.log -gpu auto &'" + "-logcat-output acloud_gf_1/instance/logcat.txt -gpu auto " + "1> acloud_gf_1/instance/kernel.log " + "2> acloud_gf_1/instance/emu_stderr.txt &'" ) def setUp(self): super().setUp() self._mock_ssh = mock.Mock() self.Patch(gf_factory.ssh, "Ssh", return_value=self._mock_ssh) - self.Patch(gf_factory.goldfish_remote_host_client, - "GoldfishRemoteHostClient") - self.Patch(gf_factory.auth, "CreateCredentials") + self._mock_remote_host_client = mock.Mock() + self.Patch(gf_factory.remote_host_client, "RemoteHostClient", + return_value=self._mock_remote_host_client) + self._mock_create_credentials = self.Patch( + gf_factory.auth, "CreateCredentials") # Emulator console self._mock_console = mock.MagicMock() self._mock_console.__enter__.return_value = self._mock_console @@ -133,10 +143,10 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): else: self._CreateImageZip(local_path) elif resource_id == "emulator-info.txt": - with open(local_path, "w") as file: + with open(local_path, "w", encoding="utf-8") as file: file.write(self._EMULATOR_INFO) else: - with open(local_path, "w") as file: + with open(local_path, "w", encoding="utf-8") as file: pass def testCreateInstanceWithCfg(self): @@ -147,11 +157,14 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.assertEqual(self._X86_64_INSTANCE_NAME, instance_name) self.assertEqual(self._X86_64_BUILD_INFO, factory.GetBuildInfoDict()) + self.assertEqual([5555], factory.GetAdbPorts()) + self.assertEqual([None], factory.GetFastbootPorts()) + self.assertEqual([None], factory.GetVncPorts()) self.assertEqual({}, factory.GetFailures()) self.assertEqual({instance_name: self._LOGS}, factory.GetLogs()) # Artifacts. self._mock_android_build_client.DownloadArtifact.assert_any_call( - "sdk_tools_linux", "111111", + "emulator-linux_x64_nolocationui", "111111", "sdk-repo-linux-emulator-111111.zip", mock.ANY, mock.ANY) self._mock_android_build_client.DownloadArtifact.assert_any_call( "sdk_x86_64-sdk", "123456", @@ -167,6 +180,18 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.assertEqual(self._mock_console.Ping.call_count, self._mock_console.Reconnect.call_count + 1) self._mock_console.Reconnect.assert_called() + # RemoteHostClient. + self._mock_remote_host_client.RecordTime.assert_has_calls([ + mock.call(constants.TIME_GCE, mock.ANY), + mock.call(constants.TIME_ARTIFACT, mock.ANY), + mock.call(constants.TIME_LAUNCH, mock.ANY)]) + self.assertEqual(3, + self._mock_remote_host_client.RecordTime.call_count) + self._mock_remote_host_client.SetStage.assert_has_calls([ + mock.call(constants.STAGE_SSH_CONNECT), + mock.call(constants.STAGE_ARTIFACT), + mock.call(constants.STAGE_BOOT_UP)]) + self.assertEqual(3, self._mock_remote_host_client.SetStage.call_count) def testCreateInstanceWithAvdSpec(self): """Test RemoteHostGoldfishDeviceFactory with command options.""" @@ -174,11 +199,12 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): self._mock_avd_spec.host_ssh_private_key_path = "key_path" self._mock_avd_spec.emulator_build_id = "999999" self._mock_avd_spec.emulator_build_target = "aarch64_sdk_tools_mac" + self._mock_avd_spec.base_instance_num = 2 self._mock_avd_spec.boot_timeout_secs = 1 self._mock_avd_spec.hw_customize = True self._mock_avd_spec.hw_property = {"disk": "4096"} - self._mock_android_build_client.DownloadArtifact.side_effect = ( - AssertionError("DownloadArtifact should not be called.")) + self._mock_create_credentials.side_effect = AssertionError( + "CreateCredentials should not be called.") # All artifacts are cached. with tempfile.TemporaryDirectory() as download_dir: self._mock_avd_spec.image_download_dir = download_dir @@ -200,6 +226,8 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.assertEqual(self._ARM64_INSTANCE_NAME, instance_name) self.assertEqual(self._ARM64_BUILD_INFO, factory.GetBuildInfoDict()) + self.assertEqual([5557], factory.GetAdbPorts()) + self.assertEqual([None], factory.GetVncPorts()) self.assertEqual({}, factory.GetFailures()) @mock.patch("acloud.public.actions.remote_host_gf_device_factory." @@ -215,7 +243,7 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): factory = gf_factory.RemoteHostGoldfishDeviceFactory( self._mock_avd_spec) - instance_name = factory.CreateInstance() + factory.CreateInstance() # Artifacts. self._mock_android_build_client.DownloadArtifact.assert_any_call( "sdk_x86_64-sdk", "123456", @@ -231,19 +259,21 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): # Images. mock_gf_utils.MixWithSystemImage.assert_called_once() self._mock_ssh.ScpPushFile.assert_called_with( - "/mixed/disk", "acloud_gf/image/x86_64/system-qemu.img") + "/mixed/disk", "acloud_gf_1/image/x86_64/system-qemu.img") - self.assertEqual(self._X86_64_INSTANCE_NAME, instance_name) + mock_gf_utils.FormatRemoteHostInstanceName.assert_called() self.assertEqual(self._X86_64_BUILD_INFO, factory.GetBuildInfoDict()) + self.assertEqual([5555], factory.GetAdbPorts()) + self.assertEqual([None], factory.GetVncPorts()) self.assertEqual({}, factory.GetFailures()) @mock.patch("acloud.public.actions.remote_host_gf_device_factory." "goldfish_utils") - def testCreateInstanceWithKernelBuild(self, mock_gf_utils): - """Test RemoteHostGoldfishDeviceFactory with kernel build.""" - self._mock_avd_spec.kernel_build_info = { + def testCreateInstanceWithBootBuild(self, mock_gf_utils): + """Test RemoteHostGoldfishDeviceFactory with boot build.""" + self._mock_avd_spec.boot_build_info = { constants.BUILD_ID: "111111", - constants.BUILD_TARGET: "aosp_x86_64-userdebug", + constants.BUILD_TARGET: "gki_x86_64-userdebug", constants.BUILD_ARTIFACT: "boot-5.10.img"} mock_gf_utils.ConvertAvdSpecToArgs.return_value = ["-gpu", "auto"] mock_gf_utils.MixWithBootImage.return_value = ( @@ -251,13 +281,13 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): factory = gf_factory.RemoteHostGoldfishDeviceFactory( self._mock_avd_spec) - instance_name = factory.CreateInstance() + factory.CreateInstance() # Artifacts. self._mock_android_build_client.DownloadArtifact.assert_any_call( "sdk_x86_64-sdk", "123456", "sdk-repo-linux-system-images-123456.zip", mock.ANY, mock.ANY) self._mock_android_build_client.DownloadArtifact.assert_any_call( - "aosp_x86_64-userdebug", "111111", + "gki_x86_64-userdebug", "111111", "boot-5.10.img", mock.ANY, mock.ANY) self._mock_android_build_client.DownloadArtifact.assert_any_call( "sdk_x86_64-sdk", "123456", @@ -267,15 +297,101 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): # Images. mock_gf_utils.MixWithBootImage.assert_called_once() self._mock_ssh.ScpPushFile.assert_any_call( - "/path/to/kernel", "acloud_gf/kernel") + "/path/to/kernel", "acloud_gf_1/kernel") self._mock_ssh.ScpPushFile.assert_any_call( - "/path/to/ramdisk", "acloud_gf/mixed_ramdisk") + "/path/to/ramdisk", "acloud_gf_1/mixed_ramdisk") - self.assertEqual(self._X86_64_INSTANCE_NAME, instance_name) + mock_gf_utils.FormatRemoteHostInstanceName.assert_called() self.assertEqual(self._X86_64_BUILD_INFO, factory.GetBuildInfoDict()) + self.assertEqual([5555], factory.GetAdbPorts()) + self.assertEqual([None], factory.GetVncPorts()) self.assertEqual({}, factory.GetFailures()) - def testCreateInstanceError(self): + @mock.patch("acloud.public.actions.remote_host_gf_device_factory." + "ota_tools") + @mock.patch("acloud.public.actions.remote_host_gf_device_factory." + "goldfish_utils") + def testCreateInstanceWithLocalFiles(self, mock_gf_utils, mock_ota_tools): + """Test RemoteHostGoldfishDeviceFactory with local files.""" + with tempfile.TemporaryDirectory() as temp_dir: + emulator_zip_path = os.path.join(temp_dir, "emulator.zip") + self._CreateSdkRepoZip(emulator_zip_path) + image_zip_path = os.path.join(temp_dir, "image.zip") + self._CreateSdkRepoZip(image_zip_path) + boot_image_path = os.path.join(temp_dir, "boot.img") + self.CreateFile(boot_image_path, b"ANDROID!") + system_image_path = os.path.join(temp_dir, "system.img") + self.CreateFile(system_image_path) + self._mock_avd_spec.emulator_zip = emulator_zip_path + self._mock_avd_spec.image_source = constants.IMAGE_SRC_LOCAL + self._mock_avd_spec.remote_image = {} + self._mock_avd_spec.local_image_artifact = image_zip_path + self._mock_avd_spec.local_kernel_image = boot_image_path + self._mock_avd_spec.local_system_image = system_image_path + self._mock_avd_spec.local_tool_dirs.append("/otatools") + mock_gf_utils.ConvertAvdSpecToArgs.return_value = ["-gpu", "auto"] + mock_gf_utils.MixWithBootImage.return_value = ( + "/path/to/kernel", "/path/to/ramdisk") + self._mock_create_credentials.side_effect = AssertionError( + "CreateCredentials should not be called.") + + factory = gf_factory.RemoteHostGoldfishDeviceFactory( + self._mock_avd_spec) + factory.CreateInstance() + + mock_gf_utils.MixWithBootImage.assert_called_once() + mock_gf_utils.MixWithSystemImage.assert_called_once() + mock_ota_tools.FindOtaToolsDir.assert_called_once() + self.assertEqual("/otatools", + mock_ota_tools.FindOtaToolsDir.call_args[0][0][0]) + + mock_gf_utils.FormatRemoteHostInstanceName.assert_called() + self.assertEqual({}, factory.GetBuildInfoDict()) + self.assertEqual([5555], factory.GetAdbPorts()) + self.assertEqual([None], factory.GetVncPorts()) + self.assertEqual({}, factory.GetFailures()) + + def testCreateInstanceInitError(self): + """Test RemoteHostGoldfishDeviceFactory with SSH error.""" + self._mock_ssh.Run.side_effect = errors.DeviceConnectionError + + factory = gf_factory.RemoteHostGoldfishDeviceFactory( + self._mock_avd_spec) + factory.CreateInstance() + + failures = factory.GetFailures() + self.assertIsInstance(failures.get(self._X86_64_INSTANCE_NAME), + errors.DeviceConnectionError) + self.assertEqual({}, factory.GetLogs()) + self._mock_remote_host_client.RecordTime.assert_called_once_with( + constants.TIME_GCE, mock.ANY) + self._mock_remote_host_client.SetStage.assert_called_once_with( + constants.STAGE_SSH_CONNECT) + + def testCreateInstanceDownloadError(self): + """Test RemoteHostGoldfishDeviceFactory with download error.""" + self._mock_android_build_client.DownloadArtifact.side_effect = ( + errors.DriverError) + + factory = gf_factory.RemoteHostGoldfishDeviceFactory( + self._mock_avd_spec) + factory.CreateInstance() + + failures = factory.GetFailures() + self.assertIsInstance(failures.get(self._X86_64_INSTANCE_NAME), + errors.DriverError) + self.assertEqual({}, factory.GetLogs()) + self._mock_remote_host_client.RecordTime.assert_has_calls([ + mock.call(constants.TIME_GCE, mock.ANY), + mock.call(constants.TIME_ARTIFACT, mock.ANY)]) + self.assertEqual(2, + self._mock_remote_host_client.RecordTime.call_count) + self._mock_remote_host_client.SetStage.assert_has_calls([ + mock.call(constants.STAGE_SSH_CONNECT), + mock.call(constants.STAGE_ARTIFACT)]) + self.assertEqual(2, self._mock_remote_host_client.SetStage.call_count) + + def testCreateInstanceBootError(self): """Test RemoteHostGoldfishDeviceFactory with boot error.""" self._mock_console.Reconnect.side_effect = ( errors.DeviceConnectionError) @@ -289,6 +405,9 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): errors.DeviceBootError) self.assertEqual({self._X86_64_INSTANCE_NAME: self._LOGS}, factory.GetLogs()) + self.assertEqual(3, + self._mock_remote_host_client.RecordTime.call_count) + self.assertEqual(3, self._mock_remote_host_client.SetStage.call_count) def testCreateInstanceTimeout(self): """Test RemoteHostGoldfishDeviceFactory with timeout.""" @@ -310,6 +429,9 @@ class RemoteHostGoldfishDeviceFactoryTest(driver_test_lib.BaseDriverTest): errors.DeviceBootTimeoutError) self.assertEqual({self._X86_64_INSTANCE_NAME: self._LOGS}, factory.GetLogs()) + self.assertEqual(3, + self._mock_remote_host_client.RecordTime.call_count) + self.assertEqual(3, self._mock_remote_host_client.SetStage.call_count) if __name__ == "__main__": diff --git a/public/actions/remote_instance_cf_device_factory.py b/public/actions/remote_instance_cf_device_factory.py index cf447743..072cb39e 100644 --- a/public/actions/remote_instance_cf_device_factory.py +++ b/public/actions/remote_instance_cf_device_factory.py @@ -16,15 +16,21 @@ device factory.""" import logging +import os +import tempfile +from acloud.create import create_common from acloud.internal import constants from acloud.internal.lib import cvd_utils +from acloud.internal.lib import ota_tools +from acloud.internal.lib import utils from acloud.public.actions import gce_device_factory from acloud.pull import pull logger = logging.getLogger(__name__) _SCREEN_CONSOLE_COMMAND = "screen ~/cuttlefish_runtime/console" +_MIXED_SUPER_IMAGE_NAME = "mixed_super.img" class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): @@ -54,18 +60,14 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): Returns: A string, representing instance name. """ - instance = self._CreateGceInstance() + instance = self.CreateGceInstance() # If instance is failed, no need to go next step. if instance in self.GetFailures(): return instance try: image_args = self._ProcessArtifacts() failures = self._compute_client.LaunchCvd( - instance, - self._avd_spec, - self._cfg.extra_data_disk_size_gb, - boot_timeout_secs=self._avd_spec.boot_timeout_secs, - extra_args=image_args) + instance, self._avd_spec, cvd_utils.GCE_BASE_DIR, image_args) for failing_instance, error_msg in failures.items(): self._SetFailures(failing_instance, error_msg) except Exception as e: @@ -88,45 +90,115 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): Returns: A list of strings, the launch_cvd arguments. """ - if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: + avd_spec = self._avd_spec + if avd_spec.image_source == constants.IMAGE_SRC_LOCAL: cvd_utils.UploadArtifacts( self._ssh, - self._local_image_artifact or self._avd_spec.local_image_dir, + cvd_utils.GCE_BASE_DIR, + self._local_image_artifact or avd_spec.local_image_dir, self._cvd_host_package_artifact) - elif self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: - self._compute_client.UpdateFetchCvd() - self._FetchBuild(self._avd_spec) + elif avd_spec.image_source == constants.IMAGE_SRC_REMOTE: + self._compute_client.UpdateFetchCvd(avd_spec.fetch_cvd_version) + self._compute_client.FetchBuild( + avd_spec.remote_image, + avd_spec.system_build_info, + avd_spec.kernel_build_info, + avd_spec.boot_build_info, + avd_spec.bootloader_build_info, + avd_spec.ota_build_info) + + launch_cvd_args = [] + if avd_spec.local_system_image or avd_spec.local_vendor_image: + with tempfile.TemporaryDirectory() as temp_dir: + super_image_path = os.path.join(temp_dir, + _MIXED_SUPER_IMAGE_NAME) + self._CreateMixedSuperImage( + super_image_path, self._GetLocalTargetFilesDir(temp_dir)) + launch_cvd_args += cvd_utils.UploadSuperImage( + self._ssh, cvd_utils.GCE_BASE_DIR, super_image_path) - if self._avd_spec.mkcert and self._avd_spec.connect_webrtc: + if avd_spec.mkcert and avd_spec.connect_webrtc: self._compute_client.UpdateCertificate() - if self._avd_spec.extra_files: - self._compute_client.UploadExtraFiles(self._avd_spec.extra_files) + if avd_spec.extra_files: + self._compute_client.UploadExtraFiles(avd_spec.extra_files) + + launch_cvd_args += cvd_utils.UploadExtraImages( + self._ssh, cvd_utils.GCE_BASE_DIR, avd_spec) + return launch_cvd_args - return cvd_utils.UploadExtraImages(self._ssh, self._avd_spec) + @utils.TimeExecute(function_description="Downloading target_files archive") + def _DownloadTargetFiles(self, download_dir): + avd_spec = self._avd_spec + build_id = avd_spec.remote_image[constants.BUILD_ID] + build_target = avd_spec.remote_image[constants.BUILD_TARGET] + create_common.DownloadRemoteArtifact( + avd_spec.cfg, build_target, build_id, + cvd_utils.GetMixBuildTargetFilename(build_target, build_id), + download_dir, decompress=True) + + def _GetLocalTargetFilesDir(self, temp_dir): + """Return a directory of extracted target_files or local images. + + Args: + temp_dir: Temporary directory to store downloaded build artifacts + and extracted target_files archive. + """ + avd_spec = self._avd_spec + if avd_spec.image_source == constants.IMAGE_SRC_LOCAL: + if self._local_image_artifact: + target_files_dir = os.path.join(temp_dir, "local_images") + os.makedirs(target_files_dir, exist_ok=True) + utils.Decompress(self._local_image_artifact, target_files_dir) + else: + target_files_dir = os.path.abspath(avd_spec.local_image_dir) + else: # must be IMAGE_SRC_REMOTE + target_files_dir = os.path.join(temp_dir, "remote_images") + os.makedirs(target_files_dir, exist_ok=True) + self._DownloadTargetFiles(target_files_dir) + return target_files_dir - def _FetchBuild(self, avd_spec): - """Download CF artifacts from android build. + def _CreateMixedSuperImage(self, super_image_path, target_files_dir): + """Create a mixed super image from device images and local system image. Args: - avd_spec: AVDSpec object that tells us what we're going to create. + super_image_path: Path to the output mixed super image. + target_files_dir: Path to extracted target_files directory + containing device images and misc_info.txt. """ - self._compute_client.FetchBuild( - avd_spec.remote_image[constants.BUILD_ID], - avd_spec.remote_image[constants.BUILD_BRANCH], - avd_spec.remote_image[constants.BUILD_TARGET], - avd_spec.system_build_info[constants.BUILD_ID], - avd_spec.system_build_info[constants.BUILD_BRANCH], - avd_spec.system_build_info[constants.BUILD_TARGET], - avd_spec.kernel_build_info[constants.BUILD_ID], - avd_spec.kernel_build_info[constants.BUILD_BRANCH], - avd_spec.kernel_build_info[constants.BUILD_TARGET], - avd_spec.bootloader_build_info[constants.BUILD_ID], - avd_spec.bootloader_build_info[constants.BUILD_BRANCH], - avd_spec.bootloader_build_info[constants.BUILD_TARGET], - avd_spec.ota_build_info[constants.BUILD_ID], - avd_spec.ota_build_info[constants.BUILD_BRANCH], - avd_spec.ota_build_info[constants.BUILD_TARGET]) + avd_spec = self._avd_spec + misc_info_path = cvd_utils.FindMiscInfo(target_files_dir) + image_dir = cvd_utils.FindImageDir(target_files_dir) + ota = ota_tools.FindOtaTools( + avd_spec.local_tool_dirs + + create_common.GetNonEmptyEnvVars( + constants.ENV_ANDROID_SOONG_HOST_OUT, + constants.ENV_ANDROID_HOST_OUT)) + + system_image_path=None + vendor_image_path=None + vendor_dlkm_image_path=None + odm_image_path=None + odm_dlkm_image_path=None + + if avd_spec.local_system_image: + system_image_path = create_common.FindSystemImage( + avd_spec.local_system_image) + + if avd_spec.local_vendor_image: + vendor_image_paths = cvd_utils.FindVendorImages( + avd_spec.local_vendor_image) + vendor_image_path = vendor_image_paths.vendor + vendor_dlkm_image_path = vendor_image_paths.vendor_dlkm + odm_image_path = vendor_image_paths.odm + odm_dlkm_image_path = vendor_image_paths.odm_dlkm + + ota.MixSuperImage(super_image_path, misc_info_path, image_dir, + system_image=system_image_path, + vendor_image=vendor_image_path, + vendor_dlkm_image=vendor_dlkm_image_path, + odm_image=odm_image_path, + odm_dlkm_image=odm_dlkm_image_path) def _FindLogFiles(self, instance, download): """Find and pull all log files from instance. @@ -136,13 +208,21 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): download: Whether to download the files to a temporary directory and show messages to the user. """ - self._all_logs[instance] = [cvd_utils.TOMBSTONES, - cvd_utils.HOST_KERNEL_LOG] + logs = [cvd_utils.HOST_KERNEL_LOG] if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: - self._all_logs[instance].append(cvd_utils.FETCHER_CONFIG_JSON) - log_files = pull.GetAllLogFilePaths(self._ssh) - self._all_logs[instance].extend(cvd_utils.ConvertRemoteLogs(log_files)) + logs.append( + cvd_utils.GetRemoteFetcherConfigJson(cvd_utils.GCE_BASE_DIR)) + logs.extend(cvd_utils.FindRemoteLogs( + self._ssh, + cvd_utils.GCE_BASE_DIR, + self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance)) + self._all_logs[instance] = logs + if download: + # To avoid long download time, fetch from the first device only. + log_files = pull.GetAllLogFilePaths(self._ssh, + constants.REMOTE_LOG_FOLDER) error_log_folder = pull.PullLogs(self._ssh, log_files, instance) self._compute_client.ExtendReportData(constants.ERROR_LOG_FOLDER, error_log_folder) @@ -158,6 +238,33 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): return {"ssh_command": self._compute_client.GetSshConnectCmd(), "screen_command": _SCREEN_CONSOLE_COMMAND} + def GetAdbPorts(self): + """Get ADB ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetAdbPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + + def GetFastbootPorts(self): + """Get Fastboot ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetFastbootPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + + def GetVncPorts(self): + """Get VNC ports of the created devices. + + Returns: + The port numbers as a list of integers. + """ + return cvd_utils.GetVncPorts(self._avd_spec.base_instance_num, + self._avd_spec.num_avds_per_instance) + def GetBuildInfoDict(self): """Get build info dictionary. diff --git a/public/actions/remote_instance_cf_device_factory_test.py b/public/actions/remote_instance_cf_device_factory_test.py index f0c52534..eaad9f80 100644 --- a/public/actions/remote_instance_cf_device_factory_test.py +++ b/public/actions/remote_instance_cf_device_factory_test.py @@ -15,6 +15,7 @@ import glob import os +import tempfile import unittest import uuid @@ -25,6 +26,7 @@ from acloud.internal import constants from acloud.internal.lib import android_build_client from acloud.internal.lib import auth from acloud.internal.lib import cvd_compute_client_multi_stage +from acloud.internal.lib import cvd_utils from acloud.internal.lib import driver_test_lib from acloud.internal.lib import utils from acloud.list import list as list_instances @@ -41,6 +43,8 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.Patch(android_build_client.AndroidBuildClient, "InitResourceHandle") self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "InitResourceHandle") self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "LaunchCvd") + self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "UpdateFetchCvd") + self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "FetchBuild") self.Patch(list_instances, "GetInstancesFromInstanceNames", return_value=mock.MagicMock()) self.Patch(list_instances, "ChooseOneRemoteInstance", return_value=mock.MagicMock()) self.Patch(utils, "GetBuildEnvironmentVariable", @@ -48,14 +52,12 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.Patch(glob, "glob", return_vale=["fake.img"]) # pylint: disable=protected-access + @staticmethod @mock.patch.object(cvd_compute_client_multi_stage.CvdComputeClient, "UpdateCertificate") - @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_FetchBuild") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." "cvd_utils") - def testProcessArtifacts(self, mock_cvd_utils, mock_download, - mock_uploadca): + def testProcessArtifacts(mock_cvd_utils, mock_uploadca): """test ProcessArtifacts.""" # Test image source type is local. args = mock.MagicMock() @@ -64,6 +66,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.flavor = "phone" args.local_image = constants.FIND_IN_BUILD_ENV args.local_system_image = None + args.local_vendor_image = None args.launch_args = None args.autoconnect = constants.INS_KEY_WEBRTC avd_spec_local_img = avd_spec.AVDSpec(args) @@ -78,7 +81,8 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_uploadca.assert_called_once() mock_uploadca.reset_mock() mock_cvd_utils.UploadArtifacts.assert_called_once_with( - mock.ANY, fake_image_name, fake_host_package_name) + mock.ANY, mock_cvd_utils.GCE_BASE_DIR, fake_image_name, + fake_host_package_name) mock_cvd_utils.UploadExtraImages.assert_called_once() # given autoconnect to vnc should not upload certificates @@ -103,11 +107,13 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.kernel_branch = "kernel_branch" args.kernel_build_target = "kernel_target" avd_spec_remote_img = avd_spec.AVDSpec(args) - self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "UpdateFetchCvd") factory_remote_img = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( avd_spec_remote_img) factory_remote_img._ProcessArtifacts() - mock_download.assert_called_once() + + compute_client = factory_remote_img.GetComputeClient() + compute_client.UpdateFetchCvd.assert_called_once() + compute_client.FetchBuild.assert_called_once() # pylint: disable=protected-access @mock.patch.dict(os.environ, {constants.ENV_BUILD_TARGET:'fake-target'}) @@ -121,6 +127,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.local_image = constants.FIND_IN_BUILD_ENV args.local_system_image = None args.adb_port = None + args.fastboot_port = None args.launch_args = None fake_avd_spec = avd_spec.AVDSpec(args) fake_avd_spec.cfg.enable_multi_stage = True @@ -138,7 +145,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec, fake_image_name, fake_host_package_name) - self.assertEqual(factory._CreateGceInstance(), "ins-1234-userbuild-aosp-cf-x86-phone") + self.assertEqual(factory.CreateGceInstance(), "ins-1234-userbuild-aosp-cf-x86-phone") # Can't get target name from zip file name. fake_image_name = "/fake/aosp_cf_x86_phone.username.zip" @@ -146,7 +153,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec, fake_image_name, fake_host_package_name) - self.assertEqual(factory._CreateGceInstance(), "ins-1234-userbuild-fake-target") + self.assertEqual(factory.CreateGceInstance(), "ins-1234-userbuild-fake-target") # No image zip path, it uses local build images. fake_image_name = "" @@ -154,7 +161,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec, fake_image_name, fake_host_package_name) - self.assertEqual(factory._CreateGceInstance(), "ins-1234-userbuild-fake-target") + self.assertEqual(factory.CreateGceInstance(), "ins-1234-userbuild-fake-target") def testReuseInstanceNameMultiStage(self): """Test reuse instance name.""" @@ -165,6 +172,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.local_image = constants.FIND_IN_BUILD_ENV args.local_system_image = None args.adb_port = None + args.fastboot_port = None args.launch_args = None fake_avd_spec = avd_spec.AVDSpec(args) fake_avd_spec.cfg.enable_multi_stage = True @@ -180,7 +188,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec, fake_image_name, fake_host_package_name) - self.assertEqual(factory._CreateGceInstance(), "fake-1234-userbuild-fake-target") + self.assertEqual(factory.CreateGceInstance(), "fake-1234-userbuild-fake-target") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." "cvd_utils") @@ -196,6 +204,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.local_image = "fake_local_image" args.local_system_image = None args.adb_port = None + args.fastboot_port = None args.cheeps_betty_image = None args.launch_args = None avd_spec_local_image = avd_spec.AVDSpec(args) @@ -220,7 +229,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_cvd_utils.GetRemoteBuildInfoDict.assert_called() @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_CreateGceInstance") + "CreateGceInstance") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory.pull") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." "cvd_utils") @@ -236,8 +245,12 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec.image_source = constants.IMAGE_SRC_LOCAL fake_avd_spec._instance_name_to_reuse = None fake_avd_spec.no_pull_log = False + fake_avd_spec.base_instance_num = None + fake_avd_spec.num_avds_per_instance = None + fake_avd_spec.local_system_image = None + fake_avd_spec.local_vendor_image = None - mock_cvd_utils.ConvertRemoteLogs.return_value = [{"path": "/logcat"}] + mock_cvd_utils.FindRemoteLogs.return_value = [{"path": "/logcat"}] mock_cvd_utils.UploadExtraImages.return_value = [ "-boot_image", "/boot/img"] @@ -252,17 +265,93 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): factory.CreateInstance() mock_create_gce_instance.assert_called_once() mock_cvd_utils.UploadArtifacts.assert_called_once() + mock_cvd_utils.FindRemoteLogs.assert_called_with( + mock.ANY, mock_cvd_utils.GCE_BASE_DIR, None, None) compute_client.LaunchCvd.assert_called_once() - self.assertEqual( - ["-boot_image", "/boot/img"], - compute_client.LaunchCvd.call_args[1].get("extra_args")) - mock_pull.GetAllLogFilePaths.assert_called_once() + self.assertIn(["-boot_image", "/boot/img"], + compute_client.LaunchCvd.call_args[0]) + mock_pull.GetAllLogFilePaths.assert_called_once_with( + mock.ANY, constants.REMOTE_LOG_FOLDER) mock_pull.PullLogs.assert_called_once() + + factory.GetAdbPorts() + mock_cvd_utils.GetAdbPorts.assert_called_with(None, None) + factory.GetVncPorts() + mock_cvd_utils.GetVncPorts.assert_called_with(None, None) + factory.GetFastbootPorts() + mock_cvd_utils.GetFastbootPorts.assert_called_with(None, None) self.assertEqual({"instance": "failure"}, factory.GetFailures()) - self.assertEqual(3, len(factory.GetLogs().get("instance"))) + self.assertEqual(2, len(factory.GetLogs().get("instance"))) + + @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." + "ota_tools") + def testLocalSystemAndVendorImageCreateInstance(self, mock_ota_tools): + """Test CreateInstance with local system image.""" + with tempfile.TemporaryDirectory() as temp_dir: + local_image_dir = os.path.join(temp_dir, "cf") + misc_info_path = os.path.join(local_image_dir, "misc_info.txt") + local_system_image_dir = os.path.join(temp_dir, "system") + local_system_image_path = os.path.join( + local_system_image_dir, "system.img") + local_vendor_image_dir = os.path.join(temp_dir, "vendor") + local_vendor_image_path = os.path.join( + local_vendor_image_dir, "vendor.img") + local_vendor_dlkm_image_path = os.path.join( + local_vendor_image_dir, "vendor_dlkm.img") + local_odm_image_path = os.path.join( + local_vendor_image_dir, "odm.img") + local_odm_dlkm_image_path = os.path.join( + local_vendor_image_dir, "odm_dlkm.img") + self.CreateFile(misc_info_path, b"key=value") + self.CreateFile(local_system_image_path) + self.CreateFile(local_vendor_image_path) + self.CreateFile(local_vendor_dlkm_image_path) + self.CreateFile(local_odm_image_path) + self.CreateFile(local_odm_dlkm_image_path) + + self.Patch(cvd_utils, "UploadArtifacts") + self.Patch(cvd_utils, "UploadSuperImage") + self.Patch(cvd_utils, "UploadExtraImages") + self.Patch(cvd_compute_client_multi_stage, "CvdComputeClient") + self.Patch( + remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, + "CreateGceInstance", return_value="instance") + self.Patch( + remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, + "_FindLogFiles") + mock_ota_tools_object = mock_ota_tools.FindOtaTools.return_value + + args = mock.MagicMock() + args.config_file = "" + args.avd_type = constants.TYPE_CF + args.flavor = "phone" + args.local_image = local_image_dir + args.local_system_image = local_system_image_dir + args.local_vendor_image = local_vendor_image_dir + args.local_tool = ["/ota/tools/dir"] + args.launch_args = None + args.no_pull_log = True + avd_spec_local_img = avd_spec.AVDSpec(args) + factory_local_img = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( + avd_spec_local_img) + compute_client = factory_local_img.GetComputeClient() + compute_client.LaunchCvd.return_value = {} + + factory_local_img.CreateInstance() + + cvd_utils.UploadArtifacts.assert_called_once_with( + mock.ANY, mock.ANY, local_image_dir, mock.ANY) + mock_ota_tools_object.MixSuperImage.assert_called_once_with( + mock.ANY, misc_info_path, local_image_dir, + system_image=local_system_image_path, + vendor_image=local_vendor_image_path, + vendor_dlkm_image=local_vendor_dlkm_image_path, + odm_image=local_odm_image_path, + odm_dlkm_image=local_odm_dlkm_image_path) + cvd_utils.UploadSuperImage.assert_called_once() @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_CreateGceInstance") + "CreateGceInstance") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory.pull") @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." "cvd_utils") @@ -278,8 +367,12 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec.image_source = constants.IMAGE_SRC_REMOTE fake_avd_spec.host_user = None fake_avd_spec.no_pull_log = True + fake_avd_spec.base_instance_num = 2 + fake_avd_spec.num_avds_per_instance = 3 + fake_avd_spec.local_system_image = None + fake_avd_spec.local_vendor_image = None - mock_cvd_utils.ConvertRemoteLogs.return_value = [{"path": "/logcat"}] + mock_cvd_utils.FindRemoteLogs.return_value = [{"path": "/logcat"}] mock_cvd_utils.UploadExtraImages.return_value = [] factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( @@ -289,10 +382,17 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): factory.CreateInstance() compute_client.FetchBuild.assert_called_once() - mock_pull.GetAllLogFilePaths.assert_called_once() + mock_cvd_utils.FindRemoteLogs.assert_called_with( + mock.ANY, mock_cvd_utils.GCE_BASE_DIR, 2, 3) + mock_pull.GetAllLogFilePaths.assert_not_called() mock_pull.PullLogs.assert_not_called() + + factory.GetAdbPorts() + mock_cvd_utils.GetAdbPorts.assert_called_with(2, 3) + factory.GetVncPorts() + mock_cvd_utils.GetVncPorts.assert_called_with(2, 3) self.assertFalse(factory.GetFailures()) - self.assertEqual(4, len(factory.GetLogs().get("instance"))) + self.assertEqual(3, len(factory.GetLogs().get("instance"))) def testGetOpenWrtInfoDict(self): """Test GetOpenWrtInfoDict.""" diff --git a/public/actions/remote_instance_fvp_device_factory.py b/public/actions/remote_instance_fvp_device_factory.py index c60235da..690fc2fe 100644 --- a/public/actions/remote_instance_fvp_device_factory.py +++ b/public/actions/remote_instance_fvp_device_factory.py @@ -34,7 +34,7 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): Returns: The instance. """ - instance = self._CreateGceInstance() + instance = self.CreateGceInstance() if instance in self.GetFailures(): return instance diff --git a/public/actions/remote_instance_fvp_device_factory_test.py b/public/actions/remote_instance_fvp_device_factory_test.py index 862c7c03..86c9ba17 100644 --- a/public/actions/remote_instance_fvp_device_factory_test.py +++ b/public/actions/remote_instance_fvp_device_factory_test.py @@ -46,7 +46,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): @staticmethod @mock.patch.object( remote_instance_fvp_device_factory.RemoteInstanceDeviceFactory, - "_CreateGceInstance") + "CreateGceInstance") @mock.patch.object(ssh, "ShellCmdWithRetry") @mock.patch.dict(os.environ, { constants.ENV_BUILD_TARGET:'fvp', @@ -65,6 +65,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.local_image = "fake_local_image" args.local_system_image = None args.adb_port = None + args.fastboot_port = None args.launch_args = None avd_spec_local_image = avd_spec.AVDSpec(args) factory = remote_instance_fvp_device_factory.RemoteInstanceDeviceFactory( diff --git a/public/avd.py b/public/avd.py index 0f3c56da..1a2961a8 100755 --- a/public/avd.py +++ b/public/avd.py @@ -39,7 +39,7 @@ class AndroidVirtualDevice(): """Represent an Android device.""" def __init__(self, instance_name, ip=None, time_info=None, stage=None, - openwrt=None): + openwrt=None, gce_hostname=None): """Initialize. Args: @@ -49,6 +49,7 @@ class AndroidVirtualDevice(): time_info: Dict of time cost information, e.g. {"launch_cvd": 5} stage: Integer of AVD in which stage, e.g. STAGE_GCE, STAGE_BOOT_UP openwrt: Boolean of the instance creates the OpenWrt device. + gce_hostname: String of the GCE hostname. """ self._ip = ip self._instance_name = instance_name @@ -77,6 +78,7 @@ class AndroidVirtualDevice(): self._build_info = {} self._stage = stage self._openwrt = openwrt + self._gce_hostname = gce_hostname @property def ip(self): @@ -115,6 +117,11 @@ class AndroidVirtualDevice(): """Getter of _openwrt.""" return self._openwrt + @property + def gce_hostname(self): + """Getter of _gce_hostname.""" + return self._gce_hostname + @build_info.setter def build_info(self, value): self._build_info = value diff --git a/public/config.py b/public/config.py index 7c976e2d..4cd432ef 100755 --- a/public/config.py +++ b/public/config.py @@ -236,6 +236,7 @@ class AcloudConfig(): self.launch_args = usr_cfg.launch_args self.oxygen_client = usr_cfg.oxygen_client self.oxygen_lease_args = usr_cfg.oxygen_lease_args + self.connect_hostname = usr_cfg.connect_hostname self.instance_name_pattern = ( usr_cfg.instance_name_pattern or internal_cfg.default_usr_cfg.instance_name_pattern) diff --git a/public/config_test.py b/public/config_test.py index 694ed6b7..d0e0aa3b 100644 --- a/public/config_test.py +++ b/public/config_test.py @@ -66,7 +66,7 @@ disk_raw_image_extension: ".img" creds_cache_file: ".fake_oauth2.dat" user_agent: "fake_user_agent" kernel_build_target: "kernel" -emulator_build_target: "sdk_tools_linux" +emulator_build_target: "emulator-linux_x64_nolocationui" default_usr_cfg { machine_type: "n1-standard-1" @@ -245,7 +245,7 @@ common_hw_property_map { "fake_stable_goldfish_host_image_name") self.assertEqual(cfg.default_usr_cfg.stable_goldfish_host_image_project, "fake_stable_goldfish_host_image_project") - self.assertEqual(cfg.emulator_build_target, "sdk_tools_linux") + self.assertEqual(cfg.emulator_build_target, "emulator-linux_x64_nolocationui") self.assertEqual(cfg.default_usr_cfg.instance_name_pattern, "fake_instance_name_pattern") diff --git a/public/data/default.config b/public/data/default.config index e36509bd..9a2327c1 100644 --- a/public/data/default.config +++ b/public/data/default.config @@ -8,17 +8,17 @@ default_extra_data_disk_device: "/dev/block/sdb" creds_cache_file: ".acloud_oauth2.dat" user_agent: "acloud" -# [GOLDFISH only] The emulator build target: "sdk_tools_linux". +# [GOLDFISH only] The emulator build target: "emulator-linux_x64_internal". # We use it to get build id if build id is not provided and It's very unlikely # that this will ever change. -emulator_build_target: "sdk_tools_linux" +emulator_build_target: "emulator-linux_x64_internal" default_usr_cfg { machine_type: "n1-standard-4" network: "default" extra_data_disk_size_gb: 0 instance_name_pattern: "ins-{uuid}-{build_id}-{build_target}" - fetch_cvd_version: "7924973" + fetch_cvd_version: "9123511" metadata_variable { key: "camera_front" @@ -57,12 +57,12 @@ default_usr_cfg { # Cuttlefish config reference: google/cuttlefish/shared/config common_hw_property_map { key: "local-phone" - value: "cpu:4,resolution:720x1280,dpi:320,memory:6g" + value: "cpu:4,resolution:720x1280,dpi:320,memory:2g" } common_hw_property_map { key: "local-auto" - value: "cpu:4,resolution:1280x800,dpi:160,memory:6g" + value: "cpu:4,resolution:1280x800,dpi:160,memory:4g" } common_hw_property_map { @@ -72,7 +72,7 @@ common_hw_property_map { common_hw_property_map { key: "local-tablet" - value: "cpu:4,resolution:2560x1800,dpi:320,memory:6g" + value: "cpu:4,resolution:2560x1800,dpi:320,memory:4g" } common_hw_property_map { @@ -82,7 +82,7 @@ common_hw_property_map { common_hw_property_map { key: "phone" - value: "cpu:4,resolution:720x1280,dpi:320,memory:4g" + value: "cpu:4,resolution:720x1280,dpi:320,memory:2g" } common_hw_property_map { diff --git a/public/device_driver.py b/public/device_driver.py index a1f1b9d6..ac680512 100755 --- a/public/device_driver.py +++ b/public/device_driver.py @@ -406,6 +406,7 @@ def CreateGCETypeAVD(cfg, target_adb_port=constants.GCE_ADB_PORT, ssh_user=_SSH_USER, client_adb_port=avd_spec.client_adb_port, + client_fastboot_port=avd_spec.client_fastboot_port, extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) device_dict[constants.VNC_PORT] = forwarded_ports.vnc_port device_dict[constants.ADB_PORT] = forwarded_ports.adb_port @@ -421,16 +422,22 @@ def CreateGCETypeAVD(cfg, else: r.SetStatus(report.Status.SUCCESS) - # Dump serial logs. - if serial_log_file: - _FetchSerialLogsFromDevices( - compute_client, - instance_names=[d.instance_name for d in device_pool.devices], - port=constants.DEFAULT_SERIAL_PORT, - output_file=serial_log_file) except errors.DriverError as e: r.AddError(str(e)) r.SetStatus(report.Status.FAIL) + finally: + # Let's do our best to obtain the serial log, even though this + # could fail in case of failed boots. + if serial_log_file: + instance_names=[d.instance_name for d in device_pool.devices] + try: + _FetchSerialLogsFromDevices( + compute_client, + instance_names=instance_names, + port=constants.DEFAULT_SERIAL_PORT, + output_file=serial_log_file) + except Exception as log_err: + logging.warning("Failed to obtain serial logs from %s", ", ".join(instance_names)) return r diff --git a/pull/pull.py b/pull/pull.py index e5eec5c4..020407d7 100644 --- a/pull/pull.py +++ b/pull/pull.py @@ -19,7 +19,6 @@ This command will pull the log files from a remote instance for AVD troubleshoot from __future__ import print_function import logging import os -import subprocess import tempfile from acloud import errors @@ -34,10 +33,6 @@ from acloud.public import report logger = logging.getLogger(__name__) -# REMOTE_LOG_FOLDER and the log files can be symbolic links. The -H flag makes -# the command skip the links except REMOTE_LOG_FOLDER. The returned logs are -# unique. -_FIND_LOG_FILE_CMD = "find -H %s -type f" % constants.REMOTE_LOG_FOLDER # Black list for log files. _KERNEL = "kernel" _IMG_FILE_EXTENSION = ".img" @@ -148,7 +143,7 @@ def SelectLogFileToPull(ssh, file_name=None): Raises: errors.CheckPathError: Can't find log files. """ - log_files = GetAllLogFilePaths(ssh) + log_files = GetAllLogFilePaths(ssh, constants.REMOTE_LOG_FOLDER) if file_name: file_path = os.path.join(constants.REMOTE_LOG_FOLDER, file_name) if file_path in log_files: @@ -167,39 +162,21 @@ def SelectLogFileToPull(ssh, file_name=None): "remote instance." % constants.REMOTE_LOG_FOLDER) -def GetAllLogFilePaths(ssh): - """Get the file paths of all log files. - - Args: - ssh: Ssh object. - - Returns: - List of all log file paths. - """ - ssh_cmd = [ssh.GetBaseCmd(constants.SSH_BIN), _FIND_LOG_FILE_CMD] - log_files = [] - try: - files_output = utils.CheckOutput(" ".join(ssh_cmd), shell=True) - log_files = FilterLogfiles(files_output.splitlines()) - except subprocess.CalledProcessError: - logger.debug("The folder(%s) that running launch_cvd doesn't exist.", - constants.REMOTE_LOG_FOLDER) - return log_files - - -def FilterLogfiles(files): - """Filter some unused files. +def GetAllLogFilePaths(ssh, remote_log_folder): + """Get all file paths under the log folder. Two rules to filter out files. 1. File name is "kernel". 2. File type is image "*.img". Args: - files: List of file paths in the remote instance. + ssh: Ssh object. + remote_log_folder: The path to the remote log folder. - Return: - List of log files. + Returns: + List of strings, the log file paths. """ + files = utils.FindRemoteFiles(ssh, [remote_log_folder]) log_files = list(files) for file_path in files: file_name = os.path.basename(file_path) diff --git a/pull/pull_test.py b/pull/pull_test.py index f1f89f78..c2ae6ab0 100644 --- a/pull/pull_test.py +++ b/pull/pull_test.py @@ -122,17 +122,22 @@ class PullTest(driver_test_lib.BaseDriverTest): with self.assertRaises(errors.CheckPathError): pull.SelectLogFileToPull(_ssh, input_file) - def testFilterLogfiles(self): - """test filer log file from black list.""" + def testGetAllLogFilePaths(self): + """test that GetAllLogFilePaths can filter logs.""" + mock_find = self.Patch(utils, "FindRemoteFiles", + return_value=["kernel.log", "logcat", "kernel"]) # Filter out file name is "kernel". - files = ["kernel.log", "logcat", "kernel"] expected_result = ["kernel.log", "logcat"] - self.assertEqual(pull.FilterLogfiles(files), expected_result) + self.assertEqual(pull.GetAllLogFilePaths(mock.Mock(), "unit/test"), + expected_result) + mock_find.assert_called_with(mock.ANY, ["unit/test"]) # Filter out file extension is ".img". - files = ["kernel.log", "system.img", "userdata.img", "launcher.log"] + mock_find.return_value = ["kernel.log", "system.img", "userdata.img", + "launcher.log"] expected_result = ["kernel.log", "launcher.log"] - self.assertEqual(pull.FilterLogfiles(files), expected_result) + self.assertEqual(pull.GetAllLogFilePaths(mock.Mock(), "unit/test"), + expected_result) @mock.patch.object(pull, "PullFileFromInstance") def testRun(self, mock_pull_file): diff --git a/reconnect/reconnect.py b/reconnect/reconnect.py index 19631831..5ab55b0f 100644 --- a/reconnect/reconnect.py +++ b/reconnect/reconnect.py @@ -28,6 +28,7 @@ from acloud.internal import constants from acloud.internal.lib import auth from acloud.internal.lib import android_compute_client from acloud.internal.lib import cvd_runtime_config +from acloud.internal.lib import gcompute_client from acloud.internal.lib import utils from acloud.internal.lib import ssh as ssh_object from acloud.internal.lib.adb_tools import AdbTools @@ -121,11 +122,12 @@ def ReconnectInstance(ssh_private_key_path, instance, reconnect_report, extra_args_ssh_tunnel=None, - autoconnect=None): + autoconnect=None, + connect_hostname=None): """Reconnect to the specified instance. It will: - - re-establish ssh tunnels for adb/vnc port forwarding + - re-establish ssh tunnels for adb/fastboot/vnc port forwarding - re-establish adb connection - restart vnc client - update device information in reconnect_report @@ -137,6 +139,7 @@ def ReconnectInstance(ssh_private_key_path, reconnect_report: Report object. extra_args_ssh_tunnel: String, extra args for ssh tunnel connection. autoconnect: String, for decide whether to launch vnc/browser or not. + connect_hostname: String, the hostname for ssh connect. Raises: errors.UnknownAvdType: Unable to reconnect to instance of unknown avd @@ -146,10 +149,14 @@ def ReconnectInstance(ssh_private_key_path, raise errors.UnknownAvdType("Unable to reconnect to instance (%s) of " "unknown avd type: %s" % (instance.name, instance.avd_type)) + # Ignore extra ssh tunnel to connect with hostname. + if connect_hostname: + extra_args_ssh_tunnel = None adb_cmd = AdbTools(instance.adb_port) vnc_port = instance.vnc_port adb_port = instance.adb_port + fastboot_port = instance.fastboot_port webrtc_port = instance.webrtc_port # ssh tunnel is up but device is disconnected on adb if instance.ssh_tunnel_is_connected and not adb_cmd.IsAdbConnectionAlive(): @@ -159,21 +166,23 @@ def ReconnectInstance(ssh_private_key_path, elif not instance.ssh_tunnel_is_connected and not instance.islocal: adb_cmd.DisconnectAdb() forwarded_ports = utils.AutoConnect( - ip_addr=instance.ip, + ip_addr=connect_hostname or instance.ip, rsa_key_file=ssh_private_key_path, target_vnc_port=utils.AVD_PORT_DICT[instance.avd_type].vnc_port, target_adb_port=utils.AVD_PORT_DICT[instance.avd_type].adb_port, + target_fastboot_port=utils.AVD_PORT_DICT[instance.avd_type].fastboot_port, ssh_user=constants.GCE_USER, extra_args_ssh_tunnel=extra_args_ssh_tunnel) vnc_port = forwarded_ports.vnc_port adb_port = forwarded_ports.adb_port + fastboot_port = forwarded_ports.fastboot_port if autoconnect is constants.INS_KEY_WEBRTC: if not instance.islocal: webrtc_port = utils.GetWebrtcPortFromSSHTunnel(instance.ip) if not webrtc_port: webrtc_port = utils.PickFreePort() utils.EstablishWebRTCSshTunnel( - ip_addr=instance.ip, + ip_addr=connect_hostname or instance.ip, webrtc_local_port=webrtc_port, rsa_key_file=ssh_private_key_path, ssh_user=constants.GCE_USER, @@ -187,13 +196,14 @@ def ReconnectInstance(ssh_private_key_path, constants.IP: instance.ip, constants.INSTANCE_NAME: instance.name, constants.VNC_PORT: vnc_port, - constants.ADB_PORT: adb_port + constants.ADB_PORT: adb_port, + constants.FASTBOOT_PORT: fastboot_port, } if adb_port and not instance.islocal: device_dict[constants.DEVICE_SERIAL] = ( constants.REMOTE_INSTANCE_ADB_SERIAL % adb_port) - if (vnc_port or webrtc_port) and adb_port: + if (vnc_port or webrtc_port) and adb_port and fastboot_port: reconnect_report.AddData(key="devices", value=device_dict) else: # We use 'ps aux' to grep adb/vnc fowarding port from ssh tunnel @@ -203,6 +213,27 @@ def ReconnectInstance(ssh_private_key_path, reconnect_report.AddError(instance.name) +def GetSshConnectHostname(cfg, instance): + """Get ssh connect hostname. + + Get GCE hostname with specific rule for cloudtop users. + + Args: + cfg: AcloudConfig object. + instance: list.Instance() object. + + Returns: + String of hostname for ssh connect. None is for not connect with + hostname such as local instance mode. + """ + if instance.islocal: + return None + if cfg.connect_hostname: + return gcompute_client.GetGCEHostName( + cfg.project, instance.name, cfg.zone) + return None + + def Run(args): """Run reconnect. @@ -232,6 +263,7 @@ def Run(args): instance, reconnect_report, cfg.extra_args_ssh_tunnel, - autoconnect=(args.autoconnect or instance.autoconnect)) + autoconnect=(args.autoconnect or instance.autoconnect), + connect_hostname=GetSshConnectHostname(cfg, instance)) utils.PrintDeviceSummary(reconnect_report) diff --git a/reconnect/reconnect_args.py b/reconnect/reconnect_args.py index 04ca32ca..bf5ec59f 100644 --- a/reconnect/reconnect_args.py +++ b/reconnect/reconnect_args.py @@ -55,7 +55,8 @@ def GetReconnectArgParser(subparser): dest="autoconnect", required=False, choices=[constants.INS_KEY_VNC, constants.INS_KEY_ADB, - constants.INS_KEY_WEBRTC], - help="If need adb only, you can pass in 'adb' here.") + constants.INS_KEY_FASTBOOT, constants.INS_KEY_WEBRTC], + help="If need adb/fastboot/vnc/webrtc only, you can pass in 'adb', 'fastboot', " + "'vnc' or 'webrtc' only here.") return reconnect_parser diff --git a/reconnect/reconnect_test.py b/reconnect/reconnect_test.py index fa5f6be8..0c768475 100644 --- a/reconnect/reconnect_test.py +++ b/reconnect/reconnect_test.py @@ -25,6 +25,7 @@ from acloud.internal.lib import auth from acloud.internal.lib import android_compute_client from acloud.internal.lib import cvd_runtime_config from acloud.internal.lib import driver_test_lib +from acloud.internal.lib import gcompute_client from acloud.internal.lib import utils from acloud.internal.lib import ssh as ssh_object from acloud.internal.lib.adb_tools import AdbTools @@ -34,7 +35,9 @@ from acloud.reconnect import reconnect ForwardedPorts = collections.namedtuple("ForwardedPorts", - [constants.VNC_PORT, constants.ADB_PORT]) + [constants.VNC_PORT, + constants.ADB_PORT, + constants.FASTBOOT_PORT]) class ReconnectTest(driver_test_lib.BaseDriverTest): @@ -50,6 +53,7 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): instance_object.ip = "1.1.1.1" instance_object.islocal = False instance_object.adb_port = "8686" + instance_object.fastboot_port = "9686" instance_object.avd_type = "cuttlefish" self.Patch(subprocess, "check_call", return_value=True) self.Patch(utils, "LaunchVncClient") @@ -62,6 +66,7 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): constants.INSTANCE_NAME: "fake_name", constants.VNC_PORT: 6666, constants.ADB_PORT: "8686", + constants.FASTBOOT_PORT: "9686", constants.DEVICE_SERIAL: "127.0.0.1:8686" } @@ -90,13 +95,14 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): instance_object.vnc_port = 5555 extra_args_ssh_tunnel = None self.Patch(utils, "AutoConnect", - return_value=ForwardedPorts(vnc_port=11111, adb_port=22222)) + return_value=ForwardedPorts(vnc_port=11111, adb_port=22222, fastboot_port=33333)) reconnect.ReconnectInstance( ssh_private_key_path, instance_object, fake_report, autoconnect="vnc") utils.AutoConnect.assert_called_with(ip_addr=instance_object.ip, rsa_key_file=ssh_private_key_path, target_vnc_port=constants.CF_VNC_PORT, target_adb_port=constants.CF_ADB_PORT, + target_fastboot_port=constants.CF_FASTBOOT_PORT, ssh_user=constants.GCE_USER, extra_args_ssh_tunnel=extra_args_ssh_tunnel) utils.LaunchVncClient.assert_called_with(11111) @@ -105,6 +111,7 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): constants.INSTANCE_NAME: "fake_name", constants.VNC_PORT: 11111, constants.ADB_PORT: 22222, + constants.FASTBOOT_PORT: 33333, constants.DEVICE_SERIAL: "127.0.0.1:22222" } fake_report.AddData.assert_called_with(key="devices", value=fake_device_dict) @@ -120,6 +127,7 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): rsa_key_file=ssh_private_key_path, target_vnc_port=constants.CF_VNC_PORT, target_adb_port=constants.CF_ADB_PORT, + target_fastboot_port=constants.CF_FASTBOOT_PORT, ssh_user=constants.GCE_USER, extra_args_ssh_tunnel=extra_args_ssh_tunnel) utils.LaunchVncClient.assert_called_with(11111, "999", "777") @@ -127,14 +135,15 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): # test fail reconnect report. self.Patch(utils, "AutoConnect", - return_value=ForwardedPorts(vnc_port=None, adb_port=None)) + return_value=ForwardedPorts(vnc_port=None, adb_port=None, fastboot_port=None)) reconnect.ReconnectInstance( ssh_private_key_path, instance_object, fake_report, autoconnect="vnc") fake_device_dict = { constants.IP: "1.1.1.1", constants.INSTANCE_NAME: "fake_name", constants.VNC_PORT: None, - constants.ADB_PORT: None + constants.ADB_PORT: None, + constants.FASTBOOT_PORT: None } fake_report.AddData.assert_called_with(key="device_failing_reconnect", value=fake_device_dict) @@ -153,7 +162,8 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): constants.IP: "1.1.1.1", constants.INSTANCE_NAME: "fake_name", constants.VNC_PORT: 5555, - constants.ADB_PORT: "8686" + constants.ADB_PORT: "8686", + constants.FASTBOOT_PORT: "9686" } fake_report.AddData.assert_called_with(key="devices", value=fake_device_dict) @@ -235,6 +245,7 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): rsa_key_file=ssh_private_key_path, target_vnc_port=constants.GCE_VNC_PORT, target_adb_port=constants.GCE_ADB_PORT, + target_fastboot_port=None, ssh_user=constants.GCE_USER, extra_args_ssh_tunnel=None) reconnect.StartVnc.assert_called_once() @@ -248,6 +259,7 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): rsa_key_file=ssh_private_key_path, target_vnc_port=constants.CF_VNC_PORT, target_adb_port=constants.CF_ADB_PORT, + target_fastboot_port=constants.CF_FASTBOOT_PORT, ssh_user=constants.GCE_USER, extra_args_ssh_tunnel=None) reconnect.StartVnc.assert_called_once() @@ -369,6 +381,20 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): reconnect.Run(fake_args) self.assertEqual(reconnect.ReconnectInstance.call_count, 1) + def testGetSshConnectHostname(self): + """Test GetSshConnectHostname.""" + self.Patch(gcompute_client, "GetGCEHostName", return_value="fake_host") + instance = mock.MagicMock() + instance.islocal = True + cfg = mock.MagicMock() + self.assertEqual(None, reconnect.GetSshConnectHostname(cfg, instance)) + + # Remote instance will get the GCE hostname. + instance.islocal = False + cfg.connect_hostname = True + self.assertEqual("fake_host", + reconnect.GetSshConnectHostname(cfg, instance)) + if __name__ == "__main__": unittest.main() diff --git a/restart/restart.py b/restart/restart.py index 5e148941..b10901e7 100644 --- a/restart/restart.py +++ b/restart/restart.py @@ -18,6 +18,7 @@ This command will restart the CF AVD from a remote instance. import logging import subprocess +import sys from acloud import errors from acloud.internal import constants @@ -32,6 +33,8 @@ from acloud.reconnect import reconnect logger = logging.getLogger(__name__) +_NOT_SUPPORT_MSG = ("Currently the restart function doesn't support local " + "instances. Please try to create one new instance.") def RestartFromInstance(cfg, instance, instance_id, powerwash_data): @@ -96,6 +99,10 @@ def Run(args): cfg, [args.instance_name]) return RestartFromInstance( cfg, instance[0], args.instance_id, args.powerwash) + if (not list_instances.GetCFRemoteInstances(cfg) + and list_instances.GetLocalInstances()): + utils.PrintColorString(_NOT_SUPPORT_MSG, utils.TextColors.FAIL) + sys.exit() return RestartFromInstance(cfg, list_instances.ChooseOneRemoteInstance(cfg), args.instance_id, diff --git a/restart/restart_test.py b/restart/restart_test.py index d74dee75..00256f4c 100644 --- a/restart/restart_test.py +++ b/restart/restart_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """Tests for restart.""" +import sys + import unittest from unittest import mock @@ -29,9 +31,9 @@ from acloud.restart import restart class RestartTest(driver_test_lib.BaseDriverTest): """Test restart.""" - + @mock.patch.object(sys, "exit") @mock.patch.object(restart, "RestartFromInstance") - def testRun(self, mock_restart): + def testRun(self, mock_restart, mock_exit): """test Run.""" cfg = mock.MagicMock() args = mock.MagicMock() @@ -49,6 +51,8 @@ class RestartTest(driver_test_lib.BaseDriverTest): # Test case for user select one instance to restart AVD. selected_instance = mock.MagicMock() + self.Patch(list_instances, "GetCFRemoteInstances", + return_value=selected_instance) self.Patch(list_instances, "ChooseOneRemoteInstance", return_value=selected_instance) args.instance_name = None @@ -56,6 +60,15 @@ class RestartTest(driver_test_lib.BaseDriverTest): mock_restart.assert_has_calls([ mock.call(cfg, selected_instance, args.instance_id, args.powerwash)]) + # Test case for not support local instances. + local_instances = mock.MagicMock() + self.Patch(list_instances, "GetCFRemoteInstances", + return_value=None) + self.Patch(list_instances, "GetLocalInstances", + return_value=local_instances) + restart.Run(args) + mock_exit.assert_called_once() + # pylint: disable=no-member def testRestartFromInstance(self): """test RestartFromInstance.""" diff --git a/run_tests.sh b/run_tests.sh index c1ee76f2..16a532dc 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -90,7 +90,7 @@ function check_env() { fi local missing_py_packages=false - for py_lib in {coverage,mock}; + for py_lib in {coverage,mock,google-api-core}; do if ! python3 -m pip list | grep $py_lib &> /dev/null; then echo "Missing required python package: $py_lib (python3 -m pip install $py_lib)" diff --git a/setup/host_setup_runner.py b/setup/host_setup_runner.py index 28cccc0d..447e33d2 100644 --- a/setup/host_setup_runner.py +++ b/setup/host_setup_runner.py @@ -39,13 +39,33 @@ logger = logging.getLogger(__name__) _CF_COMMOM_FOLDER = "cf-common" -_LIST_OF_MODULES = ["kvm_intel", "kvm"] +_INTEL = "intel" +_AMD = "amd" +_KVM_INTEL = "kvm_intel" +_KVM_AMD = "kvm_amd" +_LIST_OF_INTEL_MODULES = [_KVM_INTEL, "kvm"] +_LIST_OF_AMD_MODULES = [_KVM_AMD, "kvm"] +_DICT_MODULES = {_INTEL: _LIST_OF_INTEL_MODULES, _AMD: _LIST_OF_AMD_MODULES} +_INTEL_COMMANDS = [ + "sudo rmmod kvm_intel || true", "sudo rmmod kvm || true", + "sudo modprobe kvm", "sudo modprobe kvm_intel"] +_AMD_COMMANDS = [ + "sudo rmmod kvm_amd || true", "sudo rmmod kvm|| true", "sudo modprobe kvm", + "sudo modprobe kvm_amd"] +_DICT_SETUP_CMDS = {_INTEL: _INTEL_COMMANDS, _AMD: _AMD_COMMANDS} _UPDATE_APT_GET_CMD = "sudo apt-get update" _INSTALL_CUTTLEFISH_COMMOM_CMD = [ "git clone https://github.com/google/android-cuttlefish.git {git_folder}", - "cd {git_folder}", + "cd {git_folder}/base", "debuild -i -us -uc -b", + "cd ../frontend", + "debuild -i -us -uc -b", + "sudo dpkg -i ../cuttlefish-base_*_*64.deb || sudo apt-get install -f", + "sudo dpkg -i ../cuttlefish-user_*_*64.deb || sudo apt-get install -f", "sudo dpkg -i ../cuttlefish-common_*_*64.deb || sudo apt-get install -f"] +_INSTALL_CUTTLEFISH_COMMOM_MSG = ("\nStart to install cuttlefish-common :\n%s" + "\nEnter 'y' to continue, otherwise N or " + "enter to exit: ") class BasePkgInstaller(base_task_runner.BaseTaskRunner): @@ -134,20 +154,24 @@ class CuttlefishCommonPkgInstaller(base_task_runner.BaseTaskRunner): def _Run(self): """Install cuttlefilsh-common packages.""" + if setup_common.IsPackageInAptList(constants.CUTTLEFISH_COMMOM_PKG): + cmd = setup_common.PKG_INSTALL_CMD % constants.CUTTLEFISH_COMMOM_PKG + if not utils.GetUserAnswerYes(_INSTALL_CUTTLEFISH_COMMOM_MSG % cmd): + sys.exit(constants.EXIT_BY_USER) + setup_common.InstallPackage(constants.CUTTLEFISH_COMMOM_PKG) + return + + # Install cuttlefish-common from github. cf_common_path = os.path.join(tempfile.mkdtemp(), _CF_COMMOM_FOLDER) logger.debug("cuttlefish-common path: %s", cf_common_path) cmd = "\n".join(sub_cmd.format(git_folder=cf_common_path) for sub_cmd in _INSTALL_CUTTLEFISH_COMMOM_CMD) - - if not utils.GetUserAnswerYes("\nStart to install cuttlefish-common :\n%s" - "\nEnter 'y' to continue, otherwise N or " - "enter to exit: " % cmd): - sys.exit(constants.EXIT_BY_USER) try: + if not utils.GetUserAnswerYes(_INSTALL_CUTTLEFISH_COMMOM_MSG % cmd): + sys.exit(constants.EXIT_BY_USER) setup_common.CheckCmdOutput(cmd, shell=True) finally: shutil.rmtree(os.path.dirname(cf_common_path)) - logger.info("Cuttlefish-common package installed now.") class LocalCAHostSetup(base_task_runner.BaseTaskRunner): @@ -202,7 +226,21 @@ class CuttlefishHostSetup(base_task_runner.BaseTaskRunner): return False return not (utils.CheckUserInGroups(constants.LIST_CF_USER_GROUPS) - and self._CheckLoadedModules(_LIST_OF_MODULES)) + and self._CheckLoadedModules( + _DICT_MODULES.get(self._GetProcessorType()))) + + @staticmethod + def _GetProcessorType(): + """Get the processor type. + + Returns: + The processor type of the host. e.g. amd, intel. + """ + lsmod_output = setup_common.CheckCmdOutput("lsmod", print_cmd=False) + current_modules = [r.split()[0] for r in lsmod_output.splitlines()] + if _KVM_AMD in current_modules: + return _AMD + return _INTEL @staticmethod def _CheckLoadedModules(module_list): @@ -210,6 +248,7 @@ class CuttlefishHostSetup(base_task_runner.BaseTaskRunner): Args: module_list: The list of module name. + Returns: True if all modules are in use. """ @@ -227,11 +266,7 @@ class CuttlefishHostSetup(base_task_runner.BaseTaskRunner): """Setup host environment for local cuttlefish instance support.""" # TODO: provide --uid args to let user use prefered username username = getpass.getuser() - setup_cmds = [ - "sudo rmmod kvm_intel", - "sudo rmmod kvm", - "sudo modprobe kvm", - "sudo modprobe kvm_intel"] + setup_cmds = _DICT_SETUP_CMDS.get(self._GetProcessorType()) for group in constants.LIST_CF_USER_GROUPS: setup_cmds.append("sudo usermod -aG %s % s" % (group, username)) diff --git a/setup/host_setup_runner_test.py b/setup/host_setup_runner_test.py index d98466b6..08b631a9 100644 --- a/setup/host_setup_runner_test.py +++ b/setup/host_setup_runner_test.py @@ -74,6 +74,8 @@ lrw 16384 1 aesni_intel""" def testRun(self): """Test Run.""" self.Patch(CuttlefishHostSetup, "ShouldRun", return_value=True) + self.Patch(CuttlefishHostSetup, "_GetProcessorType", + return_value="intel") self.Patch(utils, "InteractWithQuestion", return_value="y") self.Patch(setup_common, "CheckCmdOutput") self.CuttlefishHostSetup.Run() @@ -110,6 +112,7 @@ class AvdPkgInstallerTest(driver_test_lib.BaseDriverTest): def testShouldRun(self): """Test ShouldRun.""" self.Patch(platform, "system", return_value="Linux") + self.Patch(platform, "version", return_value="Unsupport") self.assertFalse(self.AvdPkgInstaller.ShouldRun()) def testShouldNotRun(self): @@ -165,9 +168,15 @@ class CuttlefishCommonPkgInstallerTest(driver_test_lib.BaseDriverTest): self.Patch(tempfile, "mkdtemp", return_value=fake_tmp_folder) self.Patch(utils, "GetUserAnswerYes", return_value=True) self.Patch(CuttlefishCommonPkgInstaller, "ShouldRun", return_value=True) + self.Patch(setup_common, "IsPackageInAptList", return_value=False) self.CuttlefishCommonPkgInstaller.Run() self.assertEqual(mock_cmd.call_count, 1) mock_rmtree.assert_called_once_with(fake_tmp_folder) + # Install cuttlefish-common from rapture + self.Patch(setup_common, "IsPackageInAptList", return_value=True) + self.Patch(setup_common, "InstallPackage") + self.CuttlefishCommonPkgInstaller.Run() + setup_common.InstallPackage.assert_called() self.Patch(utils, "GetUserAnswerYes", return_value=False) self.Patch(sys, "exit") diff --git a/setup/setup_common.py b/setup/setup_common.py index 97ea1417..b3ce8ac6 100644 --- a/setup/setup_common.py +++ b/setup/setup_common.py @@ -27,7 +27,7 @@ from acloud.internal.lib import utils logger = logging.getLogger(__name__) PKG_INSTALL_CMD = "sudo apt-get --assume-yes install %s" -APT_CHECK_CMD = "LANG=en_US.UTF-8 apt-cache policy %s" +APT_CHECK_CMD = "LANG=en_US.UTF-8 LANGUAGE=en_US:en apt-cache policy %s" _INSTALLED_RE = re.compile(r"(.*\s*Installed:)(?P<installed_ver>.*\s?)") _CANDIDATE_RE = re.compile(r"(.*\s*Candidate:)(?P<candidate_ver>.*\s?)") @@ -78,6 +78,29 @@ def InstallPackage(pkg): pkg + "]") +def IsPackageInAptList(pkg_name): + """Check if the package is apt packages list. + + Args: + pkg_name: String, the package name. + + Returns: + True if package is in apt packages list. + """ + try: + pkg_info = CheckCmdOutput( + APT_CHECK_CMD % pkg_name, + print_cmd=False, + shell=True, + stderr=subprocess.STDOUT) + if pkg_info: + return True + return False + except subprocess.CalledProcessError as error: + # Unable locate package name on repository. + return False + + def PackageInstalled(pkg_name, compare_version=True): """Check if the package is installed or not. |