diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2022-05-09 06:24:23 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2022-05-09 06:24:23 +0000 |
commit | 53192e2a1d7f6b9e039847521b1c75e68abcb362 (patch) | |
tree | a1b03220a13a96ba0b9f35edb024e4c1d2e51a10 | |
parent | b3ddc65577ce87faf657c1a10af965c2dc6fa606 (diff) | |
parent | a4d4b1dba6318efaf89091ce80da91c5c1823e0f (diff) | |
download | acloud-android13-frc-networking-release.tar.gz |
Snap for 8558685 from a4d4b1dba6318efaf89091ce80da91c5c1823e0f to tm-frc-networking-releaset_frc_net_330443000android13-frc-networking-release
Change-Id: I950c032cadc3ed34d30669acdc882fa5a50e48b9
62 files changed, 1984 insertions, 1163 deletions
diff --git a/acloud_test.py b/acloud_test.py index ce594967..a8137bd6 100644 --- a/acloud_test.py +++ b/acloud_test.py @@ -40,6 +40,14 @@ logger.addHandler(logging.FileHandler("/dev/null")) if sys.version_info.major == 3: sys.path.insert(0, os.path.dirname(sysconfig.get_paths()['purelib'])) +# (b/219847353) Move googleapiclient to the last position of sys.path when +# existed. +for lib in sys.path: + if 'googleapiclient' in lib: + sys.path.remove(lib) + sys.path.append(lib) + break + def GetTestModules(): """Return list of testable modules. diff --git a/create/avd_spec.py b/create/avd_spec.py index 894fc6d7..9eff3fc1 100644 --- a/create/avd_spec.py +++ b/create/avd_spec.py @@ -125,6 +125,7 @@ class AVDSpec(): self._num_of_instances = None self._num_avds_per_instance = None self._no_pull_log = None + self._mkcert = None self._oxygen = None self._openwrt = None self._remote_image = None @@ -140,7 +141,9 @@ class AVDSpec(): self._host_ssh_private_key_path = None self._gpu = None self._disk_type = None + self._base_instance_num = None self._stable_host_image_name = None + self._use_launch_cvd = None # Create config instance for android_build_client to query build api. self._cfg = config.GetAcloudConfig(args) # Reporting args. @@ -154,6 +157,8 @@ class AVDSpec(): self._stable_cheeps_host_image_project = None self._username = None self._password = None + self._cheeps_betty_image = None + self._cheeps_features = None # The maximum time in seconds used to wait for the AVD to boot. self._boot_timeout_secs = None @@ -232,6 +237,14 @@ class AVDSpec(): self._image_source = constants.IMAGE_SRC_LOCAL self._ProcessLocalImageArgs(args) + if args.local_kernel_image is not None: + self._local_kernel_image = self._GetLocalImagePath( + args.local_kernel_image) + + if args.local_system_image is not None: + self._local_system_image = self._GetLocalImagePath( + args.local_system_image) + self.image_download_dir = ( args.image_download_dir if args.image_download_dir else tempfile.gettempdir()) @@ -339,13 +352,16 @@ class AVDSpec(): self._num_of_instances = args.num self._num_avds_per_instance = args.num_avds_per_instance self._no_pull_log = args.no_pull_log + self._mkcert = args.mkcert self._oxygen = args.oxygen self._openwrt = args.openwrt + self._use_launch_cvd = args.use_launch_cvd self._serial_log_file = args.serial_log_file self._emulator_build_id = args.emulator_build_id self._emulator_build_target = args.emulator_build_target self._gpu = args.gpu self._disk_type = (args.disk_type or self._cfg.disk_type) + self._base_instance_num = args.base_instance_num self._gce_metadata = create_common.ParseKeyValuePairArgs(args.gce_metadata) self._stable_host_image_name = ( args.stable_host_image_name or self._cfg.stable_host_image_name) @@ -354,6 +370,9 @@ class AVDSpec(): self._stable_cheeps_host_image_project = args.stable_cheeps_host_image_project self._username = args.username self._password = args.password + self._cheeps_betty_image = ( + args.cheeps_betty_image or self._cfg.betty_image) + self._cheeps_features = args.cheeps_features self._boot_timeout_secs = args.boot_timeout_secs self._ins_timeout_secs = args.ins_timeout_secs @@ -419,14 +438,6 @@ class AVDSpec(): "Local image doesn't support the AVD type: %s" % self._avd_type ) - if args.local_kernel_image is not None: - self._local_kernel_image = self._GetLocalImagePath( - args.local_kernel_image) - - if args.local_system_image is not None: - self._local_system_image = self._GetLocalImagePath( - args.local_system_image) - @staticmethod def _GetGceLocalImagePath(local_image_dir): """Get gce local image path. @@ -596,9 +607,6 @@ class AVDSpec(): self._remote_image[constants.BUILD_TARGET], self._remote_image[constants.BUILD_BRANCH]) - self._remote_image[constants.CHEEPS_BETTY_IMAGE] = ( - args.cheeps_betty_image or self._cfg.betty_image) - # Process system image, kernel image, bootloader, and otatools. self._system_build_info = {constants.BUILD_ID: args.system_build_id, constants.BUILD_BRANCH: args.system_branch, @@ -893,6 +901,11 @@ class AVDSpec(): return self._disk_type @property + def base_instance_num(self): + """Return base instance num.""" + return self._base_instance_num + + @property def gpu(self): """Return gpu.""" return self._gpu @@ -939,6 +952,16 @@ class AVDSpec(): return self._password @property + def cheeps_betty_image(self): + """Return cheeps_betty_image.""" + return self._cheeps_betty_image + + @property + def cheeps_features(self): + """Return cheeps_features.""" + return self._cheeps_features + + @property def boot_timeout_secs(self): """Return boot_timeout_secs.""" return self._boot_timeout_secs @@ -989,6 +1012,11 @@ class AVDSpec(): return self._no_pull_log @property + def mkcert(self): + """Return mkcert.""" + return self._mkcert + + @property def gce_metadata(self): """Return gce_metadata.""" return self._gce_metadata @@ -1004,6 +1032,11 @@ class AVDSpec(): return self._openwrt @property + def use_launch_cvd(self): + """Return use_launch_cvd.""" + return self._use_launch_cvd + + @property def launch_args(self): """Return launch_args.""" return self._launch_args diff --git a/create/avd_spec_test.py b/create/avd_spec_test.py index 62c2b8f9..71d2405f 100644 --- a/create/avd_spec_test.py +++ b/create/avd_spec_test.py @@ -49,6 +49,12 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.args.launch_args = None self.Patch(list_instances, "ChooseOneRemoteInstance", return_value=mock.MagicMock()) self.Patch(list_instances, "GetInstancesFromInstanceNames", return_value=mock.MagicMock()) + + # Setup mock Acloud config for usage in tests. + self.mock_config = mock.MagicMock() + self.mock_config.launch_args = None + self.Patch(config, 'GetAcloudConfig', return_value=self.mock_config) + self.AvdSpec = avd_spec.AVDSpec(self.args) # pylint: disable=protected-access @@ -111,45 +117,28 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.Patch(os.path, "isfile", side_effect=lambda path: path == expected_image_file) - # Specified --local-kernel-image and --local-system-image with dirs. - self.args.local_image = expected_image_dir + # Specified --local-*-image with dirs. self.args.local_kernel_image = expected_image_dir self.args.local_system_image = expected_image_dir - self.AvdSpec._avd_type = constants.TYPE_CF - self.AvdSpec._instance_type = constants.INSTANCE_TYPE_LOCAL - with mock.patch("acloud.create.avd_spec.utils." - "GetBuildEnvironmentVariable", - return_value="cf_x86_phone"): - self.AvdSpec._ProcessLocalImageArgs(self.args) - self.assertEqual(self.AvdSpec.local_image_dir, expected_image_dir) + self.AvdSpec._ProcessImageArgs(self.args) self.assertEqual(self.AvdSpec.local_kernel_image, expected_image_dir) self.assertEqual(self.AvdSpec.local_system_image, expected_image_dir) - # Specified --local-kernel-image, and --local-system-image with files. - self.args.local_image = expected_image_dir + # Specified --local-*-image with files. self.args.local_kernel_image = expected_image_file self.args.local_system_image = expected_image_file - self.AvdSpec._avd_type = constants.TYPE_CF - self.AvdSpec._instance_type = constants.INSTANCE_TYPE_LOCAL - with mock.patch("acloud.create.avd_spec.utils." - "GetBuildEnvironmentVariable", - return_value="cf_x86_phone"): - self.AvdSpec._ProcessLocalImageArgs(self.args) - self.assertEqual(self.AvdSpec.local_image_dir, expected_image_dir) + self.AvdSpec._ProcessImageArgs(self.args) self.assertEqual(self.AvdSpec.local_kernel_image, expected_image_file) self.assertEqual(self.AvdSpec.local_system_image, expected_image_file) - # Specified --avd-type=goldfish, --local_image, and - # --local-system-image without args - self.args.local_image = constants.FIND_IN_BUILD_ENV + # Specified --local-*-image without args. + self.args.local_kernel_image = constants.FIND_IN_BUILD_ENV self.args.local_system_image = constants.FIND_IN_BUILD_ENV - self.AvdSpec._avd_type = constants.TYPE_GF - self.AvdSpec._instance_type = constants.INSTANCE_TYPE_LOCAL with mock.patch("acloud.create.avd_spec.utils." "GetBuildEnvironmentVariable", return_value=expected_image_dir): - self.AvdSpec._ProcessLocalImageArgs(self.args) - self.assertEqual(self.AvdSpec.local_image_dir, expected_image_dir) + self.AvdSpec._ProcessImageArgs(self.args) + self.assertEqual(self.AvdSpec.local_kernel_image, expected_image_dir) self.assertEqual(self.AvdSpec.local_system_image, expected_image_dir) def testProcessAutoconnect(self): @@ -176,10 +165,11 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): """Test process image source.""" self.Patch(glob, "glob", return_value=["fake.img"]) # No specified local_image, image source is from remote - self.args.local_image = None self.AvdSpec._ProcessImageArgs(self.args) self.assertEqual(self.AvdSpec._image_source, constants.IMAGE_SRC_REMOTE) self.assertEqual(self.AvdSpec._local_image_dir, None) + self.assertEqual(self.AvdSpec.local_kernel_image, None) + self.assertEqual(self.AvdSpec.local_system_image, None) # Specified local_image with an arg for cf type self.Patch(os.path, "isfile", return_value=True) @@ -427,26 +417,6 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.AvdSpec._ProcessRemoteBuildArgs(self.args) self.assertTrue(self.AvdSpec.avd_type == "cuttlefish") - # Setup acloud config with betty_image spec - cfg = mock.MagicMock() - cfg.betty_image = 'foobarbaz' - cfg.launch_args = None - self.Patch(config, 'GetAcloudConfig', return_value=cfg) - self.AvdSpec = avd_spec.AVDSpec(self.args) - # --betty-image from cmdline should override config - self.args.cheeps_betty_image = 'abcdefg' - self.AvdSpec._ProcessRemoteBuildArgs(self.args) - self.assertEqual( - self.AvdSpec.remote_image[constants.CHEEPS_BETTY_IMAGE], - self.args.cheeps_betty_image) - # acloud config value is used otherwise - self.args.cheeps_betty_image = None - self.AvdSpec._ProcessRemoteBuildArgs(self.args) - self.assertEqual( - self.AvdSpec.remote_image[constants.CHEEPS_BETTY_IMAGE], - cfg.betty_image) - - def testEscapeAnsi(self): """Test EscapeAnsi.""" test_string = "\033[1;32;40m Manifest branch:" @@ -546,6 +516,22 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.AvdSpec._ProcessMiscArgs(self.args) self.assertEqual(self.AvdSpec.stable_host_image_name, "fake_host_image") + # Setup acloud config with betty_image spec + self.mock_config.betty_image = 'from-config' + # --betty-image from cmdline should override config + self.args.cheeps_betty_image = 'from-cmdline' + self.AvdSpec._ProcessMiscArgs(self.args) + self.assertEqual(self.AvdSpec.cheeps_betty_image, 'from-cmdline') + # acloud config value is used otherwise + self.args.cheeps_betty_image = None + self.AvdSpec._ProcessMiscArgs(self.args) + self.assertEqual(self.AvdSpec.cheeps_betty_image, 'from-config') + + # Verify cheeps_features is assigned from args. + self.args.cheeps_features = ['a', 'b', 'c'] + self.AvdSpec._ProcessMiscArgs(self.args) + self.assertEqual(self.args.cheeps_features, ['a', 'b', 'c']) + if __name__ == "__main__": unittest.main() diff --git a/create/cheeps_remote_image_remote_instance_test.py b/create/cheeps_remote_image_remote_instance_test.py index 22e59d71..f5dcae4c 100644 --- a/create/cheeps_remote_image_remote_instance_test.py +++ b/create/cheeps_remote_image_remote_instance_test.py @@ -25,6 +25,7 @@ class CheepsRemoteImageRemoteInstanceTest(driver_test_lib.BaseDriverTest): CHEEPS_HOST_IMAGE_PROJECT = "fake-stable-host-image-project" ANDROID_BUILD_ID = 12345 ANDROID_BUILD_TARGET = "fake-target" + DEFAULT_ADB_PORT = 9222 def setUp(self): """Set up the test.""" @@ -96,7 +97,7 @@ class CheepsRemoteImageRemoteInstanceTest(driver_test_lib.BaseDriverTest): "devices": [{ "build_id": self.ANDROID_BUILD_ID, "instance_name": self.INSTANCE, - "ip": self.IP.external, + "ip": self.IP.external + ":" + str(self.DEFAULT_ADB_PORT), },], }) self.assertEqual(report.command, "create_cheeps") diff --git a/create/create.py b/create/create.py index 335a0de3..fdf739f1 100644 --- a/create/create.py +++ b/create/create.py @@ -197,7 +197,12 @@ def _CheckForSetup(args): args.host = True logger.debug("Auto-detect to install host packages.") - if args.autoconnect == constants.INS_KEY_WEBRTC: + user_groups_setup = host_setup_runner.CuttlefishHostSetup() + if user_groups_setup.ShouldRun(): + args.host = True + logger.debug("Auto-detect to setup user groups.") + + if args.mkcert and args.autoconnect == constants.INS_KEY_WEBRTC: local_ca_setup = host_setup_runner.LocalCAHostSetup() if local_ca_setup.ShouldRun(): args.host_local_ca = True diff --git a/create/create_args.py b/create/create_args.py index 8784f525..aa3292dc 100644 --- a/create/create_args.py +++ b/create/create_args.py @@ -282,6 +282,13 @@ def AddCommonCreateArgs(parser): required=False, default=None, help="Disable auto download logs when AVD booting up failed.") + parser.add_argument( + "--no-mkcert", + dest="mkcert", + action="store_false", + required=False, + default=True, + help="Disable mkcert setup process on the host.") # TODO(147335651): Add gpu in user config. # TODO(147335651): Support "--gpu" without giving any value. parser.add_argument( @@ -533,6 +540,12 @@ def GetCreateArgParser(subparser): help="'cuttlefish only' Create OpenWrt device when launching cuttlefish " "device.") create_parser.add_argument( + "--use-launch_cvd", + action="store_true", + dest="use_launch_cvd", + required=False, + help="'cuttlefish only' Use launch_cvd to create cuttlefish devices.") + create_parser.add_argument( "--host", type=str, dest="remote_host", @@ -645,6 +658,14 @@ def GetCreateArgParser(subparser): help=("'cheeps only' The L1 betty version to use. Only makes sense " "when launching a controller image with " "stable-cheeps-host-image")) + create_parser.add_argument( + "--cheeps-feature", + type=str, + dest="cheeps_features", + required=False, + action="append", + default=[], + help=("'cheeps only' Cheeps feature to enable. Can be repeated.")) AddCommonCreateArgs(create_parser) return create_parser @@ -839,11 +860,13 @@ def VerifyArgs(args): args.stable_cheeps_host_image_project, args.username, args.password, - args.cheeps_betty_image] + args.cheeps_betty_image, + args.cheeps_features] if args.avd_type != constants.TYPE_CHEEPS and any(cheeps_only_flags): raise errors.UnsupportedCreateArgs( - "--stable-cheeps-*, --betty-image, --username and --password are " - "only valid with avd_type == %s" % constants.TYPE_CHEEPS) + "--stable-cheeps-*, --betty-image, --cheeps-feature, --username " + "and --password are only valid with avd_type == %s" + % constants.TYPE_CHEEPS) if (args.username or args.password) and not (args.username and args.password): raise ValueError("--username and --password must both be set") if not args.autoconnect and args.unlock_screen: diff --git a/create/create_args_test.py b/create/create_args_test.py index c8a5bb00..788955f4 100644 --- a/create/create_args_test.py +++ b/create/create_args_test.py @@ -36,6 +36,7 @@ def _CreateArgs(): username=None, password=None, cheeps_betty_image=None, + cheeps_features=[], local_image=None, local_kernel_image=None, local_system_image=None, diff --git a/create/create_common.py b/create/create_common.py index 1d1a5212..a5694d7b 100644 --- a/create/create_common.py +++ b/create/create_common.py @@ -146,7 +146,7 @@ def GetCvdHostPackage(package_path=None): '\n'.join(dirs_to_check)) -def FindLocalImage(path, default_name_pattern): +def FindLocalImage(path, default_name_pattern, raise_error=True): """Find an image file in the given path. Args: @@ -165,7 +165,9 @@ def FindLocalImage(path, default_name_pattern): names = [name for name in os.listdir(path) if re.fullmatch(default_name_pattern, name)] if not names: - raise errors.GetLocalImageError("No image in %s." % path) + if raise_error: + raise errors.GetLocalImageError("No image in %s." % path) + return None if len(names) != 1: raise errors.GetLocalImageError("More than one image in %s: %s" % (path, " ".join(names))) diff --git a/create/create_common_test.py b/create/create_common_test.py index 0caa97a9..14503e6a 100644 --- a/create/create_common_test.py +++ b/create/create_common_test.py @@ -152,11 +152,14 @@ class CreateCommonTest(driver_test_lib.BaseDriverTest): self.assertEqual("/dir/name", create_common.FindLocalImage("/dir/", "name")) + + self.assertIsNone(create_common.FindLocalImage("/dir", "not_exist", + raise_error=False)) with self.assertRaises(errors.GetLocalImageError): create_common.FindLocalImage("/dir", "not_exist") with self.assertRaises(errors.GetLocalImageError): - create_common.FindLocalImage("/dir", "name.?") + create_common.FindLocalImage("/dir", "name.?", raise_error=False) @mock.patch.object(utils, "Decompress") def testDownloadRemoteArtifact(self, mock_decompress): diff --git a/create/create_test.py b/create/create_test.py index 5d26c245..b203083c 100644 --- a/create/create_test.py +++ b/create/create_test.py @@ -32,7 +32,7 @@ from acloud.setup import host_setup_runner from acloud.setup import setup -# pylint: disable=invalid-name,protected-access +# pylint: disable=invalid-name,protected-access,too-many-statements class CreateTest(driver_test_lib.BaseDriverTest): """Test create functions.""" @@ -96,6 +96,9 @@ class CreateTest(driver_test_lib.BaseDriverTest): self.Patch(host_setup_runner.LocalCAHostSetup, "ShouldRun", return_value=False) + self.Patch(host_setup_runner.CuttlefishHostSetup, + "ShouldRun", + return_value=False) self.Patch(config, "AcloudConfigManager") self.Patch(config.AcloudConfigManager, "Load") self.Patch(setup, "Run") @@ -123,13 +126,18 @@ class CreateTest(driver_test_lib.BaseDriverTest): "ShouldRun") self.Patch(host_setup_runner.AvdPkgInstaller, "ShouldRun") + self.Patch(host_setup_runner.CuttlefishHostSetup, + "ShouldRun") args.local_instance = None args.local_image = None create._CheckForSetup(args) self.assertEqual(gcp_setup_runner.GcpTaskRunner.ShouldRun.call_count, 1) self.assertEqual(host_setup_runner.AvdPkgInstaller.ShouldRun.call_count, 0) + self.assertEqual( + host_setup_runner.CuttlefishHostSetup.ShouldRun.call_count, 0) gcp_setup_runner.GcpTaskRunner.ShouldRun.reset_mock() host_setup_runner.AvdPkgInstaller.ShouldRun.reset_mock() + host_setup_runner.CuttlefishHostSetup.ShouldRun.reset_mock() # Test with remote instance local image case. args.local_instance = None @@ -137,8 +145,11 @@ class CreateTest(driver_test_lib.BaseDriverTest): create._CheckForSetup(args) self.assertEqual(gcp_setup_runner.GcpTaskRunner.ShouldRun.call_count, 1) self.assertEqual(host_setup_runner.AvdPkgInstaller.ShouldRun.call_count, 0) + self.assertEqual( + host_setup_runner.CuttlefishHostSetup.ShouldRun.call_count, 0) gcp_setup_runner.GcpTaskRunner.ShouldRun.reset_mock() host_setup_runner.AvdPkgInstaller.ShouldRun.reset_mock() + host_setup_runner.CuttlefishHostSetup.ShouldRun.reset_mock() # Test with local instance remote image case. args.local_instance = 0 @@ -146,8 +157,11 @@ class CreateTest(driver_test_lib.BaseDriverTest): create._CheckForSetup(args) self.assertEqual(gcp_setup_runner.GcpTaskRunner.ShouldRun.call_count, 1) self.assertEqual(host_setup_runner.AvdPkgInstaller.ShouldRun.call_count, 1) + self.assertEqual( + host_setup_runner.CuttlefishHostSetup.ShouldRun.call_count, 1) gcp_setup_runner.GcpTaskRunner.ShouldRun.reset_mock() host_setup_runner.AvdPkgInstaller.ShouldRun.reset_mock() + host_setup_runner.CuttlefishHostSetup.ShouldRun.reset_mock() # Test with local instance local image case. args.local_instance = 0 @@ -155,8 +169,11 @@ class CreateTest(driver_test_lib.BaseDriverTest): create._CheckForSetup(args) self.assertEqual(gcp_setup_runner.GcpTaskRunner.ShouldRun.call_count, 0) self.assertEqual(host_setup_runner.AvdPkgInstaller.ShouldRun.call_count, 1) + self.assertEqual( + host_setup_runner.CuttlefishHostSetup.ShouldRun.call_count, 1) gcp_setup_runner.GcpTaskRunner.ShouldRun.reset_mock() host_setup_runner.AvdPkgInstaller.ShouldRun.reset_mock() + host_setup_runner.CuttlefishHostSetup.ShouldRun.reset_mock() # pylint: disable=no-member def testRun(self): diff --git a/create/local_image_local_instance.py b/create/local_image_local_instance.py index 077a7d78..c5913fb4 100644 --- a/create/local_image_local_instance.py +++ b/create/local_image_local_instance.py @@ -62,6 +62,7 @@ from acloud import errors from acloud.create import base_avd_create from acloud.create import create_common from acloud.internal import constants +from acloud.internal.lib import cvd_utils from acloud.internal.lib import ota_tools from acloud.internal.lib import utils from acloud.internal.lib.adb_tools import AdbTools @@ -73,18 +74,12 @@ from acloud.setup import mkcert logger = logging.getLogger(__name__) -# The boot image name pattern corresponds to the use cases: -# - In a cuttlefish build environment, ANDROID_PRODUCT_OUT conatins boot.img -# and boot-debug.img. The former is the default boot image. The latter is not -# useful for cuttlefish. -# - In an officially released GKI (Generic Kernel Image) package, the image -# name is boot-<kernel version>.img. -_BOOT_IMAGE_NAME_PATTERN = r"boot(-[\d.]+)?\.img" _SYSTEM_IMAGE_NAME_PATTERN = r"system\.img" _MISC_INFO_FILE_NAME = "misc_info.txt" _TARGET_FILES_IMAGES_DIR_NAME = "IMAGES" _TARGET_FILES_META_DIR_NAME = "META" _MIXED_SUPER_IMAGE_NAME = "mixed_super.img" +_CMD_CVD_START = " start" _CMD_LAUNCH_CVD_ARGS = ( " -daemon -config=%s -system_image_dir %s -instance_dir %s " "-undefok=report_anonymous_usage_stats,config " @@ -96,11 +91,15 @@ _CMD_LAUNCH_CVD_WEBRTC_ARGS = " -start_webrtc=true" _CMD_LAUNCH_CVD_VNC_ARG = " -start_vnc_server=true" _CMD_LAUNCH_CVD_SUPER_IMAGE_ARG = " -super_image=%s" _CMD_LAUNCH_CVD_BOOT_IMAGE_ARG = " -boot_image=%s" +_CMD_LAUNCH_CVD_VENDOR_BOOT_IMAGE_ARG = " -vendor_boot_image=%s" _CMD_LAUNCH_CVD_NO_ADB_ARG = " -run_adb_connector=false" # Connect the OpenWrt device via console file. _CMD_LAUNCH_CVD_CONSOLE_ARG = " -console=true" _CONFIG_RE = re.compile(r"^config=(?P<config>.+)") _CONSOLE_NAME = "console" +# Files to store the output when launching cvds. +_STDOUT = "stdout" +_STDERR = "stderr" _MAX_REPORTED_ERROR_LINES = 10 # In accordance with the number of network interfaces in @@ -123,7 +122,7 @@ _CONFIRM_RELAUNCH = ("\nCuttlefish AVD[id:%d] is already running. \n" ArtifactPaths = collections.namedtuple( "ArtifactPaths", ["image_dir", "host_bins", "host_artifacts", "misc_info", "ota_tools_dir", - "system_image", "boot_image"]) + "system_image", "boot_image", "vendor_boot_image"]) class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): @@ -232,9 +231,7 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): runtime_dir = instance.GetLocalInstanceRuntimeDir(local_instance_id) # TODO(b/168171781): cvd_status of list/delete via the symbolic. self.PrepareLocalCvdToolsLink(cvd_home_dir, artifact_paths.host_bins) - launch_cvd_path = os.path.join(artifact_paths.host_bins, "bin", - constants.CMD_LAUNCH_CVD) - if avd_spec.connect_webrtc: + if avd_spec.mkcert and avd_spec.connect_webrtc: self._TrustCertificatesForWebRTC(artifact_paths.host_artifacts) hw_property = None @@ -242,18 +239,17 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): hw_property = avd_spec.hw_property config = self._GetConfigFromAndroidInfo( os.path.join(artifact_paths.image_dir, constants.ANDROID_INFO_FILE)) - cmd = self.PrepareLaunchCVDCmd(launch_cvd_path, - hw_property, + cmd = self.PrepareLaunchCVDCmd(hw_property, avd_spec.connect_adb, - artifact_paths.image_dir, + artifact_paths, runtime_dir, avd_spec.connect_webrtc, avd_spec.connect_vnc, super_image_path, - artifact_paths.boot_image, avd_spec.launch_args, config or avd_spec.flavor, - avd_spec.openwrt) + avd_spec.openwrt, + avd_spec.use_launch_cvd) result_report = report.Report(command="create") instance_name = instance.GetLocalInstanceName(local_instance_id) @@ -344,7 +340,7 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): "set --local-tool to an extracted CVD host package.") @staticmethod - def _FindMiscInfo(image_dir): + def FindMiscInfo(image_dir): """Find misc info in build output dir or extracted target files. Args: @@ -369,7 +365,7 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): "Cannot find %s in %s." % (_MISC_INFO_FILE_NAME, image_dir)) @staticmethod - def _FindImageDir(image_dir): + def FindImageDir(image_dir): """Find images in build output dir or extracted target files. Args: @@ -419,8 +415,8 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): host_artifacts_path = self._FindCvdHostArtifactsPath(tool_dirs) if avd_spec.local_system_image: - misc_info_path = self._FindMiscInfo(image_dir) - image_dir = self._FindImageDir(image_dir) + misc_info_path = self.FindMiscInfo(image_dir) + image_dir = self.FindImageDir(image_dir) ota_tools_dir = os.path.abspath( ota_tools.FindOtaToolsDir(tool_dirs)) system_image_path = create_common.FindLocalImage( @@ -431,17 +427,19 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): system_image_path = None if avd_spec.local_kernel_image: - boot_image_path = create_common.FindLocalImage( - avd_spec.local_kernel_image, _BOOT_IMAGE_NAME_PATTERN) + boot_image_path, vendor_boot_image_path = cvd_utils.FindBootImages( + avd_spec.local_kernel_image) else: boot_image_path = None + vendor_boot_image_path = None return ArtifactPaths(image_dir, host_bins_path, host_artifacts=host_artifacts_path, misc_info=misc_info_path, ota_tools_dir=ota_tools_dir, system_image=system_image_path, - boot_image=boot_image_path) + boot_image=boot_image_path, + vendor_boot_image=vendor_boot_image_path) @staticmethod def _MixSuperImage(output_dir, artifact_paths): @@ -485,34 +483,38 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): return None @staticmethod - def PrepareLaunchCVDCmd(launch_cvd_path, hw_property, connect_adb, - image_dir, runtime_dir, connect_webrtc, - connect_vnc, super_image_path, boot_image_path, - launch_args, config, openwrt=False): + def PrepareLaunchCVDCmd(hw_property, connect_adb, artifact_paths, + runtime_dir, connect_webrtc, connect_vnc, + super_image_path, launch_args, config, + openwrt=False, use_launch_cvd=False): """Prepare launch_cvd command. Create the launch_cvd commands with all the required args and add in the user groups to it if necessary. Args: - launch_cvd_path: String of launch_cvd path. hw_property: dict object of hw property. - image_dir: String of local images path. + artifact_paths: ArtifactPaths object. connect_adb: Boolean flag that enables adb_connector. runtime_dir: String of runtime directory path. connect_webrtc: Boolean of connect_webrtc. connect_vnc: Boolean of connect_vnc. super_image_path: String of non-default super image path. - boot_image_path: String of non-default boot image path. launch_args: String of launch args. config: String of config name. openwrt: Boolean of enable OpenWrt devices. + use_launch_cvd: Boolean of using launch_cvd for old build cases. Returns: - String, launch_cvd cmd. + String, cvd start cmd. """ - launch_cvd_w_args = launch_cvd_path + _CMD_LAUNCH_CVD_ARGS % ( - config, image_dir, runtime_dir) + bin_dir = os.path.join(artifact_paths.host_bins, "bin") + start_cvd_cmd = (os.path.join(bin_dir, constants.CMD_CVD) + + _CMD_CVD_START) + if use_launch_cvd: + start_cvd_cmd = os.path.join(bin_dir, constants.CMD_LAUNCH_CVD) + launch_cvd_w_args = start_cvd_cmd + _CMD_LAUNCH_CVD_ARGS % ( + config, artifact_paths.image_dir, runtime_dir) if hw_property: launch_cvd_w_args = launch_cvd_w_args + _CMD_LAUNCH_CVD_HW_ARGS % ( hw_property["cpu"], hw_property["x_res"], hw_property["y_res"], @@ -535,10 +537,15 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): _CMD_LAUNCH_CVD_SUPER_IMAGE_ARG % super_image_path) - if boot_image_path: + if artifact_paths.boot_image: launch_cvd_w_args = (launch_cvd_w_args + _CMD_LAUNCH_CVD_BOOT_IMAGE_ARG % - boot_image_path) + artifact_paths.boot_image) + + if artifact_paths.vendor_boot_image: + launch_cvd_w_args = (launch_cvd_w_args + + _CMD_LAUNCH_CVD_VENDOR_BOOT_IMAGE_ARG % + artifact_paths.vendor_boot_image) if openwrt: launch_cvd_w_args = launch_cvd_w_args + _CMD_LAUNCH_CVD_CONSOLE_ARG @@ -661,29 +668,39 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): cvd_env[constants.ENV_ANDROID_HOST_OUT] = host_bins_path cvd_env[constants.ENV_CVD_HOME] = cvd_home_dir cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str(local_instance_id) + cvd_env[constants.ENV_CUTTLEFISH_CONFIG_FILE] = ( + instance.GetLocalInstanceConfigPath(local_instance_id)) + stdout_file = os.path.join(cvd_home_dir, _STDOUT) + stderr_file = os.path.join(cvd_home_dir, _STDERR) # Check the result of launch_cvd command. # An exit code of 0 is equivalent to VIRTUAL_DEVICE_BOOT_COMPLETED - try: - proc = subprocess.Popen(cmd, shell=True, env=cvd_env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, cwd=host_bins_path) - stdout, stderr = proc.communicate(timeout=timeout) - if proc.returncode == 0: - logger.info("launch_cvd stdout:\n%s", stdout) - logger.info("launch_cvd stderr:\n%s", stderr) - return - error_msg = "launch_cvd returned %d." % proc.returncode - except subprocess.TimeoutExpired: - self._StopCvd(local_instance_id, proc) - stdout, stderr = proc.communicate(timeout=5) - error_msg = "Device did not boot within %d secs." % timeout - - logger.error("launch_cvd stdout:\n%s", stdout) - logger.error("launch_cvd stderr:\n%s", stderr) - split_stderr = stderr.splitlines()[-_MAX_REPORTED_ERROR_LINES:] - raise errors.LaunchCVDFail("%s Stderr:\n%s" % - (error_msg, "\n".join(split_stderr))) + with open(stdout_file, "w+") as f_stdout, open(stderr_file, + "w+") as f_stderr: + try: + proc = subprocess.Popen( + cmd, shell=True, env=cvd_env, stdout=f_stdout, + stderr=f_stderr, text=True, cwd=host_bins_path) + proc.communicate(timeout=timeout) + f_stdout.seek(0) + f_stderr.seek(0) + if proc.returncode == 0: + logger.info("launch_cvd stdout:\n%s", f_stdout.read()) + logger.info("launch_cvd stderr:\n%s", f_stderr.read()) + return + error_msg = "launch_cvd returned %d." % proc.returncode + except subprocess.TimeoutExpired: + self._StopCvd(local_instance_id, proc) + proc.communicate(timeout=5) + error_msg = "Device did not boot within %d secs." % timeout + + f_stdout.seek(0) + f_stderr.seek(0) + stderr = f_stderr.read() + logger.error("launch_cvd stdout:\n%s", f_stdout.read()) + logger.error("launch_cvd stderr:\n%s", stderr) + split_stderr = stderr.splitlines()[-_MAX_REPORTED_ERROR_LINES:] + raise errors.LaunchCVDFail( + "%s Stderr:\n%s" % (error_msg, "\n".join(split_stderr))) @staticmethod def _FindLogs(local_instance_id): diff --git a/create/local_image_local_instance_test.py b/create/local_image_local_instance_test.py index 22dfcf5b..7baf9711 100644 --- a/create/local_image_local_instance_test.py +++ b/create/local_image_local_instance_test.py @@ -15,6 +15,7 @@ # limitations under the License. """Tests for LocalImageLocalInstance.""" +import builtins import os import subprocess import tempfile @@ -36,37 +37,37 @@ class LocalImageLocalInstanceTest(driver_test_lib.BaseDriverTest): LAUNCH_CVD_CMD_WITH_DISK = """sg group1 <<EOF sg group2 -launch_cvd -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -blank_data_image_mb fake -data_policy always_create -start_vnc_server=true +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -blank_data_image_mb fake -data_policy always_create -start_vnc_server=true EOF""" LAUNCH_CVD_CMD_NO_DISK = """sg group1 <<EOF sg group2 -launch_cvd -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -start_vnc_server=true +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -start_vnc_server=true EOF""" LAUNCH_CVD_CMD_NO_DISK_WITH_GPU = """sg group1 <<EOF sg group2 -launch_cvd -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -start_vnc_server=true +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -start_vnc_server=true EOF""" LAUNCH_CVD_CMD_WITH_WEBRTC = """sg group1 <<EOF sg group2 -launch_cvd -daemon -config=auto -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_webrtc=true +bin/cvd start -daemon -config=auto -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_webrtc=true EOF""" LAUNCH_CVD_CMD_WITH_MIXED_IMAGES = """sg group1 <<EOF sg group2 -launch_cvd -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_vnc_server=true -super_image=fake_super_image -boot_image=fake_boot_image +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_vnc_server=true -super_image=fake_super_image -boot_image=fake_boot_image -vendor_boot_image=fake_vendor_boot_image EOF""" LAUNCH_CVD_CMD_WITH_ARGS = """sg group1 <<EOF sg group2 -launch_cvd -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_vnc_server=true -setupwizard_mode=REQUIRED +bin/cvd start -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_vnc_server=true -setupwizard_mode=REQUIRED EOF""" LAUNCH_CVD_CMD_WITH_OPENWRT = """sg group1 <<EOF sg group2 -launch_cvd -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_vnc_server=true -console=true +bin/launch_cvd -daemon -config=phone -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,config -report_anonymous_usage_stats=y -start_vnc_server=true -console=true EOF""" _EXPECTED_DEVICES_IN_REPORT = [ @@ -117,7 +118,8 @@ EOF""" """Test _CreateAVD.""" mock_utils.IsSupportedPlatform.return_value = True mock_get_image.return_value = local_image_local_instance.ArtifactPaths( - "/image/path", "/host/bin/path", "host/usr/path", None, None, None, None) + "/image/path", "/host/bin/path", "host/usr/path", + None, None, None, None, None) mock_check_running_cvd.return_value = True mock_avd_spec = mock.Mock() mock_lock = mock.Mock() @@ -198,7 +200,8 @@ EOF""" "/instances/cvd/config") artifact_paths = local_image_local_instance.ArtifactPaths( "/image/path", "/host/bin/path", "/host/usr/path", "/misc/info/path", - "/ota/tools/dir", "/system/image/path", "/boot/image/path") + "/ota/tools/dir", "/system/image/path", "/boot/image/path", + "/vendor_boot/image/path") mock_ota_tools_object = mock.Mock() mock_ota_tools.OtaTools.return_value = mock_ota_tools_object mock_avd_spec = mock.Mock( @@ -287,24 +290,29 @@ EOF""" mock_avd_spec) mock_ota_tools.FindOtaToolsDir.assert_not_called() - self.assertEqual(paths, (image_dir, cvd_dir, cvd_dir, None, None, None, None)) + self.assertEqual(paths, (image_dir, cvd_dir, cvd_dir, + None, None, None, None, None)) @mock.patch("acloud.create.local_image_local_instance.ota_tools") - def testGetImageFromBuildEnvironment(self, mock_ota_tools): + @mock.patch("acloud.create.local_image_local_instance.cvd_utils") + def testGetImageFromBuildEnvironment(self, mock_cvd_utils, mock_ota_tools): """Test GetImageArtifactsPath with files in build environment.""" + boot_image_path = "/mock/boot.img" + vendor_boot_image_path = "/mock/vendor_boot.img" + mock_cvd_utils.FindBootImages.return_value = (boot_image_path, + vendor_boot_image_path) + with tempfile.TemporaryDirectory() as temp_dir: image_dir = os.path.join(temp_dir, "image") cvd_dir = os.path.join(temp_dir, "cvd-host_package") mock_ota_tools.FindOtaToolsDir.return_value = cvd_dir extra_image_dir = os.path.join(temp_dir, "extra_image") system_image_path = os.path.join(extra_image_dir, "system.img") - boot_image_path = os.path.join(extra_image_dir, "boot.img") misc_info_path = os.path.join(image_dir, "misc_info.txt") self._CreateEmptyFile(os.path.join(image_dir, "vbmeta.img")) self._CreateEmptyFile(os.path.join(cvd_dir, "bin", "launch_cvd")) self._CreateEmptyFile(os.path.join(cvd_dir, "usr/share/webrtc/certs", "server.crt")) self._CreateEmptyFile(system_image_path) - self._CreateEmptyFile(boot_image_path) self._CreateEmptyFile(os.path.join(extra_image_dir, "boot-debug.img")) self._CreateEmptyFile(misc_info_path) @@ -324,29 +332,33 @@ EOF""" mock_avd_spec) mock_ota_tools.FindOtaToolsDir.assert_called_with([cvd_dir, "/cvd"]) + mock_cvd_utils.FindBootImages.asssert_called_with(extra_image_dir) self.assertEqual(paths, (image_dir, cvd_dir, cvd_dir, misc_info_path, cvd_dir, - system_image_path, boot_image_path)) + system_image_path, boot_image_path, + vendor_boot_image_path)) @mock.patch("acloud.create.local_image_local_instance.ota_tools") - def testGetImageFromTargetFiles(self, mock_ota_tools): + @mock.patch("acloud.create.local_image_local_instance.cvd_utils") + def testGetImageFromTargetFiles(self, mock_cvd_utils, mock_ota_tools): """Test GetImageArtifactsPath with extracted target files.""" ota_tools_dir = "/mock_ota_tools" mock_ota_tools.FindOtaToolsDir.return_value = ota_tools_dir + boot_image_path = "/mock/boot.img" + mock_cvd_utils.FindBootImages.return_value = (boot_image_path, None) with tempfile.TemporaryDirectory() as temp_dir: image_dir = os.path.join(temp_dir, "image") cvd_dir = os.path.join(temp_dir, "cvd-host_package") system_image_path = os.path.join(temp_dir, "system", "test.img") misc_info_path = os.path.join(image_dir, "META", "misc_info.txt") - boot_image_path = os.path.join(temp_dir, "boot", "test.img") + self._CreateEmptyFile(os.path.join(image_dir, "IMAGES", "vbmeta.img")) self._CreateEmptyFile(os.path.join(cvd_dir, "bin", "launch_cvd")) self._CreateEmptyFile(os.path.join(cvd_dir, "usr/share/webrtc/certs", "server.crt")) self._CreateEmptyFile(system_image_path) self._CreateEmptyFile(misc_info_path) - self._CreateEmptyFile(boot_image_path) mock_avd_spec = mock.Mock( local_image_dir=image_dir, @@ -362,10 +374,11 @@ EOF""" mock_ota_tools.FindOtaToolsDir.assert_called_with( [ota_tools_dir, cvd_dir]) + mock_cvd_utils.FindBootImages.assert_called_with(boot_image_path) self.assertEqual(paths, (os.path.join(image_dir, "IMAGES"), cvd_dir, cvd_dir, misc_info_path, ota_tools_dir, system_image_path, - boot_image_path)) + boot_image_path, None)) @mock.patch.object(utils, "CheckUserInGroups") def testPrepareLaunchCVDCmd(self, mock_usergroups): @@ -374,50 +387,61 @@ EOF""" hw_property = {"cpu": "fake", "x_res": "fake", "y_res": "fake", "dpi":"fake", "memory": "fake", "disk": "fake"} constants.LIST_CF_USER_GROUPS = ["group1", "group2"] + mock_artifact_paths = mock.Mock( + spec=[], + image_dir="fake_image_dir", + host_bins="", + host_artifacts="host_artifacts", + misc_info=None, + ota_tools_dir=None, + system_image=None, + boot_image=None, + vendor_boot_image=None) launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - constants.CMD_LAUNCH_CVD, hw_property, True, "fake_image_dir", - "fake_cvd_dir", False, True, None, None, None, "phone") + hw_property, True, mock_artifact_paths, "fake_cvd_dir", False, + True, None, None, "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_DISK) # "disk" doesn't exist in hw_property. hw_property = {"cpu": "fake", "x_res": "fake", "y_res": "fake", "dpi": "fake", "memory": "fake"} launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - constants.CMD_LAUNCH_CVD, hw_property, True, "fake_image_dir", - "fake_cvd_dir", False, True, None, None, None, "phone") + hw_property, True, mock_artifact_paths, "fake_cvd_dir", False, + True, None, None, "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_NO_DISK) # "gpu" is enabled with "default" launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - constants.CMD_LAUNCH_CVD, hw_property, True, "fake_image_dir", - "fake_cvd_dir", False, True, None, None, None, "phone") + hw_property, True, mock_artifact_paths, "fake_cvd_dir", False, + True, None, None, "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_NO_DISK_WITH_GPU) # Following test with hw_property is None. launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - constants.CMD_LAUNCH_CVD, None, True, "fake_image_dir", - "fake_cvd_dir", True, False, None, None, None, "auto") + None, True, mock_artifact_paths, "fake_cvd_dir", True, False, + None, None, "auto") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_WEBRTC) + mock_artifact_paths.boot_image = "fake_boot_image" + mock_artifact_paths.vendor_boot_image = "fake_vendor_boot_image" launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - constants.CMD_LAUNCH_CVD, None, True, "fake_image_dir", - "fake_cvd_dir", False, True, "fake_super_image", "fake_boot_image", - None, "phone") + None, True, mock_artifact_paths, "fake_cvd_dir", False, True, + "fake_super_image", None, "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_MIXED_IMAGES) + mock_artifact_paths.boot_image = None + mock_artifact_paths.vendor_boot_image = None # Add args into launch command with "-setupwizard_mode=REQUIRED" launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - constants.CMD_LAUNCH_CVD, None, True, "fake_image_dir", - "fake_cvd_dir", False, True, None, None, - "-setupwizard_mode=REQUIRED", "phone") + None, True, mock_artifact_paths, "fake_cvd_dir", False, True, + None, "-setupwizard_mode=REQUIRED", "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_ARGS) - # Test with "openwrt" is enabled. + # Test with "openwrt" and "use_launch_cvd" are enabled. launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( - constants.CMD_LAUNCH_CVD, None, True, "fake_image_dir", - "fake_cvd_dir", False, True, None, None, None, - "phone", openwrt=True) + None, True, mock_artifact_paths, "fake_cvd_dir", False, True, + None, None, "phone", openwrt=True, use_launch_cvd=True) self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_OPENWRT) @mock.patch.object(utils, "GetUserAnswerYes") @@ -444,6 +468,7 @@ EOF""" @mock.patch.dict("os.environ", clear=True) def testLaunchCVD(self, mock_popen): """test _LaunchCvd should call subprocess.Popen with the env.""" + self.Patch(builtins, "open", mock.mock_open()) local_instance_id = 3 launch_cvd_cmd = "launch_cvd" host_bins_path = "host_bins_path" @@ -466,23 +491,15 @@ EOF""" cvd_home_dir, timeout) - mock_popen.assert_called_once_with(launch_cvd_cmd, - shell=True, - env=cvd_env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - cwd=host_bins_path) + mock_popen.assert_called_once() mock_proc.communicate.assert_called_once_with(timeout=timeout) @mock.patch("acloud.create.local_image_local_instance.subprocess.Popen") def testLaunchCVDFailure(self, mock_popen): """test _LaunchCvd with subprocess errors.""" + self.Patch(builtins, "open", mock.mock_open()) mock_proc = mock.Mock(returncode=9) mock_popen.return_value = mock_proc - mock_proc.communicate.side_effect = [ - ("stdout", "first line" + ("\n" * 10) + "last line\n") - ] with self.assertRaises(errors.LaunchCVDFail) as launch_cvd_failure: self.local_image_local_instance._LaunchCvd("launch_cvd", 3, @@ -491,13 +508,12 @@ EOF""" "cvd_home_dir", 100) self.assertIn("returned 9", str(launch_cvd_failure.exception)) - self.assertNotIn("first line", str(launch_cvd_failure.exception)) - self.assertIn("last line", str(launch_cvd_failure.exception)) @mock.patch("acloud.create.local_image_local_instance.list_instance") @mock.patch("acloud.create.local_image_local_instance.subprocess.Popen") def testLaunchCVDTimeout(self, mock_popen, mock_list_instance): """test _LaunchCvd with subprocess timeout.""" + self.Patch(builtins, "open", mock.mock_open()) mock_proc = mock.Mock(returncode=255) mock_popen.return_value = mock_proc mock_proc.communicate.side_effect = [ diff --git a/create/local_image_remote_host.py b/create/local_image_remote_host.py index c74f3b9e..6aecb9ce 100644 --- a/create/local_image_remote_host.py +++ b/create/local_image_remote_host.py @@ -24,7 +24,7 @@ from acloud.create import create_common from acloud.internal import constants from acloud.internal.lib import utils from acloud.public.actions import common_operations -from acloud.public.actions import remote_instance_cf_device_factory +from acloud.public.actions import remote_host_cf_device_factory class LocalImageRemoteHost(base_avd_create.BaseAVDCreate): @@ -42,7 +42,7 @@ class LocalImageRemoteHost(base_avd_create.BaseAVDCreate): Returns: A Report instance. """ - device_factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( + device_factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( avd_spec, avd_spec.local_image_artifact, create_common.GetCvdHostPackage(avd_spec.cvd_host_package)) diff --git a/create/local_image_remote_host_test.py b/create/local_image_remote_host_test.py index 15af6db8..74b198bb 100644 --- a/create/local_image_remote_host_test.py +++ b/create/local_image_remote_host_test.py @@ -25,7 +25,7 @@ from acloud.internal import constants from acloud.internal.lib import driver_test_lib from acloud.internal.lib import utils from acloud.public.actions import common_operations -from acloud.public.actions import remote_instance_cf_device_factory +from acloud.public.actions import remote_host_cf_device_factory class LocalImageRemoteHostTest(driver_test_lib.BaseDriverTest): """Test LocalImageRemoteHost method.""" @@ -46,12 +46,11 @@ class LocalImageRemoteHostTest(driver_test_lib.BaseDriverTest): spec.image_source = constants.IMAGE_SRC_LOCAL spec.connect_vnc = False self.Patch(avd_spec, "AVDSpec", return_value=spec) - self.Patch(remote_instance_cf_device_factory, - "RemoteInstanceDeviceFactory") + self.Patch(remote_host_cf_device_factory, "RemoteHostDeviceFactory") self.Patch(create_common, "GetCvdHostPackage") self.Patch(common_operations, "CreateDevices") create.Run(args) - remote_instance_cf_device_factory.RemoteInstanceDeviceFactory.assert_called_once() + remote_host_cf_device_factory.RemoteHostDeviceFactory.assert_called_once() common_operations.CreateDevices.assert_called_once() spec.connect_vnc = True diff --git a/create/remote_image_local_instance.py b/create/remote_image_local_instance.py index 8d989a8f..d0d0958f 100644 --- a/create/remote_image_local_instance.py +++ b/create/remote_image_local_instance.py @@ -25,10 +25,12 @@ import subprocess import sys from acloud import errors +from acloud.create import create_common from acloud.create import local_image_local_instance from acloud.internal import constants from acloud.internal.lib import android_build_client from acloud.internal.lib import auth +from acloud.internal.lib import ota_tools from acloud.internal.lib import utils from acloud.setup import setup_common @@ -50,6 +52,10 @@ _HOME_FOLDER = os.path.expanduser("~") # for the downloaded image artifacts. _REQUIRED_SPACE = 10 +_SYSTEM_IMAGE_NAME_PATTERN = r"system\.img" +_SYSTEM_MIX_IMAGE_DIR = "mix_image_{build_id}" +_DOWNLOAD_MIX_IMAGE_NAME = "{build_target}-target_files-{build_id}.zip" + @utils.TimeExecute(function_description="Downloading Android Build image") def DownloadAndProcessImageFiles(avd_spec): @@ -154,6 +160,21 @@ def ConfirmDownloadRemoteImageDir(download_dir): return download_dir +def GetMixBuildTargetFilename(build_target, build_id): + """Get the mix build target filename. + + Args: + build_id: String, Build id, e.g. "2263051", "P2804227" + build_target: String, the build target, e.g. cf_x86_phone-userdebug + + Returns: + String, a file name, e.g. "cf_x86_phone-target_files-2263051.zip" + """ + return _DOWNLOAD_MIX_IMAGE_NAME.format( + build_target=build_target.split('-')[0], + build_id=build_id) + + class RemoteImageLocalInstance(local_image_local_instance.LocalImageLocalInstance): """Create class for a remote image local instance AVD. @@ -189,7 +210,42 @@ class RemoteImageLocalInstance(local_image_local_instance.LocalImageLocalInstanc raise errors.GetCvdLocalHostPackageError( "No launch_cvd found. Please check downloaded artifacts dir: %s" % image_dir) + + mix_image_dir = None + if avd_spec.local_system_image: + build_id = avd_spec.remote_image[constants.BUILD_ID] + build_target = avd_spec.remote_image[constants.BUILD_TARGET] + mix_image_dir =os.path.join( + image_dir, _SYSTEM_MIX_IMAGE_DIR.format(build_id=build_id)) + if not os.path.exists(mix_image_dir): + os.makedirs(mix_image_dir) + create_common.DownloadRemoteArtifact( + avd_spec.cfg, build_target, build_id, + GetMixBuildTargetFilename(build_target, build_id), + mix_image_dir, decompress=True) + misc_info_path = super().FindMiscInfo(mix_image_dir) + mix_image_dir = super().FindImageDir(mix_image_dir) + tool_dirs = (avd_spec.local_tool_dirs + + create_common.GetNonEmptyEnvVars( + constants.ENV_ANDROID_SOONG_HOST_OUT, + constants.ENV_ANDROID_HOST_OUT)) + ota_tools_dir = os.path.abspath( + ota_tools.FindOtaToolsDir(tool_dirs)) + system_image_path = create_common.FindLocalImage( + avd_spec.local_system_image, _SYSTEM_IMAGE_NAME_PATTERN) + else: + misc_info_path = None + ota_tools_dir = None + system_image_path = None + # This method does not set the optional fields because launch_cvd loads # the paths from the fetcher config in image_dir. return local_image_local_instance.ArtifactPaths( - image_dir, image_dir, image_dir, None, None, None, None) + image_dir=mix_image_dir or image_dir, + host_bins=image_dir, + host_artifacts=image_dir, + misc_info=misc_info_path, + ota_tools_dir=ota_tools_dir, + system_image=system_image_path, + boot_image=None, + vendor_boot_image=None) diff --git a/create/remote_image_local_instance_test.py b/create/remote_image_local_instance_test.py index 8eda6391..135fafac 100644 --- a/create/remote_image_local_instance_test.py +++ b/create/remote_image_local_instance_test.py @@ -22,15 +22,18 @@ import subprocess from unittest import mock from acloud import errors +from acloud.create import create_common from acloud.create import remote_image_local_instance +from acloud.create import local_image_local_instance from acloud.internal.lib import android_build_client from acloud.internal.lib import auth from acloud.internal.lib import driver_test_lib +from acloud.internal.lib import ota_tools from acloud.internal.lib import utils from acloud.setup import setup_common -# pylint: disable=invalid-name, protected-access +# pylint: disable=invalid-name, protected-access, no-member class RemoteImageLocalInstanceTest(driver_test_lib.BaseDriverTest): """Test remote_image_local_instance methods.""" @@ -54,6 +57,7 @@ class RemoteImageLocalInstanceTest(driver_test_lib.BaseDriverTest): """Test get image artifacts path.""" mock_proc.return_value = "/unit/test" avd_spec = mock.MagicMock() + avd_spec.local_system_image = None # raise errors.NoCuttlefishCommonInstalled self.Patch(setup_common, "PackageInstalled", return_value=False) self.assertRaises(errors.NoCuttlefishCommonInstalled, @@ -70,6 +74,38 @@ class RemoteImageLocalInstanceTest(driver_test_lib.BaseDriverTest): self.assertEqual(paths.image_dir, "/unit/test") self.assertEqual(paths.host_bins, "/unit/test") + # GSI + avd_spec.local_system_image = "/test_local_system_image_dir" + avd_spec.local_tool_dirs = "/test_local_tool_dirs" + avd_spec.cfg = None + avd_spec.remote_image = self._fake_remote_image + self.Patch(os, "makedirs") + self.Patch(create_common, "DownloadRemoteArtifact") + self.Patch(os.path, "exists", side_effect=[True, False]) + self.Patch(create_common, "GetNonEmptyEnvVars") + self.Patch(local_image_local_instance.LocalImageLocalInstance, + "FindMiscInfo", return_value="/mix_image_1234/MISC") + self.Patch(local_image_local_instance.LocalImageLocalInstance, + "FindImageDir", return_value="/mix_image_1234/IMAGES") + self.Patch(ota_tools, "FindOtaToolsDir", return_value="/ota_tools_dir") + self.Patch(create_common, "FindLocalImage", return_value="/system_image_path") + paths = self.RemoteImageLocalInstance.GetImageArtifactsPath(avd_spec) + create_common.DownloadRemoteArtifact.assert_called_with( + avd_spec.cfg, "aosp_cf_x86_64_phone-userdebug", "1234", + "aosp_cf_x86_64_phone-target_files-1234.zip", "/unit/test/mix_image_1234", + decompress=True) + self.assertEqual(paths.image_dir, "/mix_image_1234/IMAGES") + self.assertEqual(paths.misc_info, "/mix_image_1234/MISC") + self.assertEqual(paths.host_bins, "/unit/test") + self.assertEqual(paths.ota_tools_dir, "/ota_tools_dir") + self.assertEqual(paths.system_image, "/system_image_path") + create_common.DownloadRemoteArtifact.reset_mock() + + self.Patch(os.path, "exists", side_effect=[True, True]) + self.RemoteImageLocalInstance.GetImageArtifactsPath(avd_spec) + create_common.DownloadRemoteArtifact.assert_not_called() + + @mock.patch.object(shutil, "rmtree") def testDownloadAndProcessImageFiles(self, mock_rmtree): """Test process remote cuttlefish image.""" diff --git a/create/remote_image_remote_host.py b/create/remote_image_remote_host.py index 5ca2ae0d..96ba658a 100644 --- a/create/remote_image_remote_host.py +++ b/create/remote_image_remote_host.py @@ -25,7 +25,7 @@ from acloud.create import base_avd_create from acloud.internal import constants from acloud.internal.lib import utils from acloud.public.actions import common_operations -from acloud.public.actions import remote_instance_cf_device_factory +from acloud.public.actions import remote_host_cf_device_factory logger = logging.getLogger(__name__) @@ -44,7 +44,7 @@ class RemoteImageRemoteHost(base_avd_create.BaseAVDCreate): Returns: A Report instance. """ - device_factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( + device_factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( avd_spec=avd_spec) report = common_operations.CreateDevices( "create_cf", avd_spec.cfg, device_factory, num=1, diff --git a/create/remote_image_remote_host_test.py b/create/remote_image_remote_host_test.py index 1f665e74..8c0150c0 100644 --- a/create/remote_image_remote_host_test.py +++ b/create/remote_image_remote_host_test.py @@ -24,7 +24,7 @@ from acloud.internal import constants from acloud.internal.lib import driver_test_lib from acloud.internal.lib import utils from acloud.public.actions import common_operations -from acloud.public.actions import remote_instance_cf_device_factory +from acloud.public.actions import remote_host_cf_device_factory class RemoteImageRemoteHostTest(driver_test_lib.BaseDriverTest): """Test RemoteImageRemoteHost method.""" @@ -45,11 +45,10 @@ class RemoteImageRemoteHostTest(driver_test_lib.BaseDriverTest): spec.image_source = constants.IMAGE_SRC_REMOTE spec.connect_vnc = False self.Patch(avd_spec, "AVDSpec", return_value=spec) - self.Patch(remote_instance_cf_device_factory, - "RemoteInstanceDeviceFactory") + self.Patch(remote_host_cf_device_factory, "RemoteHostDeviceFactory") self.Patch(common_operations, "CreateDevices") create.Run(args) - remote_instance_cf_device_factory.RemoteInstanceDeviceFactory.assert_called_once() + remote_host_cf_device_factory.RemoteHostDeviceFactory.assert_called_once() common_operations.CreateDevices.assert_called_once() spec.connect_vnc = True diff --git a/create/remote_image_remote_instance.py b/create/remote_image_remote_instance.py index de01f7e3..225df208 100644 --- a/create/remote_image_remote_instance.py +++ b/create/remote_image_remote_instance.py @@ -36,7 +36,6 @@ from acloud.public import report logger = logging.getLogger(__name__) _DEVICE = "device" _DEVICES = "devices" -_DEVICE_KEY_MAPPING = {"serverUrl": "ip", "sessionId": "instance_name"} _LAUNCH_CVD_TIME = "launch_cvd_time" _RE_SESSION_ID = re.compile(r".*session_id:\"(?P<session_id>[^\"]+)") _RE_SERVER_URL = re.compile(r".*server_url:\"(?P<server_url>[^\"]+)") @@ -98,6 +97,10 @@ class RemoteImageRemoteInstance(base_avd_create.BaseAVDCreate): avd_spec.remote_image[constants.BUILD_TARGET], avd_spec.remote_image[constants.BUILD_ID], avd_spec.remote_image[constants.BUILD_BRANCH], + avd_spec.system_build_info[constants.BUILD_TARGET], + avd_spec.system_build_info[constants.BUILD_ID], + avd_spec.kernel_build_info[constants.BUILD_TARGET], + avd_spec.kernel_build_info[constants.BUILD_ID], avd_spec.cfg.oxygen_client, avd_spec.cfg.oxygen_lease_args) session_id, server_url = self._GetDeviceInfoFromResponse(response) @@ -153,21 +156,3 @@ class RemoteImageRemoteInstance(base_avd_create.BaseAVDCreate): server_url = server_url_match.group("server_url") break return session_id, server_url - - @staticmethod - def _ReplaceDeviceDataKeys(device_data): - """Replace keys of device data from oxygen response. - - To keep the device data using the same keys in Acloud report. Before - writing data to report, it needs to update the keys. - - Values: - device_data: Dict of device data. e.g. {'sessionId': 'b01ead68', - 'serverUrl': '10.1.1.1'} - """ - for key, val in _DEVICE_KEY_MAPPING.items(): - if key in device_data: - device_data[val] = device_data[key] - del device_data[key] - else: - logger.debug("There is no '%s' data in response.", key) diff --git a/create/remote_image_remote_instance_test.py b/create/remote_image_remote_instance_test.py index 33bba172..68af4f58 100644 --- a/create/remote_image_remote_instance_test.py +++ b/create/remote_image_remote_instance_test.py @@ -91,6 +91,12 @@ class RemoteImageRemoteInstanceTest(driver_test_lib.BaseDriverTest): avd_spec.remote_image = {constants.BUILD_TARGET: "fake_target", constants.BUILD_ID: "fake_id", constants.BUILD_BRANCH: "fake_branch"} + avd_spec.system_build_info = {constants.BUILD_TARGET: "fake_target", + constants.BUILD_ID: "fake_id", + constants.BUILD_BRANCH: "fake_branch"} + avd_spec.kernel_build_info = {constants.BUILD_TARGET: "fake_target", + constants.BUILD_ID: "fake_id", + constants.BUILD_BRANCH: "fake_branch"} response_fail = "Lease device fail." self.Patch(oxygen_client.OxygenClient, "LeaseDevice", side_effect=[ONE_LINE_LEASE_RESPONSE, response_fail]) @@ -109,6 +115,12 @@ class RemoteImageRemoteInstanceTest(driver_test_lib.BaseDriverTest): avd_spec.remote_image = {constants.BUILD_TARGET: "fake_target", constants.BUILD_ID: "fake_id", constants.BUILD_BRANCH: "fake_branch"} + avd_spec.system_build_info = {constants.BUILD_TARGET: "fake_target", + constants.BUILD_ID: "fake_id", + constants.BUILD_BRANCH: "fake_branch"} + avd_spec.kernel_build_info = {constants.BUILD_TARGET: "fake_target", + constants.BUILD_ID: "fake_id", + constants.BUILD_BRANCH: "fake_branch"} response_fail = "Lease device fail." self.Patch(oxygen_client.OxygenClient, "LeaseDevice", side_effect=[LEASE_FAILURE_RESPONSE, response_fail]) @@ -132,13 +144,6 @@ class RemoteImageRemoteInstanceTest(driver_test_lib.BaseDriverTest): MULTIPLE_LINES_LEASE_RESPONSE), (expect_session_id, expect_server_url)) - def testReplaceDeviceDataKeys(self): - """test ReplaceDeviceDataKeys.""" - device_data = {"sessionId": "fake_device", "serverUrl": "10.1.1.1"} - expected_result = {"instance_name": "fake_device", "ip": "10.1.1.1"} - self.remote_image_remote_instance._ReplaceDeviceDataKeys(device_data) - self.assertEqual(device_data, expected_result) - if __name__ == '__main__': unittest.main() diff --git a/delete/delete.py b/delete/delete.py index e8aaaaf6..98ee79e7 100644 --- a/delete/delete.py +++ b/delete/delete.py @@ -25,12 +25,12 @@ import subprocess from acloud import errors from acloud.internal import constants -from acloud.internal.lib import auth from acloud.internal.lib import cvd_compute_client_multi_stage +from acloud.internal.lib import cvd_utils from acloud.internal.lib import emulator_console from acloud.internal.lib import goldfish_remote_host_client from acloud.internal.lib import oxygen_client -from acloud.internal.lib import ssh as ssh_object +from acloud.internal.lib import ssh from acloud.internal.lib import utils from acloud.list import list as list_instances from acloud.public import config @@ -279,17 +279,13 @@ def CleanUpRemoteHost(cfg, remote_host, host_user, Returns: delete_report. """ - credentials = auth.CreateCredentials(cfg) - compute_client = cvd_compute_client_multi_stage.CvdComputeClient( - acloud_config=cfg, - oauth2_credentials=credentials) - ssh = ssh_object.Ssh( - ip=ssh_object.IP(ip=remote_host), + ssh_obj = ssh.Ssh( + ip=ssh.IP(ip=remote_host), user=host_user, ssh_private_key_path=( host_ssh_private_key_path or cfg.ssh_private_key_path)) try: - compute_client.InitRemoteHost(ssh, remote_host, host_user) + cvd_utils.CleanUpRemoteCvd(ssh_obj, raise_error=True) delete_report.SetStatus(report.Status.SUCCESS) device_driver.AddDeletionResultToReport( delete_report, [remote_host], failed=[], diff --git a/delete/delete_test.py b/delete/delete_test.py index d2166698..6baa8b91 100644 --- a/delete/delete_test.py +++ b/delete/delete_test.py @@ -208,17 +208,14 @@ class DeleteTest(driver_test_lib.BaseDriverTest): self.assertEqual(delete_report.status, "FAIL") self.assertEqual(len(delete_report.errors), 1) - @mock.patch.object(delete, "auth") - @mock.patch.object(delete, "cvd_compute_client_multi_stage") - @mock.patch.object(delete, "ssh_object") - def testCleanUpRemoteHost(self, mock_ssh, mock_client, mock_auth): + @mock.patch.object(delete, "ssh") + @mock.patch.object(delete, "cvd_utils") + def testCleanUpRemoteHost(self, mock_cvd_utils, mock_ssh): """Test CleanUpRemoteHost.""" mock_ssh_ip = mock.Mock() mock_ssh.IP.return_value = mock_ssh_ip mock_ssh_obj = mock.Mock() mock_ssh.Ssh.return_value = mock_ssh_obj - mock_client_obj = mock.Mock() - mock_client.CvdComputeClient.return_value = mock_client_obj cfg_attrs = {"ssh_private_key_path": "cfg_key_path"} mock_cfg = mock.Mock(spec_set=list(cfg_attrs.keys()), **cfg_attrs) delete_report = report.Report(command="delete") @@ -230,8 +227,8 @@ class DeleteTest(driver_test_lib.BaseDriverTest): ip=mock_ssh_ip, user="vsoc-01", ssh_private_key_path="cfg_key_path") - mock_client_obj.InitRemoteHost.assert_called_with( - mock_ssh_obj, "192.0.2.1", "vsoc-01") + mock_cvd_utils.CleanUpRemoteCvd.assert_called_with(mock_ssh_obj, + raise_error=True) self.assertEqual(delete_report.status, "SUCCESS") self.assertEqual(delete_report.data, { "deleted": [ @@ -244,8 +241,8 @@ class DeleteTest(driver_test_lib.BaseDriverTest): mock_ssh_ip.reset_mock() mock_ssh_obj.reset_mock() - mock_client_obj.InitRemoteHost.reset_mock() - mock_client_obj.InitRemoteHost.side_effect = ( + mock_cvd_utils.reset_mock() + mock_cvd_utils.CleanUpRemoteCvd.side_effect = ( subprocess.CalledProcessError(cmd="test", returncode=1)) delete_report = report.Report(command="delete") @@ -256,8 +253,8 @@ class DeleteTest(driver_test_lib.BaseDriverTest): ip=mock_ssh_ip, user="user", ssh_private_key_path="key_path") - mock_client_obj.InitRemoteHost.assert_called_with( - mock_ssh_obj, "192.0.2.2", "user") + mock_cvd_utils.CleanUpRemoteCvd.assert_called_with(mock_ssh_obj, + raise_error=True) self.assertEqual(delete_report.status, "FAIL") self.assertEqual(len(delete_report.errors), 1) diff --git a/internal/constants.py b/internal/constants.py index 8b7bd6fd..c0a87328 100755 --- a/internal/constants.py +++ b/internal/constants.py @@ -114,6 +114,7 @@ ADB_PORT = "adb_port" WEBRTC_PORT = "webrtc_port" DEVICE_SERIAL = "device_serial" LOGS = "logs" +BASE_INSTANCE_NUM = "base_instance_num" # For cuttlefish remote instances CF_ADB_PORT = 6520 CF_VNC_PORT = 6444 @@ -132,6 +133,7 @@ FVP_ADB_PORT = 5555 MAX_PORT = 65535 COMMAND_PS = ["ps", "aux"] +CMD_CVD = "cvd" CMD_LAUNCH_CVD = "launch_cvd" CMD_PGREP = "pgrep" CMD_STOP_CVD = "stop_cvd" @@ -205,16 +207,14 @@ SSL_CA_NAME = "ACloud-webRTC-CA" SSL_TRUST_CA_DIR = "/usr/local/share/ca-certificates" # Remote Log -REMOTE_LOG_FOLDER = f"/home/{GCE_USER}/cuttlefish_runtime" - -# Cheeps specific stuff. -CHEEPS_BETTY_IMAGE = "betty_image" +REMOTE_LOG_FOLDER = "cuttlefish_runtime" # Key name in report ERROR_LOG_FOLDER = "error_log_folder" # Type of "logs" entries in report. # The values must be consistent with LogDataType in TradeFed. +LOG_TYPE_DIR = "DIR" LOG_TYPE_KERNEL_LOG = "KERNEL_LOG" LOG_TYPE_LOGCAT = "LOGCAT" LOG_TYPE_TEXT = "TEXT" diff --git a/internal/lib/adb_tools_test.py b/internal/lib/adb_tools_test.py index a3ebf441..cc46ca37 100644 --- a/internal/lib/adb_tools_test.py +++ b/internal/lib/adb_tools_test.py @@ -17,7 +17,6 @@ import subprocess import unittest from unittest import mock -from six import b from acloud import errors from acloud.internal.lib import adb_tools @@ -26,16 +25,16 @@ from acloud.internal.lib import driver_test_lib class AdbToolsTest(driver_test_lib.BaseDriverTest): """Test adb functions.""" - DEVICE_ALIVE = b("List of devices attached\n" - "127.0.0.1:48451 device product:aosp_cf_x86_phone " - "model:Cuttlefish_x86_phone device:vsoc_x86 " - "transport_id:98") - DEVICE_OFFLINE = b("List of devices attached\n" - "127.0.0.1:48451 offline") - DEVICE_STATE_ONLY = b("List of devices attached\n" - "127.0.0.1:48451\toffline\n" - "emulator-5554\tdevice\n") - DEVICE_NONE = b("List of devices attached") + DEVICE_ALIVE = ("List of devices attached\n" + "127.0.0.1:48451 device product:aosp_cf_x86_phone " + "model:Cuttlefish_x86_phone device:vsoc_x86 " + "transport_id:98").encode() + DEVICE_OFFLINE = ("List of devices attached\n" + "127.0.0.1:48451 offline").encode() + DEVICE_STATE_ONLY = ("List of devices attached\n" + "127.0.0.1:48451\toffline\n" + "emulator-5554\tdevice\n").encode() + DEVICE_NONE = b"List of devices attached" def setUp(self): """Patch the path to adb.""" diff --git a/internal/lib/android_build_client_test.py b/internal/lib/android_build_client_test.py index bb21b254..f589dc3b 100644 --- a/internal/lib/android_build_client_test.py +++ b/internal/lib/android_build_client_test.py @@ -22,7 +22,6 @@ import time import unittest from unittest import mock -import six import apiclient @@ -233,9 +232,10 @@ class AndroidBuildClientTest(driver_test_lib.BaseDriverTest): "}" ) expected_arg = "-credential_source=fake_token" - self.Patch(six.moves.builtins, "open", mock.mock_open(read_data=certification)) - cert_arg = self.client.GetFetchCertArg(cert_file_path) - self.assertEqual(expected_arg, cert_arg) + with mock.patch("builtins.open", + mock.mock_open(read_data=certification)): + cert_arg = self.client.GetFetchCertArg(cert_file_path) + self.assertEqual(expected_arg, cert_arg) def testProcessBuild(self): """Test creating "cuttlefish build" strings.""" diff --git a/internal/lib/base_cloud_client.py b/internal/lib/base_cloud_client.py index 8fdbe329..6bdce743 100755 --- a/internal/lib/base_cloud_client.py +++ b/internal/lib/base_cloud_client.py @@ -20,10 +20,7 @@ BasicCloudApiCliend does basic setup for a cloud API. import logging import socket import ssl - -import six -from six.moves import http_client - +import http # pylint: disable=import-error import httplib2 from apiclient import errors as gerrors @@ -57,7 +54,7 @@ class BaseCloudApiClient(): 502, # Bad Gateway 503, # Service Unavailable ] - RETRIABLE_ERRORS = (http_client.HTTPException, httplib2.HttpLib2Error, + RETRIABLE_ERRORS = (http.client.HTTPException, httplib2.HttpLib2Error, socket.error, ssl.SSLError) RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, ) @@ -249,7 +246,7 @@ class BaseCloudApiClient(): results[request_id] = (response, self._TranslateError(exception)) batch = self._service.new_batch_http_request() - for request_id, request in six.iteritems(requests): + for request_id, request in requests.items(): batch.add( request=request, callback=_CallBack, request_id=request_id) batch.execute() diff --git a/internal/lib/cheeps_compute_client.py b/internal/lib/cheeps_compute_client.py index 31f7dfb5..f6f4aa42 100644 --- a/internal/lib/cheeps_compute_client.py +++ b/internal/lib/cheeps_compute_client.py @@ -90,7 +90,8 @@ class CheepsComputeClient(android_compute_client.AndroidComputeClient): metadata["android_build_id"] = avd_spec.remote_image[constants.BUILD_ID] metadata["android_build_target"] = avd_spec.remote_image[constants.BUILD_TARGET] - metadata["betty_image"] = avd_spec.remote_image[constants.CHEEPS_BETTY_IMAGE] + metadata["betty_image"] = avd_spec.cheeps_betty_image + metadata["cheeps_features"] = ','.join(avd_spec.cheeps_features) gcompute_client.ComputeClient.CreateInstance( self, diff --git a/internal/lib/cheeps_compute_client_test.py b/internal/lib/cheeps_compute_client_test.py index 8f46f44e..8e515924 100644 --- a/internal/lib/cheeps_compute_client_test.py +++ b/internal/lib/cheeps_compute_client_test.py @@ -45,6 +45,7 @@ class CheepsComputeClientTest(driver_test_lib.BaseDriverTest): USER = "test_user" PASSWORD = "test_password" CHEEPS_BETTY_IMAGE = 'abcasdf' + CHEEPS_FEATURES = ['a', 'b', 'c'] def _GetFakeConfig(self): """Create a fake configuration object. @@ -88,6 +89,7 @@ class CheepsComputeClientTest(driver_test_lib.BaseDriverTest): 'android_build_target': self.ANDROID_BUILD_TARGET, 'avd_type': "cheeps", 'betty_image': self.CHEEPS_BETTY_IMAGE, + 'cheeps_features': ','.join(self.CHEEPS_FEATURES), 'cvd_01_dpi': str(self.DPI), 'cvd_01_x_res': str(self.X_RES), 'cvd_01_y_res': str(self.Y_RES), @@ -109,8 +111,9 @@ class CheepsComputeClientTest(driver_test_lib.BaseDriverTest): avd_spec.remote_image = { constants.BUILD_ID: self.ANDROID_BUILD_ID, constants.BUILD_TARGET: self.ANDROID_BUILD_TARGET, - constants.CHEEPS_BETTY_IMAGE: self.CHEEPS_BETTY_IMAGE, } + avd_spec.cheeps_betty_image = self.CHEEPS_BETTY_IMAGE + avd_spec.cheeps_features = self.CHEEPS_FEATURES self.cheeps_compute_client.CreateInstance( self.INSTANCE, @@ -136,6 +139,7 @@ class CheepsComputeClientTest(driver_test_lib.BaseDriverTest): 'android_build_target': self.ANDROID_BUILD_TARGET, 'avd_type': "cheeps", 'betty_image': None, + 'cheeps_features': "", 'cvd_01_dpi': str(self.DPI), 'cvd_01_x_res': str(self.X_RES), 'cvd_01_y_res': str(self.Y_RES), @@ -156,8 +160,9 @@ class CheepsComputeClientTest(driver_test_lib.BaseDriverTest): avd_spec.remote_image = { constants.BUILD_ID: self.ANDROID_BUILD_ID, constants.BUILD_TARGET: self.ANDROID_BUILD_TARGET, - constants.CHEEPS_BETTY_IMAGE: None, } + avd_spec.cheeps_betty_image = None + avd_spec.cheeps_features = [] self.cheeps_compute_client.CreateInstance( self.INSTANCE, diff --git a/internal/lib/cvd_compute_client.py b/internal/lib/cvd_compute_client.py index cdedbe2e..e4245e76 100644 --- a/internal/lib/cvd_compute_client.py +++ b/internal/lib/cvd_compute_client.py @@ -56,7 +56,7 @@ _METADATA_TO_UNSET = ["cvd_01_launch", "cvd_01_fetch_kernel_bid", "cvd_01_fetch_kernel_build_target"] - +# TODO(228405515): Delete CvdComputeClient class. class CvdComputeClient(android_compute_client.AndroidComputeClient): """Client that manages Anadroid Virtual Device.""" diff --git a/internal/lib/cvd_compute_client_multi_stage.py b/internal/lib/cvd_compute_client_multi_stage.py index d075fdcb..5464dee3 100644 --- a/internal/lib/cvd_compute_client_multi_stage.py +++ b/internal/lib/cvd_compute_client_multi_stage.py @@ -46,11 +46,10 @@ from acloud import errors from acloud.internal import constants from acloud.internal.lib import android_build_client from acloud.internal.lib import android_compute_client +from acloud.internal.lib import cvd_utils from acloud.internal.lib import gcompute_client from acloud.internal.lib import utils from acloud.internal.lib.ssh import Ssh -from acloud.public import report -from acloud.pull import pull from acloud.setup import mkcert @@ -81,7 +80,7 @@ _NO_RETRY = 0 _LAUNCH_CVD_COMMAND = "launch_cvd_command" _CONFIG_RE = re.compile(r"^config=(?P<config>.+)") _TRUST_REMOTE_INSTANCE_COMMAND = ( - f"\"sudo cp ~/{constants.WEBRTC_CERTS_PATH}/{constants.SSL_CA_NAME}.pem " + f"\"sudo cp -p ~/{constants.WEBRTC_CERTS_PATH}/{constants.SSL_CA_NAME}.pem " f"{constants.SSL_TRUST_CA_DIR}/{constants.SSL_CA_NAME}.crt;" "sudo update-ca-certificates;\"") # Remote host instance name @@ -129,9 +128,8 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): self._report_internal_ip = report_internal_ip self._gpu = gpu # Store all failures result when creating one or multiple instances. + # This attribute is only used by the deprecated create_cf command. self._all_failures = {} - # Map from instance names to lists of report.LogFile. - self._all_logs = {} self._extra_args_ssh_tunnel = acloud_config.extra_args_ssh_tunnel self._ssh = None self._ip = None @@ -188,8 +186,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): self._ip = ip self._user = user self._ssh.WaitForSsh(timeout=self._ins_timeout_secs) - self.StopCvd() - self.CleanUp() + cvd_utils.CleanUpRemoteCvd(self._ssh, raise_error=False) # TODO(171376263): Refactor CreateInstance() args with avd_spec. # pylint: disable=arguments-differ,too-many-locals,broad-except @@ -262,8 +259,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): self._ssh.WaitForSsh(timeout=self._ins_timeout_secs) if avd_spec: if avd_spec.instance_name_to_reuse: - self.StopCvd() - self.CleanUp() + cvd_utils.CleanUpRemoteCvd(self._ssh, raise_error=False) return instance # TODO: Remove following code after create_cf deprecated. @@ -274,10 +270,12 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): kernel_branch, kernel_build_target, bootloader_build_id, bootloader_branch, bootloader_build_target, ota_build_id, ota_branch, ota_build_target) - self.LaunchCvd(instance, - blank_data_disk_size_gb=blank_data_disk_size_gb, - boot_timeout_secs=self._boot_timeout_secs) - + failures = self.LaunchCvd( + instance, + blank_data_disk_size_gb=blank_data_disk_size_gb, + boot_timeout_secs=self._boot_timeout_secs, + extra_args=[]) + self._all_failures.update(failures) return instance except Exception as e: self._all_failures[instance] = e @@ -354,6 +352,9 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): if avd_spec.num_avds_per_instance > 1: launch_cvd_args.append( _NUM_AVDS_ARG % {"num_AVD": avd_spec.num_avds_per_instance}) + if avd_spec.base_instance_num: + launch_cvd_args.append( + "--base-instance-num=%s" % avd_spec.base_instance_num) if avd_spec.launch_args: launch_cvd_args.append(avd_spec.launch_args) else: @@ -372,39 +373,13 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): launch_cvd_args.append(_AGREEMENT_PROMPT_ARG) return launch_cvd_args - # pylint: disable=broad-except - def StopCvd(self): - """Stop CVD. - - If stop_cvd fails, assume that it's because there was no previously - running device. - """ - ssh_command = "./bin/stop_cvd" - try: - self._ssh.Run(ssh_command) - except Exception as e: - logger.debug("Failed to stop_cvd (possibly no running device): %s", e) - - def CleanUp(self): - """Clean up the files/folders on the existing instance. - - If previous AVD have these files/folders, reusing the instance may have - side effects if not cleaned. The path in the instance is /home/vsoc-01/* - if the GCE user is vsoc-01. - """ - - ssh_command = "'/bin/rm -rf /home/%s/*'" % self._user - try: - self._ssh.Run(ssh_command) - except subprocess.CalledProcessError as e: - logger.debug("Failed to clean up the files/folders: %s", e) - @utils.TimeExecute(function_description="Launching AVD(s) and waiting for boot up", result_evaluator=utils.BootEvaluator) def LaunchCvd(self, instance, avd_spec=None, blank_data_disk_size_gb=None, decompress_kernel=None, - boot_timeout_secs=None): + boot_timeout_secs=None, + extra_args=()): """Launch CVD. Launch AVD with launch_cvd. If the process is failed, acloud would show @@ -417,6 +392,8 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): decompress_kernel: Boolean, if true decompress the kernel. boot_timeout_secs: Integer, the maximum time to wait for the command to respond. + extra_args: Collection of strings, the extra arguments generated by + acloud. e.g., remote image paths. Returns: dict of faliures, return this dict for BootEvaluator to handle @@ -425,14 +402,16 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): self.SetStage(constants.STAGE_BOOT_UP) timestart = time.time() error_msg = "" - launch_cvd_args = self._GetLaunchCvdArgs(avd_spec, - blank_data_disk_size_gb, - decompress_kernel, - instance) + launch_cvd_args = list(extra_args) + launch_cvd_args.extend( + self._GetLaunchCvdArgs(avd_spec, blank_data_disk_size_gb, + decompress_kernel, instance)) boot_timeout_secs = self._GetBootTimeout( boot_timeout_secs or constants.DEFAULT_CF_BOOT_TIMEOUT) ssh_command = "./bin/launch_cvd -daemon " + " ".join(launch_cvd_args) try: + if avd_spec and avd_spec.base_instance_num: + self.ExtendReportData(constants.BASE_INSTANCE_NUM, avd_spec.base_instance_num) self.ExtendReportData(_LAUNCH_CVD_COMMAND, ssh_command) self._ssh.Run(ssh_command, boot_timeout_secs, retry=_NO_RETRY) self._UpdateOpenWrtStatus(avd_spec) @@ -448,11 +427,8 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): error_msg = ( "WEBRTC is not supported in the current build. Please try VNC such " "as '$acloud create --autoconnect vnc'") - self._all_failures[instance] = error_msg utils.PrintColorString(str(e), utils.TextColors.FAIL) - self._FindLogFiles(instance, - error_msg and avd_spec and not avd_spec.no_pull_log) self._execution_time[_LAUNCH_CVD] = round(time.time() - timestart, 2) return {instance: error_msg} if error_msg else {} @@ -471,33 +447,6 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): logger.debug("Timeout for boot: %s secs", boot_timeout_secs) return boot_timeout_secs - def _FindLogFiles(self, instance, download): - """Find and pull all log files from instance. - - Args: - instance: String, instance name. - download: Whether to download the files to a temporary directory - and show messages to the user. - """ - self._all_logs[instance] = [ - report.LogFile("/var/log/kern.log", constants.LOG_TYPE_KERNEL_LOG, - "host_kernel.log")] - log_files = pull.GetAllLogFilePaths(self._ssh) - for log_file in log_files: - log = report.LogFile(log_file, constants.LOG_TYPE_TEXT) - if log_file.endswith("kernel.log"): - log = report.LogFile(log_file, constants.LOG_TYPE_KERNEL_LOG) - if log_file.endswith("logcat"): - log = report.LogFile(log_file, constants.LOG_TYPE_LOGCAT, - "full_gce_logcat") - self._all_logs[instance].append(log) - - if not download: - return - error_log_folder = pull.GetDownloadLogFolder(instance) - pull.PullLogs(self._ssh, log_files, error_log_folder) - self.ExtendReportData(constants.ERROR_LOG_FOLDER, error_log_folder) - @utils.TimeExecute(function_description="Reusing GCE instance") def _ReusingGceInstance(self, avd_spec): """Reusing a cuttlefish existing instance. @@ -750,11 +699,6 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): return self._all_failures @property - def all_logs(self): - """Return all_logs""" - return self._all_logs - - @property def execution_time(self): """Return execution_time""" return self._execution_time diff --git a/internal/lib/cvd_compute_client_multi_stage_test.py b/internal/lib/cvd_compute_client_multi_stage_test.py index 81e80820..d9646714 100644 --- a/internal/lib/cvd_compute_client_multi_stage_test.py +++ b/internal/lib/cvd_compute_client_multi_stage_test.py @@ -117,6 +117,7 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): self.args.avd_type = constants.TYPE_CF self.args.flavor = "phone" self.args.adb_port = None + self.args.base_instance_num = None self.args.hw_property = "cpu:2,resolution:1080x1920,dpi:240,memory:4g,disk:10g" self.args.num_avds_per_instance = 2 self.args.remote_host = False @@ -173,11 +174,10 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): @mock.patch.object(gcompute_client.ComputeClient, "CreateInstance") @mock.patch.object(cvd_compute_client_multi_stage.CvdComputeClient, "_GetDiskArgs", return_value=[{"fake_arg": "fake_value"}]) - @mock.patch("acloud.internal.lib.cvd_compute_client_multi_stage.pull") @mock.patch("getpass.getuser", return_value="fake_user") - def testCreateInstance(self, _get_user, mock_pull, _get_disk_args, - mock_create, _get_image, _compare_machine_size, - mock_check_img, _mock_env): + def testCreateInstance(self, _get_user, _get_disk_args, mock_create, + _get_image, _compare_machine_size, mock_check_img, + _mock_env): """Test CreateInstance.""" expected_metadata = dict() expected_metadata_local_image = dict() @@ -187,24 +187,6 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): expected_disk_args = [{"fake_arg": "fake_value"}] fake_avd_spec = avd_spec.AVDSpec(self.args) fake_avd_spec._instance_name_to_reuse = None - expected_logs = { - self.INSTANCE: [ - { - "path": "/var/log/kern.log", - "type": constants.LOG_TYPE_KERNEL_LOG, - "name": "host_kernel.log" - }, - {"path": "/kernel.log", "type": constants.LOG_TYPE_KERNEL_LOG}, - { - "path": "/logcat", - "type": constants.LOG_TYPE_LOGCAT, - "name": "full_gce_logcat" - }, - {"path": "/launcher.log", "type": constants.LOG_TYPE_TEXT} - ] - } - mock_pull.GetAllLogFilePaths.return_value = [ - "/kernel.log", "/logcat", "/launcher.log"] created_subprocess = mock.MagicMock() created_subprocess.stdout = mock.MagicMock() @@ -237,8 +219,6 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): gpu=self.GPU, disk_type=None, disable_external_ip=False) - self.assertEqual(self.cvd_compute_client_multi_stage.all_logs, - expected_logs) mock_check_img.return_value = True #test use local image in the remote instance. diff --git a/internal/lib/cvd_runtime_config_test.py b/internal/lib/cvd_runtime_config_test.py index 05f5fcc4..42bd48b1 100644 --- a/internal/lib/cvd_runtime_config_test.py +++ b/internal/lib/cvd_runtime_config_test.py @@ -19,7 +19,6 @@ import os import unittest from unittest import mock -import six from acloud import errors from acloud.internal.lib import cvd_runtime_config as cf_cfg @@ -90,7 +89,7 @@ class CvdRuntimeconfigTest(driver_test_lib.BaseDriverTest): } mock_open = mock.mock_open(read_data=self.CF_RUNTIME_CONFIG) cf_cfg_path = "/fake-path/local-instance-2/fake.config" - with mock.patch.object(six.moves.builtins, "open", mock_open): + with mock.patch("builtins.open", mock_open): fake_cvd_runtime_config = cf_cfg.CvdRuntimeConfig(cf_cfg_path) self.assertEqual(fake_cvd_runtime_config._config_dict, expected_dict) self.assertEqual(fake_cvd_runtime_config.enable_webrtc, None) diff --git a/internal/lib/cvd_utils.py b/internal/lib/cvd_utils.py new file mode 100644 index 00000000..5194806e --- /dev/null +++ b/internal/lib/cvd_utils.py @@ -0,0 +1,334 @@ +# Copyright 2022 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility functions that process cuttlefish images.""" + +import glob +import logging +import os +import posixpath as remote_path +import subprocess + +from acloud import errors +from acloud.create import create_common +from acloud.internal import constants +from acloud.internal.lib import ssh +from acloud.internal.lib import utils +from acloud.public import report + + +logger = logging.getLogger(__name__) + +# bootloader and kernel are files required to launch AVD. +_ARTIFACT_FILES = ["*.img", "bootloader", "kernel"] +_REMOTE_IMAGE_DIR = "acloud_cf" +# The boot image name pattern corresponds to the use cases: +# - In a cuttlefish build environment, ANDROID_PRODUCT_OUT conatins boot.img +# and boot-debug.img. The former is the default boot image. The latter is not +# useful for cuttlefish. +# - In an officially released GKI (Generic Kernel Image) package, the image +# name is boot-<kernel version>.img. +_BOOT_IMAGE_NAME_PATTERN = r"boot(-[\d.]+)?\.img" +_VENDOR_BOOT_IMAGE_NAME = "vendor_boot.img" +_KERNEL_IMAGE_NAMES = ("kernel", "bzImage", "Image") +_INITRAMFS_IMAGE_NAME = "initramfs.img" +_REMOTE_BOOT_IMAGE_PATH = remote_path.join(_REMOTE_IMAGE_DIR, "boot.img") +_REMOTE_VENDOR_BOOT_IMAGE_PATH = remote_path.join( + _REMOTE_IMAGE_DIR, _VENDOR_BOOT_IMAGE_NAME) +_REMOTE_KERNEL_IMAGE_PATH = remote_path.join( + _REMOTE_IMAGE_DIR, _KERNEL_IMAGE_NAMES[0]) +_REMOTE_INITRAMFS_IMAGE_PATH = remote_path.join( + _REMOTE_IMAGE_DIR, _INITRAMFS_IMAGE_NAME) + +_ANDROID_BOOT_IMAGE_MAGIC = b"ANDROID!" + +HOST_KERNEL_LOG = report.LogFile( + "/var/log/kern.log", constants.LOG_TYPE_KERNEL_LOG, "host_kernel.log") +TOMBSTONES = report.LogFile( + constants.REMOTE_LOG_FOLDER + "/tombstones", constants.LOG_TYPE_DIR, + "tombstones-zip") +FETCHER_CONFIG_JSON = report.LogFile( + "fetcher_config.json", constants.LOG_TYPE_TEXT) + + +def _UploadImageZip(ssh_obj, image_zip): + """Upload an image zip to a remote host and a GCE instance. + + Args: + ssh_obj: An Ssh object. + image_zip: The path to the image zip. + """ + remote_cmd = f"/usr/bin/install_zip.sh . < {image_zip}" + logger.debug("remote_cmd:\n %s", remote_cmd) + ssh_obj.Run(remote_cmd) + + +def _UploadImageDir(ssh_obj, image_dir): + """Upload an image directory to a remote host or a GCE instance. + + The images are compressed for faster upload. + + Args: + ssh_obj: An Ssh object. + image_dir: The directory containing the files to be uploaded. + """ + try: + images_path = os.path.join(image_dir, "required_images") + with open(images_path, "r", encoding="utf-8") as images: + artifact_files = images.read().splitlines() + except IOError: + # Older builds may not have a required_images file. In this case + # we fall back to *.img. + artifact_files = [] + for file_name in _ARTIFACT_FILES: + artifact_files.extend( + os.path.basename(image) for image in glob.glob( + os.path.join(image_dir, file_name))) + # Upload android-info.txt to parse config value. + artifact_files.append(constants.ANDROID_INFO_FILE) + cmd = (f"tar -cf - --lzop -S -C {image_dir} {' '.join(artifact_files)} | " + f"{ssh_obj.GetBaseCmd(constants.SSH_BIN)} -- tar -xf - --lzop -S") + logger.debug("cmd:\n %s", cmd) + ssh.ShellCmdWithRetry(cmd) + + +def _UploadCvdHostPackage(ssh_obj, cvd_host_package): + """Upload a CVD host package to a remote host or a GCE instance. + + Args: + ssh_obj: An Ssh object. + cvd_host_package: The path to the CVD host package. + """ + remote_cmd = f"tar -x -z -f - < {cvd_host_package}" + logger.debug("remote_cmd:\n %s", remote_cmd) + ssh_obj.Run(remote_cmd) + + +@utils.TimeExecute(function_description="Processing and uploading local images") +def UploadArtifacts(ssh_obj, image_path, cvd_host_package): + """Upload images and a CVD host package to a remote host or a GCE instance. + + Args: + ssh_obj: An Ssh object. + image_path: A string, the path to the image zip built by `m dist` or + the directory containing the images built by `m`. + cvd_host_package: A string, the path to the CVD host package in gzip. + """ + if os.path.isdir(image_path): + _UploadImageDir(ssh_obj, image_path) + else: + _UploadImageZip(ssh_obj, image_path) + _UploadCvdHostPackage(ssh_obj, cvd_host_package) + + +def _IsBootImage(image_path): + """Check if a file is an Android boot image by reading the magic bytes. + + Args: + image_path: The file path. + + Returns: + A boolean, whether the file is a boot image. + """ + if not os.path.isfile(image_path): + return False + with open(image_path, "rb") as image_file: + return image_file.read(8) == _ANDROID_BOOT_IMAGE_MAGIC + + +def FindBootImages(search_path): + """Find boot and vendor_boot images in a path. + + Args: + search_path: A path to an image file or an image directory. + + Returns: + The boot image path and the vendor_boot image path. Each value can be + None if the path doesn't exist. + + Raises: + errors.GetLocalImageError if search_path contains more than one boot + image or the file format is not correct. + """ + boot_image_path = create_common.FindLocalImage( + search_path, _BOOT_IMAGE_NAME_PATTERN, raise_error=False) + if boot_image_path and not _IsBootImage(boot_image_path): + raise errors.GetLocalImageError( + f"{boot_image_path} is not a boot image.") + + vendor_boot_image_path = os.path.join(search_path, _VENDOR_BOOT_IMAGE_NAME) + if not os.path.isfile(vendor_boot_image_path): + vendor_boot_image_path = None + + return boot_image_path, vendor_boot_image_path + + +def _FindKernelImages(search_path): + """Find kernel and initramfs images in a path. + + Args: + search_path: A path to an image directory. + + Returns: + The kernel image path and the initramfs image path. Each value can be + None if the path doesn't exist. + """ + paths = [os.path.join(search_path, name) for name in _KERNEL_IMAGE_NAMES] + kernel_image_path = next((path for path in paths if os.path.isfile(path)), + None) + + initramfs_image_path = os.path.join(search_path, _INITRAMFS_IMAGE_NAME) + if not os.path.isfile(initramfs_image_path): + initramfs_image_path = None + + return kernel_image_path, initramfs_image_path + + +@utils.TimeExecute(function_description="Uploading local kernel images.") +def _UploadKernelImages(ssh_obj, search_path): + """Find and upload kernel images to a remote host or a GCE instance. + + Args: + ssh_obj: An Ssh object. + search_path: A path to an image file or an image directory. + + Returns: + A list of strings, the launch_cvd arguments including the remote paths. + + Raises: + errors.GetLocalImageError if search_path does not contain kernel + images. + """ + # Assume that the caller cleaned up the remote home directory. + ssh_obj.Run("mkdir -p " + _REMOTE_IMAGE_DIR) + + boot_image_path, vendor_boot_image_path = FindBootImages(search_path) + if boot_image_path: + ssh_obj.ScpPushFile(boot_image_path, _REMOTE_BOOT_IMAGE_PATH) + launch_cvd_args = ["-boot_image", _REMOTE_BOOT_IMAGE_PATH] + if vendor_boot_image_path: + ssh_obj.ScpPushFile(vendor_boot_image_path, + _REMOTE_VENDOR_BOOT_IMAGE_PATH) + launch_cvd_args.extend(["-vendor_boot_image", + _REMOTE_VENDOR_BOOT_IMAGE_PATH]) + return launch_cvd_args + + kernel_image_path, initramfs_image_path = _FindKernelImages(search_path) + if kernel_image_path and initramfs_image_path: + ssh_obj.ScpPushFile(kernel_image_path, _REMOTE_KERNEL_IMAGE_PATH) + ssh_obj.ScpPushFile(initramfs_image_path, _REMOTE_INITRAMFS_IMAGE_PATH) + return ["-kernel_path", _REMOTE_KERNEL_IMAGE_PATH, + "-initramfs_path", _REMOTE_INITRAMFS_IMAGE_PATH] + + raise errors.GetLocalImageError( + f"{search_path} is not a boot image or a directory containing images.") + + +def UploadExtraImages(ssh_obj, avd_spec): + """Find and upload the images specified in avd_spec. + + Args: + ssh_obj: An Ssh object. + avd_spec: An AvdSpec object containing extra image paths. + + Returns: + A list of strings, the launch_cvd arguments including the remote paths. + + Raises: + errors.GetLocalImageError if any specified image path does not exist. + """ + if avd_spec.local_kernel_image: + return _UploadKernelImages(ssh_obj, avd_spec.local_kernel_image) + return [] + + +def CleanUpRemoteCvd(ssh_obj, raise_error): + """Call stop_cvd and delete the files on a remote host or a GCE instance. + + Args: + ssh_obj: An Ssh object. + raise_error: Whether to raise an error if the remote instance is not + running. + + Raises: + subprocess.CalledProcessError if any command fails. + """ + stop_cvd_cmd = "./bin/stop_cvd" + if raise_error: + ssh_obj.Run(stop_cvd_cmd) + else: + try: + ssh_obj.Run(stop_cvd_cmd, retry=0) + except subprocess.CalledProcessError as e: + logger.debug( + "Failed to stop_cvd (possibly no running device): %s", e) + + # This command deletes all files except hidden files under HOME. + # It does not raise an error if no files can be deleted. + ssh_obj.Run("'rm -rf ./*'") + + +def ConvertRemoteLogs(log_paths): + """Convert paths on a remote host or a GCE instance to log objects. + + Args: + log_paths: A collection of strings, the remote paths to the logs. + + Returns: + A list of report.LogFile objects. + """ + logs = [] + for log_path in log_paths: + log = report.LogFile(log_path, constants.LOG_TYPE_TEXT) + if log_path.endswith("kernel.log"): + log = report.LogFile(log_path, constants.LOG_TYPE_KERNEL_LOG) + elif log_path.endswith("logcat"): + log = report.LogFile(log_path, constants.LOG_TYPE_LOGCAT, + "full_gce_logcat") + elif not (log_path.endswith(".log") or + log_path.endswith("cuttlefish_config.json")): + continue + logs.append(log) + return logs + + +def GetRemoteBuildInfoDict(avd_spec): + """Convert remote build infos to a dictionary for reporting. + + Args: + avd_spec: An AvdSpec object containing the build infos. + + Returns: + A dict containing the build infos. + """ + build_info_dict = { + key: val for key, val in avd_spec.remote_image.items() if val} + + # kernel_target has a default value. If the user provides kernel_build_id + # or kernel_branch, then convert kernel build info. + if (avd_spec.kernel_build_info.get(constants.BUILD_ID) or + avd_spec.kernel_build_info.get(constants.BUILD_BRANCH)): + build_info_dict.update( + {"kernel_" + key: val + for key, val in avd_spec.kernel_build_info.items() if val} + ) + build_info_dict.update( + {"system_" + key: val + for key, val in avd_spec.system_build_info.items() if val} + ) + build_info_dict.update( + {"bootloader_" + key: val + for key, val in avd_spec.bootloader_build_info.items() if val} + ) + return build_info_dict diff --git a/internal/lib/cvd_utils_test.py b/internal/lib/cvd_utils_test.py new file mode 100644 index 00000000..28dc4410 --- /dev/null +++ b/internal/lib/cvd_utils_test.py @@ -0,0 +1,216 @@ +# Copyright 2022 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for cvd_utils.""" + +import os +import subprocess +import tempfile +import unittest +from unittest import mock + +from acloud import errors +from acloud.internal import constants +from acloud.internal.lib import cvd_utils + + +class CvdUtilsTest(unittest.TestCase): + """Test the functions in cvd_utils.""" + + @staticmethod + def _CreateFile(path, data=b""): + """Create and write binary data to a file.""" + with open(path, "wb") as file_obj: + file_obj.write(data) + + @staticmethod + @mock.patch("acloud.internal.lib.cvd_utils.os.path.isdir", + return_value=False) + def testUploadImageZip(_mock_isdir): + """Test UploadArtifacts with image zip.""" + mock_ssh = mock.Mock() + cvd_utils.UploadArtifacts(mock_ssh, "/mock/img.zip", "/mock/cvd.tgz") + mock_ssh.Run.assert_any_call("/usr/bin/install_zip.sh . < " + "/mock/img.zip") + mock_ssh.Run.assert_any_call("tar -x -z -f - < /mock/cvd.tgz") + + @staticmethod + @mock.patch("acloud.internal.lib.cvd_utils.glob") + @mock.patch("acloud.internal.lib.cvd_utils.os.path.isdir", + return_value=True) + @mock.patch("acloud.internal.lib.cvd_utils.ssh.ShellCmdWithRetry") + def testUploadImageDir(mock_shell, _mock_isdir, mock_glob): + """Test UploadArtifacts with image directory.""" + mock_ssh = mock.Mock() + mock_ssh.GetBaseCmd.return_value = "/mock/ssh" + expected_shell_cmd = ("tar -cf - --lzop -S -C /mock/dir " + "super.img bootloader kernel android-info.txt | " + "/mock/ssh -- tar -xf - --lzop -S") + expected_ssh_cmd = "tar -x -z -f - < /mock/cvd.tgz" + + # Test with required_images file. + mock_open = mock.mock_open(read_data="super.img\nbootloader\nkernel") + with mock.patch("acloud.internal.lib.cvd_utils.open", mock_open): + cvd_utils.UploadArtifacts(mock_ssh, "/mock/dir", "/mock/cvd.tgz") + mock_open.assert_called_with("/mock/dir/required_images", "r", + encoding="utf-8") + mock_glob.glob.assert_not_called() + mock_shell.assert_called_with(expected_shell_cmd) + mock_ssh.Run.assert_called_with(expected_ssh_cmd) + + # Test with glob. + mock_ssh.reset_mock() + mock_shell.reset_mock() + mock_glob.glob.side_effect = ( + lambda path: [path.replace("*", "super")]) + with mock.patch("acloud.internal.lib.cvd_utils.open", + side_effect=IOError("file does not exist")): + cvd_utils.UploadArtifacts(mock_ssh, "/mock/dir", "/mock/cvd.tgz") + mock_glob.glob.assert_called() + mock_shell.assert_called_with(expected_shell_cmd) + mock_ssh.Run.assert_called_with(expected_ssh_cmd) + + def testUploadBootImages(self): + """Test FindBootImages and UploadExtraImages.""" + mock_ssh = mock.Mock() + with tempfile.TemporaryDirectory(prefix="cvd_utils") as image_dir: + boot_image_path = os.path.join(image_dir, "boot.img") + self._CreateFile(boot_image_path, b"ANDROID!test") + self._CreateFile(os.path.join(image_dir, "vendor_boot.img")) + + mock_avd_spec = mock.Mock(local_kernel_image=boot_image_path) + args = cvd_utils.UploadExtraImages(mock_ssh, mock_avd_spec) + self.assertEqual(["-boot_image", "acloud_cf/boot.img"], args) + mock_ssh.Run.assert_called_once_with("mkdir -p acloud_cf") + mock_ssh.ScpPushFile.assert_called_once() + + mock_ssh.reset_mock() + mock_avd_spec.local_kernel_image = image_dir + args = cvd_utils.UploadExtraImages(mock_ssh, mock_avd_spec) + self.assertEqual( + ["-boot_image", "acloud_cf/boot.img", + "-vendor_boot_image", "acloud_cf/vendor_boot.img"], + args) + mock_ssh.Run.assert_called_once() + self.assertEqual(2, mock_ssh.ScpPushFile.call_count) + + def testUploadKernelImages(self): + """Test UploadExtraImages with kernel images.""" + mock_ssh = mock.Mock() + with tempfile.TemporaryDirectory(prefix="cvd_utils") as image_dir: + kernel_image_path = os.path.join(image_dir, "Image") + self._CreateFile(kernel_image_path) + self._CreateFile(os.path.join(image_dir, "initramfs.img")) + + mock_avd_spec = mock.Mock(local_kernel_image=kernel_image_path) + with self.assertRaises(errors.GetLocalImageError): + cvd_utils.UploadExtraImages(mock_ssh, mock_avd_spec) + + mock_ssh.reset_mock() + mock_avd_spec.local_kernel_image = image_dir + args = cvd_utils.UploadExtraImages(mock_ssh, mock_avd_spec) + self.assertEqual( + ["-kernel_path", "acloud_cf/kernel", + "-initramfs_path", "acloud_cf/initramfs.img"], + args) + mock_ssh.Run.assert_called_once() + self.assertEqual(2, mock_ssh.ScpPushFile.call_count) + + + def testCleanUpRemoteCvd(self): + """Test CleanUpRemoteCvd.""" + mock_ssh = mock.Mock() + cvd_utils.CleanUpRemoteCvd(mock_ssh, raise_error=True) + mock_ssh.Run.assert_any_call("./bin/stop_cvd") + mock_ssh.Run.assert_any_call("'rm -rf ./*'") + + mock_ssh.reset_mock() + mock_ssh.Run.side_effect = [ + subprocess.CalledProcessError(cmd="should raise", returncode=1)] + with self.assertRaises(subprocess.CalledProcessError): + cvd_utils.CleanUpRemoteCvd(mock_ssh, raise_error=True) + + mock_ssh.reset_mock() + mock_ssh.Run.side_effect = [ + subprocess.CalledProcessError(cmd="should ignore", returncode=1), + None] + cvd_utils.CleanUpRemoteCvd(mock_ssh, raise_error=False) + mock_ssh.Run.assert_any_call("./bin/stop_cvd", retry=0) + mock_ssh.Run.assert_any_call("'rm -rf ./*'") + + def testConvertRemoteLogs(self): + """Test ConvertRemoteLogs.""" + logs = cvd_utils.ConvertRemoteLogs( + ["/kernel.log", "/logcat", "/launcher.log", "/access-kregistry"]) + expected_logs = [ + {"path": "/kernel.log", "type": constants.LOG_TYPE_KERNEL_LOG}, + { + "path": "/logcat", + "type": constants.LOG_TYPE_LOGCAT, + "name": "full_gce_logcat" + }, + {"path": "/launcher.log", "type": constants.LOG_TYPE_TEXT} + ] + self.assertEqual(expected_logs, logs) + + def testGetRemoteBuildInfoDict(self): + """Test GetRemoteBuildInfoDict.""" + remote_image = { + "branch": "aosp-android-12-gsi", + "build_id": "100000", + "build_target": "aosp_cf_x86_64_phone-userdebug"} + mock_avd_spec = mock.Mock( + spec=[], + remote_image=remote_image, + kernel_build_info={"build_target": "kernel"}, + system_build_info={}, + bootloader_build_info={}) + self.assertEqual(remote_image, + cvd_utils.GetRemoteBuildInfoDict(mock_avd_spec)) + + kernel_build_info = { + "branch": "aosp_kernel-common-android12-5.10", + "build_id": "200000", + "build_target": "kernel_virt_x86_64"} + system_build_info = { + "branch": "aosp-android-12-gsi", + "build_id": "300000", + "build_target": "aosp_x86_64-userdebug"} + bootloader_build_info = { + "branch": "aosp_u-boot-mainline", + "build_id": "400000", + "build_target": "u-boot_crosvm_x86_64"} + all_build_info = { + "kernel_branch": "aosp_kernel-common-android12-5.10", + "kernel_build_id": "200000", + "kernel_build_target": "kernel_virt_x86_64", + "system_branch": "aosp-android-12-gsi", + "system_build_id": "300000", + "system_build_target": "aosp_x86_64-userdebug", + "bootloader_branch": "aosp_u-boot-mainline", + "bootloader_build_id": "400000", + "bootloader_build_target": "u-boot_crosvm_x86_64"} + all_build_info.update(remote_image) + mock_avd_spec = mock.Mock( + spec=[], + remote_image=remote_image, + kernel_build_info=kernel_build_info, + system_build_info=system_build_info, + bootloader_build_info=bootloader_build_info) + self.assertEqual(all_build_info, + cvd_utils.GetRemoteBuildInfoDict(mock_avd_spec)) + + +if __name__ == "__main__": + unittest.main() diff --git a/internal/lib/engprod_client.py b/internal/lib/engprod_client.py deleted file mode 100644 index 5f9ffe10..00000000 --- a/internal/lib/engprod_client.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2021 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A client that talks to EngProd APIs.""" - -import json -import subprocess - -from urllib.parse import urljoin - - -class EngProdClient(): - """Client that manages EngProd api.""" - - @staticmethod - def LeaseDevice(build_target, build_id, api_key, api_url): - """Lease one cuttlefish device. - - Args: - build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" - build_id: Build ID, a string, e.g. "2263051", "P2804227" - api_key: String of api key. - api_url: String of api url. - - Returns: - The response of curl command. - """ - request_data = "{\"target\": \"%s\", \"build_id\": \"%s\"}" % ( - build_target, build_id) - lease_url = urljoin(api_url, "lease?key=%s" % api_key) - response = subprocess.check_output([ - "curl", "--request", "POST", lease_url, "-H", - "Accept: application/json", "-H", "Content-Type: application/json", - "-d", request_data - ]) - return json.loads(response) diff --git a/internal/lib/gcompute_client.py b/internal/lib/gcompute_client.py index b22dd7fd..b30b6868 100755 --- a/internal/lib/gcompute_client.py +++ b/internal/lib/gcompute_client.py @@ -33,8 +33,6 @@ import logging import os import re -import six - from acloud import errors from acloud.internal import constants from acloud.internal.lib import base_cloud_client @@ -1050,14 +1048,14 @@ class ComputeClient(base_cloud_client.BaseCloudApiClient): # Initialize return values failed = [] error_msgs = [] - for resource_name, (_, error) in six.iteritems(results): + for resource_name, (_, error) in results.items(): if error is not None: failed.append(resource_name) error_msgs.append(str(error)) done = [] # Wait for the executing operations to finish. logger.info("Waiting for executing operations") - for resource_name in six.iterkeys(requests): + for resource_name in requests.keys(): operation, _ = results[resource_name] if operation: try: @@ -1312,7 +1310,7 @@ class ComputeClient(base_cloud_client.BaseCloudApiClient): metadata_list = [{ _METADATA_KEY: key, _METADATA_KEY_VALUE: val - } for key, val in six.iteritems(metadata)] + } for key, val in metadata.items()] body[_METADATA] = {_ITEMS: metadata_list} logger.info("Creating instance: project %s, zone %s, body:%s", self._project, zone, body) diff --git a/internal/lib/gcompute_client_test.py b/internal/lib/gcompute_client_test.py index 73e5d502..27eff81d 100644 --- a/internal/lib/gcompute_client_test.py +++ b/internal/lib/gcompute_client_test.py @@ -22,7 +22,6 @@ import os import unittest from unittest import mock -import six # pylint: disable=import-error from acloud import errors @@ -151,7 +150,7 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): self._SetupMocksForGetOperationStatus( {"error": {"errors": ["error1", "error2"]}}, gcompute_client.OperationScope.GLOBAL) - six.assertRaisesRegex(self, + self.assertRaisesRegex( errors.DriverError, "Get operation state failed.*error1.*error2", self.compute_client._GetOperationStatus, @@ -342,8 +341,7 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): "source": GS_IMAGE_SOURCE_URI, }, } - six.assertRaisesRegex( - self, + self.assertRaisesRegex( errors.DriverError, "Expected fake error", self.compute_client.CreateImage, @@ -1125,8 +1123,7 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): instance=self.INSTANCE, zone=self.ZONE) self.assertEqual(result, "fake contents") else: - six.assertRaisesRegex( - self, + self.assertRaisesRegex( errors.DriverError, "Malformed response.*", self.compute_client.GetSerialPortOutput, @@ -1243,18 +1240,17 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): """Test the rsa key path not exists.""" fake_ssh_rsa_path = "/path/to/test_rsa.pub" self.Patch(os.path, "exists", return_value=False) - six.assertRaisesRegex(self, - errors.DriverError, - "RSA file %s does not exist." % fake_ssh_rsa_path, - gcompute_client.GetRsaKey, - ssh_rsa_path=fake_ssh_rsa_path) + self.assertRaisesRegex(errors.DriverError, + "RSA file %s does not exist." % fake_ssh_rsa_path, + gcompute_client.GetRsaKey, + ssh_rsa_path=fake_ssh_rsa_path) def testGetRsaKey(self): """Test get the rsa key.""" fake_ssh_rsa_path = "/path/to/test_rsa.pub" self.Patch(os.path, "exists", return_value=True) m = mock.mock_open(read_data=self.SSHKEY) - with mock.patch.object(six.moves.builtins, "open", m): + with mock.patch("builtins.open", m): result = gcompute_client.GetRsaKey(fake_ssh_rsa_path) self.assertEqual(self.SSHKEY, result) @@ -1386,7 +1382,7 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): self.Patch( gcompute_client.ComputeClient, "GetInstance", return_value=instance_metadata_key_not_exist) - with mock.patch.object(six.moves.builtins, "open", m): + with mock.patch("builtins.open", m): self.compute_client.AddSshRsaInstanceMetadata( fake_user, "/path/to/test_rsa.pub", @@ -1402,7 +1398,7 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): self.Patch( gcompute_client.ComputeClient, "GetInstance", return_value=instance_metadata_key_exist) - with mock.patch.object(six.moves.builtins, "open", m): + with mock.patch("builtins.open", m): self.compute_client.AddSshRsaInstanceMetadata( fake_user, "/path/to/test_rsa.pub", diff --git a/internal/lib/ota_tools.py b/internal/lib/ota_tools.py index 8d613c88..edfe23fa 100644 --- a/internal/lib/ota_tools.py +++ b/internal/lib/ota_tools.py @@ -17,9 +17,6 @@ import logging import os import tempfile -from six import b - - from acloud import errors from acloud.internal.lib import utils @@ -166,18 +163,18 @@ class OtaTools: if split_line[0] == "dynamic_partition_list": partition_names = split_line[1].split() elif split_line[0] == "lpmake": - output_file.write(b("lpmake=%s\n" % lpmake_path)) + output_file.write("lpmake=%s\n" % lpmake_path) continue elif split_line[0].endswith("_image"): continue - output_file.write(b(line)) + output_file.write(line) if not partition_names: logger.w("No dynamic partition list in misc info.") for partition_name in partition_names: - output_file.write(b("%s_image=%s\n" % - (partition_name, get_image(partition_name)))) + output_file.write("%s_image=%s\n" % + (partition_name, get_image(partition_name))) @utils.TimeExecute(function_description="Build super image") @utils.TimeoutException(_BUILD_SUPER_IMAGE_TIMEOUT_SECS) @@ -199,7 +196,7 @@ class OtaTools: with open(misc_info_path, "r") as misc_info: with tempfile.NamedTemporaryFile( prefix="misc_info_", suffix=".txt", - delete=False) as new_misc_info: + delete=False, mode="w") as new_misc_info: new_misc_info_path = new_misc_info.name self._RewriteMiscInfo(new_misc_info, misc_info, lpmake, get_image) @@ -247,11 +244,11 @@ class OtaTools: for line in input_file: split_line = line.split() if len(split_line) == 3: - output_file.write(b("%s %s %s\n" % (get_image(split_line[1]), - split_line[1], - split_line[2]))) + output_file.write("%s %s %s\n" % (get_image(split_line[1]), + split_line[1], + split_line[2])) else: - output_file.write(b(line)) + output_file.write(line) @utils.TimeExecute(function_description="Make combined image") @utils.TimeoutException(_MK_COMBINED_IMG_TIMEOUT_SECS) @@ -274,7 +271,7 @@ class OtaTools: with open(system_qemu_config_path, "r") as config: with tempfile.NamedTemporaryFile( prefix="system-qemu-config_", suffix=".txt", - delete=False) as new_config: + delete=False, mode="w") as new_config: new_config_path = new_config.name self._RewriteSystemQemuConfig(new_config, config, get_image) diff --git a/internal/lib/oxygen_client.py b/internal/lib/oxygen_client.py index 862cd4ed..24daa153 100644 --- a/internal/lib/oxygen_client.py +++ b/internal/lib/oxygen_client.py @@ -26,14 +26,19 @@ class OxygenClient(): """Client that manages Oxygen proxy api.""" @staticmethod - def LeaseDevice(build_target, build_id, build_branch, oxygen_client, - cmd_args): + def LeaseDevice(build_target, build_id, build_branch, system_build_target, + system_build_id, kernel_build_target, kernel_build_id, + oxygen_client, cmd_args): """Lease one cuttlefish device. Args: build_target: Target name, e.g. "aosp_cf_x86_64_phone-userdebug" build_id: Build ID, a string, e.g. "2263051", "P2804227" build_branch: Build branch, e.g. "aosp-master" + system_build_target: Target name of system build + system_build_id: Build ID of system build + kernel_build_target: Target name of kernel build + kernel_build_id: Build ID of kernel build oxygen_client: String of oxygen client path. cmd_args: String of lease command args. e.g. "-user user_mail" @@ -44,6 +49,13 @@ class OxygenClient(): build_target, "-build_branch", build_branch] if cmd_args: cmd.extend(shlex.split(cmd_args)) + if system_build_id: + cmd.extend(["-system_build_id", system_build_id]) + cmd.extend(["-system_build_target", system_build_target]) + if kernel_build_id: + cmd.extend(["-kernel_build_id", kernel_build_id]) + cmd.extend(["-kernel_build_target", kernel_build_target]) + logger.debug("Command to oxygen client: %s", cmd) response = subprocess.check_output( cmd, stderr=subprocess.STDOUT, encoding='utf-8') logger.debug("The response from oxygen client: %s", response) diff --git a/internal/lib/oxygen_client_test.py b/internal/lib/oxygen_client_test.py index 45601f6b..0a3425bc 100644 --- a/internal/lib/oxygen_client_test.py +++ b/internal/lib/oxygen_client_test.py @@ -36,13 +36,19 @@ class OxygenClentTest(driver_test_lib.BaseDriverTest): build_id = "fake_id" oxygen_client_path = "oxygen_client_path" build_branch = "master-branch" + + # Test mixed build lease request. lease_args = "" expected_cmd = [oxygen_client_path, "-lease", "-build_id", build_id, "-build_target", build_target, "-build_branch", - build_branch] + build_branch, "-system_build_id", "system_build_id1", + "-system_build_target", "system_build_target1", + "-kernel_build_id", "kernel_build_id2", + "-kernel_build_target", "kernel_build_target2"] oxygen_client.OxygenClient.LeaseDevice( - build_target, build_id, build_branch, oxygen_client_path, - lease_args) + build_target, build_id, build_branch, "system_build_target1", + "system_build_id1", "kernel_build_target2", "kernel_build_id2", + oxygen_client_path, lease_args) mock_subprocess.assert_called_with(expected_cmd, stderr=subprocess.STDOUT, encoding='utf-8') @@ -53,8 +59,8 @@ class OxygenClentTest(driver_test_lib.BaseDriverTest): "-build_target", build_target, "-build_branch", build_branch, "-user", "user@gmail.com"] oxygen_client.OxygenClient.LeaseDevice( - build_target, build_id, build_branch, oxygen_client_path, - lease_args) + build_target, build_id, build_branch, "", "", "", "", + oxygen_client_path, lease_args) mock_subprocess.assert_called_with(expected_cmd, stderr=subprocess.STDOUT, encoding='utf-8') diff --git a/internal/lib/utils.py b/internal/lib/utils.py index 05655e2e..0295b292 100755 --- a/internal/lib/utils.py +++ b/internal/lib/utils.py @@ -40,8 +40,6 @@ import uuid import webbrowser import zipfile -import six - from acloud import errors from acloud.internal import constants @@ -330,7 +328,7 @@ def MakeTarFile(src_dict, dest): """ logger.info("Compressing %s into %s.", src_dict.keys(), dest) with tarfile.open(dest, "w:gz") as tar: - for src, arcname in six.iteritems(src_dict): + for src, arcname in src_dict.items(): tar.add(src, arcname=arcname) def CreateSshKeyPairIfNotExist(private_key_path, public_key_path): @@ -427,7 +425,7 @@ def VerifyRsaPubKey(rsa): key_type, data, _ = elements try: - binary_data = base64.decodebytes(six.b(data)) + binary_data = base64.decodebytes(data.encode()) # number of bytes of int type int_length = 4 # binary_data is like "7ssh-key..." in a binary format. @@ -437,7 +435,7 @@ def VerifyRsaPubKey(rsa): # We will verify that the rsa conforms to this format. # ">I" in the following line means "big-endian unsigned integer". type_length = struct.unpack(">I", binary_data[:int_length])[0] - if binary_data[int_length:int_length + type_length] != six.b(key_type): + if binary_data[int_length:int_length + type_length] != key_type.encode(): raise errors.DriverError("rsa key is invalid: %s" % rsa) except (struct.error, binascii.Error) as e: raise errors.DriverError( @@ -514,7 +512,7 @@ def InteractWithQuestion(question, colors=TextColors.WARNING): Returns: String, input from user. """ - return str(six.moves.input(colors + question + TextColors.ENDC).strip()) + return str(input(colors + question + TextColors.ENDC).strip()) def GetUserAnswerYes(question): @@ -602,7 +600,7 @@ class BatchHttpRequestExecutor: self._final_results.update(results) # Clear pending_requests self._pending_requests.clear() - for request_id, result in six.iteritems(results): + for request_id, result in results.items(): exception = result[1] if exception is not None and self._ShoudRetry(exception): # If this is a retriable exception, put it in pending_requests @@ -888,6 +886,11 @@ def EstablishWebRTCSshTunnel(ip_addr, webrtc_local_port, rsa_key_file, ssh_user, """ webrtc_server_port = GetWebRTCServerPort( ip_addr, rsa_key_file, ssh_user, extra_args_ssh_tunnel) + + # TODO(b/209502647): design a better way to forward webrtc ports. + if extra_args_ssh_tunnel: + for webrtc_port in WEBRTC_PORTS_MAPPING: + ReleasePort(webrtc_port.local) port_mapping = (WEBRTC_PORTS_MAPPING + [PortMapping(webrtc_local_port, webrtc_server_port)]) try: @@ -1032,7 +1035,7 @@ def GetAnswerFromList(answer_list, enable_choose_all=False): while True: try: - choice = six.moves.input("Enter your choice[0-%d]: " % max_choice) + choice = input("Enter your choice[0-%d]: " % max_choice) choice = int(choice) except ValueError: print("'%s' is not a valid integer.", choice) @@ -1466,11 +1469,9 @@ def GetDictItems(namedtuple_object): namedtuple_object: namedtuple object. Returns: - collections.namedtuple.__dict__.items() when using python2. collections.namedtuple._asdict().items() when using python3. """ - return (namedtuple_object.__dict__.items() if six.PY2 - else namedtuple_object._asdict().items()) + return namedtuple_object._asdict().items() def CleanupSSVncviewer(vnc_port): @@ -1588,4 +1589,4 @@ def SetCvdPorts(base_instance_num): AVD_PORT_DICT[constants.TYPE_CF] = ForwardedPorts( constants.CF_VNC_PORT + offset, constants.CF_ADB_PORT + offset) - # TODO: adjust WebRTC ports
\ No newline at end of file + # TODO: adjust WebRTC ports diff --git a/internal/lib/utils_test.py b/internal/lib/utils_test.py index ed5bf4a2..ccbca330 100644 --- a/internal/lib/utils_test.py +++ b/internal/lib/utils_test.py @@ -29,7 +29,6 @@ import webbrowser import unittest from unittest import mock -import six from acloud import errors from acloud.internal.lib import driver_test_lib @@ -191,7 +190,7 @@ class UtilsTest(driver_test_lib.BaseDriverTest): mock_open = mock.mock_open(read_data=public_key) self.Patch(subprocess, "check_output") self.Patch(os, "rename") - with mock.patch.object(six.moves.builtins, "open", mock_open): + with mock.patch("builtins.open", mock_open): utils.CreateSshKeyPairIfNotExist(private_key, public_key) self.assertEqual(subprocess.check_output.call_count, 1) #pylint: disable=no-member subprocess.check_output.assert_called_with( #pylint: disable=no-member @@ -262,7 +261,7 @@ class UtilsTest(driver_test_lib.BaseDriverTest): mock.call(16) ]) - @mock.patch.object(six.moves, "input") + @mock.patch("builtins.input") def testGetAnswerFromList(self, mock_raw_input): """Test GetAnswerFromList.""" answer_list = ["image1.zip", "image2.zip", "image3.zip"] diff --git a/list/instance.py b/list/instance.py index f856d276..5b238357 100644 --- a/list/instance.py +++ b/list/instance.py @@ -28,6 +28,7 @@ The details include: import collections import datetime +import json import logging import os import re @@ -53,7 +54,14 @@ _ACLOUD_CVD_TEMP = os.path.join(tempfile.gettempdir(), "acloud_cvd_temp") _CVD_CONFIG_FOLDER = "%(cvd_runtime)s/instances/cvd-%(id)d" _CVD_LOG_FOLDER = _CVD_CONFIG_FOLDER + "/logs" _CVD_RUNTIME_FOLDER_NAME = "cuttlefish_runtime" +_CVD_BIN = "cvd" +_CVD_BIN_FOLDER = "host_bins/bin" _CVD_STATUS_BIN = "cvd_status" +_CVD_SERVER = "cvd_server" +_CVD_STOP_ERROR_KEYWORDS = "cvd_internal_stop E" +# Default timeout 30 secs for cvd commands. +_CVD_TIMEOUT = 30 +_INSTANCE_ASSEMBLY_DIR = "cuttlefish_assembly" _LOCAL_INSTANCE_NAME_FORMAT = "local-instance-%(id)d" _LOCAL_INSTANCE_NAME_PATTERN = re.compile(r"^local-instance-(?P<id>\d+)$") _ACLOUDWEB_INSTANCE_START_STRING = "cf-" @@ -65,6 +73,7 @@ _RE_SSH_TUNNEL_PATTERN = (r"((.*\s*-L\s)(?P<%s>\d+):127.0.0.1:%s)" r"((.*\s*-L\s)(?P<%s>\d+):127.0.0.1:%s)" r"(.+%s)") _RE_TIMEZONE = re.compile(r"^(?P<time>[0-9\-\.:T]*)(?P<timezone>[+-]\d+:\d+)$") +_RE_DEVICE_INFO = re.compile(r"(?s).*(?P<device_info>[{][\s\w\W]+})") _COMMAND_PS_LAUNCH_CVD = ["ps", "-wweo", "lstart,cmd"] _RE_RUN_CVD = re.compile(r"(?P<date_str>^[^/]+)(.*run_cvd)") @@ -119,7 +128,7 @@ def GetLocalInstanceIdByName(name): return None -def GetLocalInstanceConfig(local_instance_id): +def GetLocalInstanceConfigPath(local_instance_id): """Get the path of instance config. Args: @@ -128,15 +137,23 @@ def GetLocalInstanceConfig(local_instance_id): Return: String, path of cf runtime config. """ - ins_runtime_dir = GetLocalInstanceRuntimeDir(local_instance_id) - cfg_dirs = [ins_runtime_dir] - cfg_dirs.append(_CVD_CONFIG_FOLDER % { - "cvd_runtime": ins_runtime_dir, - "id": local_instance_id}) - for cfg_dir in cfg_dirs: - cfg_path = os.path.join(cfg_dir, constants.CUTTLEFISH_CONFIG_FILE) - if os.path.isfile(cfg_path): - return cfg_path + ins_assembly_dir = os.path.join(GetLocalInstanceHomeDir(local_instance_id), + _INSTANCE_ASSEMBLY_DIR) + return os.path.join(ins_assembly_dir, constants.CUTTLEFISH_CONFIG_FILE) + + +def GetLocalInstanceConfig(local_instance_id): + """Get the path of existed config from local instance. + + Args: + local_instance_id: Integer of instance id. + + Return: + String, path of cf runtime config. None for config not exist. + """ + cfg_path = GetLocalInstanceConfigPath(local_instance_id) + if os.path.isfile(cfg_path): + return cfg_path return None @@ -263,6 +280,20 @@ def _GetElapsedTime(start_time): logger.debug(("Can't parse datetime string(%s)."), start_time) return _MSG_UNABLE_TO_CALCULATE +def _IsProcessRunning(process): + """Check if this process is running. + + Returns: + Boolean, True for this process is running. + """ + match_pattern = re.compile(f"(.+)({process} )(.+)") + process_output = utils.CheckOutput(constants.COMMAND_PS) + for line in process_output.splitlines(): + process_match = match_pattern.match(line) + if process_match: + return True + return False + # pylint: disable=useless-object-inheritance class Instance(object): @@ -449,7 +480,6 @@ class LocalInstance(Instance): self._instance_dir = self._cf_runtime_cfg.instance_dir self._virtual_disk_paths = self._cf_runtime_cfg.virtual_disk_paths self._local_instance_id = int(self._cf_runtime_cfg.instance_id) - display = _DISPLAY_STRING % {"x_res": self._cf_runtime_cfg.x_res, "y_res": self._cf_runtime_cfg.y_res, "dpi": self._cf_runtime_cfg.dpi} @@ -463,6 +493,10 @@ class LocalInstance(Instance): adb_device = AdbTools(device_serial="0.0.0.0:%s" % self._cf_runtime_cfg.adb_port) webrtc_port = local_image_local_instance.LocalImageLocalInstance.GetWebrtcSigServerPort( self._local_instance_id) + cvd_fleet_info = self.GetDevidInfoFromCvdFleet() + if cvd_fleet_info: + display = cvd_fleet_info.get("displays") + device_information = None if adb_device.IsAdbConnected(): device_information = adb_device.device_information @@ -481,6 +515,76 @@ class LocalInstance(Instance): instance_home = "%s instance home: %s" % (_INDENT, self._instance_dir) return "%s\n%s" % (super().Summary(), instance_home) + def _GetCvdEnv(self): + """Get the environment to run cvd commands. + + Returns: + os.environ with cuttlefish variables updated. + """ + cvd_env = os.environ.copy() + cvd_env[constants.ENV_ANDROID_SOONG_HOST_OUT] = os.path.dirname( + self._cf_runtime_cfg.cvd_tools_path) + cvd_env[constants.ENV_CUTTLEFISH_CONFIG_FILE] = self._cf_runtime_cfg.config_path + cvd_env[constants.ENV_CVD_HOME] = GetLocalInstanceHomeDir(self._local_instance_id) + cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str(self._local_instance_id) + return cvd_env + + def GetDevidInfoFromCvdFleet(self): + """Get device information from 'cvd fleet'. + + Execute 'cvd fleet' cmd to get device information. + + Returns + Output of 'cvd fleet'. None for fail to run 'cvd fleet'. + """ + ins_home_dir = GetLocalInstanceHomeDir(self._local_instance_id) + try: + cvd_tool = os.path.join(ins_home_dir, _CVD_BIN_FOLDER, _CVD_BIN) + cvd_fleet_cmd = f"{cvd_tool} fleet" + if not os.path.exists(cvd_tool): + logger.warning("Cvd tools path doesn't exist:%s", cvd_tool) + return None + if not _IsProcessRunning(_CVD_SERVER): + logger.warning("The %s is not active.", _CVD_SERVER) + return None + logger.debug("Running cmd [%s] to get device info.", cvd_fleet_cmd) + process = subprocess.Popen(cvd_fleet_cmd, shell=True, text=True, + env=self._GetCvdEnv(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, _ = process.communicate(timeout=_CVD_TIMEOUT) + logger.debug("Output of cvd fleet: %s", stdout) + return json.loads(self._ParsingCvdFleetOutput(stdout)) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, + json.JSONDecodeError) as error: + logger.error("Failed to run 'cvd fleet': %s", str(error)) + return None + + @staticmethod + def _ParsingCvdFleetOutput(output): + """Parsing the output of cvd fleet. + + The output example: + WARNING: cvd_server client version (8245608) does not match. + { + "adb_serial" : "0.0.0.0:6520", + "assembly_dir" : "/home/cuttlefish_runtime/assembly", + "displays" : ["720 x 1280 ( 320 )"], + "instance_dir" : "/home/cuttlefish_runtime/instances/cvd-1", + "instance_name" : "cvd-1", + "status" : "Running", + "web_access" : "https://0.0.0.0:8443/client.html?deviceId=cvd-1", + "webrtc_port" : "8443" + } + + Returns: + Parsed output filtered warning message. + """ + device_match = _RE_DEVICE_INFO.match(output) + if device_match: + return device_match.group("device_info") + return "" + def CvdStatus(self): """check if local instance is active. @@ -493,12 +597,6 @@ class LocalInstance(Instance): logger.debug("No cvd tools path found from config:%s", self._cf_runtime_cfg.config_path) return False - cvd_env = os.environ.copy() - cvd_env[constants.ENV_ANDROID_SOONG_HOST_OUT] = os.path.dirname( - self._cf_runtime_cfg.cvd_tools_path) - cvd_env[constants.ENV_CUTTLEFISH_CONFIG_FILE] = self._cf_runtime_cfg.config_path - cvd_env[constants.ENV_CVD_HOME] = GetLocalInstanceHomeDir(self._local_instance_id) - cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str(self._local_instance_id) try: cvd_status_cmd = os.path.join(self._cf_runtime_cfg.cvd_tools_path, _CVD_STATUS_BIN) @@ -521,7 +619,7 @@ class LocalInstance(Instance): stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - env=cvd_env) + env=self._GetCvdEnv()) stdout, _ = process.communicate() if process.returncode != 0: if stdout: @@ -534,37 +632,52 @@ class LocalInstance(Instance): return False def Delete(self): - """Execute stop_cvd to stop local cuttlefish instance. + """Execute "cvd stop" to stop local cuttlefish instance. - - We should get the same host tool used to launch cvd to delete instance - , So get stop_cvd bin from the cvd runtime config. - - Add CUTTLEFISH_CONFIG_FILE env variable to tell stop_cvd which cvd - need to be deleted. + - We should get the same host tool used to delete instance. + - Add CUTTLEFISH_CONFIG_FILE env variable to tell cvd which cvd need to + be deleted. - Stop adb since local instance use the fixed adb port and could be reused again soon. """ - stop_cvd_cmd = os.path.join(self.cf_runtime_cfg.cvd_tools_path, - constants.CMD_STOP_CVD) + ins_home_dir = GetLocalInstanceHomeDir(self._local_instance_id) + cvd_tool = os.path.join(ins_home_dir, _CVD_BIN_FOLDER, _CVD_BIN) + stop_cvd_cmd = f"{cvd_tool} stop" logger.debug("Running cmd[%s] to delete local cvd", stop_cvd_cmd) - cvd_env = os.environ.copy() - if self.instance_dir: - cvd_env[constants.ENV_CUTTLEFISH_CONFIG_FILE] = self._cf_runtime_cfg.config_path - cvd_env[constants.ENV_CVD_HOME] = GetLocalInstanceHomeDir( - self._local_instance_id) - cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str(self._local_instance_id) - else: + if not self.instance_dir: logger.error("instance_dir is null!! instance[%d] might not be" " deleted", self._local_instance_id) - subprocess.check_call( - utils.AddUserGroupsToCmd(stop_cvd_cmd, - constants.LIST_CF_USER_GROUPS), - stderr=subprocess.STDOUT, shell=True, env=cvd_env) + try: + output = subprocess.check_output( + utils.AddUserGroupsToCmd(stop_cvd_cmd, + constants.LIST_CF_USER_GROUPS), + stderr=subprocess.STDOUT, shell=True, env=self._GetCvdEnv(), + text=True, timeout=_CVD_TIMEOUT) + # TODO: Remove workaround of stop_cvd when 'cvd stop' is stable. + if _CVD_STOP_ERROR_KEYWORDS in output: + logger.debug("Fail to stop cvd: %s", output) + self._ExecuteStopCvd(os.path.join(ins_home_dir, _CVD_BIN_FOLDER)) + except (subprocess.TimeoutExpired, subprocess.CalledProcessError) as e: + logger.debug("'cvd stop' error: %s", str(e)) + self._ExecuteStopCvd(os.path.join(ins_home_dir, _CVD_BIN_FOLDER)) adb_cmd = AdbTools(self.adb_port) # When relaunch a local instance, we need to pass in retry=True to make # sure adb device is completely gone since it will use the same adb port adb_cmd.DisconnectAdb(retry=True) + def _ExecuteStopCvd(self, dir_path): + """Execute "stop_cvd" to stop local cuttlefish instance. + + Args: + bin_dir: String, directory path of "stop_cvd". + """ + stop_cvd_cmd = os.path.join(dir_path, constants.CMD_STOP_CVD) + subprocess.check_call( + utils.AddUserGroupsToCmd( + stop_cvd_cmd, constants.LIST_CF_USER_GROUPS), + stderr=subprocess.STDOUT, shell=True, env=self._GetCvdEnv()) + def GetLock(self): """Return the LocalInstanceLock for this object.""" return GetLocalInstanceLock(self._local_instance_id) diff --git a/list/instance_test.py b/list/instance_test.py index 5aef584d..8475653d 100644 --- a/list/instance_test.py +++ b/list/instance_test.py @@ -22,7 +22,6 @@ import subprocess import unittest from unittest import mock -from six import b # pylint: disable=import-error import dateutil.parser @@ -38,12 +37,12 @@ from acloud.list import instance class InstanceTest(driver_test_lib.BaseDriverTest): """Test instance.""" - PS_SSH_TUNNEL = b("/fake_ps_1 --fake arg \n" - "/fake_ps_2 --fake arg \n" - "/usr/bin/ssh -i ~/.ssh/acloud_rsa " - "-o UserKnownHostsFile=/dev/null " - "-o StrictHostKeyChecking=no -L 54321:127.0.0.1:6520 " - "-L 12345:127.0.0.1:6444 -N -f -l user 1.1.1.1") + PS_SSH_TUNNEL = ("/fake_ps_1 --fake arg \n" + "/fake_ps_2 --fake arg \n" + "/usr/bin/ssh -i ~/.ssh/acloud_rsa " + "-o UserKnownHostsFile=/dev/null " + "-o StrictHostKeyChecking=no -L 54321:127.0.0.1:6520 " + "-L 12345:127.0.0.1:6444 -N -f -l user 1.1.1.1").encode() GCE_INSTANCE = { constants.INS_KEY_NAME: "fake_ins_name", constants.INS_KEY_CREATETIME: "fake_create_time", @@ -85,6 +84,8 @@ class InstanceTest(driver_test_lib.BaseDriverTest): mock_adb_tools.return_value = mock_adb_tools_object self.Patch(cvd_runtime_config, "CvdRuntimeConfig", return_value=self._MockCvdRuntimeConfig()) + self.Patch(instance.LocalInstance, "GetDevidInfoFromCvdFleet", + return_value=None) local_instance = instance.LocalInstance("fake_config_path") self.assertEqual("local-instance-2", local_instance.name) @@ -99,9 +100,50 @@ class InstanceTest(driver_test_lib.BaseDriverTest): self.assertEqual(6445, local_instance.vnc_port) self.assertEqual(8444, local_instance.webrtc_port) + # pylint: disable=protected-access + def testGetCvdEnv(self): + """Test GetCvdEnv.""" + self.Patch(cvd_runtime_config, "CvdRuntimeConfig", + return_value=self._MockCvdRuntimeConfig()) + self.Patch(instance, "_IsProcessRunning", return_value=False) + local_instance = instance.LocalInstance("fake_config_path") + cvd_env = local_instance._GetCvdEnv() + self.assertEqual(cvd_env[constants.ENV_CUTTLEFISH_INSTANCE], "2") + self.assertEqual(cvd_env[constants.ENV_CUTTLEFISH_CONFIG_FILE], + "fake_config_path") + + # pylint: disable=protected-access + def testParsingCvdFleetOutput(self): + """Test ParsingCvdFleetOutput.""" + cvd_fleet_output = """WARNING: cvd_server client version does not match +{ +"adb_serial" : "0.0.0.0:6520", +"instance_name" : "cvd-1", +}""" + + expected_result = """{ +"adb_serial" : "0.0.0.0:6520", +"instance_name" : "cvd-1", +}""" + + self.assertEqual( + instance.LocalInstance._ParsingCvdFleetOutput(cvd_fleet_output), + expected_result) + + # pylint: disable=protected-access + def testIsProcessRunning(self): + """Test IsProcessRunning.""" + process = "cvd_server" + self.Patch(utils, "CheckOutput", + return_value="/bin/cvd_server -server_fd=4") + self.assertEqual(instance._IsProcessRunning(process), True) + + self.Patch(utils, "CheckOutput", return_value="/bin/cvd start") + self.assertEqual(instance._IsProcessRunning(process), False) + @mock.patch("acloud.list.instance.AdbTools") def testDeleteLocalInstance(self, mock_adb_tools): - """Test executing stop_cvd command.""" + """Test executing 'cvd stop' command.""" self.Patch(cvd_runtime_config, "CvdRuntimeConfig", return_value=self._MockCvdRuntimeConfig()) mock_adb_tools_object = mock.Mock(device_information={}) @@ -109,20 +151,30 @@ class InstanceTest(driver_test_lib.BaseDriverTest): mock_adb_tools.return_value = mock_adb_tools_object self.Patch(utils, "AddUserGroupsToCmd", side_effect=lambda cmd, groups: cmd) + self.Patch(instance.LocalInstance, "GetDevidInfoFromCvdFleet", + return_value=None) mock_check_call = self.Patch(subprocess, "check_call") + mock_check_output = self.Patch( + subprocess, "check_output", + return_value="cvd_internal_stop E stop cvd failed") local_instance = instance.LocalInstance("fake_config_path") with mock.patch.dict("acloud.list.instance.os.environ", clear=True): local_instance.Delete() expected_env = { - 'CUTTLEFISH_INSTANCE': '2', - 'HOME': '/tmp/acloud_cvd_temp/local-instance-2', - 'CUTTLEFISH_CONFIG_FILE': 'fake_config_path', + "CUTTLEFISH_INSTANCE": "2", + "HOME": "/tmp/acloud_cvd_temp/local-instance-2", + "CUTTLEFISH_CONFIG_FILE": "fake_config_path", + "ANDROID_SOONG_HOST_OUT": "", } + mock_check_output.assert_called_with( + "/tmp/acloud_cvd_temp/local-instance-2/host_bins/bin/cvd stop", + stderr=subprocess.STDOUT, shell=True, env=expected_env, text=True, + timeout=instance._CVD_TIMEOUT) mock_check_call.assert_called_with( - 'fake_cvd_tools_path/stop_cvd', stderr=subprocess.STDOUT, - shell=True, env=expected_env) + "/tmp/acloud_cvd_temp/local-instance-2/host_bins/bin/stop_cvd", + stderr=subprocess.STDOUT, shell=True, env=expected_env) mock_adb_tools_object.DisconnectAdb.assert_called() @mock.patch("acloud.list.instance.tempfile") @@ -226,6 +278,8 @@ class InstanceTest(driver_test_lib.BaseDriverTest): instance.RemoteInstance, "GetAdbVncPortFromSSHTunnel", return_value=forwarded_ports(vnc_port=fake_vnc, adb_port=fake_adb)) + self.Patch(utils, "GetWebrtcPortFromSSHTunnel", + return_value="fake_webrtc_port") self.Patch(instance, "_GetElapsedTime", return_value="fake_time") self.Patch(AdbTools, "IsAdbConnected", return_value=True) @@ -329,15 +383,15 @@ class InstanceTest(driver_test_lib.BaseDriverTest): def testGetLocalInstanceConfig(self): """Test GetLocalInstanceConfig.""" - self.Patch(instance, "GetLocalInstanceRuntimeDir", - return_value="ins_runtime_dir") + self.Patch(instance, "GetLocalInstanceHomeDir", + return_value="ins_home") self.Patch(os.path, "isfile", return_value=False) instance_id = 1 self.assertEqual(instance.GetLocalInstanceConfig(instance_id), None) # Test config in new folder path. - self.Patch(os.path, "isfile", side_effect=[False, True]) - expected_result = "ins_runtime_dir/instances/cvd-1/cuttlefish_config.json" + self.Patch(os.path, "isfile", return_value=True) + expected_result = "ins_home/cuttlefish_assembly/cuttlefish_config.json" self.assertEqual( instance.GetLocalInstanceConfig(instance_id), expected_result) diff --git a/list/list_test.py b/list/list_test.py index ef4b09dd..6e5d031c 100644 --- a/list/list_test.py +++ b/list/list_test.py @@ -202,6 +202,8 @@ class ListTest(driver_test_lib.BaseDriverTest): ) self.Patch(cvd_runtime_config, "CvdRuntimeConfig", return_value=cf_config) + self.Patch(instance.LocalInstance, "GetDevidInfoFromCvdFleet", + return_value=None) ins = instance.LocalInstance("fake_cf_path") list_instance.PrintInstancesDetails([ins], verbose=True) diff --git a/public/acloud_main.py b/public/acloud_main.py index 55f7989b..96648d6e 100644 --- a/public/acloud_main.py +++ b/public/acloud_main.py @@ -70,9 +70,7 @@ Try $acloud [cmd] --help for further details. from __future__ import print_function import argparse import logging -import os import sys -import sysconfig import traceback if sys.version_info.major == 2: @@ -82,12 +80,13 @@ if sys.version_info.major == 2: sys.version_info.micro)) sys.exit(1) -# This is a workaround to put '/usr/lib/python3.X' ahead of googleapiclient of -# build system path list to fix python3 issue of http.client(b/144743252) -# that googleapiclient existed http.py conflict with python3 build-in lib. -# Using embedded_launcher(b/135639220) perhaps work whereas it didn't solve yet. -if sys.version_info.major == 3: - sys.path.insert(0, os.path.dirname(sysconfig.get_paths()['purelib'])) +# (b/219847353) Move googleapiclient to the last position of sys.path when +# existed. +for lib in sys.path: + if 'googleapiclient' in lib: + sys.path.remove(lib) + sys.path.append(lib) + break # By Default silence root logger's stream handler since 3p lib may initial # root logger no matter what level we're using. The acloud logger behavior will @@ -105,7 +104,6 @@ from acloud.create import create_args from acloud.delete import delete from acloud.delete import delete_args from acloud.internal import constants -from acloud.internal.lib import utils from acloud.reconnect import reconnect from acloud.reconnect import reconnect_args from acloud.list import list as list_instances @@ -116,7 +114,6 @@ from acloud.powerwash import powerwash_args from acloud.public import acloud_common from acloud.public import config from acloud.public import report -from acloud.public.actions import create_cuttlefish_action from acloud.public.actions import create_goldfish_action from acloud.pull import pull from acloud.pull import pull_args @@ -135,12 +132,10 @@ NO_ERROR_MESSAGE = "" PROG = "acloud" # Commands -CMD_CREATE_CUTTLEFISH = "create_cf" CMD_CREATE_GOLDFISH = "create_gf" # Config requires fields. _CREATE_REQUIRE_FIELDS = ["project", "zone", "machine_type"] -_CREATE_CF_REQUIRE_FIELDS = ["resolution"] # show contact info to user. _CONTACT_INFO = ("If you have any question or need acloud team support, " "please feel free to contact us by email at " @@ -178,13 +173,6 @@ def _ParseArgs(args): subparsers = parser.add_subparsers(metavar="{" + usage + "}") subparser_list = [] - # Command "create_cf", create cuttlefish instances - create_cf_parser = subparsers.add_parser(CMD_CREATE_CUTTLEFISH) - create_cf_parser.required = False - create_cf_parser.set_defaults(which=CMD_CREATE_CUTTLEFISH) - create_args.AddCommonCreateArgs(create_cf_parser) - subparser_list.append(create_cf_parser) - # Command "create_gf", create goldfish instances # In order to create a goldfish device we need the following parameters: # 1. The emulator build we wish to use, this is the binary that emulates @@ -281,10 +269,6 @@ def _VerifyArgs(parsed_args): create_args.VerifyArgs(parsed_args) if parsed_args.which == setup_args.CMD_SETUP: setup_args.VerifyArgs(parsed_args) - if parsed_args.which == CMD_CREATE_CUTTLEFISH: - if not parsed_args.build_id and not parsed_args.branch: - raise errors.CommandArgError( - "Must specify --build_id or --branch") if parsed_args.which == CMD_CREATE_GOLDFISH: if not parsed_args.emulator_build_id and not parsed_args.build_id and ( not parsed_args.emulator_branch and not parsed_args.branch): @@ -300,9 +284,7 @@ def _VerifyArgs(parsed_args): "--system-* args are not supported for AVD type: %s" % constants.TYPE_GF) - if parsed_args.which in [ - create_args.CMD_CREATE, CMD_CREATE_CUTTLEFISH, CMD_CREATE_GOLDFISH - ]: + if parsed_args.which in [create_args.CMD_CREATE, CMD_CREATE_GOLDFISH]: if (parsed_args.serial_log_file and not parsed_args.serial_log_file.endswith(".tar.gz")): raise errors.CommandArgError( @@ -322,8 +304,6 @@ def _ParsingConfig(args, cfg): missing_fields = [] if args.which == create_args.CMD_CREATE and args.local_instance is None: missing_fields = cfg.GetMissingFields(_CREATE_REQUIRE_FIELDS) - if args.which == CMD_CREATE_CUTTLEFISH: - missing_fields.extend(cfg.GetMissingFields(_CREATE_CF_REQUIRE_FIELDS)) if missing_fields: return ( "Config file (%s) missing required fields: %s, please add these " @@ -421,31 +401,6 @@ def main(argv=None): constants.ACLOUD_UNKNOWN_ARGS_ERROR) elif args.which == create_args.CMD_CREATE: reporter = create.Run(args) - elif args.which == CMD_CREATE_CUTTLEFISH: - # Set ports offset when base_instance_num is specified - utils.SetCvdPorts(args.base_instance_num) - - reporter = create_cuttlefish_action.CreateDevices( - cfg=cfg, - build_target=args.build_target, - build_id=args.build_id, - branch=args.branch, - kernel_build_id=args.kernel_build_id, - kernel_branch=args.kernel_branch, - kernel_build_target=args.kernel_build_target, - system_branch=args.system_branch, - system_build_id=args.system_build_id, - system_build_target=args.system_build_target, - bootloader_branch=args.bootloader_branch, - bootloader_build_id=args.bootloader_build_id, - bootloader_build_target=args.bootloader_build_target, - gpu=args.gpu, - num=args.num, - serial_log_file=args.serial_log_file, - autoconnect=args.autoconnect, - report_internal_ip=args.report_internal_ip, - boot_timeout_secs=args.boot_timeout_secs, - ins_timeout_secs=args.ins_timeout_secs) elif args.which == CMD_CREATE_GOLDFISH: reporter = create_goldfish_action.CreateDevices( cfg=cfg, diff --git a/public/actions/common_operations.py b/public/actions/common_operations.py index e08d0edc..2b64ec54 100644 --- a/public/actions/common_operations.py +++ b/public/actions/common_operations.py @@ -208,6 +208,20 @@ def _GetErrorType(error): return constants.GCE_QUOTA_ERROR return constants.ACLOUD_UNKNOWN_ERROR +def _GetAdbPort(avd_type, base_instance_num): + """Get Adb port according to avd_type and device offset. + + Args: + avd_type: String, the AVD type(cuttlefish, goldfish...). + base_instance_num: int, device offset. + + Returns: + int, adb port. + """ + if avd_type in utils.AVD_PORT_DICT: + return utils.AVD_PORT_DICT[avd_type].adb_port + base_instance_num - 1 + return None + # pylint: disable=too-many-locals,unused-argument,too-many-branches def CreateDevices(command, cfg, device_factory, num, avd_type, report_internal_ip=False, autoconnect=False, @@ -274,8 +288,15 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, for device in device_pool.devices: ip = (device.ip.internal if report_internal_ip else device.ip.external) + base_instance_num = 1 + if constants.BASE_INSTANCE_NUM in device_pool._compute_client.dict_report: + base_instance_num = device_pool._compute_client.dict_report[constants.BASE_INSTANCE_NUM] + adb_port = _GetAdbPort( + avd_type, + base_instance_num + ) device_dict = { - "ip": ip, + "ip": ip + (":" + str(adb_port) if adb_port else ""), "instance_name": device.instance_name } if device.build_info: @@ -290,7 +311,7 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, rsa_key_file=(ssh_private_key_path or cfg.ssh_private_key_path), target_vnc_port=utils.AVD_PORT_DICT[avd_type].vnc_port, - target_adb_port=utils.AVD_PORT_DICT[avd_type].adb_port, + target_adb_port=adb_port, ssh_user=ssh_user, client_adb_port=client_adb_port, extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) diff --git a/public/actions/common_operations_test.py b/public/actions/common_operations_test.py index fd26fe3e..b06a60cd 100644 --- a/public/actions/common_operations_test.py +++ b/public/actions/common_operations_test.py @@ -144,7 +144,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): self.assertEqual( _report.data, {"devices": [{ - "ip": self.IP.external, + "ip": self.IP.external + ":6520", "instance_name": self.INSTANCE, "branch": self.BRANCH, "build_id": self.BUILD_ID, diff --git a/public/actions/create_cuttlefish_action_test.py b/public/actions/create_cuttlefish_action_test.py index b00a5af2..0ddef21e 100644 --- a/public/actions/create_cuttlefish_action_test.py +++ b/public/actions/create_cuttlefish_action_test.py @@ -56,6 +56,7 @@ class CreateCuttlefishActionTest(driver_test_lib.BaseDriverTest): STABLE_HOST_IMAGE_PROJECT = "fake-stable-host-image-project" EXTRA_DATA_DISK_GB = 4 EXTRA_SCOPES = ["scope1", "scope2"] + DEFAULT_ADB_PORT = 6520 def setUp(self): """Set up the test.""" @@ -174,7 +175,7 @@ class CreateCuttlefishActionTest(driver_test_lib.BaseDriverTest): "bootloader_build_id": self.BOOTLOADER_BUILD_ID, "bootloader_build_target": self.BOOTLOADER_BUILD_TARGET, "instance_name": self.INSTANCE, - "ip": self.IP.external, + "ip": self.IP.external + ":" + str(self.DEFAULT_ADB_PORT), }, ], }) diff --git a/public/actions/create_goldfish_action_test.py b/public/actions/create_goldfish_action_test.py index 77d04a97..dbc47b5b 100644 --- a/public/actions/create_goldfish_action_test.py +++ b/public/actions/create_goldfish_action_test.py @@ -50,6 +50,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): EXTRA_DATA_DISK_GB = 4 EXTRA_SCOPES = None LAUNCH_ARGS = "fake-args" + DEFAULT_ADB_PORT = 5555 def setUp(self): """Sets up the test.""" @@ -156,7 +157,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): "devices": [ { "instance_name": self.INSTANCE, - "ip": self.IP.external, + "ip": self.IP.external + ":" + str(self.DEFAULT_ADB_PORT), "branch": self.BRANCH, "build_id": self.BUILD_ID, "build_target": self.BUILD_TARGET, @@ -277,7 +278,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): self.assertEqual(report.data, { "devices": [{ "instance_name": self.INSTANCE, - "ip": self.IP.external, + "ip": self.IP.external + ":" + str(self.DEFAULT_ADB_PORT), "branch": self.BRANCH, "build_id": self.BUILD_ID, "build_target": self.BUILD_TARGET, @@ -390,7 +391,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): self.assertEqual(report.data, { "devices": [{ "instance_name": self.INSTANCE, - "ip": self.IP.external, + "ip": self.IP.external + ":" + str(self.DEFAULT_ADB_PORT), "branch": self.BRANCH, "build_id": self.BUILD_ID, "build_target": self.BUILD_TARGET, diff --git a/public/actions/gce_device_factory.py b/public/actions/gce_device_factory.py index 08da69e9..5502f296 100644 --- a/public/actions/gce_device_factory.py +++ b/public/actions/gce_device_factory.py @@ -39,6 +39,7 @@ class GCEDeviceFactory(base_device_factory.BaseDeviceFactory): self._cfg = avd_spec.cfg self._local_image_artifact = local_image_artifact self._report_internal_ip = avd_spec.report_internal_ip + self._all_failures = {} self.credentials = auth.CreateCredentials(avd_spec.cfg) # Control compute_client with enable_multi_stage compute_client = cvd_compute_client_multi_stage.CvdComputeClient( @@ -102,9 +103,9 @@ class GCEDeviceFactory(base_device_factory.BaseDeviceFactory): Returns: A dictionary that contains all the failures. The key is the name of the instance that fails to boot, - and the value is an errors.DeviceBootError object. + and the value is a string or an errors.DeviceBootError object. """ - return self._compute_client.all_failures + return self._all_failures def _SetFailures(self, instance, error_msg): """Set failures from this device. @@ -115,4 +116,4 @@ class GCEDeviceFactory(base_device_factory.BaseDeviceFactory): instance: String of instance name. error_msg: String of error message. """ - self._compute_client.all_failures[instance] = error_msg + self._all_failures[instance] = error_msg diff --git a/public/actions/remote_host_cf_device_factory.py b/public/actions/remote_host_cf_device_factory.py new file mode 100644 index 00000000..4b4181e7 --- /dev/null +++ b/public/actions/remote_host_cf_device_factory.py @@ -0,0 +1,284 @@ +# Copyright 2022 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""RemoteHostDeviceFactory implements the device factory interface and creates +cuttlefish instances on a remote host.""" + +import glob +import logging +import os +import shutil +import subprocess +import tempfile + +from acloud import errors +from acloud.internal import constants +from acloud.internal.lib import auth +from acloud.internal.lib import cvd_compute_client_multi_stage +from acloud.internal.lib import cvd_utils +from acloud.internal.lib import utils +from acloud.internal.lib import ssh +from acloud.public.actions import base_device_factory +from acloud.pull import pull + + +logger = logging.getLogger(__name__) +_ALL_FILES = "*" +_HOME_FOLDER = os.path.expanduser("~") +_SCREEN_CONSOLE_COMMAND = "screen ~/cuttlefish_runtime/console" + + +class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): + """A class that can produce a cuttlefish device. + + Attributes: + avd_spec: AVDSpec object that tells us what we're going to create. + local_image_artifact: A string, path to local image. + cvd_host_package_artifact: A string, path to cvd host package. + all_failures: A dictionary mapping instance names to errors. + all_logs: A dictionary mapping instance names to lists of + report.LogFile. + compute_client: An object of cvd_compute_client.CvdComputeClient. + ssh: An Ssh object. + """ + + _USER_BUILD = "userbuild" + + def __init__(self, avd_spec, local_image_artifact=None, + cvd_host_package_artifact=None): + """Initialize attributes.""" + self._avd_spec = avd_spec + self._local_image_artifact = local_image_artifact + self._cvd_host_package_artifact = cvd_host_package_artifact + self._all_failures = {} + self._all_logs = {} + credentials = auth.CreateCredentials(avd_spec.cfg) + compute_client = cvd_compute_client_multi_stage.CvdComputeClient( + acloud_config=avd_spec.cfg, + oauth2_credentials=credentials, + ins_timeout_secs=avd_spec.ins_timeout_secs, + report_internal_ip=avd_spec.report_internal_ip, + gpu=avd_spec.gpu) + super().__init__(compute_client) + self._ssh = None + + def CreateInstance(self): + """Create a single configured cuttlefish device. + + Returns: + A string, representing instance name. + """ + instance = self._InitRemotehost() + image_args = self._ProcessRemoteHostArtifacts() + failures = self._compute_client.LaunchCvd( + instance, + self._avd_spec, + self._avd_spec.cfg.extra_data_disk_size_gb, + boot_timeout_secs=self._avd_spec.boot_timeout_secs, + extra_args=image_args) + self._all_failures.update(failures) + self._FindLogFiles( + instance, instance in failures and not self._avd_spec.no_pull_log) + return instance + + def _InitRemotehost(self): + """Initialize remote host. + + Determine the remote host instance name, and activate ssh. It need to + get the IP address in the common_operation. So need to pass the IP and + ssh to compute_client. + + build_target: The format is like "aosp_cf_x86_phone". We only get info + from the user build image file name. If the file name is + not custom format (no "-"), we will use $TARGET_PRODUCT + from environment variable as build_target. + + Returns: + A string, representing instance name. + """ + image_name = os.path.basename( + self._local_image_artifact) if self._local_image_artifact else "" + build_target = (os.environ.get(constants.ENV_BUILD_TARGET) + if "-" not in image_name else + image_name.split("-", maxsplit=1)[0]) + build_id = self._USER_BUILD + if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: + build_id = self._avd_spec.remote_image[constants.BUILD_ID] + + instance = self._compute_client.FormatRemoteHostInstanceName( + self._avd_spec.remote_host, build_id, build_target) + ip = ssh.IP(ip=self._avd_spec.remote_host) + self._ssh = ssh.Ssh( + ip=ip, + user=self._avd_spec.host_user, + ssh_private_key_path=(self._avd_spec.host_ssh_private_key_path or + self._avd_spec.cfg.ssh_private_key_path), + extra_args_ssh_tunnel=self._avd_spec.cfg.extra_args_ssh_tunnel, + report_internal_ip=self._avd_spec.report_internal_ip) + self._compute_client.InitRemoteHost( + self._ssh, ip, self._avd_spec.host_user) + return instance + + def _ProcessRemoteHostArtifacts(self): + """Process remote host artifacts. + + - If images source is local, tool will upload images from local site to + remote host. + - If images source is remote, tool will download images from android + build to local and unzip it then upload to remote host, because there + is no permission to fetch build rom on the remote host. + + Returns: + A list of strings, the launch_cvd arguments. + """ + self._compute_client.SetStage(constants.STAGE_ARTIFACT) + if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: + cvd_utils.UploadArtifacts( + self._ssh, + self._local_image_artifact or self._avd_spec.local_image_dir, + self._cvd_host_package_artifact) + else: + try: + artifacts_path = tempfile.mkdtemp() + logger.debug("Extracted path of artifacts: %s", artifacts_path) + self._DownloadArtifacts(artifacts_path) + self._UploadRemoteImageArtifacts(artifacts_path) + finally: + shutil.rmtree(artifacts_path) + + return cvd_utils.UploadExtraImages(self._ssh, self._avd_spec) + + @utils.TimeExecute(function_description="Downloading Android Build artifact") + def _DownloadArtifacts(self, extract_path): + """Download the CF image artifacts and process them. + + - Download images from the Android Build system. + - Download cvd host package from the Android Build system. + + Args: + extract_path: String, a path include extracted files. + + Raises: + errors.GetRemoteImageError: Fails to download rom images. + """ + cfg = self._avd_spec.cfg + build_id = self._avd_spec.remote_image[constants.BUILD_ID] + build_branch = self._avd_spec.remote_image[constants.BUILD_BRANCH] + build_target = self._avd_spec.remote_image[constants.BUILD_TARGET] + + # Download images with fetch_cvd + fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) + self._compute_client.build_api.DownloadFetchcvd(fetch_cvd, + cfg.fetch_cvd_version) + fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( + build_id, build_branch, build_target, + self._avd_spec.system_build_info.get(constants.BUILD_ID), + self._avd_spec.system_build_info.get(constants.BUILD_BRANCH), + self._avd_spec.system_build_info.get(constants.BUILD_TARGET), + self._avd_spec.kernel_build_info.get(constants.BUILD_ID), + self._avd_spec.kernel_build_info.get(constants.BUILD_BRANCH), + self._avd_spec.kernel_build_info.get(constants.BUILD_TARGET), + self._avd_spec.bootloader_build_info.get(constants.BUILD_ID), + self._avd_spec.bootloader_build_info.get(constants.BUILD_BRANCH), + self._avd_spec.bootloader_build_info.get(constants.BUILD_TARGET), + self._avd_spec.ota_build_info.get(constants.BUILD_ID), + self._avd_spec.ota_build_info.get(constants.BUILD_BRANCH), + self._avd_spec.ota_build_info.get(constants.BUILD_TARGET)) + creds_cache_file = os.path.join(_HOME_FOLDER, cfg.creds_cache_file) + fetch_cvd_cert_arg = self._compute_client.build_api.GetFetchCertArg( + creds_cache_file) + fetch_cvd_args = [fetch_cvd, f"-directory={extract_path}", + fetch_cvd_cert_arg] + fetch_cvd_args.extend(fetch_cvd_build_args) + logger.debug("Download images command: %s", fetch_cvd_args) + try: + subprocess.check_call(fetch_cvd_args) + except subprocess.CalledProcessError as e: + raise errors.GetRemoteImageError(f"Fails to download images: {e}") + + @utils.TimeExecute(function_description="Uploading remote image artifacts") + def _UploadRemoteImageArtifacts(self, images_dir): + """Upload remote image artifacts to instance. + + Args: + images_dir: String, directory of local artifacts downloaded by + fetch_cvd. + """ + artifact_files = [ + os.path.basename(image) + for image in glob.glob(os.path.join(images_dir, _ALL_FILES)) + ] + ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) + # TODO(b/182259589): Refactor upload image command into a function. + cmd = (f"tar -cf - --lzop -S -C {images_dir} " + f"{' '.join(artifact_files)} | " + f"{ssh_cmd} -- tar -xf - --lzop -S") + logger.debug("cmd:\n %s", cmd) + ssh.ShellCmdWithRetry(cmd) + + def _FindLogFiles(self, instance, download): + """Find and pull all log files from instance. + + Args: + instance: String, instance name. + download: Whether to download the files to a temporary directory + and show messages to the user. + """ + self._all_logs[instance] = [cvd_utils.TOMBSTONES] + log_files = pull.GetAllLogFilePaths(self._ssh) + self._all_logs[instance].extend(cvd_utils.ConvertRemoteLogs(log_files)) + + if download: + error_log_folder = pull.PullLogs(self._ssh, log_files, instance) + self._compute_client.ExtendReportData(constants.ERROR_LOG_FOLDER, + error_log_folder) + + def GetOpenWrtInfoDict(self): + """Get openwrt info dictionary. + + Returns: + A openwrt info dictionary. None for the case is not openwrt device. + """ + if not self._avd_spec.openwrt: + return None + return {"ssh_command": self._compute_client.GetSshConnectCmd(), + "screen_command": _SCREEN_CONSOLE_COMMAND} + + def GetBuildInfoDict(self): + """Get build info dictionary. + + Returns: + A build info dictionary. None for local image case. + """ + if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: + return None + return cvd_utils.GetRemoteBuildInfoDict(self._avd_spec) + + def GetFailures(self): + """Get failures from all devices. + + Returns: + A dictionary that contains all the failures. + The key is the name of the instance that fails to boot, + and the value is a string or an errors.DeviceBootError object. + """ + return self._all_failures + + def GetLogs(self): + """Get all device logs. + + Returns: + A dictionary that maps instance names to lists of report.LogFile. + """ + return self._all_logs diff --git a/public/actions/remote_host_cf_device_factory_test.py b/public/actions/remote_host_cf_device_factory_test.py new file mode 100644 index 00000000..4b9c3e68 --- /dev/null +++ b/public/actions/remote_host_cf_device_factory_test.py @@ -0,0 +1,178 @@ +# Copyright 2022 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for remote_host_cf_device_factory.""" + +import unittest +from unittest import mock + +from acloud.internal import constants +from acloud.internal.lib import auth +from acloud.internal.lib import driver_test_lib +from acloud.internal.lib import cvd_compute_client_multi_stage +from acloud.public.actions import remote_host_cf_device_factory + +class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): + """Test RemoteHostDeviceFactory.""" + + def setUp(self): + """Set up the test.""" + super().setUp() + self.Patch(auth, "CreateCredentials") + self.Patch(cvd_compute_client_multi_stage, "CvdComputeClient") + + @staticmethod + def _CreateMockAvdSpec(): + """Create a mock AvdSpec with necessary attributes.""" + mock_cfg = mock.Mock(spec=[], + extra_data_disk_size_gb=10, + ssh_private_key_path="/mock/id_rsa", + extra_args_ssh_tunnel="extra args", + fetch_cvd_version="123456", + creds_cache_file="credential") + return mock.Mock(spec=[], + remote_image={ + "branch": "aosp-android12-gsi", + "build_id": "100000", + "build_target": "aosp_cf_x86_64_phone-userdebug"}, + system_build_info={}, + kernel_build_info={}, + bootloader_build_info={}, + ota_build_info={}, + remote_host="192.0.2.100", + host_user="user1", + host_ssh_private_key_path=None, + report_internal_ip=False, + image_source=constants.IMAGE_SRC_REMOTE, + local_image_dir=None, + ins_timeout_secs=200, + boot_timeout_secs=100, + gpu="auto", + no_pull_log=False, + cfg=mock_cfg) + + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_compute_client_multi_stage") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.ssh") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_utils") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.pull") + def testCreateInstanceWithImageDir(self, mock_pull, mock_cvd_utils, + mock_ssh, _mock_client): + """Test CreateInstance with local image directory.""" + mock_avd_spec = self._CreateMockAvdSpec() + mock_avd_spec.image_source = constants.IMAGE_SRC_LOCAL + mock_avd_spec.local_image_dir = "/mock/img" + factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( + mock_avd_spec, cvd_host_package_artifact="/mock/cvd.tar.gz") + + mock_client_obj = factory.GetComputeClient() + mock_client_obj.FormatRemoteHostInstanceName.return_value = "inst" + mock_client_obj.LaunchCvd.return_value = {"inst": "failure"} + + log = {"path": "/log.txt"} + tombstones = {"path": "/tombstones"} + mock_cvd_utils.TOMBSTONES = tombstones + mock_cvd_utils.UploadExtraImages.return_value = ["extra"] + mock_cvd_utils.ConvertRemoteLogs.return_value = [log] + + self.assertEqual("inst", factory.CreateInstance()) + mock_ssh.Ssh.assert_called_once() + mock_client_obj.InitRemoteHost.assert_called_once() + mock_cvd_utils.UploadArtifacts.assert_called_with( + mock.ANY, "/mock/img", "/mock/cvd.tar.gz") + mock_client_obj.LaunchCvd.assert_called_with( + "inst", + mock_avd_spec, + mock_avd_spec.cfg.extra_data_disk_size_gb, + boot_timeout_secs=mock_avd_spec.boot_timeout_secs, + extra_args=["extra"]) + mock_pull.GetAllLogFilePaths.assert_called_once() + mock_pull.PullLogs.assert_called_once() + self.assertEqual({"inst": "failure"}, factory.GetFailures()) + self.assertEqual({"inst": [tombstones, log]}, factory.GetLogs()) + + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_compute_client_multi_stage") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.ssh") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_utils") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.pull") + def testCreateInstanceWithImageZip(self, mock_pull, mock_cvd_utils, + mock_ssh, _mock_client): + """Test CreateInstance with local image zip.""" + mock_avd_spec = self._CreateMockAvdSpec() + mock_avd_spec.image_source = constants.IMAGE_SRC_LOCAL + factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( + mock_avd_spec, local_image_artifact="/mock/img.zip", + cvd_host_package_artifact="/mock/cvd.tar.gz") + + mock_client_obj = factory.GetComputeClient() + mock_client_obj.FormatRemoteHostInstanceName.return_value = "inst" + mock_client_obj.LaunchCvd.return_value = {} + + self.assertEqual("inst", factory.CreateInstance()) + mock_ssh.Ssh.assert_called_once() + mock_client_obj.InitRemoteHost.assert_called_once() + mock_cvd_utils.UploadArtifacts.assert_called_with( + mock.ANY, "/mock/img.zip", "/mock/cvd.tar.gz") + mock_client_obj.LaunchCvd.assert_called() + mock_pull.GetAllLogFilePaths.assert_called_once() + mock_pull.PullLogs.assert_not_called() + self.assertFalse(factory.GetFailures()) + self.assertEqual(1, len(factory.GetLogs()["inst"])) + + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_compute_client_multi_stage") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.ssh") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "cvd_utils") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory." + "subprocess.check_call") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.glob") + @mock.patch("acloud.public.actions.remote_host_cf_device_factory.pull") + def testCreateInstanceWithRemoteImages(self, mock_pull, mock_glob, + mock_check_call, _mock_cvd_utils, + mock_ssh, _mock_client): + """Test CreateInstance with remote images.""" + mock_avd_spec = self._CreateMockAvdSpec() + mock_avd_spec.image_source = constants.IMAGE_SRC_REMOTE + mock_ssh_obj = mock.Mock() + mock_ssh.Ssh.return_value = mock_ssh_obj + mock_ssh_obj.GetBaseCmd.return_value = "/mock/ssh" + mock_glob.glob.return_value = ["/mock/super.img"] + factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( + mock_avd_spec) + + mock_client_obj = factory.GetComputeClient() + mock_client_obj.FormatRemoteHostInstanceName.return_value = "inst" + mock_client_obj.LaunchCvd.return_value = {} + + self.assertEqual("inst", factory.CreateInstance()) + mock_ssh.Ssh.assert_called_once() + mock_client_obj.InitRemoteHost.assert_called_once() + mock_check_call.assert_called_once() + mock_ssh.ShellCmdWithRetry.assert_called_once() + self.assertRegex(mock_ssh.ShellCmdWithRetry.call_args[0][0], + r"^tar -cf - --lzop -S -C \S+ super\.img \| " + r"/mock/ssh -- tar -xf - --lzop -S$") + mock_client_obj.LaunchCvd.assert_called() + mock_pull.GetAllLogFilePaths.assert_called_once() + mock_pull.PullLogs.assert_not_called() + self.assertFalse(factory.GetFailures()) + self.assertEqual(1, len(factory.GetLogs()["inst"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/public/actions/remote_instance_cf_device_factory.py b/public/actions/remote_instance_cf_device_factory.py index dfa45c11..cf447743 100644 --- a/public/actions/remote_instance_cf_device_factory.py +++ b/public/actions/remote_instance_cf_device_factory.py @@ -15,27 +15,15 @@ """RemoteInstanceDeviceFactory provides basic interface to create a cuttlefish device factory.""" -import glob import logging -import os -import shutil -import subprocess -import tempfile -from acloud import errors from acloud.internal import constants -from acloud.internal.lib import utils -from acloud.internal.lib import ssh +from acloud.internal.lib import cvd_utils from acloud.public.actions import gce_device_factory +from acloud.pull import pull logger = logging.getLogger(__name__) -_ALL_FILES = "*" -# bootloader and kernel are files required to launch AVD. -_BOOTLOADER = "bootloader" -_KERNEL = "kernel" -_ARTIFACT_FILES = ["*.img", _BOOTLOADER, _KERNEL] -_HOME_FOLDER = os.path.expanduser("~") _SCREEN_CONSOLE_COMMAND = "screen ~/cuttlefish_runtime/console" @@ -56,155 +44,39 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): def __init__(self, avd_spec, local_image_artifact=None, cvd_host_package_artifact=None): super().__init__(avd_spec, local_image_artifact) + self._all_logs = {} self._cvd_host_package_artifact = cvd_host_package_artifact # pylint: disable=broad-except def CreateInstance(self): """Create a single configured cuttlefish device. - GCE: - 1. Create gcp instance. - 2. Upload local built artifacts to remote instance or fetch build on - remote instance. - 3. Launch CVD. - - Remote host: - 1. Init remote host. - 2. Download the artifacts to local and upload the artifacts to host - 3. Launch CVD. - - Returns: - A string, representing instance name. - """ - if self._avd_spec.instance_type == constants.INSTANCE_TYPE_HOST: - instance = self._InitRemotehost() - self._ProcessRemoteHostArtifacts() - self._LaunchCvd(instance=instance, - decompress_kernel=None, - boot_timeout_secs=self._avd_spec.boot_timeout_secs) - else: - instance = self._CreateGceInstance() - # If instance is failed, no need to go next step. - if instance in self.GetFailures(): - return instance - try: - self._ProcessArtifacts(self._avd_spec.image_source) - self._LaunchCvd(instance=instance, - boot_timeout_secs=self._avd_spec.boot_timeout_secs) - except Exception as e: - self._SetFailures(instance, e) - - return instance - - def _InitRemotehost(self): - """Initialize remote host. - - Determine the remote host instance name, and activate ssh. It need to - get the IP address in the common_operation. So need to pass the IP and - ssh to compute_client. - - build_target: The format is like "aosp_cf_x86_phone". We only get info - from the user build image file name. If the file name is - not custom format (no "-"), we will use $TARGET_PRODUCT - from environment variable as build_target. - Returns: A string, representing instance name. """ - image_name = os.path.basename( - self._local_image_artifact) if self._local_image_artifact else "" - build_target = (os.environ.get(constants.ENV_BUILD_TARGET) if "-" not - in image_name else image_name.split("-")[0]) - build_id = self._USER_BUILD - if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: - build_id = self._avd_spec.remote_image[constants.BUILD_ID] - - instance = self._compute_client.FormatRemoteHostInstanceName( - self._avd_spec.remote_host, build_id, build_target) - ip = ssh.IP(ip=self._avd_spec.remote_host) - self._ssh = ssh.Ssh( - ip=ip, - user=self._avd_spec.host_user, - ssh_private_key_path=(self._avd_spec.host_ssh_private_key_path or - self._cfg.ssh_private_key_path), - extra_args_ssh_tunnel=self._cfg.extra_args_ssh_tunnel, - report_internal_ip=self._report_internal_ip) - self._compute_client.InitRemoteHost( - self._ssh, ip, self._avd_spec.host_user) - return instance - - @utils.TimeExecute(function_description="Downloading Android Build artifact") - def _DownloadArtifacts(self, extract_path): - """Download the CF image artifacts and process them. - - - Download images from the Android Build system. - - Download cvd host package from the Android Build system. - - Args: - extract_path: String, a path include extracted files. - - Raises: - errors.GetRemoteImageError: Fails to download rom images. - """ - cfg = self._avd_spec.cfg - build_id = self._avd_spec.remote_image[constants.BUILD_ID] - build_branch = self._avd_spec.remote_image[constants.BUILD_BRANCH] - build_target = self._avd_spec.remote_image[constants.BUILD_TARGET] - - # Download images with fetch_cvd - fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) - self._compute_client.build_api.DownloadFetchcvd(fetch_cvd, - cfg.fetch_cvd_version) - fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( - build_id, build_branch, build_target, - self._avd_spec.system_build_info.get(constants.BUILD_ID), - self._avd_spec.system_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.system_build_info.get(constants.BUILD_TARGET), - self._avd_spec.kernel_build_info.get(constants.BUILD_ID), - self._avd_spec.kernel_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.kernel_build_info.get(constants.BUILD_TARGET), - self._avd_spec.bootloader_build_info.get(constants.BUILD_ID), - self._avd_spec.bootloader_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.bootloader_build_info.get(constants.BUILD_TARGET), - self._avd_spec.ota_build_info.get(constants.BUILD_ID), - self._avd_spec.ota_build_info.get(constants.BUILD_BRANCH), - self._avd_spec.ota_build_info.get(constants.BUILD_TARGET)) - creds_cache_file = os.path.join(_HOME_FOLDER, cfg.creds_cache_file) - fetch_cvd_cert_arg = self._compute_client.build_api.GetFetchCertArg( - creds_cache_file) - fetch_cvd_args = [fetch_cvd, "-directory=%s" % extract_path, - fetch_cvd_cert_arg] - fetch_cvd_args.extend(fetch_cvd_build_args) - logger.debug("Download images command: %s", fetch_cvd_args) + instance = self._CreateGceInstance() + # If instance is failed, no need to go next step. + if instance in self.GetFailures(): + return instance try: - subprocess.check_call(fetch_cvd_args) - except subprocess.CalledProcessError as e: - raise errors.GetRemoteImageError("Fails to download images: %s" % e) - - def _ProcessRemoteHostArtifacts(self): - """Process remote host artifacts. - - - If images source is local, tool will upload images from local site to - remote host. - - If images source is remote, tool will download images from android - build to local and unzip it then upload to remote host, because there - is no permission to fetch build rom on the remote host. - """ - self._compute_client.SetStage(constants.STAGE_ARTIFACT) - if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: - self._UploadLocalImageArtifacts( - self._local_image_artifact, self._cvd_host_package_artifact, - self._avd_spec.local_image_dir) - else: - try: - artifacts_path = tempfile.mkdtemp() - logger.debug("Extracted path of artifacts: %s", artifacts_path) - self._DownloadArtifacts(artifacts_path) - self._UploadRemoteImageArtifacts(artifacts_path) - finally: - shutil.rmtree(artifacts_path) + image_args = self._ProcessArtifacts() + failures = self._compute_client.LaunchCvd( + instance, + self._avd_spec, + self._cfg.extra_data_disk_size_gb, + boot_timeout_secs=self._avd_spec.boot_timeout_secs, + extra_args=image_args) + for failing_instance, error_msg in failures.items(): + self._SetFailures(failing_instance, error_msg) + except Exception as e: + self._SetFailures(instance, e) + + self._FindLogFiles( + instance, + instance in self.GetFailures() and not self._avd_spec.no_pull_log) + return instance - def _ProcessArtifacts(self, image_source): + def _ProcessArtifacts(self): """Process artifacts. - If images source is local, tool will upload images from local site to @@ -213,23 +85,25 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): build to remote instance. Before download images, we have to update fetch_cvd to remote instance. - Args: - image_source: String, the type of image source is remote or local. + Returns: + A list of strings, the launch_cvd arguments. """ - if image_source == constants.IMAGE_SRC_LOCAL: - self._UploadLocalImageArtifacts(self._local_image_artifact, - self._cvd_host_package_artifact, - self._avd_spec.local_image_dir) - elif image_source == constants.IMAGE_SRC_REMOTE: + if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: + cvd_utils.UploadArtifacts( + self._ssh, + self._local_image_artifact or self._avd_spec.local_image_dir, + self._cvd_host_package_artifact) + elif self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: self._compute_client.UpdateFetchCvd() self._FetchBuild(self._avd_spec) - if self._avd_spec.connect_webrtc: + if self._avd_spec.mkcert and self._avd_spec.connect_webrtc: self._compute_client.UpdateCertificate() if self._avd_spec.extra_files: self._compute_client.UploadExtraFiles(self._avd_spec.extra_files) + return cvd_utils.UploadExtraImages(self._ssh, self._avd_spec) def _FetchBuild(self, avd_spec): """Download CF artifacts from android build. @@ -254,94 +128,24 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): avd_spec.ota_build_info[constants.BUILD_BRANCH], avd_spec.ota_build_info[constants.BUILD_TARGET]) - @utils.TimeExecute(function_description="Processing and uploading local images") - def _UploadLocalImageArtifacts(self, - local_image_zip, - cvd_host_package_artifact, - images_dir): - """Upload local images and avd local host package to instance. - - There are two ways to upload local images. - 1. Using local image zip, it would be decompressed by install_zip.sh. - 2. Using local image directory, this directory contains all images. - Images are compressed/decompressed by lzop during upload process. - - Args: - local_image_zip: String, path to zip of local images which - build from 'm dist'. - cvd_host_package_artifact: String, path to cvd host package. - images_dir: String, directory of local images which build - from 'm'. - """ - if local_image_zip: - remote_cmd = ("/usr/bin/install_zip.sh . < %s" % local_image_zip) - logger.debug("remote_cmd:\n %s", remote_cmd) - self._ssh.Run(remote_cmd) - else: - # Compress image files for faster upload. - try: - images_path = os.path.join(images_dir, "required_images") - with open(images_path, "r") as images: - artifact_files = images.read().splitlines() - except IOError: - # Older builds may not have a required_images file. In this case - # we fall back to *.img. - artifact_files = [] - for file_name in _ARTIFACT_FILES: - artifact_files.extend( - os.path.basename(image) for image in glob.glob( - os.path.join(images_dir, file_name))) - # Upload android-info.txt to parse config value. - artifact_files.append(constants.ANDROID_INFO_FILE) - cmd = ("tar -cf - --lzop -S -C {images_dir} {artifact_files} | " - "{ssh_cmd} -- tar -xf - --lzop -S".format( - images_dir=images_dir, - artifact_files=" ".join(artifact_files), - ssh_cmd=self._ssh.GetBaseCmd(constants.SSH_BIN))) - logger.debug("cmd:\n %s", cmd) - ssh.ShellCmdWithRetry(cmd) - - # host_package - remote_cmd = ("tar -x -z -f - < %s" % cvd_host_package_artifact) - logger.debug("remote_cmd:\n %s", remote_cmd) - self._ssh.Run(remote_cmd) - - @utils.TimeExecute(function_description="Uploading remote image artifacts") - def _UploadRemoteImageArtifacts(self, images_dir): - """Upload remote image artifacts to instance. - - Args: - images_dir: String, directory of local artifacts downloaded by fetch_cvd. - """ - artifact_files = [ - os.path.basename(image) - for image in glob.glob(os.path.join(images_dir, _ALL_FILES)) - ] - # TODO(b/182259589): Refactor upload image command into a function. - cmd = ("tar -cf - --lzop -S -C {images_dir} {artifact_files} | " - "{ssh_cmd} -- tar -xf - --lzop -S".format( - images_dir=images_dir, - artifact_files=" ".join(artifact_files), - ssh_cmd=self._ssh.GetBaseCmd(constants.SSH_BIN))) - logger.debug("cmd:\n %s", cmd) - ssh.ShellCmdWithRetry(cmd) - - def _LaunchCvd(self, instance, decompress_kernel=None, - boot_timeout_secs=None): - """Launch CVD. + def _FindLogFiles(self, instance, download): + """Find and pull all log files from instance. Args: instance: String, instance name. - boot_timeout_secs: Integer, the maximum time to wait for the - command to respond. + download: Whether to download the files to a temporary directory + and show messages to the user. """ - # TODO(b/140076771) Support kernel image for local image mode. - self._compute_client.LaunchCvd( - instance, - self._avd_spec, - self._cfg.extra_data_disk_size_gb, - decompress_kernel, - boot_timeout_secs) + self._all_logs[instance] = [cvd_utils.TOMBSTONES, + cvd_utils.HOST_KERNEL_LOG] + if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: + self._all_logs[instance].append(cvd_utils.FETCHER_CONFIG_JSON) + log_files = pull.GetAllLogFilePaths(self._ssh) + self._all_logs[instance].extend(cvd_utils.ConvertRemoteLogs(log_files)) + if download: + error_log_folder = pull.PullLogs(self._ssh, log_files, instance) + self._compute_client.ExtendReportData(constants.ERROR_LOG_FOLDER, + error_log_folder) def GetOpenWrtInfoDict(self): """Get openwrt info dictionary. @@ -362,26 +166,7 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): """ if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: return None - build_info_dict = { - key: val for key, val in self._avd_spec.remote_image.items() if val} - - # kernel_target have default value "kernel". If user provide kernel_build_id - # or kernel_branch, then start to process kernel image. - if (self._avd_spec.kernel_build_info[constants.BUILD_ID] - or self._avd_spec.kernel_build_info[constants.BUILD_BRANCH]): - build_info_dict.update( - {"kernel_%s" % key: val - for key, val in self._avd_spec.kernel_build_info.items() if val} - ) - build_info_dict.update( - {"system_%s" % key: val - for key, val in self._avd_spec.system_build_info.items() if val} - ) - build_info_dict.update( - {"bootloader_%s" % key: val - for key, val in self._avd_spec.bootloader_build_info.items() if val} - ) - return build_info_dict + return cvd_utils.GetRemoteBuildInfoDict(self._avd_spec) def GetLogs(self): """Get all device logs. @@ -389,4 +174,4 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): Returns: A dictionary that maps instance names to lists of report.LogFile. """ - return self._compute_client.all_logs + return self._all_logs diff --git a/public/actions/remote_instance_cf_device_factory_test.py b/public/actions/remote_instance_cf_device_factory_test.py index 1aecf0f1..f0c52534 100644 --- a/public/actions/remote_instance_cf_device_factory_test.py +++ b/public/actions/remote_instance_cf_device_factory_test.py @@ -15,23 +15,17 @@ import glob import os -import shutil -import subprocess -import tempfile import unittest import uuid from unittest import mock -import six - from acloud.create import avd_spec from acloud.internal import constants from acloud.internal.lib import android_build_client from acloud.internal.lib import auth from acloud.internal.lib import cvd_compute_client_multi_stage from acloud.internal.lib import driver_test_lib -from acloud.internal.lib import ssh from acloud.internal.lib import utils from acloud.list import list as list_instances from acloud.public.actions import remote_instance_cf_device_factory @@ -46,6 +40,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.Patch(auth, "CreateCredentials", return_value=mock.MagicMock()) self.Patch(android_build_client.AndroidBuildClient, "InitResourceHandle") self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "InitResourceHandle") + self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "LaunchCvd") self.Patch(list_instances, "GetInstancesFromInstanceNames", return_value=mock.MagicMock()) self.Patch(list_instances, "ChooseOneRemoteInstance", return_value=mock.MagicMock()) self.Patch(utils, "GetBuildEnvironmentVariable", @@ -57,9 +52,10 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): "UpdateCertificate") @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, "_FetchBuild") - @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_UploadLocalImageArtifacts") - def testProcessArtifacts(self, mock_upload, mock_download, mock_uploadca): + @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." + "cvd_utils") + def testProcessArtifacts(self, mock_cvd_utils, mock_download, + mock_uploadca): """test ProcessArtifacts.""" # Test image source type is local. args = mock.MagicMock() @@ -77,11 +73,13 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): avd_spec_local_img, fake_image_name, fake_host_package_name) - factory_local_img._ProcessArtifacts(constants.IMAGE_SRC_LOCAL) - self.assertEqual(mock_upload.call_count, 1) + factory_local_img._ProcessArtifacts() # cf default autoconnect webrtc and should upload certificates - self.assertEqual(mock_uploadca.call_count, 1) + mock_uploadca.assert_called_once() mock_uploadca.reset_mock() + mock_cvd_utils.UploadArtifacts.assert_called_once_with( + mock.ANY, fake_image_name, fake_host_package_name) + mock_cvd_utils.UploadExtraImages.assert_called_once() # given autoconnect to vnc should not upload certificates args.autoconnect = constants.INS_KEY_VNC @@ -90,8 +88,8 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): avd_spec_local_img, fake_image_name, fake_host_package_name) - factory_local_img._ProcessArtifacts(constants.IMAGE_SRC_LOCAL) - self.assertEqual(mock_uploadca.call_count, 0) + factory_local_img._ProcessArtifacts() + mock_uploadca.assert_not_called() # Test image source type is remote. args.local_image = None @@ -108,8 +106,8 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "UpdateFetchCvd") factory_remote_img = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( avd_spec_remote_img) - factory_remote_img._ProcessArtifacts(constants.IMAGE_SRC_REMOTE) - self.assertEqual(mock_download.call_count, 1) + factory_remote_img._ProcessArtifacts() + mock_download.assert_called_once() # pylint: disable=protected-access @mock.patch.dict(os.environ, {constants.ENV_BUILD_TARGET:'fake-target'}) @@ -158,42 +156,6 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_host_package_name) self.assertEqual(factory._CreateGceInstance(), "ins-1234-userbuild-fake-target") - @mock.patch.dict(os.environ, {constants.ENV_BUILD_TARGET:'fake-target'}) - def testRemoteHostInstanceName(self): - """Test Remote host instance name.""" - args = mock.MagicMock() - args.config_file = "" - args.avd_type = constants.TYPE_CF - args.flavor = "phone" - args.remote_host = "1.1.1.1" - args.local_image = constants.FIND_IN_BUILD_ENV - args.local_system_image = None - args.adb_port = None - args.launch_args = None - fake_avd_spec = avd_spec.AVDSpec(args) - fake_avd_spec.cfg.enable_multi_stage = True - fake_avd_spec._instance_name_to_reuse = None - fake_uuid = mock.MagicMock(hex="1234") - self.Patch(uuid, "uuid4", return_value=fake_uuid) - self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "CreateInstance") - self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "InitRemoteHost") - fake_host_package_name = "/fake/host_package.tar.gz" - - fake_image_name = "/fake/aosp_cf_x86_phone-img-eng.username.zip" - factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( - fake_avd_spec, - fake_image_name, - fake_host_package_name) - self.assertEqual(factory._InitRemotehost(), "host-1.1.1.1-userbuild-aosp_cf_x86_phone") - - # No image zip path, it uses local build images. - fake_image_name = "" - factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( - fake_avd_spec, - fake_image_name, - fake_host_package_name) - self.assertEqual(factory._InitRemotehost(), "host-1.1.1.1-userbuild-fake-target") - def testReuseInstanceNameMultiStage(self): """Test reuse instance name.""" args = mock.MagicMock() @@ -220,7 +182,9 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_host_package_name) self.assertEqual(factory._CreateGceInstance(), "fake-1234-userbuild-fake-target") - def testGetBuildInfoDict(self): + @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." + "cvd_utils") + def testGetBuildInfoDict(self, mock_cvd_utils): """Test GetBuildInfoDict.""" fake_host_package_name = "/fake/host_package.tar.gz" fake_image_name = "/fake/aosp_cf_x86_phone-img-eng.username.zip" @@ -240,263 +204,95 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_image_name, fake_host_package_name) self.assertEqual(factory.GetBuildInfoDict(), None) + mock_cvd_utils.assert_not_called() # Test image source type is remote. args.local_image = None args.build_id = "123" args.branch = "fake_branch" args.build_target = "fake_target" - args.system_build_id = "234" - args.system_branch = "sys_branch" - args.system_build_target = "sys_target" - args.kernel_build_id = "345" - args.kernel_branch = "kernel_branch" - args.kernel_build_target = "kernel_target" - args.kernel_artifact = None - args.bootloader_build_id = "456" - args.bootloader_branch = "bootloader_branch" - args.bootloader_build_target = "bootloader_target" avd_spec_remote_image = avd_spec.AVDSpec(args) factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( avd_spec_remote_image, fake_image_name, fake_host_package_name) - expected_build_info = { - "build_id": "123", - "branch": "fake_branch", - "build_target": "fake_target", - "system_build_id": "234", - "system_branch": "sys_branch", - "system_build_target": "sys_target", - "kernel_build_id": "345", - "kernel_branch": "kernel_branch", - "kernel_build_target": "kernel_target", - "bootloader_build_id": "456", - "bootloader_branch": "bootloader_branch", - "bootloader_build_target": "bootloader_target" - } - self.assertEqual(factory.GetBuildInfoDict(), expected_build_info) - - @mock.patch.object(ssh, "ShellCmdWithRetry") - @mock.patch.object(ssh.Ssh, "Run") - def testUploadArtifacts(self, mock_ssh_run, mock_shell): - """Test UploadArtifacts.""" - fake_host_package = "/fake/host_package.tar.gz" - fake_image = "/fake/aosp_cf_x86_phone-img-eng.username.zip" - fake_local_image_dir = "/fake_image" - fake_ip = ssh.IP(external="1.1.1.1", internal="10.1.1.1") - args = mock.MagicMock() - # Test local image extract from image zip case. - args.config_file = "" - args.avd_type = constants.TYPE_CF - args.flavor = "phone" - args.local_image = "fake_local_image" - args.local_system_image = None - args.adb_port = None - args.launch_args = None - avd_spec_local_image = avd_spec.AVDSpec(args) - factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( - avd_spec_local_image, - fake_image, - fake_host_package) - factory._ssh = ssh.Ssh(ip=fake_ip, - user=constants.GCE_USER, - ssh_private_key_path="/fake/acloud_rea") - factory._UploadLocalImageArtifacts(fake_image, - fake_host_package, - fake_local_image_dir) - expected_cmd1 = ("/usr/bin/install_zip.sh . < %s" % fake_image) - expected_cmd2 = ("tar -x -z -f - < %s" % fake_host_package) - mock_ssh_run.assert_has_calls([ - mock.call(expected_cmd1), - mock.call(expected_cmd2)]) - - # Test local image get from local folder case. - fake_image = None - self.Patch(glob, "glob", side_effect=[["fake.img"], ["bootloader"], ["kernel"]]) - factory._UploadLocalImageArtifacts(fake_image, - fake_host_package, - fake_local_image_dir) - expected_cmd = ( - "tar -cf - --lzop -S -C %s fake.img bootloader kernel android-info.txt | " - "%s -- tar -xf - --lzop -S" % - (fake_local_image_dir, factory._ssh.GetBaseCmd(constants.SSH_BIN))) - mock_shell.assert_called_once_with(expected_cmd) - - mock_shell.reset_mock() - required_images = mock.mock_open(read_data=( - "boot.img\n" - "cache.img\n" - "super.img\n" - "userdata.img\n" - "bootloader\n")) - with mock.patch.object(six.moves.builtins, "open", required_images): - factory._UploadLocalImageArtifacts(fake_image, - fake_host_package, - fake_local_image_dir) - expected_cmd = ( - "tar -cf - --lzop -S -C %s boot.img cache.img super.img userdata.img " - "bootloader android-info.txt | %s -- tar -xf - --lzop -S" % - (fake_local_image_dir, factory._ssh.GetBaseCmd(constants.SSH_BIN))) - mock_shell.assert_called_once_with(expected_cmd) - - @mock.patch.object(ssh, "ShellCmdWithRetry") - def testUploadRemoteImageArtifacts(self, mock_shell): - """Test UploadRemoteImageArtifacts.""" - fake_host_package = "/fake/host_package.tar.gz" - fake_image_zip = None - fake_local_image_dir = "/fake_image" - fake_ip = ssh.IP(external="1.1.1.1", internal="10.1.1.1") - args = mock.MagicMock() - # Test local image extract from image zip case. - args.config_file = "" - args.avd_type = constants.TYPE_CF - args.flavor = "phone" - args.local_image = "fake_local_image" - args.local_system_image = None - args.adb_port = None - args.launch_args = None - avd_spec_local_image = avd_spec.AVDSpec(args) - factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( - avd_spec_local_image, - fake_image_zip, - fake_host_package) - factory._ssh = ssh.Ssh(ip=fake_ip, - user=constants.GCE_USER, - ssh_private_key_path="/fake/acloud_rea") - - self.Patch(glob, "glob", return_value=["fake.img", "bootloader", "kernel"]) - factory._UploadRemoteImageArtifacts(fake_local_image_dir) - - expected_cmd = ( - "tar -cf - --lzop -S -C %s fake.img bootloader kernel | " - "%s -- tar -xf - --lzop -S" % - (fake_local_image_dir, factory._ssh.GetBaseCmd(constants.SSH_BIN))) - mock_shell.assert_called_once_with(expected_cmd) - - @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_InitRemotehost") - @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_UploadLocalImageArtifacts") - @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_LaunchCvd") - def testLocalImageRemoteHost(self, mock_launchcvd, mock_upload, mock_init_remote_host): - """Test local image with remote host.""" - self.Patch( - cvd_compute_client_multi_stage, - "CvdComputeClient", - return_value=mock.MagicMock()) - fake_avd_spec = mock.MagicMock() - fake_avd_spec.instance_type = constants.INSTANCE_TYPE_HOST - fake_avd_spec.image_source = constants.IMAGE_SRC_LOCAL - fake_avd_spec._instance_name_to_reuse = None - fake_host_package_name = "/fake/host_package.tar.gz" - fake_image_name = "" - factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( - fake_avd_spec, - fake_image_name, - fake_host_package_name) - factory.CreateInstance() - self.assertEqual(mock_init_remote_host.call_count, 1) - self.assertEqual(mock_upload.call_count, 1) - self.assertEqual(mock_launchcvd.call_count, 1) + factory.GetBuildInfoDict() + mock_cvd_utils.GetRemoteBuildInfoDict.assert_called() @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, "_CreateGceInstance") - @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_UploadLocalImageArtifacts") - @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_LaunchCvd") - def testLocalImageCreateInstance(self, mock_launchcvd, mock_upload, mock_create_gce_instance): - """Test local image with create instance.""" + @mock.patch("acloud.public.actions.remote_instance_cf_device_factory.pull") + @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." + "cvd_utils") + def testLocalImageCreateInstance(self, mock_cvd_utils, mock_pull, + mock_create_gce_instance): + """Test CreateInstance with local images.""" self.Patch( cvd_compute_client_multi_stage, "CvdComputeClient", return_value=mock.MagicMock()) + mock_create_gce_instance.return_value = "instance" fake_avd_spec = mock.MagicMock() - fake_avd_spec.instance_type = constants.INSTANCE_TYPE_REMOTE fake_avd_spec.image_source = constants.IMAGE_SRC_LOCAL fake_avd_spec._instance_name_to_reuse = None + fake_avd_spec.no_pull_log = False + + mock_cvd_utils.ConvertRemoteLogs.return_value = [{"path": "/logcat"}] + mock_cvd_utils.UploadExtraImages.return_value = [ + "-boot_image", "/boot/img"] + fake_host_package_name = "/fake/host_package.tar.gz" fake_image_name = "" factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( fake_avd_spec, fake_image_name, fake_host_package_name) + compute_client = factory.GetComputeClient() + compute_client.LaunchCvd.return_value = {"instance": "failure"} factory.CreateInstance() - self.assertEqual(mock_create_gce_instance.call_count, 1) - self.assertEqual(mock_upload.call_count, 1) - self.assertEqual(mock_launchcvd.call_count, 1) - - # pylint: disable=no-member - @mock.patch.object(subprocess, "check_call") - def testDownloadArtifacts(self, mock_check_call): - """Test process remote cuttlefish image.""" - extract_path = "/tmp/1111/" - fake_remote_image = {"build_target" : "aosp_cf_x86_phone-userdebug", - "branch" : "aosp-master", - "build_id": "1234"} - self.Patch( - cvd_compute_client_multi_stage, - "CvdComputeClient", - return_value=mock.MagicMock()) - self.Patch(tempfile, "mkdtemp", return_value="/tmp/1111/") - self.Patch(shutil, "rmtree") - fake_avd_spec = mock.MagicMock() - fake_avd_spec.cfg = mock.MagicMock() - fake_avd_spec.cfg.creds_cache_file = "cache.file" - fake_avd_spec.remote_image = fake_remote_image - fake_avd_spec.image_download_dir = "/tmp" - self.Patch(os.path, "exists", return_value=False) - self.Patch(os, "makedirs") - factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( - fake_avd_spec) + mock_create_gce_instance.assert_called_once() + mock_cvd_utils.UploadArtifacts.assert_called_once() + compute_client.LaunchCvd.assert_called_once() + self.assertEqual( + ["-boot_image", "/boot/img"], + compute_client.LaunchCvd.call_args[1].get("extra_args")) + mock_pull.GetAllLogFilePaths.assert_called_once() + mock_pull.PullLogs.assert_called_once() + self.assertEqual({"instance": "failure"}, factory.GetFailures()) + self.assertEqual(3, len(factory.GetLogs().get("instance"))) - factory._DownloadArtifacts(extract_path) - self.assertEqual(mock_check_call.call_count, 1) - - @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_UploadLocalImageArtifacts") @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_UploadRemoteImageArtifacts") - def testProcessRemoteHostArtifacts(self, - mock_upload_remote_image, - mock_upload_local_image): - """Test process remote host artifacts.""" + "_CreateGceInstance") + @mock.patch("acloud.public.actions.remote_instance_cf_device_factory.pull") + @mock.patch("acloud.public.actions.remote_instance_cf_device_factory." + "cvd_utils") + def testRemoteImageCreateInstance(self, mock_cvd_utils, mock_pull, + mock_create_gce_instance): + """Test CreateInstance with remote images.""" self.Patch( cvd_compute_client_multi_stage, "CvdComputeClient", return_value=mock.MagicMock()) + mock_create_gce_instance.return_value = "instance" fake_avd_spec = mock.MagicMock() + fake_avd_spec.image_source = constants.IMAGE_SRC_REMOTE + fake_avd_spec.host_user = None + fake_avd_spec.no_pull_log = True - # Test process remote host artifacts with local images. - fake_avd_spec.instance_type = constants.INSTANCE_TYPE_HOST - fake_avd_spec.image_source = constants.IMAGE_SRC_LOCAL - fake_avd_spec._instance_name_to_reuse = None - fake_avd_spec.cfg = mock.MagicMock() - fake_avd_spec.cfg.creds_cache_file = "cache.file" - fake_host_package_name = "/fake/host_package.tar.gz" - fake_image_name = "" - factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( - fake_avd_spec, - fake_image_name, - fake_host_package_name) - factory._ProcessRemoteHostArtifacts() - self.assertEqual(mock_upload_local_image.call_count, 1) + mock_cvd_utils.ConvertRemoteLogs.return_value = [{"path": "/logcat"}] + mock_cvd_utils.UploadExtraImages.return_value = [] - # Test process remote host artifacts with remote images. - fake_tmp_folder = "/tmp/1111/" - self.Patch(tempfile, "mkdtemp", return_value=fake_tmp_folder) - self.Patch(shutil, "rmtree") - self.Patch(subprocess, "check_call") - fake_avd_spec.instance_type = constants.INSTANCE_TYPE_HOST - fake_avd_spec.image_source = constants.IMAGE_SRC_REMOTE - fake_avd_spec._instance_name_to_reuse = None factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( fake_avd_spec) - factory._ProcessRemoteHostArtifacts() - self.assertEqual(mock_upload_remote_image.call_count, 1) - shutil.rmtree.assert_called_once_with(fake_tmp_folder) + compute_client = factory.GetComputeClient() + compute_client.LaunchCvd.return_value = {} + factory.CreateInstance() + + compute_client.FetchBuild.assert_called_once() + mock_pull.GetAllLogFilePaths.assert_called_once() + mock_pull.PullLogs.assert_not_called() + self.assertFalse(factory.GetFailures()) + self.assertEqual(4, len(factory.GetLogs().get("instance"))) def testGetOpenWrtInfoDict(self): """Test GetOpenWrtInfoDict.""" diff --git a/public/actions/remote_instance_fvp_device_factory_test.py b/public/actions/remote_instance_fvp_device_factory_test.py index 74330733..862c7c03 100644 --- a/public/actions/remote_instance_fvp_device_factory_test.py +++ b/public/actions/remote_instance_fvp_device_factory_test.py @@ -19,8 +19,6 @@ import unittest from unittest import mock -import six - from acloud.create import avd_spec from acloud.internal import constants from acloud.internal.lib import android_build_client @@ -80,7 +78,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): "fip.bin\n" "system-qemu.img\n" "userdata.img\n")) - with mock.patch.object(six.moves.builtins, "open", mock_open): + with mock.patch("builtins.open", mock_open): factory.CreateInstance() mock_create_gce.assert_called_once() diff --git a/public/config.py b/public/config.py index a8b6dc86..7c976e2d 100755 --- a/public/config.py +++ b/public/config.py @@ -46,8 +46,6 @@ TODO: import logging import os -import six - from google.protobuf import text_format # pylint: disable=no-name-in-module,import-error @@ -163,15 +161,15 @@ class AcloudConfig(): self.ssh_public_key_path = usr_cfg.ssh_public_key_path self.storage_bucket_name = usr_cfg.storage_bucket_name self.metadata_variable = dict( - six.iteritems(internal_cfg.default_usr_cfg.metadata_variable)) + internal_cfg.default_usr_cfg.metadata_variable.items()) self.metadata_variable.update(usr_cfg.metadata_variable) self.device_resolution_map = dict( - six.iteritems(internal_cfg.device_resolution_map)) + internal_cfg.device_resolution_map.items()) self.device_default_orientation_map = dict( - six.iteritems(internal_cfg.device_default_orientation_map)) + internal_cfg.device_default_orientation_map.items()) self.no_project_access_msg_map = dict( - six.iteritems(internal_cfg.no_project_access_msg_map)) + internal_cfg.no_project_access_msg_map.items()) self.min_machine_size = internal_cfg.min_machine_size self.disk_image_name = internal_cfg.disk_image_name self.disk_image_mime_type = internal_cfg.disk_image_mime_type @@ -179,9 +177,9 @@ class AcloudConfig(): self.disk_raw_image_name = internal_cfg.disk_raw_image_name self.disk_raw_image_extension = internal_cfg.disk_raw_image_extension self.valid_branch_and_min_build_id = dict( - six.iteritems(internal_cfg.valid_branch_and_min_build_id)) + internal_cfg.valid_branch_and_min_build_id.items()) self.precreated_data_image_map = dict( - six.iteritems(internal_cfg.precreated_data_image)) + internal_cfg.precreated_data_image.items()) self.extra_data_disk_size_gb = ( usr_cfg.extra_data_disk_size_gb or internal_cfg.default_usr_cfg.extra_data_disk_size_gb) @@ -236,8 +234,6 @@ class AcloudConfig(): self.hw_property = usr_cfg.hw_property self.launch_args = usr_cfg.launch_args - self.api_key = usr_cfg.api_key - self.api_url = usr_cfg.api_url self.oxygen_client = usr_cfg.oxygen_client self.oxygen_lease_args = usr_cfg.oxygen_lease_args self.instance_name_pattern = ( diff --git a/public/config_test.py b/public/config_test.py index 28cfbdb2..694ed6b7 100644 --- a/public/config_test.py +++ b/public/config_test.py @@ -21,8 +21,6 @@ import tempfile from unittest import mock -import six - # pylint: disable=no-name-in-module,import-error from acloud import errors from acloud.internal.proto import internal_config_pb2 @@ -152,7 +150,7 @@ common_hw_property_map { self.assertEqual(cfg.client_secret, "fake_client_secret") self.assertEqual(cfg.extra_args_ssh_tunnel, "fake_extra_args_ssh_tunnel") self.assertEqual( - dict(six.iteritems(cfg.metadata_variable)), + dict(cfg.metadata_variable.items()), {"metadata_1": "metadata_value_1"}) self.assertEqual(cfg.hw_property, "cpu:3,resolution:1080x1920,dpi:480,memory:4g," @@ -224,17 +222,17 @@ common_hw_property_map { self.assertEqual(cfg.default_usr_cfg.machine_type, "n1-standard-1") self.assertEqual(cfg.default_usr_cfg.network, "default") self.assertEqual( - dict(six.iteritems(cfg.default_usr_cfg.metadata_variable)), + dict(cfg.default_usr_cfg.metadata_variable.items()), {"metadata_1": "metadata_value_1", "metadata_2": "metadata_value_2"}) self.assertEqual( - dict(six.iteritems(cfg.device_resolution_map)), + dict(cfg.device_resolution_map.items()), {"nexus5": "1080x1920x32x480"}) self.assertEqual( - dict(six.iteritems(cfg.device_default_orientation_map)), + dict(cfg.device_default_orientation_map.items()), {"nexus5": "portrait"}) self.assertEqual( - dict(six.iteritems(cfg.valid_branch_and_min_build_id)), + dict(cfg.valid_branch_and_min_build_id.items()), {"aosp-master": 0}) self.assertEqual(cfg.default_usr_cfg.stable_host_image_name, "fake_stable_host_image_name") @@ -259,7 +257,7 @@ common_hw_property_map { # hw property self.assertEqual( - dict(six.iteritems(cfg.common_hw_property_map)), + dict(cfg.common_hw_property_map.items()), {"phone": "cpu:2,resolution:1080x1920,dpi:420,memory:4g,disk:8g", "auto": "cpu:4,resolution:1280x800,dpi:160,memory:4g"}) diff --git a/pull/pull.py b/pull/pull.py index 7f118da4..e5eec5c4 100644 --- a/pull/pull.py +++ b/pull/pull.py @@ -34,7 +34,10 @@ from acloud.public import report logger = logging.getLogger(__name__) -_FIND_LOG_FILE_CMD = "find -L %s -type f" % constants.REMOTE_LOG_FOLDER +# REMOTE_LOG_FOLDER and the log files can be symbolic links. The -H flag makes +# the command skip the links except REMOTE_LOG_FOLDER. The returned logs are +# unique. +_FIND_LOG_FILE_CMD = "find -H %s -type f" % constants.REMOTE_LOG_FOLDER # Black list for log files. _KERNEL = "kernel" _IMG_FILE_EXTENSION = ".img" @@ -61,25 +64,30 @@ def PullFileFromInstance(cfg, instance, file_name=None, no_prompts=False): ssh_private_key_path=cfg.ssh_private_key_path, extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) log_files = SelectLogFileToPull(ssh, file_name) - download_folder = GetDownloadLogFolder(instance.name) - PullLogs(ssh, log_files, download_folder) + PullLogs(ssh, log_files, instance.name) if len(log_files) == 1: DisplayLog(ssh, log_files[0], no_prompts) return report.Report(command="pull") -def PullLogs(ssh, log_files, download_folder): +def PullLogs(ssh, log_files, instance_name): """Pull log files from remote instance. Args: ssh: Ssh object. log_files: List of file path in the remote instance. - download_folder: String of download folder path. + instance_name: The instance name that is used to create the download + folder. + + Returns: + The download folder path. """ + download_folder = _GetDownloadLogFolder(instance_name) for log_file in log_files: target_file = os.path.join(download_folder, os.path.basename(log_file)) ssh.ScpPullFile(log_file, target_file) _DisplayPullResult(download_folder) + return download_folder def DisplayLog(ssh, log_file, no_prompts=False): @@ -108,7 +116,7 @@ def _DisplayPullResult(download_folder): "AVD issues." % download_folder) -def GetDownloadLogFolder(instance): +def _GetDownloadLogFolder(instance): """Get the download log folder accroding to instance name. Args: diff --git a/pull/pull_test.py b/pull/pull_test.py index b5e7e77b..f1f89f78 100644 --- a/pull/pull_test.py +++ b/pull/pull_test.py @@ -43,7 +43,6 @@ class PullTest(driver_test_lib.BaseDriverTest): # Multiple selected files case. selected_files = ["file1.log", "file2.log"] self.Patch(pull, "SelectLogFileToPull", return_value=selected_files) - self.Patch(pull, "GetDownloadLogFolder", return_value="fake_folder") self.Patch(pull, "PullLogs") self.Patch(pull, "DisplayLog") pull.PullFileFromInstance(cfg, instance) @@ -55,14 +54,18 @@ class PullTest(driver_test_lib.BaseDriverTest): pull.PullFileFromInstance(cfg, instance) self.assertEqual(pull.DisplayLog.call_count, 1) - # pylint: disable=no-member def testPullLogs(self): """test PullLogs.""" + self.Patch(tempfile, "gettempdir", return_value="/tmp") + self.Patch(os.path, "exists", return_value=False) + mock_makedirs = self.Patch(os, "makedirs") _ssh = mock.MagicMock() self.Patch(utils, "PrintColorString") + log_files = ["file1.log", "file2.log"] - download_folder = "/fake_folder" - pull.PullLogs(_ssh, log_files, download_folder) + download_folder = pull.PullLogs(_ssh, log_files, "instance") + self.assertEqual(download_folder, "/tmp/instance") + mock_makedirs.assert_called_once_with("/tmp/instance") self.assertEqual(_ssh.ScpPullFile.call_count, 2) utils.PrintColorString.assert_called_once() @@ -80,14 +83,6 @@ class PullTest(driver_test_lib.BaseDriverTest): mock_ssh_run.assert_has_calls([ mock.call(expected_cmd, show_output=True)]) - def testGetDownloadLogFolder(self): - """test GetDownloadLogFolder.""" - self.Patch(tempfile, "gettempdir", return_value="/tmp") - self.Patch(os.path, "exists", return_value=True) - instance = "instance" - expected_path = "/tmp/instance" - self.assertEqual(pull.GetDownloadLogFolder(instance), expected_path) - def testSelectLogFileToPull(self): """test choose log files from the remote instance.""" _ssh = mock.MagicMock() @@ -112,16 +107,16 @@ class PullTest(driver_test_lib.BaseDriverTest): self.assertEqual(pull.SelectLogFileToPull(_ssh), expected_result) # Test user provided file name exist. - log_files = ["/home/vsoc-01/cuttlefish_runtime/file1.log", - "/home/vsoc-01/cuttlefish_runtime/file2.log"] + log_files = ["cuttlefish_runtime/file1.log", + "cuttlefish_runtime/file2.log"] input_file = "file1.log" self.Patch(pull, "GetAllLogFilePaths", return_value=log_files) - expected_result = ["/home/vsoc-01/cuttlefish_runtime/file1.log"] + expected_result = ["cuttlefish_runtime/file1.log"] self.assertEqual(pull.SelectLogFileToPull(_ssh, input_file), expected_result) # Test user provided file name not exist. - log_files = ["/home/vsoc-01/cuttlefish_runtime/file1.log", - "/home/vsoc-01/cuttlefish_runtime/file2.log"] + log_files = ["cuttlefish_runtime/file1.log", + "cuttlefish_runtime/file2.log"] input_file = "not_exist.log" self.Patch(pull, "GetAllLogFilePaths", return_value=log_files) with self.assertRaises(errors.CheckPathError): diff --git a/setup/mkcert.py b/setup/mkcert.py index ded7556f..cc8065cd 100644 --- a/setup/mkcert.py +++ b/setup/mkcert.py @@ -22,6 +22,7 @@ import logging import os import platform import shutil +import stat from acloud.internal import constants from acloud.internal.lib import utils @@ -90,6 +91,9 @@ def Install(): UnInstall() utils.Popen(_CA_CMD, shell=True) + # The rootCA.pem file should grant READ permission to others. + if not os.stat(_CA_CRT_PATH).st_mode & stat.S_IROTH: + os.chmod(_CA_CRT_PATH, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) utils.Popen(_TRUST_CA_COPY_CMD, shell=True) utils.Popen(_UPDATE_TRUST_CA_CMD, shell=True) utils.Popen(_TRUST_CHROME_CMD, shell=True) @@ -130,6 +134,9 @@ def IsRootCAReady(): logger.debug("Root SSL Certificate: %s, does not exist", cert_file_name) return False + # TODO: this check can be delete when the mkcert mechanism is stable. + if not os.stat(_TRUST_CA_PATH).st_mode & stat.S_IROTH: + return False if not filecmp.cmp(_CA_CRT_PATH, _TRUST_CA_PATH): logger.debug("The trusted CA %s file is not the same with %s ", diff --git a/setup/mkcert_test.py b/setup/mkcert_test.py index 98872cf8..04898788 100644 --- a/setup/mkcert_test.py +++ b/setup/mkcert_test.py @@ -35,7 +35,11 @@ class MkcertTest(driver_test_lib.BaseDriverTest): self.Patch(mkcert, "UnInstall") self.Patch(utils, "Popen") self.Patch(shutil, "rmtree") + self.Patch(os, "stat") + self.Patch(os, "chmod") + os.stat().st_mode = 33188 mkcert.Install() + os.chmod.assert_not_called() shutil.rmtree.assert_not_called() mkcert.UnInstall.assert_not_called() self.assertEqual(4, utils.Popen.call_count) @@ -43,7 +47,9 @@ class MkcertTest(driver_test_lib.BaseDriverTest): self.Patch(os.path, "isdir", return_value=True) self.Patch(os.path, "exists", return_value=True) + os.stat().st_mode = 33184 mkcert.Install() + os.chmod.assert_called_once() shutil.rmtree.assert_called_once() mkcert.UnInstall.assert_called_once() self.assertEqual(4, utils.Popen.call_count) |