diff options
author | Hsin-Yi Chen <hsinyichen@google.com> | 2024-03-11 02:29:42 +0000 |
---|---|---|
committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2024-03-11 02:29:42 +0000 |
commit | c46b3fbc57382b4b7497f0399cd96c684ab6708d (patch) | |
tree | e5f369a64cbccb38f5004ce3740b29a6c464b3f3 | |
parent | 76b9f10d4982e32162077e5ae744b04f73f06832 (diff) | |
parent | ec2c103f7beef53ac2f4656732456ab3ab45f6da (diff) | |
download | acloud-c46b3fbc57382b4b7497f0399cd96c684ab6708d.tar.gz |
Merge changes I073167c5,Ia55f9898 into main
* changes:
Copy --remote-image-dir to instance directory
Split the options and the paths returned by UploadExtraImages
-rw-r--r-- | internal/lib/cvd_utils.py | 58 | ||||
-rw-r--r-- | internal/lib/cvd_utils_test.py | 35 | ||||
-rw-r--r-- | public/actions/remote_host_cf_device_factory.py | 53 | ||||
-rw-r--r-- | public/actions/remote_host_cf_device_factory_test.py | 23 | ||||
-rw-r--r-- | public/actions/remote_instance_cf_device_factory.py | 2 | ||||
-rw-r--r-- | public/actions/remote_instance_cf_device_factory_test.py | 2 |
6 files changed, 104 insertions, 69 deletions
diff --git a/internal/lib/cvd_utils.py b/internal/lib/cvd_utils.py index 14f51178..40197aa0 100644 --- a/internal/lib/cvd_utils.py +++ b/internal/lib/cvd_utils.py @@ -316,7 +316,8 @@ def _UploadKernelImages(ssh_obj, remote_image_dir, search_path): search_path: A path to an image file or an image directory. Returns: - A list of strings, the launch_cvd arguments including the remote paths. + A list of string pairs. Each pair consists of a launch_cvd option and a + remote path. Raises: errors.GetLocalImageError if search_path does not contain kernel @@ -334,22 +335,22 @@ def _UploadKernelImages(ssh_obj, remote_image_dir, search_path): remote_image_dir, _REMOTE_INITRAMFS_IMAGE_PATH) ssh_obj.ScpPushFile(kernel_image_path, remote_kernel_image_path) ssh_obj.ScpPushFile(initramfs_image_path, remote_initramfs_image_path) - return ["-kernel_path", remote_kernel_image_path, - "-initramfs_path", remote_initramfs_image_path] + return [("-kernel_path", remote_kernel_image_path), + ("-initramfs_path", remote_initramfs_image_path)] boot_image_path, vendor_boot_image_path = FindBootImages(search_path) if boot_image_path: remote_boot_image_path = remote_path.join( remote_image_dir, _REMOTE_BOOT_IMAGE_PATH) ssh_obj.ScpPushFile(boot_image_path, remote_boot_image_path) - launch_cvd_args = ["-boot_image", remote_boot_image_path] + launch_cvd_args = [("-boot_image", remote_boot_image_path)] if vendor_boot_image_path: remote_vendor_boot_image_path = remote_path.join( remote_image_dir, _REMOTE_VENDOR_BOOT_IMAGE_PATH) ssh_obj.ScpPushFile(vendor_boot_image_path, remote_vendor_boot_image_path) - launch_cvd_args.extend(["-vendor_boot_image", - remote_vendor_boot_image_path]) + launch_cvd_args.append(("-vendor_boot_image", + remote_vendor_boot_image_path)) return launch_cvd_args raise errors.GetLocalImageError( @@ -444,12 +445,12 @@ def _UploadVbmetaImage(ssh_obj, remote_image_dir, vbmeta_image_path): vbmeta_image_path: The path to the vbmeta image. Returns: - A list of strings, the launch_cvd arguments including the remote paths. + A pair of strings, the launch_cvd option and the remote path. """ remote_vbmeta_image_path = remote_path.join(remote_image_dir, _REMOTE_VBMETA_IMAGE_PATH) ssh_obj.ScpPushFile(vbmeta_image_path, remote_vbmeta_image_path) - return ["-vbmeta_image", remote_vbmeta_image_path] + return "-vbmeta_image", remote_vbmeta_image_path def AreTargetFilesRequired(avd_spec): @@ -472,7 +473,8 @@ def UploadExtraImages(ssh_obj, remote_image_dir, avd_spec, target_files_dir): avd_spec requires building a super image. Returns: - A list of strings, the launch_cvd arguments including the remote paths. + A list of string pairs. Each pair consists of a launch_cvd option and a + remote path. Raises: errors.GetLocalImageError if any specified image path does not exist. @@ -500,13 +502,13 @@ def UploadExtraImages(ssh_obj, remote_image_dir, avd_spec, target_files_dir): with tempfile.TemporaryDirectory() as super_image_dir: _MixSuperImage(os.path.join(super_image_dir, _SUPER_IMAGE_NAME), avd_spec, target_files_dir, ota) - extra_img_args += _UploadSuperImage(ssh_obj, remote_image_dir, - super_image_dir) + extra_img_args.append(_UploadSuperImage(ssh_obj, remote_image_dir, + super_image_dir)) vbmeta_image_path = os.path.join(super_image_dir, "vbmeta.img") ota.MakeDisabledVbmetaImage(vbmeta_image_path) - extra_img_args += _UploadVbmetaImage(ssh_obj, remote_image_dir, - vbmeta_image_path) + extra_img_args.append(_UploadVbmetaImage(ssh_obj, remote_image_dir, + vbmeta_image_path)) return extra_img_args @@ -521,7 +523,7 @@ def _UploadSuperImage(ssh_obj, remote_image_dir, super_image_dir): super_image_dir: The path to the directory containing the super image. Returns: - A list of strings, the launch_cvd arguments including the remote paths. + A pair of strings, the launch_cvd option and the remote path. """ remote_super_image_path = remote_path.join(remote_image_dir, _REMOTE_SUPER_IMAGE_PATH) @@ -530,8 +532,7 @@ def _UploadSuperImage(ssh_obj, remote_image_dir, super_image_dir): f"{ssh_obj.GetBaseCmd(constants.SSH_BIN)} -- " f"tar -xf - --lzop -S -C {remote_super_image_dir}") ssh.ShellCmdWithRetry(cmd) - launch_cvd_args = ["-super_image", remote_super_image_path] - return launch_cvd_args + return "-super_image", remote_super_image_path def CleanUpRemoteCvd(ssh_obj, remote_dir, raise_error): @@ -546,7 +547,8 @@ def CleanUpRemoteCvd(ssh_obj, remote_dir, raise_error): Raises: subprocess.CalledProcessError if any command fails. """ - # TODO(b/293966645): Find stop_cvd in --remote-image-dir. + # FIXME: Use the images and launch_cvd in --remote-image-dir when + # cuttlefish can reliably share images. home = remote_path.join("$HOME", remote_dir) stop_cvd_path = remote_path.join(remote_dir, "bin", "stop_cvd") stop_cvd_cmd = f"'HOME={home} {stop_cvd_path}'" @@ -620,10 +622,10 @@ def LoadRemoteImageArgs(ssh_obj, remote_args_path): Args: ssh_obj: An Ssh object. - remote_image_dir: The remote path containing the arguments. + remote_args_path: The remote path containing the arguments. Returns: - A list of strings, the launch_cvd arguments. + A list of string pairs, the arguments generated by UploadExtraImages. None if the directory has not been initialized. """ # If the file doesn't exist, the command returns 0 and outputs nothing. @@ -646,7 +648,8 @@ def SaveRemoteImageArgs(ssh_obj, remote_args_path, launch_cvd_args): Args: ssh_obj: An Ssh object. remote_args_path: The remote path containing the arguments. - launch_cvd_args: A list of strings, the launch_cvd arguments. + launch_cvd_args: A list of string pairs, the arguments generated by + UploadExtraImages. """ # The json string is interpreted twice by SSH client and remote shell. args_str = shlex.quote(json.dumps(launch_cvd_args)) @@ -754,16 +757,11 @@ def GetRemoteLaunchCvdCmd(remote_dir, avd_spec, config, extra_args): Returns: A string, the launch_cvd command. """ - # launch_cvd requires ANDROID_HOST_OUT to be absolute. - cmd = ([f"{constants.ENV_ANDROID_HOST_OUT}=" - f"$(readlink -n -m {avd_spec.remote_image_dir})", - f"{constants.ENV_ANDROID_PRODUCT_OUT}=" - f"${constants.ENV_ANDROID_HOST_OUT}"] - if avd_spec.remote_image_dir else []) - cmd.extend(["HOME=" + remote_path.join("$HOME", remote_dir), - remote_path.join(avd_spec.remote_image_dir or remote_dir, - "bin", "launch_cvd"), - "-daemon"]) + # FIXME: Use the images and launch_cvd in avd_spec.remote_image_dir when + # cuttlefish can reliably share images. + cmd = ["HOME=" + remote_path.join("$HOME", remote_dir), + remote_path.join(remote_dir, "bin", "launch_cvd"), + "-daemon"] cmd.extend(extra_args) cmd.extend(_GetLaunchCvdArgs(avd_spec, config)) return " ".join(cmd) diff --git a/internal/lib/cvd_utils_test.py b/internal/lib/cvd_utils_test.py index edb0ea04..fec013cd 100644 --- a/internal/lib/cvd_utils_test.py +++ b/internal/lib/cvd_utils_test.py @@ -159,7 +159,7 @@ class CvdUtilsTest(driver_test_lib.BaseDriverTest): local_vendor_image=None) args = cvd_utils.UploadExtraImages(mock_ssh, "dir", mock_avd_spec, None) - self.assertEqual(["-boot_image", "dir/acloud_image/boot.img"], + self.assertEqual([("-boot_image", "dir/acloud_image/boot.img")], args) mock_ssh.Run.assert_called_once_with("mkdir -p dir/acloud_image") mock_ssh.ScpPushFile.assert_called_once_with( @@ -170,8 +170,8 @@ class CvdUtilsTest(driver_test_lib.BaseDriverTest): args = cvd_utils.UploadExtraImages(mock_ssh, "dir", mock_avd_spec, None) self.assertEqual( - ["-boot_image", "dir/acloud_image/boot.img", - "-vendor_boot_image", "dir/acloud_image/vendor_boot.img"], + [("-boot_image", "dir/acloud_image/boot.img"), + ("-vendor_boot_image", "dir/acloud_image/vendor_boot.img")], args) mock_ssh.Run.assert_called_once() self.assertEqual(2, mock_ssh.ScpPushFile.call_count) @@ -198,8 +198,8 @@ class CvdUtilsTest(driver_test_lib.BaseDriverTest): args = cvd_utils.UploadExtraImages(mock_ssh, "dir", mock_avd_spec, None) self.assertEqual( - ["-kernel_path", "dir/acloud_image/kernel", - "-initramfs_path", "dir/acloud_image/initramfs.img"], + [("-kernel_path", "dir/acloud_image/kernel"), + ("-initramfs_path", "dir/acloud_image/initramfs.img")], args) mock_ssh.Run.assert_called_once() self.assertEqual(2, mock_ssh.ScpPushFile.call_count) @@ -232,8 +232,8 @@ class CvdUtilsTest(driver_test_lib.BaseDriverTest): target_files_dir) self.assertEqual( - ["-super_image", "dir/acloud_image/super.img", - "-vbmeta_image", "dir/acloud_image/vbmeta.img"], + [("-super_image", "dir/acloud_image/super.img"), + ("-vbmeta_image", "dir/acloud_image/vbmeta.img")], args) mock_find_ota_tools.assert_called_once_with([]) mock_ssh.Run.assert_called_once_with("mkdir -p dir/acloud_image") @@ -331,12 +331,11 @@ class CvdUtilsTest(driver_test_lib.BaseDriverTest): self.assertFalse(os.path.exists(args_path)) # Test with an initialized directory. - self.CreateFile(args_path, b'["ok"]') + self.CreateFile(args_path, b'[["arg", "1"]]') - args = cvd_utils.LoadRemoteImageArgs( - mock_ssh, args_path) + args = cvd_utils.LoadRemoteImageArgs(mock_ssh, args_path) - self.assertEqual(args, ["ok"]) + self.assertEqual(args, [["arg", "1"]]) def testSaveRemoteImageArgs(self): """Test SaveRemoteImageArgs.""" @@ -350,13 +349,12 @@ class CvdUtilsTest(driver_test_lib.BaseDriverTest): "sh -c " + cmd, shell=True, cwd=temp_dir, env=env, text=True) args_path = os.path.join(temp_dir, "args.txt") - cvd_utils.SaveRemoteImageArgs(mock_ssh, args_path, ["ok"]) + cvd_utils.SaveRemoteImageArgs(mock_ssh, args_path, [("arg", "1")]) mock_ssh.Run.assert_called_with( - f"""'echo '"'"'["ok"]'"'"' > {args_path}'""") + f"""'echo '"'"'[["arg", "1"]]'"'"' > {args_path}'""") with open(args_path, "r", encoding="utf-8") as args_file: - self.assertEqual(args_file.read().strip(), '["ok"]') - + self.assertEqual(args_file.read().strip(), '[["arg", "1"]]') def testGetConfigFromRemoteAndroidInfo(self): """Test GetConfigFromRemoteAndroidInfo.""" @@ -383,7 +381,6 @@ class CvdUtilsTest(driver_test_lib.BaseDriverTest): cfg=mock_cfg, hw_customize=False, hw_property=hw_property, - remote_image_dir=None, connect_webrtc=False, connect_vnc=False, openwrt=False, @@ -413,7 +410,6 @@ class CvdUtilsTest(driver_test_lib.BaseDriverTest): cfg=mock_cfg, hw_customize=True, hw_property=hw_property, - remote_image_dir="img_dir", connect_webrtc=True, webrtc_device_id="pet-name", connect_vnc=True, @@ -422,10 +418,7 @@ class CvdUtilsTest(driver_test_lib.BaseDriverTest): base_instance_num=3, launch_args="--setupwizard_mode=REQUIRED") expected_cmd = ( - "ANDROID_HOST_OUT=$(readlink -n -m img_dir) " - "ANDROID_PRODUCT_OUT=$ANDROID_HOST_OUT " - "HOME=$HOME/dir " - "img_dir/bin/launch_cvd -daemon --extra args " + "HOME=$HOME/dir dir/bin/launch_cvd -daemon --extra args " "-data_policy=create_if_missing -blank_data_image_mb=20480 " "-config=phone -x_res=1080 -y_res=1920 -dpi=240 " "-data_policy=always_create -blank_data_image_mb=10240 " diff --git a/public/actions/remote_host_cf_device_factory.py b/public/actions/remote_host_cf_device_factory.py index bfb43c84..7d6f88ab 100644 --- a/public/actions/remote_host_cf_device_factory.py +++ b/public/actions/remote_host_cf_device_factory.py @@ -184,6 +184,7 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): A list of strings, the launch_cvd arguments. """ remote_image_dir = self._avd_spec.remote_image_dir + reuse_remote_image_dir = False if remote_image_dir: remote_args_path = remote_path.join(remote_image_dir, _IMAGE_ARGS_FILE_NAME) @@ -191,15 +192,23 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): self._ssh, remote_args_path) if launch_cvd_args is not None: logger.info("Reuse the images in %s", remote_image_dir) - return launch_cvd_args + reuse_remote_image_dir = True logger.info("Create images in %s", remote_image_dir) - launch_cvd_args = self._InitRemoteImageDir() + if not reuse_remote_image_dir: + launch_cvd_args = self._InitRemoteImageDir() if remote_image_dir: - cvd_utils.SaveRemoteImageArgs(self._ssh, remote_args_path, - launch_cvd_args) - return launch_cvd_args + if not reuse_remote_image_dir: + cvd_utils.SaveRemoteImageArgs(self._ssh, remote_args_path, + launch_cvd_args) + # FIXME: Use the images in remote_image_dir when cuttlefish can + # reliably share images. + launch_cvd_args = self._ReplaceRemoteImageArgs( + launch_cvd_args, remote_image_dir, self._GetInstancePath()) + self._CopyRemoteImageDir(remote_image_dir, self._GetInstancePath()) + + return [arg for arg_pair in launch_cvd_args for arg in arg_pair] def _InitRemoteImageDir(self): """Create remote host artifacts. @@ -211,7 +220,8 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): is no permission to fetch build rom on the remote host. Returns: - A list of strings, the launch_cvd arguments. + A list of string pairs, the launch_cvd arguments generated by + UploadExtraImages. """ self._ssh.Run(f"mkdir -p {self._GetArtifactPath()}") @@ -432,6 +442,37 @@ class RemoteHostDeviceFactory(base_device_factory.BaseDeviceFactory): logger.debug("cmd:\n %s", cmd) ssh.ShellCmdWithRetry(cmd) + @staticmethod + def _ReplaceRemoteImageArgs(launch_cvd_args, old_dir, new_dir): + """Replace the prefix of launch_cvd path arguments. + + Args: + launch_cvd_args: A list of string pairs. Each pair consists of a + launch_cvd option and a remote path. + old_dir: The prefix of the paths to be replaced. + new_dir: The new prefix of the paths. + + Returns: + A list of string pairs, the replaced arguments. + """ + if any(remote_path.isabs(path) != remote_path.isabs(old_dir) for + _, path in launch_cvd_args): + raise ValueError(f"Cannot convert {launch_cvd_args} to relative " + f"paths under {old_dir}") + return [(option, + remote_path.join(new_dir, remote_path.relpath(path, old_dir))) + for option, path in launch_cvd_args] + + @utils.TimeExecute(function_description="Copying images") + def _CopyRemoteImageDir(self, remote_src_dir, remote_dst_dir): + """Copy a remote directory recursively. + + Args: + remote_src_dir: The source directory. + remote_dst_dir: The destination directory. + """ + self._ssh.Run(f"cp -frT {remote_src_dir} {remote_dst_dir}") + @utils.TimeExecute( function_description="Launching AVD(s) and waiting for boot up", result_evaluator=utils.BootEvaluator) diff --git a/public/actions/remote_host_cf_device_factory_test.py b/public/actions/remote_host_cf_device_factory_test.py index 9ee010de..4b6fa5e5 100644 --- a/public/actions/remote_host_cf_device_factory_test.py +++ b/public/actions/remote_host_cf_device_factory_test.py @@ -98,7 +98,7 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_cvd_utils.GetRemoteHostBaseDir.return_value = "acloud_cf_2" mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" mock_cvd_utils.AreTargetFilesRequired.return_value = True - mock_cvd_utils.UploadExtraImages.return_value = ["extra"] + mock_cvd_utils.UploadExtraImages.return_value = [("-extra", "image")] mock_cvd_utils.ExecuteRemoteLaunchCvd.return_value = "failure" mock_cvd_utils.FindRemoteLogs.return_value = [log] @@ -119,7 +119,7 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_ssh_obj, "acloud_cf_2") # LaunchCvd mock_cvd_utils.GetRemoteLaunchCvdCmd.assert_called_with( - "acloud_cf_2", mock_avd_spec, mock.ANY, ["extra"]) + "acloud_cf_2", mock_avd_spec, mock.ANY, ["-extra", "image"]) mock_cvd_utils.ExecuteRemoteLaunchCvd.assert_called() # FindLogFiles mock_cvd_utils.FindRemoteLogs.assert_called_with( @@ -401,9 +401,7 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_ssh_obj = mock.Mock() mock_ssh.Ssh.return_value = mock_ssh_obj - mock_ssh_obj.GetBaseCmd.return_value = "/mock/ssh" # Test initializing the remote image dir. - mock_ssh_obj.Run.return_value = "" mock_glob.glob.return_value = ["/mock/super.img"] factory = remote_host_cf_device_factory.RemoteHostDeviceFactory( mock_avd_spec) @@ -412,7 +410,8 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_cvd_utils.FormatRemoteHostInstanceName.return_value = "inst" mock_cvd_utils.LoadRemoteImageArgs.return_value = None mock_cvd_utils.AreTargetFilesRequired.return_value = False - mock_cvd_utils.UploadExtraImages.return_value = ["arg1"] + mock_cvd_utils.UploadExtraImages.return_value = [ + ("arg", "mock_img_dir/1")] mock_cvd_utils.ExecuteRemoteLaunchCvd.return_value = "" mock_cvd_utils.FindRemoteLogs.return_value = [] @@ -422,21 +421,25 @@ class RemoteHostDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_cvd_utils.LoadRemoteImageArgs.assert_called_once_with( mock_ssh_obj, "mock_img_dir/acloud_image_args.txt") mock_cvd_utils.SaveRemoteImageArgs.assert_called_once_with( - mock_ssh_obj, "mock_img_dir/acloud_image_args.txt", ["arg1"]) + mock_ssh_obj, "mock_img_dir/acloud_image_args.txt", + [("arg", "mock_img_dir/1")]) + mock_ssh_obj.Run.assert_called_with("cp -frT mock_img_dir acloud_cf_1") self._mock_build_api.DownloadFetchcvd.assert_called_once() - print(mock_cvd_utils.GetRemoteLaunchCvdCmd.call_args[0][3]) - self.assertEqual(["arg1"], + self.assertEqual(["arg", "acloud_cf_1/1"], mock_cvd_utils.GetRemoteLaunchCvdCmd.call_args[0][3]) # Test reusing the remote image dir. - mock_cvd_utils.LoadRemoteImageArgs.return_value = ["arg2"] + mock_cvd_utils.LoadRemoteImageArgs.return_value = [ + ["arg", "mock_img_dir/2"]] mock_cvd_utils.SaveRemoteImageArgs.reset_mock() + mock_ssh_obj.reset_mock() self._mock_build_api.DownloadFetchcvd.reset_mock() self.assertEqual("inst", factory.CreateInstance()) mock_cvd_utils.SaveRemoteImageArgs.assert_not_called() + mock_ssh_obj.Run.assert_called_with("cp -frT mock_img_dir acloud_cf_1") self._mock_build_api.DownloadFetchcvd.assert_not_called() - self.assertEqual(["arg2"], + self.assertEqual(["arg", "acloud_cf_1/2"], mock_cvd_utils.GetRemoteLaunchCvdCmd.call_args[0][3]) diff --git a/public/actions/remote_instance_cf_device_factory.py b/public/actions/remote_instance_cf_device_factory.py index b6259997..5dc91a6b 100644 --- a/public/actions/remote_instance_cf_device_factory.py +++ b/public/actions/remote_instance_cf_device_factory.py @@ -126,7 +126,7 @@ class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): if avd_spec.extra_files: self._compute_client.UploadExtraFiles(avd_spec.extra_files) - return launch_cvd_args + return [arg for arg_pair in launch_cvd_args for arg in arg_pair] @utils.TimeExecute(function_description="Downloading target_files archive") def _DownloadTargetFiles(self, download_dir): diff --git a/public/actions/remote_instance_cf_device_factory_test.py b/public/actions/remote_instance_cf_device_factory_test.py index ecd24857..9d736a29 100644 --- a/public/actions/remote_instance_cf_device_factory_test.py +++ b/public/actions/remote_instance_cf_device_factory_test.py @@ -243,7 +243,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): mock_cvd_utils.AreTargetFilesRequired.return_value = False mock_cvd_utils.FindRemoteLogs.return_value = [{"path": "/logcat"}] mock_cvd_utils.UploadExtraImages.return_value = [ - "-boot_image", "/boot/img"] + ("-boot_image", "/boot/img")] fake_host_package_name = "/fake/host_package.tar.gz" fake_image_name = "" |