diff options
106 files changed, 6398 insertions, 1565 deletions
@@ -12,16 +12,36 @@ // See the License for the specific language governing permissions and // limitations under the License. +package { + default_applicable_licenses: ["tools_acloud_license"], +} + +// Added automatically by a large-scale-change +// http://go/android-license-faq +license { + name: "tools_acloud_license", + visibility: [":__subpackages__"], + license_kinds: [ + "SPDX-license-identifier-Apache-2.0", + ], + license_text: [ + "LICENSE", + ], +} + python_defaults { name: "acloud_default", pkg_path: "acloud", version: { py2: { - enabled: true, + enabled: false, embedded_launcher: false, + libs: [ + "py-pyopenssl", + ] }, py3: { - enabled: false, + enabled: true, embedded_launcher: false, }, }, @@ -29,7 +49,7 @@ python_defaults { python_binary_host { name: "acloud", - // Make acloud's built name to acloud-dev + // Make acloud's built name to acloud-dev default build python3 binary. stem: "acloud-dev", defaults: ["acloud_default"], main: "public/acloud_main.py", @@ -39,6 +59,7 @@ python_binary_host { ], data: [ "public/data/default.config", + ":acloud_version", ], libs: [ "acloud_create", @@ -47,15 +68,16 @@ python_binary_host { "acloud_internal", "acloud_list", "acloud_pull", + "acloud_powerwash", "acloud_metrics", "acloud_proto", "acloud_public", + "acloud_restart", "acloud_setup", "py-apitools", "py-dateutil", "py-google-api-python-client", "py-oauth2client", - "py-pyopenssl", "py-six", ], dist: { @@ -84,19 +106,25 @@ python_test_host { "acloud_reconnect", "acloud_internal", "acloud_list", + "acloud_powerwash", + "acloud_public", "acloud_pull", "acloud_proto", - "acloud_public", + "acloud_restart", "acloud_setup", "asuite_cc_client", "py-apitools", "py-dateutil", "py-google-api-python-client", - "py-mock", "py-oauth2client", ], test_config: "acloud_unittest.xml", - test_suites: ["general-tests"], + test_suites: [ + "general-tests", + ], + test_options: { + unit_test: true, + } } python_library_host { @@ -188,6 +216,22 @@ python_library_host{ } python_library_host{ + name: "acloud_powerwash", + defaults: ["acloud_default"], + srcs: [ + "powerwash/*.py", + ], +} + +python_library_host{ + name: "acloud_restart", + defaults: ["acloud_default"], + srcs: [ + "restart/*.py", + ], +} + +python_library_host{ name: "acloud_metrics", defaults: ["acloud_default"], srcs: [ @@ -198,3 +242,10 @@ python_library_host{ "asuite_metrics", ], } + +genrule { + name: "acloud_version", + tool_files: ["gen_version.sh"], + cmd: "$(location gen_version.sh) $(out)", + out: ["public/data/VERSION"], +} @@ -1,2 +1,3 @@ +herbertxue@google.com kevcheng@google.com samchiu@google.com @@ -51,12 +51,10 @@ instance (running on a virtual machine in the cloud) and local instance (running on your local host) use cases. You also have the option to use a locally built image or an image from the Android Build servers. -**Disclaimer: Creation of a cuttlefish local instance is not formally supported, please use at your own risk.** - Here's a quick cheat-sheet for the 4 use cases: * Remote instance using an Android Build image (LKGB (Last Known Good Build) -for cuttlefish phone target in the branch of your repo, default aosp master +for cuttlefish phone target in the branch of your repo, default aosp main (master) if we can't determine it) > $ acloud create @@ -79,10 +77,10 @@ target and/or build id (e.g. `--branch my_branch`). Acloud will assume the following if they're not specified: * `--branch`: The branch of the repo you're running the acloud command in, e.g. -in an aosp repo on the master branch, acloud will infer the aosp-master branch. +in an aosp repo on the master branch, acloud will infer the aosp-main (aosp-master) branch. * `--build-target`: Defaults to the phone target for cuttlefish (e.g. -aosp\_cf\_x86\_phone-userdebug in aosp-master). +aosp\_cf\_x86\_phone-userdebug in aosp-main (aosp-master)). * `--build-id`: Default to the Last Known Good Build (LKGB) id for the branch and target set from above. @@ -223,4 +221,4 @@ Cheatsheet: If you have any questions or feedback, contact [acloud@google.com](mailto:acloud@google.com). -If you have any bugs or feature requests, [go/acloud-bug](http://go/acloud-bug). +If you have any bugs or feature requests email them to [buganizer-system+419709@google.com](mailto:buganizer-system+419709@google.com)
\ No newline at end of file diff --git a/TEST_MAPPING b/TEST_MAPPING deleted file mode 100644 index 61a80b2c..00000000 --- a/TEST_MAPPING +++ /dev/null @@ -1,8 +0,0 @@ -{ - "presubmit" : [ - { - "name" : "acloud_test", - "host" : true - } - ] -} diff --git a/acloud_test.py b/acloud_test.py index a7fc680d..ce594967 100644 --- a/acloud_test.py +++ b/acloud_test.py @@ -19,6 +19,7 @@ from importlib import import_module import logging import os import sys +import sysconfig import unittest @@ -36,6 +37,9 @@ logger = logging.getLogger(ACLOUD_LOGGER) logger.setLevel(logging.CRITICAL) logger.addHandler(logging.FileHandler("/dev/null")) +if sys.version_info.major == 3: + sys.path.insert(0, os.path.dirname(sysconfig.get_paths()['purelib'])) + def GetTestModules(): """Return list of testable modules. diff --git a/create/avd_spec.py b/create/avd_spec.py index b34ad033..7be4ece5 100644 --- a/create/avd_spec.py +++ b/create/avd_spec.py @@ -45,7 +45,7 @@ _BRANCH_RE = re.compile(r"^Manifest branch: (?P<branch>.+)") _COMMAND_REPO_INFO = "repo info platform/tools/acloud" _REPO_TIMEOUT = 3 _CF_ZIP_PATTERN = "*img*.zip" -_DEFAULT_BUILD_BITNESS = "x86" +_DEFAULT_BUILD_BITNESS = "x86_64" _DEFAULT_BUILD_TYPE = "userdebug" _ENV_ANDROID_PRODUCT_OUT = "ANDROID_PRODUCT_OUT" _ENV_ANDROID_BUILD_TOP = "ANDROID_BUILD_TOP" @@ -90,7 +90,7 @@ def EscapeAnsi(line): # pylint: disable=too-many-public-methods -class AVDSpec: +class AVDSpec(): """Class to store data on the type of AVD to create.""" def __init__(self, args): @@ -110,18 +110,26 @@ class AVDSpec: self._flavor = None self._image_source = None self._instance_type = None + self._launch_args = None self._local_image_dir = None self._local_image_artifact = None - self._local_system_image_dir = None + self._local_instance_dir = None + self._local_kernel_image = None + self._local_system_image = None self._local_tool_dirs = None self._image_download_dir = None self._num_of_instances = None + self._num_avds_per_instance = None self._no_pull_log = None + self._oxygen = None self._remote_image = None self._system_build_info = None self._kernel_build_info = None + self._bootloader_build_info = None self._hw_property = None + self._hw_customize = False self._remote_host = None + self._gce_metadata = None self._host_user = None self._host_ssh_private_key_path = None # Create config instance for android_build_client to query build api. @@ -197,7 +205,7 @@ class AVDSpec: args: Namespace object from argparse.parse_args. """ # If user didn't specify --local-image, infer remote image args - if args.local_image == "": + if args.local_image is None: self._image_source = constants.IMAGE_SRC_REMOTE if (self._avd_type == constants.TYPE_GF and self._instance_type != constants.INSTANCE_TYPE_REMOTE): @@ -227,7 +235,7 @@ class AVDSpec: Raises: error.MalformedHWPropertyError: If hw_property_str is malformed. """ - hw_dict = create_common.ParseHWPropertyArgs(hw_property_str) + hw_dict = create_common.ParseKeyValuePairArgs(hw_property_str) arg_hw_properties = {} for key, value in hw_dict.items(): # Parsing HW properties int to avdspec. @@ -262,19 +270,27 @@ class AVDSpec: This method will initialize _hw_property in the following manner: - 1. Get default hw properties from config. - 2. Override by hw_property args. + 1. Get default hw properties from flavor. + 2. Override hw properties from config. + 3. Override by hw_property args. Args: args: Namespace object from argparse.parse_args. """ - self._cfg.OverrideHwPropertyWithFlavor(self._flavor) self._hw_property = {} - self._hw_property = self._ParseHWPropertyStr(self._cfg.hw_property) + default_property = self._cfg.GetDefaultHwProperty(self._flavor, + self._instance_type) + self._hw_property = self._ParseHWPropertyStr(default_property) logger.debug("Default hw property for [%s] flavor: %s", self._flavor, self._hw_property) + if self._cfg.hw_property: + self._hw_customize = True + cfg_hw_property = self._ParseHWPropertyStr(self._cfg.hw_property) + logger.debug("Hw property from config: %s", cfg_hw_property) + self._hw_property.update(cfg_hw_property) if args.hw_property: + self._hw_customize = True arg_hw_property = self._ParseHWPropertyStr(args.hw_property) logger.debug("Use custom hw property: %s", arg_hw_property) self._hw_property.update(arg_hw_property) @@ -293,19 +309,23 @@ class AVDSpec: if args.remote_host: self._instance_type = constants.INSTANCE_TYPE_HOST else: - self._instance_type = (constants.INSTANCE_TYPE_LOCAL - if args.local_instance else - constants.INSTANCE_TYPE_REMOTE) + self._instance_type = (constants.INSTANCE_TYPE_REMOTE + if args.local_instance is None else + constants.INSTANCE_TYPE_LOCAL) self._remote_host = args.remote_host self._host_user = args.host_user self._host_ssh_private_key_path = args.host_ssh_private_key_path self._local_instance_id = args.local_instance + self._local_instance_dir = args.local_instance_dir self._local_tool_dirs = args.local_tool self._num_of_instances = args.num + self._num_avds_per_instance = args.num_avds_per_instance self._no_pull_log = args.no_pull_log + self._oxygen = args.oxygen self._serial_log_file = args.serial_log_file self._emulator_build_id = args.emulator_build_id self._gpu = args.gpu + self._gce_metadata = create_common.ParseKeyValuePairArgs(args.gce_metadata) self._stable_cheeps_host_image_name = args.stable_cheeps_host_image_name self._stable_cheeps_host_image_project = args.stable_cheeps_host_image_project @@ -314,6 +334,8 @@ class AVDSpec: self._boot_timeout_secs = args.boot_timeout_secs self._ins_timeout_secs = args.ins_timeout_secs + self._launch_args = " ".join( + list(filter(None, [self._cfg.launch_args, args.launch_args]))) if args.reuse_gce: if args.reuse_gce != constants.SELECT_ONE_GCE_INSTANCE: @@ -358,12 +380,14 @@ class AVDSpec: """ if self._avd_type == constants.TYPE_CF: self._ProcessCFLocalImageArgs(args.local_image, args.flavor) + elif self._avd_type == constants.TYPE_FVP: + self._ProcessFVPLocalImageArgs() elif self._avd_type == constants.TYPE_GF: - self._local_image_dir = self._ProcessGFLocalImageArgs( + self._local_image_dir = self._GetLocalImagePath( args.local_image) - if args.local_system_image != "": - self._local_system_image_dir = self._ProcessGFLocalImageArgs( - args.local_system_image) + if not os.path.isdir(self._local_image_dir): + raise errors.GetLocalImageError("%s is not a directory." % + args.local_image) elif self._avd_type == constants.TYPE_GCE: self._local_image_artifact = self._GetGceLocalImagePath( args.local_image) @@ -372,6 +396,14 @@ class AVDSpec: "Local image doesn't support the AVD type: %s" % self._avd_type ) + if args.local_kernel_image is not None: + self._local_kernel_image = self._GetLocalImagePath( + args.local_kernel_image) + + if args.local_system_image is not None: + self._local_system_image = self._GetLocalImagePath( + args.local_system_image) + @staticmethod def _GetGceLocalImagePath(local_image_dir): """Get gce local image path. @@ -408,31 +440,30 @@ class AVDSpec: ", ".join(_GCE_LOCAL_IMAGE_CANDIDATES)) @staticmethod - def _ProcessGFLocalImageArgs(local_image_arg): - """Get local built image path for goldfish. + def _GetLocalImagePath(local_image_arg): + """Get local image path from argument or environment variable. Args: - local_image_arg: The path to the unzipped update package or SDK - repository, i.e., <target>-img-<build>.zip or - sdk-repo-<os>-system-images-<build>.zip. - If the value is empty, this method returns + local_image_arg: The path to the unzipped image package. If the + value is empty, this method returns ANDROID_PRODUCT_OUT in build environment. Returns: - String, the path to the image directory. + String, the path to the image file or directory. Raises: - errors.GetLocalImageError if the directory is not found. + errors.GetLocalImageError if the path does not exist. """ - image_dir = (local_image_arg if local_image_arg else - utils.GetBuildEnvironmentVariable( - constants.ENV_ANDROID_PRODUCT_OUT)) - - if not os.path.isdir(image_dir): - raise errors.GetLocalImageError( - "%s is not a directory." % image_dir) + if local_image_arg == constants.FIND_IN_BUILD_ENV: + image_path = utils.GetBuildEnvironmentVariable( + constants.ENV_ANDROID_PRODUCT_OUT) + else: + image_path = local_image_arg - return image_dir + if not os.path.exists(image_path): + raise errors.GetLocalImageError("%s does not exist." % + local_image_arg) + return image_path def _ProcessCFLocalImageArgs(self, local_image_arg, flavor_arg): """Get local built image path for cuttlefish-type AVD. @@ -449,8 +480,19 @@ class AVDSpec: """ flavor_from_build_string = None - local_image_path = local_image_arg or utils.GetBuildEnvironmentVariable( - _ENV_ANDROID_PRODUCT_OUT) + if local_image_arg == constants.FIND_IN_BUILD_ENV: + self._CheckCFBuildTarget(self._instance_type) + local_image_path = utils.GetBuildEnvironmentVariable( + _ENV_ANDROID_PRODUCT_OUT) + # Since dir is provided, check that any images exist to ensure user + # didn't forget to 'make' before launch AVD. + image_list = glob.glob(os.path.join(local_image_path, "*.img")) + if not image_list: + raise errors.GetLocalImageError( + "No image found(Did you choose a lunch target and run `m`?)" + ": %s.\n " % local_image_path) + else: + local_image_path = local_image_arg if os.path.isfile(local_image_path): self._local_image_artifact = local_image_arg @@ -462,14 +504,6 @@ class AVDSpec: utils.TextColors.WARNING) else: self._local_image_dir = local_image_path - # Since dir is provided, so checking that any images exist to ensure - # user didn't forget to 'make' before launch AVD. - image_list = glob.glob(os.path.join(self.local_image_dir, "*.img")) - if not image_list: - raise errors.GetLocalImageError( - "No image found(Did you choose a lunch target and run `m`?)" - ": %s.\n " % self.local_image_dir) - try: flavor_from_build_string = self._GetFlavorFromString( utils.GetBuildEnvironmentVariable(constants.ENV_BUILD_TARGET)) @@ -480,6 +514,26 @@ class AVDSpec: if flavor_from_build_string and not flavor_arg: self._flavor = flavor_from_build_string + def _ProcessFVPLocalImageArgs(self): + """Get local built image path for FVP-type AVD.""" + build_target = utils.GetBuildEnvironmentVariable( + constants.ENV_BUILD_TARGET) + if build_target != "fvp": + utils.PrintColorString( + "%s is not an fvp target (Try lunching fvp-eng " + "and running 'm')" % build_target, + utils.TextColors.WARNING) + self._local_image_dir = utils.GetBuildEnvironmentVariable( + _ENV_ANDROID_PRODUCT_OUT) + + # Since dir is provided, so checking that any images exist to ensure + # user didn't forget to 'make' before launch AVD. + image_list = glob.glob(os.path.join(self.local_image_dir, "*.img")) + if not image_list: + raise errors.GetLocalImageError( + "No image found(Did you choose a lunch target and run `m`?)" + ": %s.\n " % self._local_image_dir) + def _ProcessRemoteBuildArgs(self, args): """Get the remote build args. @@ -519,6 +573,9 @@ class AVDSpec: self._remote_image[constants.BUILD_TARGET], self._remote_image[constants.BUILD_BRANCH]) + self._remote_image[constants.CHEEPS_BETTY_IMAGE] = ( + args.cheeps_betty_image or self._cfg.betty_image) + # Process system image and kernel image. self._system_build_info = {constants.BUILD_ID: args.system_build_id, constants.BUILD_BRANCH: args.system_branch, @@ -526,6 +583,30 @@ class AVDSpec: self._kernel_build_info = {constants.BUILD_ID: args.kernel_build_id, constants.BUILD_BRANCH: args.kernel_branch, constants.BUILD_TARGET: args.kernel_build_target} + self._bootloader_build_info = { + constants.BUILD_ID: args.bootloader_build_id, + constants.BUILD_BRANCH: args.bootloader_branch, + constants.BUILD_TARGET: args.bootloader_build_target} + + @staticmethod + def _CheckCFBuildTarget(instance_type): + """Check build target for the given instance type + + Args: + instance_type: String of instance type + + Raises: + errors.GetLocalImageError if the pattern is not match with + current build target. + """ + build_target = utils.GetBuildEnvironmentVariable( + constants.ENV_BUILD_TARGET) + pattern = constants.CF_AVD_BUILD_TARGET_PATTERN_MAPPING[instance_type] + if pattern not in build_target: + utils.PrintColorString( + "%s is not a %s target (Try lunching a proper cuttlefish " + "target and running 'm')" % (build_target, pattern), + utils.TextColors.WARNING) @staticmethod def _GetGitRemote(): @@ -539,16 +620,15 @@ class AVDSpec: """ try: android_build_top = os.environ[constants.ENV_ANDROID_BUILD_TOP] - except KeyError: + except KeyError as e: raise errors.GetAndroidBuildEnvVarError( "Could not get environment var: %s\n" "Try to run '#source build/envsetup.sh && lunch <target>'" - % _ENV_ANDROID_BUILD_TOP - ) + % _ENV_ANDROID_BUILD_TOP) from e acloud_project = os.path.join(android_build_top, "tools", "acloud") - return EscapeAnsi(subprocess.check_output(_COMMAND_GIT_REMOTE, - cwd=acloud_project).strip()) + return EscapeAnsi(utils.CheckOutput(_COMMAND_GIT_REMOTE, + cwd=acloud_project).strip()) def _GetBuildBranch(self, build_id, build_target): """Infer build branch if user didn't specify branch name. @@ -583,9 +663,11 @@ class AVDSpec: env = os.environ.copy() env.pop("PYTHONPATH", None) logger.info("Running command \"%s\"", _COMMAND_REPO_INFO) + # TODO(154173071): Migrate acloud to py3, then apply Popen to append with encoding process = subprocess.Popen(_COMMAND_REPO_INFO, shell=True, stdin=None, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, env=env) + stderr=subprocess.STDOUT, env=env, + universal_newlines=True) timer = threading.Timer(_REPO_TIMEOUT, process.kill) timer.start() stdout, _ = process.communicate() @@ -609,7 +691,7 @@ class AVDSpec: Target = {REPO_PREFIX}{avd_type}_{bitness}_{flavor}- {DEFAULT_BUILD_TARGET_TYPE}. - Example target: aosp_cf_x86_phone-userdebug + Example target: aosp_cf_x86_64_phone-userdebug Args: args: Namespace object from argparse.parse_args. @@ -640,6 +722,11 @@ class AVDSpec: return self._hw_property @property + def hw_customize(self): + """Return the hw_customize.""" + return self._hw_customize + + @property def local_image_dir(self): """Return local image dir.""" return self._local_image_dir @@ -650,9 +737,19 @@ class AVDSpec: return self._local_image_artifact @property - def local_system_image_dir(self): - """Return local system image dir.""" - return self._local_system_image_dir + def local_instance_dir(self): + """Return local instance directory.""" + return self._local_instance_dir + + @property + def local_kernel_image(self): + """Return local kernel image path.""" + return self._local_kernel_image + + @property + def local_system_image(self): + """Return local system image path.""" + return self._local_system_image @property def local_tool_dirs(self): @@ -714,6 +811,11 @@ class AVDSpec: return self._num_of_instances @property + def num_avds_per_instance(self): + """Return num_avds_per_instance.""" + return self._num_avds_per_instance + + @property def report_internal_ip(self): """Return report internal ip.""" return self._report_internal_ip @@ -724,6 +826,11 @@ class AVDSpec: return self._kernel_build_info @property + def bootloader_build_info(self): + """Return bootloader build info.""" + return self._bootloader_build_info + + @property def flavor(self): """Return flavor.""" return self._flavor @@ -828,3 +935,18 @@ class AVDSpec: def no_pull_log(self): """Return no_pull_log.""" return self._no_pull_log + + @property + def gce_metadata(self): + """Return gce_metadata.""" + return self._gce_metadata + + @property + def oxygen(self): + """Return oxygen.""" + return self._oxygen + + @property + def launch_args(self): + """Return launch_args.""" + return self._launch_args diff --git a/create/avd_spec_test.py b/create/avd_spec_test.py index 22c1db5d..4c480068 100644 --- a/create/avd_spec_test.py +++ b/create/avd_spec_test.py @@ -17,7 +17,8 @@ import glob import os import subprocess import unittest -import mock + +from unittest import mock from acloud import errors from acloud.create import avd_spec @@ -27,6 +28,7 @@ from acloud.internal.lib import auth from acloud.internal.lib import driver_test_lib from acloud.internal.lib import utils from acloud.list import list as list_instances +from acloud.public import config # pylint: disable=invalid-name,protected-access @@ -35,13 +37,16 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): def setUp(self): """Initialize new avd_spec.AVDSpec.""" - super(AvdSpecTest, self).setUp() + super().setUp() self.args = mock.MagicMock() self.args.flavor = "" - self.args.local_image = "" + self.args.local_image = None + self.args.local_kernel_image = None + self.args.local_system_image = None self.args.config_file = "" self.args.build_target = "fake_build_target" self.args.adb_port = None + self.args.launch_args = None self.Patch(list_instances, "ChooseOneRemoteInstance", return_value=mock.MagicMock()) self.Patch(list_instances, "GetInstancesFromInstanceNames", return_value=mock.MagicMock()) self.AvdSpec = avd_spec.AVDSpec(self.args) @@ -52,9 +57,15 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.Patch(glob, "glob", return_value=["fake.img"]) expected_image_artifact = "/path/cf_x86_phone-img-eng.user.zip" expected_image_dir = "/path-to-image-dir" + self.Patch(os.path, "exists", + side_effect=lambda path: path in (expected_image_artifact, + expected_image_dir)) + self.Patch(os.path, "isdir", + side_effect=lambda path: path == expected_image_dir) + self.Patch(os.path, "isfile", + side_effect=lambda path: path == expected_image_artifact) # Specified --local-image to a local zipped image file - self.Patch(os.path, "isfile", return_value=True) self.args.local_image = "/path/cf_x86_phone-img-eng.user.zip" self.AvdSpec._avd_type = constants.TYPE_CF self.AvdSpec._instance_type = constants.INSTANCE_TYPE_REMOTE @@ -64,8 +75,7 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): # Specified --local-image to a dir contains images self.Patch(utils, "GetBuildEnvironmentVariable", - return_value="test_environ") - self.Patch(os.path, "isfile", return_value=False) + return_value="test_cf_x86") self.args.local_image = "/path-to-image-dir" self.AvdSpec._avd_type = constants.TYPE_CF self.AvdSpec._instance_type = constants.INSTANCE_TYPE_REMOTE @@ -73,9 +83,9 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.assertEqual(self.AvdSpec._local_image_dir, expected_image_dir) # Specified local_image without arg - self.args.local_image = None + self.args.local_image = constants.FIND_IN_BUILD_ENV self.Patch(utils, "GetBuildEnvironmentVariable", - return_value="test_environ") + side_effect=["cf_x86_auto", "test_environ", "test_environ"]) self.AvdSpec._ProcessLocalImageArgs(self.args) self.assertEqual(self.AvdSpec._local_image_dir, "test_environ") self.assertEqual(self.AvdSpec.local_image_artifact, expected_image_artifact) @@ -83,28 +93,70 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): # Specified --avd-type=goldfish --local-image with a dir self.Patch(utils, "GetBuildEnvironmentVariable", return_value="test_environ") - self.Patch(os.path, "isdir", return_value=True) self.args.local_image = "/path-to-image-dir" self.AvdSpec._avd_type = constants.TYPE_GF self.AvdSpec._instance_type = constants.INSTANCE_TYPE_LOCAL self.AvdSpec._ProcessLocalImageArgs(self.args) self.assertEqual(self.AvdSpec._local_image_dir, expected_image_dir) - # Specified --avd-type=goldfish --local_image without arg - self.Patch(utils, "GetBuildEnvironmentVariable", - return_value="test_environ") - self.Patch(os.path, "isdir", return_value=True) - self.args.local_image = None + def testProcessLocalMixedImageArgs(self): + """Test process args.local_kernel_image and args.local_system_image.""" + expected_image_dir = "/path-to-image-dir" + expected_image_file = "/path-to-image-file" + self.Patch(os.path, "exists", + side_effect=lambda path: path in (expected_image_file, + expected_image_dir)) + self.Patch(os.path, "isdir", + side_effect=lambda path: path == expected_image_dir) + self.Patch(os.path, "isfile", + side_effect=lambda path: path == expected_image_file) + + # Specified --local-kernel-image and --local-system-image with dirs. + self.args.local_image = expected_image_dir + self.args.local_kernel_image = expected_image_dir + self.args.local_system_image = expected_image_dir + self.AvdSpec._avd_type = constants.TYPE_CF + self.AvdSpec._instance_type = constants.INSTANCE_TYPE_LOCAL + with mock.patch("acloud.create.avd_spec.utils." + "GetBuildEnvironmentVariable", + return_value="cf_x86_phone"): + self.AvdSpec._ProcessLocalImageArgs(self.args) + self.assertEqual(self.AvdSpec.local_image_dir, expected_image_dir) + self.assertEqual(self.AvdSpec.local_kernel_image, expected_image_dir) + self.assertEqual(self.AvdSpec.local_system_image, expected_image_dir) + + # Specified --local-kernel-image, and --local-system-image with files. + self.args.local_image = expected_image_dir + self.args.local_kernel_image = expected_image_file + self.args.local_system_image = expected_image_file + self.AvdSpec._avd_type = constants.TYPE_CF + self.AvdSpec._instance_type = constants.INSTANCE_TYPE_LOCAL + with mock.patch("acloud.create.avd_spec.utils." + "GetBuildEnvironmentVariable", + return_value="cf_x86_phone"): + self.AvdSpec._ProcessLocalImageArgs(self.args) + self.assertEqual(self.AvdSpec.local_image_dir, expected_image_dir) + self.assertEqual(self.AvdSpec.local_kernel_image, expected_image_file) + self.assertEqual(self.AvdSpec.local_system_image, expected_image_file) + + # Specified --avd-type=goldfish, --local_image, and + # --local-system-image without args + self.args.local_image = constants.FIND_IN_BUILD_ENV + self.args.local_system_image = constants.FIND_IN_BUILD_ENV self.AvdSpec._avd_type = constants.TYPE_GF self.AvdSpec._instance_type = constants.INSTANCE_TYPE_LOCAL - self.AvdSpec._ProcessLocalImageArgs(self.args) - self.assertEqual(self.AvdSpec._local_image_dir, "test_environ") + with mock.patch("acloud.create.avd_spec.utils." + "GetBuildEnvironmentVariable", + return_value=expected_image_dir): + self.AvdSpec._ProcessLocalImageArgs(self.args) + self.assertEqual(self.AvdSpec.local_image_dir, expected_image_dir) + self.assertEqual(self.AvdSpec.local_system_image, expected_image_dir) def testProcessImageArgs(self): """Test process image source.""" self.Patch(glob, "glob", return_value=["fake.img"]) # No specified local_image, image source is from remote - self.args.local_image = "" + self.args.local_image = None self.AvdSpec._ProcessImageArgs(self.args) self.assertEqual(self.AvdSpec._image_source, constants.IMAGE_SRC_REMOTE) self.assertEqual(self.AvdSpec._local_image_dir, None) @@ -138,19 +190,19 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): fake_subprocess.stdout.readline = mock.MagicMock(return_value='') fake_subprocess.poll = mock.MagicMock(return_value=0) fake_subprocess.returncode = 0 - return_value = "Manifest branch: master" + return_value = "Manifest branch: fake_branch" fake_subprocess.communicate = mock.MagicMock(return_value=(return_value, '')) self.Patch(subprocess, "Popen", return_value=fake_subprocess) mock_gitremote.return_value = "aosp" - self.assertEqual(self.AvdSpec._GetBranchFromRepo(), "aosp-master") + self.assertEqual(self.AvdSpec._GetBranchFromRepo(), "aosp-fake_branch") # Check default repo gets default branch prefix. mock_gitremote.return_value = "" - return_value = "Manifest branch: master" + return_value = "Manifest branch: fake_branch" fake_subprocess.communicate = mock.MagicMock(return_value=(return_value, '')) self.Patch(subprocess, "Popen", return_value=fake_subprocess) - self.assertEqual(self.AvdSpec._GetBranchFromRepo(), "git_master") + self.assertEqual(self.AvdSpec._GetBranchFromRepo(), "git_fake_branch") # Can't get branch from repo info, set it as default branch. return_value = "Manifest branch:" @@ -187,21 +239,21 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.args.avd_type = constants.TYPE_GCE self.assertEqual( self.AvdSpec._GetBuildTarget(self.args), - "gce_x86_iot-userdebug") + "gce_x86_64_iot-userdebug") self.AvdSpec._remote_image[constants.BUILD_BRANCH] = "aosp-master" self.AvdSpec._flavor = constants.FLAVOR_PHONE self.args.avd_type = constants.TYPE_CF self.assertEqual( self.AvdSpec._GetBuildTarget(self.args), - "aosp_cf_x86_phone-userdebug") + "aosp_cf_x86_64_phone-userdebug") self.AvdSpec._remote_image[constants.BUILD_BRANCH] = "git_branch" self.AvdSpec._flavor = constants.FLAVOR_PHONE self.args.avd_type = constants.TYPE_CF self.assertEqual( self.AvdSpec._GetBuildTarget(self.args), - "cf_x86_phone-userdebug") + "cf_x86_64_phone-userdebug") # pylint: disable=protected-access def testProcessHWPropertyWithInvalidArgs(self): @@ -232,6 +284,26 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.AvdSpec._ProcessHWPropertyArgs(args) # pylint: disable=protected-access + @mock.patch.object(utils, "PrintColorString") + def testCheckCFBuildTarget(self, print_warning): + """Test _CheckCFBuildTarget.""" + # patch correct env variable. + self.Patch(utils, "GetBuildEnvironmentVariable", + return_value="cf_x86_phone-userdebug") + self.AvdSpec._CheckCFBuildTarget(constants.INSTANCE_TYPE_REMOTE) + self.AvdSpec._CheckCFBuildTarget(constants.INSTANCE_TYPE_LOCAL) + + self.Patch(utils, "GetBuildEnvironmentVariable", + return_value="aosp_cf_arm64_auto-userdebug") + self.AvdSpec._CheckCFBuildTarget(constants.INSTANCE_TYPE_HOST) + # patch wrong env variable. + self.Patch(utils, "GetBuildEnvironmentVariable", + return_value="test_environ") + self.AvdSpec._CheckCFBuildTarget(constants.INSTANCE_TYPE_REMOTE) + + print_warning.assert_called_once() + + # pylint: disable=protected-access def testParseHWPropertyStr(self): """Test _ParseHWPropertyStr.""" expected_dict = {"cpu": "2", "x_res": "1080", "y_res": "1920", @@ -266,6 +338,7 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): """Test _ProcessRemoteBuildArgs.""" self.args.branch = "git_master" self.args.build_id = "1234" + self.args.launch_args = None # Verify auto-assigned avd_type if build_targe contains "_gce_". self.args.build_target = "aosp_gce_x86_phone-userdebug" @@ -305,6 +378,26 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.AvdSpec._ProcessRemoteBuildArgs(self.args) self.assertTrue(self.AvdSpec.avd_type == "cuttlefish") + # Setup acloud config with betty_image spec + cfg = mock.MagicMock() + cfg.betty_image = 'foobarbaz' + cfg.launch_args = None + self.Patch(config, 'GetAcloudConfig', return_value=cfg) + self.AvdSpec = avd_spec.AVDSpec(self.args) + # --betty-image from cmdline should override config + self.args.cheeps_betty_image = 'abcdefg' + self.AvdSpec._ProcessRemoteBuildArgs(self.args) + self.assertEqual( + self.AvdSpec.remote_image[constants.CHEEPS_BETTY_IMAGE], + self.args.cheeps_betty_image) + # acloud config value is used otherwise + self.args.cheeps_betty_image = None + self.AvdSpec._ProcessRemoteBuildArgs(self.args) + self.assertEqual( + self.AvdSpec.remote_image[constants.CHEEPS_BETTY_IMAGE], + cfg.betty_image) + + def testEscapeAnsi(self): """Test EscapeAnsi.""" test_string = "\033[1;32;40m Manifest branch:" @@ -352,7 +445,7 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.assertEqual(self.AvdSpec._instance_type, constants.INSTANCE_TYPE_REMOTE) self.args.remote_host = None - self.args.local_instance = True + self.args.local_instance = 0 self.AvdSpec._ProcessMiscArgs(self.args) self.assertEqual(self.AvdSpec._instance_type, constants.INSTANCE_TYPE_LOCAL) @@ -362,10 +455,14 @@ class AvdSpecTest(driver_test_lib.BaseDriverTest): self.assertEqual(self.AvdSpec._instance_type, constants.INSTANCE_TYPE_HOST) self.args.remote_host = "1.1.1.1" - self.args.local_instance = True + self.args.local_instance = 1 self.AvdSpec._ProcessMiscArgs(self.args) self.assertEqual(self.AvdSpec._instance_type, constants.INSTANCE_TYPE_HOST) + self.args.oxygen = True + self.AvdSpec._ProcessMiscArgs(self.args) + self.assertTrue(self.AvdSpec._oxygen) + # Test avd_spec.autoconnect self.args.autoconnect = False self.AvdSpec._ProcessMiscArgs(self.args) diff --git a/create/cheeps_remote_image_remote_instance_test.py b/create/cheeps_remote_image_remote_instance_test.py index 2b0bba27..17d917e9 100644 --- a/create/cheeps_remote_image_remote_instance_test.py +++ b/create/cheeps_remote_image_remote_instance_test.py @@ -2,7 +2,7 @@ import unittest import uuid -import mock +from unittest import mock from acloud.create import cheeps_remote_image_remote_instance from acloud.internal import constants diff --git a/create/create.py b/create/create.py index 37b73519..3953a468 100644 --- a/create/create.py +++ b/create/create.py @@ -48,6 +48,7 @@ from acloud.setup import host_setup_runner _MAKE_CMD = "build/soong/soong_ui.bash" _MAKE_ARG = "--make-mode" +_YES = "y" _CREATOR_CLASS_DICT = { # GCE types @@ -76,6 +77,9 @@ _CREATOR_CLASS_DICT = { goldfish_remote_image_remote_instance.GoldfishRemoteImageRemoteInstance, (constants.TYPE_GF, constants.IMAGE_SRC_LOCAL, constants.INSTANCE_TYPE_LOCAL): goldfish_local_image_local_instance.GoldfishLocalImageLocalInstance, + # FVP types + (constants.TYPE_FVP, constants.IMAGE_SRC_LOCAL, constants.INSTANCE_TYPE_REMOTE): + local_image_remote_instance.LocalImageRemoteInstance, } @@ -119,7 +123,7 @@ def _CheckForAutoconnect(args): return disable_autoconnect = False - answer = utils.InteractWithQuestion( + answer = _YES if args.no_prompt else utils.InteractWithQuestion( "adb is required for autoconnect, without it autoconnect will be " "disabled, would you like acloud to build it[y/N]? ") if answer in constants.USER_ANSWER_YES: @@ -166,8 +170,9 @@ def _CheckForSetup(args): args.host = False args.host_base = False args.force = False + args.update_config = None # Remote image/instance requires the GCP config setup. - if not args.local_instance or args.local_image == "": + if args.local_instance is None or args.local_image is None: gcp_setup = gcp_setup_runner.GcpTaskRunner(args.config_file) if gcp_setup.ShouldRun(): args.gcp_init = True @@ -178,7 +183,7 @@ def _CheckForSetup(args): # The following local instance create will trigger this if statment and go # through the whole setup again even though it's already done because the # user groups aren't set until the user logs out and back in. - if args.local_instance: + if args.local_instance is not None: host_pkg_setup = host_setup_runner.AvdPkgInstaller() if host_pkg_setup.ShouldRun(): args.host = True diff --git a/create/create_args.py b/create/create_args.py index 5bbab572..8071f558 100644 --- a/create/create_args.py +++ b/create/create_args.py @@ -17,6 +17,7 @@ Defines the create arg parser that holds create specific args. """ import argparse +import logging import os from acloud import errors @@ -24,7 +25,8 @@ from acloud.create import create_common from acloud.internal import constants from acloud.internal.lib import utils - +logger = logging.getLogger(__name__) +_DEFAULT_GPU = "default" CMD_CREATE = "create" @@ -130,6 +132,24 @@ def AddCommonCreateArgs(parser): dest="build_id", help="Android build id, e.g. 2145099, P2804227") parser.add_argument( + "--bootloader-branch", + type=str, + dest="bootloader_branch", + help="'cuttlefish only' Branch to consume the bootloader from.", + required=False) + parser.add_argument( + "--bootloader-build-id", + type=str, + dest="bootloader_build_id", + help="'cuttlefish only' Bootloader build id, e.g. P2804227", + required=False) + parser.add_argument( + "--bootloader-build-target", + type=str, + dest="bootloader_build_target", + help="'cuttlefish only' Bootloader build target.", + required=False) + parser.add_argument( "--kernel-build-id", type=str, dest="kernel_build_id", @@ -177,6 +197,12 @@ def AddCommonCreateArgs(parser): help="'cuttlefish only' System image build target, specify if different " "from --build-target", required=False) + parser.add_argument( + "--launch-args", + type=str, + dest="launch_args", + help="'cuttlefish only' Add extra args to launch_cvd command.", + required=False) # TODO(146314062): Remove --multi-stage-launch after infra don't use this # args. parser.add_argument( @@ -205,10 +231,39 @@ def AddCommonCreateArgs(parser): parser.add_argument( "--gpu", type=str, + const=_DEFAULT_GPU, + nargs="?", dest="gpu", required=False, default=None, - help="GPU accelerator to use if any. e.g. nvidia-tesla-k80.") + help="GPU accelerator to use if any. e.g. nvidia-tesla-k80. For local " + "instances, this arg without assigning any value is to enable " + "local gpu support.") + # Hide following args for users, it is only used in infra. + parser.add_argument( + "--local-instance-dir", + dest="local_instance_dir", + required=False, + help=argparse.SUPPRESS) + parser.add_argument( + "--num-avds-per-instance", + type=int, + dest="num_avds_per_instance", + required=False, + default=1, + help=argparse.SUPPRESS) + parser.add_argument( + "--oxygen", + action="store_true", + dest="oxygen", + required=False, + help=argparse.SUPPRESS) + parser.add_argument( + "--zone", + type=str, + dest="zone", + required=False, + help=argparse.SUPPRESS) # TODO(b/118439885): Old arg formats to support transition, delete when # transistion is done. @@ -266,6 +321,24 @@ def AddCommonCreateArgs(parser): dest="kernel_build_target", default="kernel", help=argparse.SUPPRESS) + parser.add_argument( + "--bootloader_branch", + type=str, + dest="bootloader_branch", + help=argparse.SUPPRESS, + required=False) + parser.add_argument( + "--bootloader_build_id", + type=str, + dest="bootloader_build_id", + help=argparse.SUPPRESS, + required=False) + parser.add_argument( + "--bootloader_build_target", + type=str, + dest="bootloader_build_target", + help=argparse.SUPPRESS, + required=False) def GetCreateArgParser(subparser): @@ -280,17 +353,19 @@ def GetCreateArgParser(subparser): create_parser = subparser.add_parser(CMD_CREATE) create_parser.required = False create_parser.set_defaults(which=CMD_CREATE) - # Use default=0 to distinguish remote instance or local. The instance type - # will be remote if arg --local-instance is not provided. + # Use default=None to distinguish remote instance or local. The instance + # type will be remote if the arg is not provided. create_parser.add_argument( "--local-instance", - type=int, - const=1, + type=_PositiveInteger, + const=0, + metavar="ID", nargs="?", dest="local_instance", required=False, - help="Create a local AVD instance with the option to specify the local " - "instance ID (primarily for infra usage).") + help="Create a local AVD instance using the resources associated with " + "the ID. Choose an unused ID automatically if the value is " + "not specified (primarily for infra usage).") create_parser.add_argument( "--adb-port", "-p", type=int, @@ -303,34 +378,49 @@ def GetCreateArgParser(subparser): type=str, dest="avd_type", default=constants.TYPE_CF, - choices=[constants.TYPE_GCE, constants.TYPE_CF, constants.TYPE_GF, constants.TYPE_CHEEPS], + choices=[constants.TYPE_GCE, constants.TYPE_CF, constants.TYPE_GF, constants.TYPE_CHEEPS, + constants.TYPE_FVP], help="Android Virtual Device type (default %s)." % constants.TYPE_CF) create_parser.add_argument( - "--flavor", + "--config", "--flavor", type=str, dest="flavor", - help="The device flavor of the AVD (default %s)." % constants.FLAVOR_PHONE) + help="The device flavor of the AVD (default %s). e.g. phone, tv, foldable." + % constants.FLAVOR_PHONE) create_parser.add_argument( "--local-image", + const=constants.FIND_IN_BUILD_ENV, type=str, dest="local_image", nargs="?", - default="", required=False, help="Use the locally built image for the AVD. Look for the image " "artifact in $ANDROID_PRODUCT_OUT if no args value is provided." "e.g --local-image or --local-image /path/to/dir or --local-image " "/path/to/file") create_parser.add_argument( + "--local-kernel-image", + const=constants.FIND_IN_BUILD_ENV, + type=str, + dest="local_kernel_image", + nargs="?", + required=False, + help="Use the locally built kernel image for the AVD. Look for " + "boot.img or boot-*.img if the argument is a directory. Look for the " + "image in $ANDROID_PRODUCT_OUT if no argument is provided. e.g., " + "--local-kernel-image, --local-kernel-image /path/to/dir, or " + "--local-kernel-image /path/to/img") + create_parser.add_argument( "--local-system-image", + const=constants.FIND_IN_BUILD_ENV, type=str, dest="local_system_image", nargs="?", - default="", required=False, help="Use the locally built system images for the AVD. Look for the " "images in $ANDROID_PRODUCT_OUT if no args value is provided. " - "e.g., --local-system-image or --local-system-image /path/to/dir") + "e.g., --local-system-image, --local-system-image /path/to/dir, or " + "--local-system-image /path/to/img") create_parser.add_argument( "--local-tool", type=str, @@ -366,6 +456,13 @@ def GetCreateArgParser(subparser): "provided. Select one gce instance to reuse if --reuse-gce is " "provided.") create_parser.add_argument( + "--gce-metadata", + type=str, + dest="gce_metadata", + default=None, + help="'GCE instance only' Record data into GCE instance metadata with " + "key-value pair format. e.g. id:12,name:unknown.") + create_parser.add_argument( "--host", type=str, dest="remote_host", @@ -447,11 +544,31 @@ def GetCreateArgParser(subparser): required=False, default=None, help="'cheeps only' password to log in to Chrome OS with.") + create_parser.add_argument( + "--betty-image", + type=str, + dest="cheeps_betty_image", + required=False, + default=None, + help=("'cheeps only' The L1 betty version to use. Only makes sense " + "when launching a controller image with " + "stable-cheeps-host-image")) AddCommonCreateArgs(create_parser) return create_parser +def _PositiveInteger(arg): + """Convert an argument from a string to a positive integer.""" + try: + value = int(arg) + except ValueError as e: + raise argparse.ArgumentTypeError(arg + " is not an integer.") from e + if value <= 0: + raise argparse.ArgumentTypeError(arg + " is not positive.") + return value + + def _VerifyLocalArgs(args): """Verify args starting with --local. @@ -468,8 +585,12 @@ def _VerifyLocalArgs(args): raise errors.CheckPathError( "Specified path doesn't exist: %s" % args.local_image) - # TODO(b/133211308): Support TYPE_CF. - if args.local_system_image != "" and args.avd_type != constants.TYPE_GF: + if args.local_instance_dir and not os.path.exists(args.local_instance_dir): + raise errors.CheckPathError( + "Specified path doesn't exist: %s" % args.local_instance_dir) + + if not (args.local_system_image is None or + args.avd_type in (constants.TYPE_CF, constants.TYPE_GF)): raise errors.UnsupportedCreateArgs("%s instance does not support " "--local-system-image" % args.avd_type) @@ -479,11 +600,6 @@ def _VerifyLocalArgs(args): raise errors.CheckPathError( "Specified path doesn't exist: %s" % args.local_system_image) - if args.local_instance is not None and args.local_instance < 1: - raise errors.UnsupportedLocalInstanceId("Local instance id can not be " - "less than 1. Actually passed:%d" - % args.local_instance) - for tool_dir in args.local_tool: if not os.path.exists(tool_dir): raise errors.CheckPathError( @@ -529,7 +645,6 @@ def VerifyArgs(args): args: Namespace object from argparse.parse_args. Raises: - errors.UnsupportedFlavor: Flavor doesn't support. errors.UnsupportedMultiAdbPort: multi adb port doesn't support. errors.UnsupportedCreateArgs: When a create arg is specified but unsupported for a particular avd type. @@ -539,9 +654,9 @@ def VerifyArgs(args): # We don't use argparse's builtin validation because we need to be able to # tell when a user doesn't specify a flavor. if args.flavor and args.flavor not in constants.ALL_FLAVORS: - raise errors.UnsupportedFlavor( - "Flavor[%s] isn't in support list: %s" % (args.flavor, - constants.ALL_FLAVORS)) + logger.debug("Flavor[%s] isn't in default support list: %s", + args.flavor, constants.ALL_FLAVORS) + if args.avd_type != constants.TYPE_CF: if args.system_branch or args.system_build_id or args.system_build_target: raise errors.UnsupportedCreateArgs( @@ -556,23 +671,30 @@ def VerifyArgs(args): raise errors.UnsupportedCreateArgs( "--num is not supported for local instance.") + if args.local_instance is None and args.gpu == _DEFAULT_GPU: + raise errors.UnsupportedCreateArgs( + "Please assign one gpu model for GCE instance. Reference: " + "https://cloud.google.com/compute/docs/gpus") + if args.adb_port: utils.CheckPortFree(args.adb_port) - hw_properties = create_common.ParseHWPropertyArgs(args.hw_property) + hw_properties = create_common.ParseKeyValuePairArgs(args.hw_property) for key in hw_properties: if key not in constants.HW_PROPERTIES: raise errors.InvalidHWPropertyError( "[%s] is an invalid hw property, supported values are:%s. " % (key, constants.HW_PROPERTIES)) - if args.avd_type != constants.TYPE_CHEEPS and ( - args.stable_cheeps_host_image_name or - args.stable_cheeps_host_image_project or - args.username or args.password): + cheeps_only_flags = [args.stable_cheeps_host_image_name, + args.stable_cheeps_host_image_project, + args.username, + args.password, + args.cheeps_betty_image] + if args.avd_type != constants.TYPE_CHEEPS and any(cheeps_only_flags): raise errors.UnsupportedCreateArgs( - "--stable-cheeps-*, --username and --password are only valid with " - "avd_type == %s" % constants.TYPE_CHEEPS) + "--stable-cheeps-*, --betty-image, --username and --password are " + "only valid with avd_type == %s" % constants.TYPE_CHEEPS) if (args.username or args.password) and not (args.username and args.password): raise ValueError("--username and --password must both be set") if not args.autoconnect and args.unlock_screen: diff --git a/create/create_args_test.py b/create/create_args_test.py index 8cca9ee2..fe5794e6 100644 --- a/create/create_args_test.py +++ b/create/create_args_test.py @@ -14,7 +14,8 @@ """Tests for create.""" import unittest -import mock + +from unittest import mock from acloud import errors from acloud.create import create_args @@ -26,15 +27,17 @@ def _CreateArgs(): """set default pass in arguments.""" mock_args = mock.MagicMock( flavor=None, - num=None, + num=1, adb_port=None, hw_property=None, stable_cheeps_host_image_name=None, stable_cheeps_host_image_project=None, username=None, password=None, - local_image="", - local_system_image="", + cheeps_betty_image=None, + local_image=None, + local_kernel_image=None, + local_system_image=None, system_branch=None, system_build_id=None, system_build_target=None, diff --git a/create/create_common.py b/create/create_common.py index 2846e318..a27bf314 100644 --- a/create/create_common.py +++ b/create/create_common.py @@ -17,6 +17,8 @@ import logging import os +import re +import shutil from acloud import errors from acloud.internal import constants @@ -28,7 +30,7 @@ from acloud.internal.lib import utils logger = logging.getLogger(__name__) -def ParseHWPropertyArgs(dict_str, item_separator=",", key_value_separator=":"): +def ParseKeyValuePairArgs(dict_str, item_separator=",", key_value_separator=":"): """Helper function to initialize a dict object from string. e.g. @@ -46,9 +48,9 @@ def ParseHWPropertyArgs(dict_str, item_separator=",", key_value_separator=":"): Raises: error.MalformedDictStringError: If dict_str is malformed. """ - hw_dict = {} + args_dict = {} if not dict_str: - return hw_dict + return args_dict for item in dict_str.split(item_separator): if key_value_separator not in item: @@ -58,9 +60,9 @@ def ParseHWPropertyArgs(dict_str, item_separator=",", key_value_separator=":"): if not value or not key: raise errors.MalformedDictStringError( "Missing key or value in %s, expecting form of 'a:b'" % item) - hw_dict[key.strip()] = value.strip() + args_dict[key.strip()] = value.strip() - return hw_dict + return args_dict def GetCvdHostPackage(): @@ -75,7 +77,11 @@ def GetCvdHostPackage(): Raises: errors.GetCvdLocalHostPackageError: Can't find cvd host package. """ - dirs_to_check = list(filter(None, [os.environ.get(constants.ENV_ANDROID_HOST_OUT)])) + dirs_to_check = list( + filter(None, [ + os.environ.get(constants.ENV_ANDROID_SOONG_HOST_OUT), + os.environ.get(constants.ENV_ANDROID_HOST_OUT) + ])) dist_dir = utils.GetDistDir() if dist_dir: dirs_to_check.append(dist_dir) @@ -91,6 +97,35 @@ def GetCvdHostPackage(): '\n'.join(dirs_to_check)) +def FindLocalImage(path, default_name_pattern): + """Find an image file in the given path. + + Args: + path: The path to the file or the parent directory. + default_name_pattern: A regex string, the file to look for if the path + is a directory. + + Returns: + The absolute path to the image file. + + Raises: + errors.GetLocalImageError if this method cannot find exactly one image. + """ + path = os.path.abspath(path) + if os.path.isdir(path): + names = [name for name in os.listdir(path) if + re.fullmatch(default_name_pattern, name)] + if not names: + raise errors.GetLocalImageError("No image in %s." % path) + if len(names) != 1: + raise errors.GetLocalImageError("More than one image in %s: %s" % + (path, " ".join(names))) + path = os.path.join(path, names[0]) + if os.path.isfile(path): + return path + raise errors.GetLocalImageError("%s is not a file." % path) + + def DownloadRemoteArtifact(cfg, build_target, build_id, artifact, extract_path, decompress=False): """Download remote artifact. @@ -118,3 +153,28 @@ def DownloadRemoteArtifact(cfg, build_target, build_id, artifact, extract_path, logger.debug("Deleted temporary file %s", temp_file) except OSError as e: logger.error("Failed to delete temporary file: %s", str(e)) + + +def PrepareLocalInstanceDir(instance_dir, avd_spec): + """Create a directory for a local cuttlefish or goldfish instance. + + If avd_spec has the local instance directory, this method creates a + symbolic link from instance_dir to the directory. Otherwise, it creates an + empty directory at instance_dir. + + Args: + instance_dir: The absolute path to the default instance directory. + avd_spec: AVDSpec object that provides the instance directory. + """ + if os.path.islink(instance_dir): + os.remove(instance_dir) + else: + shutil.rmtree(instance_dir, ignore_errors=True) + + if avd_spec.local_instance_dir: + abs_instance_dir = os.path.abspath(avd_spec.local_instance_dir) + if instance_dir != abs_instance_dir: + os.symlink(abs_instance_dir, instance_dir) + return + if not os.path.exists(instance_dir): + os.makedirs(instance_dir) diff --git a/create/create_common_test.py b/create/create_common_test.py index adfd0c27..b19547be 100644 --- a/create/create_common_test.py +++ b/create/create_common_test.py @@ -14,9 +14,11 @@ """Tests for create_common.""" import os +import shutil +import tempfile import unittest -import mock +from unittest import mock from acloud import errors from acloud.create import create_common @@ -26,7 +28,7 @@ from acloud.internal.lib import driver_test_lib from acloud.internal.lib import utils -class FakeZipFile(object): +class FakeZipFile: """Fake implementation of ZipFile()""" # pylint: disable=invalid-name,unused-argument,no-self-use @@ -46,23 +48,23 @@ class CreateCommonTest(driver_test_lib.BaseDriverTest): # pylint: disable=protected-access def testProcessHWPropertyWithInvalidArgs(self): - """Test ParseHWPropertyArgs with invalid args.""" + """Test ParseKeyValuePairArgs with invalid args.""" # Checking wrong property value. args_str = "cpu:3,disk:" with self.assertRaises(errors.MalformedDictStringError): - create_common.ParseHWPropertyArgs(args_str) + create_common.ParseKeyValuePairArgs(args_str) # Checking wrong property format. args_str = "cpu:3,disk" with self.assertRaises(errors.MalformedDictStringError): - create_common.ParseHWPropertyArgs(args_str) + create_common.ParseKeyValuePairArgs(args_str) def testParseHWPropertyStr(self): - """Test ParseHWPropertyArgs.""" + """Test ParseKeyValuePairArgs.""" expected_dict = {"cpu": "2", "resolution": "1080x1920", "dpi": "240", "memory": "4g", "disk": "4g"} args_str = "cpu:2,resolution:1080x1920,dpi:240,memory:4g,disk:4g" - result_dict = create_common.ParseHWPropertyArgs(args_str) + result_dict = create_common.ParseKeyValuePairArgs(args_str) self.assertTrue(expected_dict == result_dict) def testGetCvdHostPackage(self): @@ -76,9 +78,9 @@ class CreateCommonTest(driver_test_lib.BaseDriverTest): self.Patch(os.environ, "get", return_value="/fake_dir2") self.Patch(utils, "GetDistDir", return_value="/fake_dir1") - # First path is host out dir, 2nd path is dist dir. + # First and 2nd path are host out dirs, 3rd path is dist dir. self.Patch(os.path, "exists", - side_effect=[False, True]) + side_effect=[False, False, True]) # Find cvd host in dist dir. self.assertEqual( @@ -94,6 +96,27 @@ class CreateCommonTest(driver_test_lib.BaseDriverTest): create_common.GetCvdHostPackage(), "/fake_dir2/cvd-host_package.tar.gz") + @mock.patch("acloud.create.create_common.os.path.isfile", + side_effect=lambda path: path == "/dir/name") + @mock.patch("acloud.create.create_common.os.path.isdir", + side_effect=lambda path: path == "/dir") + @mock.patch("acloud.create.create_common.os.listdir", + return_value=["name", "name2"]) + def testFindLocalImage(self, _mock_listdir, _mock_isdir, _mock_isfile): + """Test FindLocalImage.""" + self.assertEqual( + "/dir/name", + create_common.FindLocalImage("/test/../dir/name", "not_exist")) + + self.assertEqual("/dir/name", + create_common.FindLocalImage("/dir/", "name")) + + with self.assertRaises(errors.GetLocalImageError): + create_common.FindLocalImage("/dir", "not_exist") + + with self.assertRaises(errors.GetLocalImageError): + create_common.FindLocalImage("/dir", "name.?") + @mock.patch.object(utils, "Decompress") def testDownloadRemoteArtifact(self, mock_decompress): """Test Download cuttlefish package.""" @@ -146,6 +169,25 @@ class CreateCommonTest(driver_test_lib.BaseDriverTest): "%s/%s" % (extract_path, checkfile2)) self.assertEqual(mock_decompress.call_count, 0) + def testPrepareLocalInstanceDir(self): + """test PrepareLocalInstanceDir.""" + temp_dir = tempfile.mkdtemp() + try: + cvd_home_dir = os.path.join(temp_dir, "local-instance-1") + mock_avd_spec = mock.Mock(local_instance_dir=None) + create_common.PrepareLocalInstanceDir(cvd_home_dir, mock_avd_spec) + self.assertTrue(os.path.isdir(cvd_home_dir) and + not os.path.islink(cvd_home_dir)) + + link_target_dir = os.path.join(temp_dir, "cvd_home") + os.mkdir(link_target_dir) + mock_avd_spec.local_instance_dir = link_target_dir + create_common.PrepareLocalInstanceDir(cvd_home_dir, mock_avd_spec) + self.assertTrue(os.path.islink(cvd_home_dir) and + os.path.samefile(cvd_home_dir, link_target_dir)) + finally: + shutil.rmtree(temp_dir) + if __name__ == "__main__": unittest.main() diff --git a/create/create_test.py b/create/create_test.py index d648cdb0..da69da1e 100644 --- a/create/create_test.py +++ b/create/create_test.py @@ -16,7 +16,8 @@ import os import subprocess import unittest -import mock + +from unittest import mock from acloud import errors from acloud.create import avd_spec @@ -57,6 +58,7 @@ class CreateTest(driver_test_lib.BaseDriverTest): """Test CheckForAutoconnect.""" args = mock.MagicMock() args.autoconnect = True + args.no_prompt = False self.Patch(utils, "InteractWithQuestion", return_value="Y") self.Patch(utils, "FindExecutable", return_value=None) @@ -110,6 +112,47 @@ class CreateTest(driver_test_lib.BaseDriverTest): create._CheckForSetup(args) setup.Run.assert_called_once() + # Should or not run gcp_setup or install packages. + # Test with remote instance remote image case. + self.Patch(gcp_setup_runner.GcpTaskRunner, + "ShouldRun") + self.Patch(host_setup_runner.AvdPkgInstaller, + "ShouldRun") + args.local_instance = None + args.local_image = None + create._CheckForSetup(args) + self.assertEqual(gcp_setup_runner.GcpTaskRunner.ShouldRun.call_count, 1) + self.assertEqual(host_setup_runner.AvdPkgInstaller.ShouldRun.call_count, 0) + gcp_setup_runner.GcpTaskRunner.ShouldRun.reset_mock() + host_setup_runner.AvdPkgInstaller.ShouldRun.reset_mock() + + # Test with remote instance local image case. + args.local_instance = None + args.local_image = "" + create._CheckForSetup(args) + self.assertEqual(gcp_setup_runner.GcpTaskRunner.ShouldRun.call_count, 1) + self.assertEqual(host_setup_runner.AvdPkgInstaller.ShouldRun.call_count, 0) + gcp_setup_runner.GcpTaskRunner.ShouldRun.reset_mock() + host_setup_runner.AvdPkgInstaller.ShouldRun.reset_mock() + + # Test with local instance remote image case. + args.local_instance = 0 + args.local_image = None + create._CheckForSetup(args) + self.assertEqual(gcp_setup_runner.GcpTaskRunner.ShouldRun.call_count, 1) + self.assertEqual(host_setup_runner.AvdPkgInstaller.ShouldRun.call_count, 1) + gcp_setup_runner.GcpTaskRunner.ShouldRun.reset_mock() + host_setup_runner.AvdPkgInstaller.ShouldRun.reset_mock() + + # Test with local instance local image case. + args.local_instance = 0 + args.local_image = "" + create._CheckForSetup(args) + self.assertEqual(gcp_setup_runner.GcpTaskRunner.ShouldRun.call_count, 0) + self.assertEqual(host_setup_runner.AvdPkgInstaller.ShouldRun.call_count, 1) + gcp_setup_runner.GcpTaskRunner.ShouldRun.reset_mock() + host_setup_runner.AvdPkgInstaller.ShouldRun.reset_mock() + # pylint: disable=no-member def testRun(self): """Test Run.""" diff --git a/create/goldfish_local_image_local_instance.py b/create/goldfish_local_image_local_instance.py index b98242a3..db839621 100644 --- a/create/goldfish_local_image_local_instance.py +++ b/create/goldfish_local_image_local_instance.py @@ -30,8 +30,7 @@ required. should be an unzipped SDK image repository, i.e., sdk-repo-<os>-system-images-<build>.zip. - If the instance requires mixed images, the local image directory should - contain both the unzipped update package and the unzipped extra image - package, i.e., <target>-img-<build>.zip and + be an unzipped extra image package, i.e., emu-extra-<os>-system-images-<build>.zip. - If the instance requires mixed images, one of the local tool directories should be an unzipped OTA tools package, i.e., otatools.zip. @@ -45,8 +44,8 @@ import sys from acloud import errors from acloud.create import base_avd_create +from acloud.create import create_common from acloud.internal import constants -from acloud.internal.lib import adb_tools from acloud.internal.lib import ota_tools from acloud.internal.lib import utils from acloud.list import instance @@ -57,19 +56,16 @@ logger = logging.getLogger(__name__) # Input and output file names _EMULATOR_BIN_NAME = "emulator" +_EMULATOR_BIN_DIR_NAMES = ("bin64", "qemu") _SDK_REPO_EMULATOR_DIR_NAME = "emulator" _SYSTEM_IMAGE_NAME = "system.img" +_SYSTEM_IMAGE_NAME_PATTERN = r"system\.img" _SYSTEM_QEMU_IMAGE_NAME = "system-qemu.img" _NON_MIXED_BACKUP_IMAGE_EXT = ".bak-non-mixed" _BUILD_PROP_FILE_NAME = "build.prop" _MISC_INFO_FILE_NAME = "misc_info.txt" _SYSTEM_QEMU_CONFIG_FILE_NAME = "system-qemu-config.txt" -# Partition names -_SYSTEM_PARTITION_NAME = "system" -_SUPER_PARTITION_NAME = "super" -_VBMETA_PARTITION_NAME = "vbmeta" - # Timeout _DEFAULT_EMULATOR_TIMEOUT_SECS = 150 _EMULATOR_TIMEOUT_ERROR = "Emulator did not boot within %(timeout)d secs." @@ -85,62 +81,9 @@ _MISSING_EMULATOR_MSG = ("Emulator binary is not found. Check " "or set --local-tool to an unzipped SDK emulator " "repository.") - -def _GetImageForLogicalPartition(partition_name, system_image_path, image_dir): - """Map a logical partition name to an image path. - - Args: - partition_name: String. On emulator, the logical partitions include - "system", "vendor", and "product". - system_image_path: String. The path to system image. - image_dir: String. The directory containing the other images. - - Returns: - system_image_path if the partition is "system". - Otherwise, this method returns the path under image_dir. - - Raises - errors.GetLocalImageError if the image does not exist. - """ - if partition_name == _SYSTEM_PARTITION_NAME: - image_path = system_image_path - else: - image_path = os.path.join(image_dir, partition_name + ".img") - if not os.path.isfile(image_path): - raise errors.GetLocalImageError( - "Cannot find image for logical partition %s" % partition_name) - return image_path - - -def _GetImageForPhysicalPartition(partition_name, super_image_path, - vbmeta_image_path, image_dir): - """Map a physical partition name to an image path. - - Args: - partition_name: String. On emulator, the physical partitions include - "super" and "vbmeta". - super_image_path: String. The path to super image. - vbmeta_image_path: String. The path to vbmeta image. - image_dir: String. The directory containing the other images. - - Returns: - super_image_path if the partition is "super". - vbmeta_image_path if the partition is "vbmeta". - Otherwise, this method returns the path under image_dir. - - Raises: - errors.GetLocalImageError if the image does not exist. - """ - if partition_name == _SUPER_PARTITION_NAME: - image_path = super_image_path - elif partition_name == _VBMETA_PARTITION_NAME: - image_path = vbmeta_image_path - else: - image_path = os.path.join(image_dir, partition_name + ".img") - if not os.path.isfile(image_path): - raise errors.GetLocalImageError( - "Unexpected physical partition: %s" % partition_name) - return image_path +_INSTANCES_IN_USE_MSG = ("All instances are in use. Try resetting an instance " + "by specifying --local-instance and an id between 1 " + "and %(max_id)d.") class GoldfishLocalImageLocalInstance(base_avd_create.BaseAVDCreate): @@ -155,29 +98,86 @@ class GoldfishLocalImageLocalInstance(base_avd_create.BaseAVDCreate): Returns: A Report instance. + """ + if not utils.IsSupportedPlatform(print_warning=True): + result_report = report.Report(command="create") + result_report.SetStatus(report.Status.FAIL) + return result_report + + try: + ins_id, ins_lock = self._LockInstance(avd_spec) + except errors.CreateError as e: + result_report = report.Report(command="create") + result_report.AddError(str(e)) + result_report.SetStatus(report.Status.FAIL) + return result_report + + try: + ins = instance.LocalGoldfishInstance(ins_id, + avd_flavor=avd_spec.flavor) + if not self._CheckRunningEmulator(ins.adb, no_prompts): + # Mark as in-use so that it won't be auto-selected again. + ins_lock.SetInUse(True) + sys.exit(constants.EXIT_BY_USER) + + result_report = self._CreateAVDForInstance(ins, avd_spec) + # The infrastructure is able to delete the instance only if the + # instance name is reported. This method changes the state to + # in-use after creating the report. + ins_lock.SetInUse(True) + return result_report + finally: + ins_lock.Unlock() + + @staticmethod + def _LockInstance(avd_spec): + """Select an id and lock the instance. + + Args: + avd_spec: AVDSpec for the device. + + Returns: + The instance id and the LocalInstanceLock that is locked by this + process. + + Raises: + errors.CreateError if fails to select or lock the instance. + """ + if avd_spec.local_instance_id: + ins_id = avd_spec.local_instance_id + ins_lock = instance.LocalGoldfishInstance.GetLockById(ins_id) + if ins_lock.Lock(): + return ins_id, ins_lock + raise errors.CreateError("Instance %d is locked by another " + "process." % ins_id) + + max_id = instance.LocalGoldfishInstance.GetMaxNumberOfInstances() + for ins_id in range(1, max_id + 1): + ins_lock = instance.LocalGoldfishInstance.GetLockById(ins_id) + if ins_lock.LockIfNotInUse(timeout_secs=0): + logger.info("Selected instance id: %d", ins_id) + return ins_id, ins_lock + raise errors.CreateError(_INSTANCES_IN_USE_MSG % {"max_id": max_id}) + + def _CreateAVDForInstance(self, ins, avd_spec): + """Create an emulator process for the goldfish instance. + + Args: + ins: LocalGoldfishInstance to be initialized. + avd_spec: AVDSpec for the device. + + Returns: + A Report instance. Raises: errors.GetSdkRepoPackageError if emulator binary is not found. errors.GetLocalImageError if the local image directory does not contain required files. - errors.CreateError if an instance exists and cannot be deleted. errors.CheckPathError if OTA tools are not found. """ - if not utils.IsSupportedPlatform(print_warning=True): - result_report = report.Report(command="create") - result_report.SetStatus(report.Status.FAIL) - return result_report - emulator_path = self._FindEmulatorBinary(avd_spec.local_tool_dirs) - emulator_path = os.path.abspath(emulator_path) - image_dir = os.path.abspath(avd_spec.local_image_dir) - - if not (os.path.isfile(os.path.join(image_dir, _SYSTEM_IMAGE_NAME)) or - os.path.isfile(os.path.join(image_dir, - _SYSTEM_QEMU_IMAGE_NAME))): - raise errors.GetLocalImageError("No system image in %s." % - image_dir) + image_dir = self._FindImageDir(avd_spec.local_image_dir) # TODO(b/141898893): In Android build environment, emulator gets build # information from $ANDROID_PRODUCT_OUT/system/build.prop. @@ -186,54 +186,44 @@ class GoldfishLocalImageLocalInstance(base_avd_create.BaseAVDCreate): # image_dir/system/build.prop. self._CopyBuildProp(image_dir) - instance_id = avd_spec.local_instance_id - inst = instance.LocalGoldfishInstance(instance_id, - avd_flavor=avd_spec.flavor) - adb = adb_tools.AdbTools(adb_port=inst.adb_port, - device_serial=inst.device_serial) - - self._CheckRunningEmulator(adb, no_prompts) - - instance_dir = inst.instance_dir - shutil.rmtree(instance_dir, ignore_errors=True) - os.makedirs(instance_dir) + instance_dir = ins.instance_dir + create_common.PrepareLocalInstanceDir(instance_dir, avd_spec) extra_args = self._ConvertAvdSpecToArgs(avd_spec, instance_dir) logger.info("Instance directory: %s", instance_dir) proc = self._StartEmulatorProcess(emulator_path, instance_dir, - image_dir, inst.console_port, - inst.adb_port, extra_args) + image_dir, ins.console_port, + ins.adb_port, extra_args) boot_timeout_secs = (avd_spec.boot_timeout_secs or _DEFAULT_EMULATOR_TIMEOUT_SECS) result_report = report.Report(command="create") try: - self._WaitForEmulatorToStart(adb, proc, boot_timeout_secs) + self._WaitForEmulatorToStart(ins.adb, proc, boot_timeout_secs) except (errors.DeviceBootTimeoutError, errors.SubprocessFail) as e: result_report.SetStatus(report.Status.BOOT_FAIL) - result_report.AddDeviceBootFailure(inst.name, inst.ip, - inst.adb_port, vnc_port=None, - error=str(e)) + result_report.AddDeviceBootFailure(ins.name, ins.ip, + ins.adb_port, vnc_port=None, + error=str(e), + device_serial=ins.device_serial) else: result_report.SetStatus(report.Status.SUCCESS) - result_report.AddDevice(inst.name, inst.ip, inst.adb_port, - vnc_port=None) - - if proc.poll() is None: - inst.WriteCreationTimestamp() + result_report.AddDevice(ins.name, ins.ip, ins.adb_port, + vnc_port=None, + device_serial=ins.device_serial) return result_report @staticmethod - def _MixImages(output_dir, image_dir, system_image_dir, ota): + def _MixImages(output_dir, image_dir, system_image_path, ota): """Mix emulator images and a system image into a disk image. Args: output_dir: The path to the output directory. image_dir: The input directory that provides images except system.img. - system_image_dir: The input directory that provides system.img. + system_image_path: The path to the system image. ota: An instance of ota_tools.OtaTools. Returns: @@ -241,11 +231,11 @@ class GoldfishLocalImageLocalInstance(base_avd_create.BaseAVDCreate): """ # Create the super image. mixed_super_image_path = os.path.join(output_dir, "mixed_super.img") - system_image_path = os.path.join(system_image_dir, _SYSTEM_IMAGE_NAME) - ota.BuildSuperImage(mixed_super_image_path, - os.path.join(image_dir, _MISC_INFO_FILE_NAME), - lambda partition: _GetImageForLogicalPartition( - partition, system_image_path, image_dir)) + ota.BuildSuperImage( + mixed_super_image_path, + os.path.join(image_dir, _MISC_INFO_FILE_NAME), + lambda partition: ota_tools.GetImageForPartition( + partition, image_dir, system=system_image_path)) # Create the vbmeta image. disabled_vbmeta_image_path = os.path.join(output_dir, @@ -254,37 +244,96 @@ class GoldfishLocalImageLocalInstance(base_avd_create.BaseAVDCreate): # Create the disk image. combined_image = os.path.join(output_dir, "combined.img") - ota.MkCombinedImg(combined_image, - os.path.join(image_dir, - _SYSTEM_QEMU_CONFIG_FILE_NAME), - lambda partition: _GetImageForPhysicalPartition( - partition, mixed_super_image_path, - disabled_vbmeta_image_path, image_dir)) + ota.MkCombinedImg( + combined_image, + os.path.join(image_dir, _SYSTEM_QEMU_CONFIG_FILE_NAME), + lambda partition: ota_tools.GetImageForPartition( + partition, image_dir, super=mixed_super_image_path, + vbmeta=disabled_vbmeta_image_path)) return combined_image @staticmethod def _FindEmulatorBinary(search_paths): - """Return the path to the emulator binary.""" + """Find emulator binary in the directories. + + The directories may be extracted from zip archives without preserving + file permissions. When this method finds the emulator binary and its + dependencies, it sets the files to be executable. + + Args: + search_paths: Collection of strings, the directories to search for + emulator binary. + + Returns: + The path to the emulator binary. + + Raises: + errors.GetSdkRepoPackageError if emulator binary is not found. + """ + emulator_dir = None # Find in unzipped sdk-repo-*.zip. for search_path in search_paths: - path = os.path.join(search_path, _EMULATOR_BIN_NAME) - if os.path.isfile(path): - return path + if os.path.isfile(os.path.join(search_path, _EMULATOR_BIN_NAME)): + emulator_dir = search_path + break - path = os.path.join(search_path, _SDK_REPO_EMULATOR_DIR_NAME, - _EMULATOR_BIN_NAME) - if os.path.isfile(path): - return path + sdk_repo_dir = os.path.join(search_path, + _SDK_REPO_EMULATOR_DIR_NAME) + if os.path.isfile(os.path.join(sdk_repo_dir, _EMULATOR_BIN_NAME)): + emulator_dir = sdk_repo_dir + break # Find in build environment. - prebuilt_emulator_dir = os.environ.get( - constants.ENV_ANDROID_EMULATOR_PREBUILTS) - if prebuilt_emulator_dir: - path = os.path.join(prebuilt_emulator_dir, _EMULATOR_BIN_NAME) - if os.path.isfile(path): - return path + if not emulator_dir: + prebuilt_emulator_dir = os.environ.get( + constants.ENV_ANDROID_EMULATOR_PREBUILTS) + if (prebuilt_emulator_dir and os.path.isfile( + os.path.join(prebuilt_emulator_dir, _EMULATOR_BIN_NAME))): + emulator_dir = prebuilt_emulator_dir + + if not emulator_dir: + raise errors.GetSdkRepoPackageError(_MISSING_EMULATOR_MSG) + + emulator_dir = os.path.abspath(emulator_dir) + # Set the binaries to be executable. + for subdir_name in _EMULATOR_BIN_DIR_NAMES: + subdir_path = os.path.join(emulator_dir, subdir_name) + if os.path.isdir(subdir_path): + utils.SetDirectoryTreeExecutable(subdir_path) + + emulator_path = os.path.join(emulator_dir, _EMULATOR_BIN_NAME) + utils.SetExecutable(emulator_path) + return emulator_path - raise errors.GetSdkRepoPackageError(_MISSING_EMULATOR_MSG) + @staticmethod + def _FindImageDir(image_dir): + """Find emulator images in the directory. + + In build environment, the images are in $ANDROID_PRODUCT_OUT. + In an extracted SDK repository, the images are in the subdirectory + named after the CPU architecture. + + Args: + image_dir: The path given by the environment variable or the user. + + Returns: + The directory containing the emulator images. + + Raises: + errors.GetLocalImageError if the images are not found. + """ + image_dir = os.path.abspath(image_dir) + entries = os.listdir(image_dir) + if len(entries) == 1: + first_entry = os.path.join(image_dir, entries[0]) + if os.path.isdir(first_entry): + image_dir = first_entry + + if (os.path.isfile(os.path.join(image_dir, _SYSTEM_QEMU_IMAGE_NAME)) or + os.path.isfile(os.path.join(image_dir, _SYSTEM_IMAGE_NAME))): + return image_dir + + raise errors.GetLocalImageError("No device image in %s." % image_dir) @staticmethod def _IsEmulatorRunning(adb): @@ -305,18 +354,21 @@ class GoldfishLocalImageLocalInstance(base_avd_create.BaseAVDCreate): adb: adb_tools.AdbTools initialized with the emulator's serial. no_prompts: Boolean, True to skip all prompts. + Returns: + Whether the user wants to continue. + Raises: errors.CreateError if the emulator isn't deleted. """ if not self._IsEmulatorRunning(adb): - return + return True logger.info("Goldfish AVD is already running.") if no_prompts or utils.GetUserAnswerYes(_CONFIRM_RELAUNCH): if adb.EmuCommand("kill") != 0: raise errors.CreateError("Cannot kill emulator.") self._WaitForEmulatorToStop(adb) - else: - sys.exit(constants.EXIT_BY_USER) + return True + return False @staticmethod def _CopyBuildProp(image_dir): @@ -335,7 +387,7 @@ class GoldfishLocalImageLocalInstance(base_avd_create.BaseAVDCreate): build_prop_src_path = os.path.join(image_dir, _BUILD_PROP_FILE_NAME) if not os.path.isfile(build_prop_src_path): raise errors.GetLocalImageError("No %s in %s." % - _BUILD_PROP_FILE_NAME, image_dir) + (_BUILD_PROP_FILE_NAME, image_dir)) build_prop_dir = os.path.dirname(build_prop_path) logger.info("Copy %s to %s", _BUILD_PROP_FILE_NAME, build_prop_path) if not os.path.exists(build_prop_dir): @@ -394,18 +446,22 @@ class GoldfishLocalImageLocalInstance(base_avd_create.BaseAVDCreate): if not avd_spec.autoconnect: args.append("-no-window") - if avd_spec.local_system_image_dir: + if avd_spec.local_system_image: mixed_image_dir = os.path.join(instance_dir, "mixed_images") + if os.path.exists(mixed_image_dir): + shutil.rmtree(mixed_image_dir) os.mkdir(mixed_image_dir) - image_dir = os.path.abspath(avd_spec.local_image_dir) + image_dir = self._FindImageDir(avd_spec.local_image_dir) + + system_image_path = create_common.FindLocalImage( + avd_spec.local_system_image, _SYSTEM_IMAGE_NAME_PATTERN) ota_tools_dir = ota_tools.FindOtaTools(avd_spec.local_tool_dirs) ota_tools_dir = os.path.abspath(ota_tools_dir) mixed_image = self._MixImages( - mixed_image_dir, image_dir, - os.path.abspath(avd_spec.local_system_image_dir), + mixed_image_dir, image_dir, system_image_path, ota_tools.OtaTools(ota_tools_dir)) # TODO(b/142228085): Use -system instead of modifying image_dir. diff --git a/create/goldfish_local_image_local_instance_test.py b/create/goldfish_local_image_local_instance_test.py index 1af0efc4..900197bf 100644 --- a/create/goldfish_local_image_local_instance_test.py +++ b/create/goldfish_local_image_local_instance_test.py @@ -17,7 +17,8 @@ import os import shutil import tempfile import unittest -import mock + +from unittest import mock from acloud import errors import acloud.create.goldfish_local_image_local_instance as instance_module @@ -30,7 +31,8 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): { "instance_name": "local-goldfish-instance", "ip": "127.0.0.1:5555", - "adb_port": 5555 + "adb_port": 5555, + "device_serial": "unittest" } ] @@ -41,6 +43,9 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): self._tool_dir = os.path.join(self._temp_dir, "tool") self._instance_dir = os.path.join(self._temp_dir, "instance") self._emulator_is_running = False + self._mock_lock = mock.Mock() + self._mock_lock.Lock.return_value = True + self._mock_lock.LockIfNotInUse.side_effect = (False, True) self._mock_proc = mock.Mock() self._mock_proc.poll.side_effect = ( lambda: None if self._emulator_is_running else 0) @@ -82,22 +87,23 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): raise ValueError("Unexpected arguments " + str(args)) - def _SetUpMocks(self, mock_popen, mock_adb_tools, mock_utils, - mock_instance): + def _SetUpMocks(self, mock_popen, mock_utils, mock_instance): mock_utils.IsSupportedPlatform.return_value = True + mock_adb_tools = mock.Mock(side_effect=self._MockEmuCommand) + mock_instance_object = mock.Mock(ip="127.0.0.1", adb_port=5555, console_port="5554", device_serial="unittest", - instance_dir=self._instance_dir) + instance_dir=self._instance_dir, + adb=mock_adb_tools) # name is a positional argument of Mock(). mock_instance_object.name = "local-goldfish-instance" - mock_instance.return_value = mock_instance_object - mock_adb_tools_object = mock.Mock() - mock_adb_tools_object.EmuCommand.side_effect = self._MockEmuCommand - mock_adb_tools.return_value = mock_adb_tools_object + mock_instance.return_value = mock_instance_object + mock_instance.GetLockById.return_value = self._mock_lock + mock_instance.GetMaxNumberOfInstances.return_value = 2 mock_popen.side_effect = self._MockPopen @@ -118,13 +124,11 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): "LocalGoldfishInstance") @mock.patch("acloud.create.goldfish_local_image_local_instance.utils") @mock.patch("acloud.create.goldfish_local_image_local_instance." - "adb_tools.AdbTools") - @mock.patch("acloud.create.goldfish_local_image_local_instance." "subprocess.Popen") - def testCreateAVDInBuildEnvironment(self, mock_popen, mock_adb_tools, - mock_utils, mock_instance): + def testCreateAVDInBuildEnvironment(self, mock_popen, mock_utils, + mock_instance): """Test _CreateAVD with build environment variables and files.""" - self._SetUpMocks(mock_popen, mock_adb_tools, mock_utils, mock_instance) + self._SetUpMocks(mock_popen, mock_utils, mock_instance) self._CreateEmptyFile(os.path.join(self._image_dir, "system-qemu.img")) @@ -139,8 +143,9 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): gpu=None, autoconnect=True, local_instance_id=1, + local_instance_dir=None, local_image_dir=self._image_dir, - local_system_image_dir=None, + local_system_image=None, local_tool_dirs=[]) # Test deleting an existing instance. @@ -154,10 +159,15 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): self.assertEqual(report.data.get("devices"), self._EXPECTED_DEVICES_IN_REPORT) + self._mock_lock.Lock.assert_called_once() + self._mock_lock.SetInUse.assert_called_once_with(True) + self._mock_lock.Unlock.assert_called_once() + mock_instance.assert_called_once_with(1, avd_flavor="phone") self.assertTrue(os.path.isdir(self._instance_dir)) + mock_utils.SetExecutable.assert_called_with(self._emulator_path) mock_popen.assert_called_once() self.assertEqual(mock_popen.call_args[0][0], self._GetExpectedEmulatorArgs()) @@ -168,24 +178,28 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): "LocalGoldfishInstance") @mock.patch("acloud.create.goldfish_local_image_local_instance.utils") @mock.patch("acloud.create.goldfish_local_image_local_instance." - "adb_tools.AdbTools") - @mock.patch("acloud.create.goldfish_local_image_local_instance." "subprocess.Popen") - def testCreateAVDFromSdkRepository(self, mock_popen, mock_adb_tools, + def testCreateAVDFromSdkRepository(self, mock_popen, mock_utils, mock_instance): """Test _CreateAVD with SDK repository files.""" - self._SetUpMocks(mock_popen, mock_adb_tools, mock_utils, mock_instance) + self._SetUpMocks(mock_popen, mock_utils, mock_instance) - self._CreateEmptyFile(os.path.join(self._image_dir, "system.img")) - self._CreateEmptyFile(os.path.join(self._image_dir, "build.prop")) + self._CreateEmptyFile(os.path.join(self._image_dir, "x86", + "system.img")) + self._CreateEmptyFile(os.path.join(self._image_dir, "x86", + "build.prop")) + + instance_dir = os.path.join(self._temp_dir, "local_instance_dir") + os.mkdir(instance_dir) mock_avd_spec = mock.Mock(flavor="phone", boot_timeout_secs=None, gpu=None, autoconnect=True, local_instance_id=2, + local_instance_dir=instance_dir, local_image_dir=self._image_dir, - local_system_image_dir=None, + local_system_image=None, local_tool_dirs=[self._tool_dir]) with mock.patch.dict("acloud.create." @@ -196,30 +210,33 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): self.assertEqual(report.data.get("devices"), self._EXPECTED_DEVICES_IN_REPORT) + self._mock_lock.Lock.assert_called_once() + self._mock_lock.SetInUse.assert_called_once_with(True) + self._mock_lock.Unlock.assert_called_once() + mock_instance.assert_called_once_with(2, avd_flavor="phone") - self.assertTrue(os.path.isdir(self._instance_dir)) + self.assertTrue(os.path.isdir(self._instance_dir) and + os.path.islink(self._instance_dir)) + mock_utils.SetExecutable.assert_called_with(self._emulator_path) mock_popen.assert_called_once() self.assertEqual(mock_popen.call_args[0][0], self._GetExpectedEmulatorArgs()) self._mock_proc.poll.assert_called() self.assertTrue(os.path.isfile( - os.path.join(self._image_dir, "system", "build.prop"))) + os.path.join(self._image_dir, "x86", "system", "build.prop"))) # pylint: disable=protected-access @mock.patch("acloud.create.goldfish_local_image_local_instance.instance." "LocalGoldfishInstance") @mock.patch("acloud.create.goldfish_local_image_local_instance.utils") @mock.patch("acloud.create.goldfish_local_image_local_instance." - "adb_tools.AdbTools") - @mock.patch("acloud.create.goldfish_local_image_local_instance." "subprocess.Popen") - def testCreateAVDTimeout(self, mock_popen, mock_adb_tools, - mock_utils, mock_instance): + def testCreateAVDTimeout(self, mock_popen, mock_utils, mock_instance): """Test _CreateAVD with SDK repository files and timeout error.""" - self._SetUpMocks(mock_popen, mock_adb_tools, mock_utils, mock_instance) + self._SetUpMocks(mock_popen, mock_utils, mock_instance) mock_utils.PollAndWait.side_effect = errors.DeviceBootTimeoutError( "timeout") @@ -231,8 +248,9 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): gpu=None, autoconnect=True, local_instance_id=2, + local_instance_dir=None, local_image_dir=self._image_dir, - local_system_image_dir=None, + local_system_image=None, local_tool_dirs=[self._tool_dir]) with mock.patch.dict("acloud.create." @@ -240,6 +258,10 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): dict(), clear=True): report = self._goldfish._CreateAVD(mock_avd_spec, no_prompts=True) + self._mock_lock.Lock.assert_called_once() + self._mock_lock.SetInUse.assert_called_once_with(True) + self._mock_lock.Unlock.assert_called_once() + self.assertEqual(report.data.get("devices_failing_boot"), self._EXPECTED_DEVICES_IN_REPORT) self.assertEqual(report.errors, ["timeout"]) @@ -249,13 +271,111 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): "LocalGoldfishInstance") @mock.patch("acloud.create.goldfish_local_image_local_instance.utils") @mock.patch("acloud.create.goldfish_local_image_local_instance." - "adb_tools.AdbTools") + "subprocess.Popen") + def testCreateAVDWithoutReport(self, mock_popen, mock_utils, + mock_instance): + """Test _CreateAVD with SDK repository files and no report.""" + self._SetUpMocks(mock_popen, mock_utils, mock_instance) + + mock_avd_spec = mock.Mock(flavor="phone", + boot_timeout_secs=None, + gpu=None, + autoconnect=True, + local_instance_id=0, + local_instance_dir=None, + local_image_dir=self._image_dir, + local_system_image=None, + local_tool_dirs=[self._tool_dir]) + + with mock.patch.dict("acloud.create." + "goldfish_local_image_local_instance.os.environ", + dict(), clear=True): + with self.assertRaises(errors.GetLocalImageError): + self._goldfish._CreateAVD(mock_avd_spec, no_prompts=True) + + self._mock_lock.Lock.assert_not_called() + self.assertEqual(2, self._mock_lock.LockIfNotInUse.call_count) + self._mock_lock.SetInUse.assert_not_called() + self._mock_lock.Unlock.assert_called_once() + + # pylint: disable=protected-access + @mock.patch("acloud.create.goldfish_local_image_local_instance.instance." + "LocalGoldfishInstance") + @mock.patch("acloud.create.goldfish_local_image_local_instance.utils") @mock.patch("acloud.create.goldfish_local_image_local_instance." "subprocess.Popen") @mock.patch("acloud.create.goldfish_local_image_local_instance.ota_tools") def testCreateAVDWithMixedImages(self, mock_ota_tools, mock_popen, - mock_adb_tools, mock_utils, - mock_instance): + mock_utils, mock_instance): + """Test _CreateAVD with mixed images and SDK repository files.""" + mock_ota_tools.FindOtaTools.return_value = self._tool_dir + mock_ota_tools_object = mock.Mock() + mock_ota_tools.OtaTools.return_value = mock_ota_tools_object + mock_ota_tools_object.MkCombinedImg.side_effect = ( + lambda out_path, _conf, _get_img: self._CreateEmptyFile(out_path)) + + self._SetUpMocks(mock_popen, mock_utils, mock_instance) + + self._CreateEmptyFile(os.path.join(self._image_dir, "x86", + "system.img")) + self._CreateEmptyFile(os.path.join(self._image_dir, "x86", "system", + "build.prop")) + + mock_avd_spec = mock.Mock(flavor="phone", + boot_timeout_secs=None, + gpu="auto", + autoconnect=False, + local_instance_id=3, + local_instance_dir=None, + local_image_dir=self._image_dir, + local_system_image=os.path.join( + self._image_dir, "x86", "system.img"), + local_tool_dirs=[self._tool_dir]) + + with mock.patch.dict("acloud.create." + "goldfish_local_image_local_instance.os.environ", + dict(), clear=True): + report = self._goldfish._CreateAVD(mock_avd_spec, no_prompts=True) + + self.assertEqual(report.data.get("devices"), + self._EXPECTED_DEVICES_IN_REPORT) + + mock_instance.assert_called_once_with(3, avd_flavor="phone") + + self.assertTrue(os.path.isdir(self._instance_dir)) + + mock_ota_tools.FindOtaTools.assert_called_once() + mock_ota_tools.OtaTools.assert_called_with(self._tool_dir) + + mock_ota_tools_object.BuildSuperImage.assert_called_once() + self.assertEqual(mock_ota_tools_object.BuildSuperImage.call_args[0][1], + os.path.join(self._image_dir, "x86", "misc_info.txt")) + + mock_ota_tools_object.MakeDisabledVbmetaImage.assert_called_once() + + mock_ota_tools_object.MkCombinedImg.assert_called_once() + self.assertEqual( + mock_ota_tools_object.MkCombinedImg.call_args[0][1], + os.path.join(self._image_dir, "x86", "system-qemu-config.txt")) + + mock_utils.SetExecutable.assert_called_with(self._emulator_path) + mock_popen.assert_called_once() + self.assertEqual( + mock_popen.call_args[0][0], + self._GetExpectedEmulatorArgs( + "-gpu", "auto", "-no-window", "-qemu", "-append", + "androidboot.verifiedbootstate=orange")) + self._mock_proc.poll.assert_called() + + # pylint: disable=protected-access + @mock.patch("acloud.create.goldfish_local_image_local_instance.instance." + "LocalGoldfishInstance") + @mock.patch("acloud.create.goldfish_local_image_local_instance.utils") + @mock.patch("acloud.create.goldfish_local_image_local_instance." + "subprocess.Popen") + @mock.patch("acloud.create.goldfish_local_image_local_instance.ota_tools") + def testCreateAVDWithMixedImageDirs(self, mock_ota_tools, mock_popen, + mock_utils, mock_instance): """Test _CreateAVD with mixed images in build environment.""" mock_ota_tools.FindOtaTools.return_value = self._tool_dir mock_ota_tools_object = mock.Mock() @@ -263,26 +383,26 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): mock_ota_tools_object.MkCombinedImg.side_effect = ( lambda out_path, _conf, _get_img: self._CreateEmptyFile(out_path)) - self._SetUpMocks(mock_popen, mock_adb_tools, mock_utils, mock_instance) + self._SetUpMocks(mock_popen, mock_utils, mock_instance) self._CreateEmptyFile(os.path.join(self._image_dir, "system-qemu.img")) + self._CreateEmptyFile(os.path.join(self._image_dir, + "system.img")) self._CreateEmptyFile(os.path.join(self._image_dir, "system", "build.prop")) mock_environ = {"ANDROID_EMULATOR_PREBUILTS": os.path.join(self._tool_dir, "emulator")} - mock_utils.GetBuildEnvironmentVariable.side_effect = ( - lambda key: mock_environ[key]) - mock_avd_spec = mock.Mock(flavor="phone", boot_timeout_secs=None, gpu="auto", autoconnect=False, local_instance_id=3, + local_instance_dir=None, local_image_dir=self._image_dir, - local_system_image_dir="/unit/test", + local_system_image=self._image_dir, local_tool_dirs=[]) with mock.patch.dict("acloud.create." @@ -311,6 +431,7 @@ class GoldfishLocalImageLocalInstance(unittest.TestCase): mock_ota_tools_object.MkCombinedImg.call_args[0][1], os.path.join(self._image_dir, "system-qemu-config.txt")) + mock_utils.SetExecutable.assert_called_with(self._emulator_path) mock_popen.assert_called_once() self.assertEqual( mock_popen.call_args[0][0], diff --git a/create/local_image_local_instance.py b/create/local_image_local_instance.py index 0ee5ee8d..c8c38cdc 100644 --- a/create/local_image_local_instance.py +++ b/create/local_image_local_instance.py @@ -22,7 +22,8 @@ The cuttlefish tool requires 3 variables: - HOME: To specify the temporary folder of launch_cvd. - CUTTLEFISH_INSTANCE: To specify the instance id. Acloud user must either set ANDROID_HOST_OUT or run acloud with --local-tool. -Acloud sets the other 2 variables for each local instance. +The user can optionally specify the folder by --local-instance-dir and the +instance id by --local-instance. The adb port and vnc port of local instance will be decided according to instance id. The rule of adb port will be '6520 + [instance id] - 1' and the vnc @@ -32,18 +33,34 @@ If instance id = 3 the adb port will be 6522 and vnc port will be 6446. To delete the local instance, we will call stop_cvd with the environment variable [CUTTLEFISH_CONFIG_FILE] which is pointing to the runtime cuttlefish json. + +To run this program outside of a build environment, the following setup is +required. +- One of the local tool directories is a decompressed cvd host package, + i.e., cvd-host_package.tar.gz. +- If the instance doesn't require mixed images, the local image directory + should be an unzipped update package, i.e., <target>-img-<build>.zip, + which contains a super image. +- If the instance requires mixing system image, the local image directory + should be an unzipped target files package, i.e., + <target>-target_files-<build>.zip, + which contains misc info and images not packed into a super image. +- If the instance requires mixing system image, one of the local tool + directories should be an unzipped OTA tools package, i.e., otatools.zip. """ +import collections +import glob import logging import os -import shutil import subprocess -import threading import sys from acloud import errors from acloud.create import base_avd_create +from acloud.create import create_common from acloud.internal import constants +from acloud.internal.lib import ota_tools from acloud.internal.lib import utils from acloud.internal.lib.adb_tools import AdbTools from acloud.list import list as list_instance @@ -53,18 +70,53 @@ from acloud.public import report logger = logging.getLogger(__name__) -_CMD_LAUNCH_CVD_ARGS = (" -daemon -cpus %s -x_res %s -y_res %s -dpi %s " - "-memory_mb %s -run_adb_connector=%s " - "-system_image_dir %s -instance_dir %s") +# The boot image name pattern corresponds to the use cases: +# - In a cuttlefish build environment, ANDROID_PRODUCT_OUT conatins boot.img +# and boot-debug.img. The former is the default boot image. The latter is not +# useful for cuttlefish. +# - In an officially released GKI (Generic Kernel Image) package, the image +# name is boot-<kernel version>.img. +_BOOT_IMAGE_NAME_PATTERN = r"boot(-[\d.]+)?\.img" +_SYSTEM_IMAGE_NAME_PATTERN = r"system\.img" +_MISC_INFO_FILE_NAME = "misc_info.txt" +_TARGET_FILES_IMAGES_DIR_NAME = "IMAGES" +_TARGET_FILES_META_DIR_NAME = "META" +_MIXED_SUPER_IMAGE_NAME = "mixed_super.img" +_CMD_LAUNCH_CVD_ARGS = ( + " -daemon -config=%s -run_adb_connector=%s " + "-system_image_dir %s -instance_dir %s " + "-undefok=report_anonymous_usage_stats,enable_sandbox,config " + "-report_anonymous_usage_stats=y " + "-enable_sandbox=false") +_CMD_LAUNCH_CVD_HW_ARGS = " -cpus %s -x_res %s -y_res %s -dpi %s -memory_mb %s" _CMD_LAUNCH_CVD_DISK_ARGS = (" -blank_data_image_mb %s " "-data_policy always_create") +_CMD_LAUNCH_CVD_WEBRTC_ARGS = (" -guest_enforce_security=false " + "-vm_manager=crosvm " + "-start_webrtc=true " + "-webrtc_public_ip=%s" % constants.LOCALHOST) +_CMD_LAUNCH_CVD_VNC_ARG = " -start_vnc_server=true" +_CMD_LAUNCH_CVD_SUPER_IMAGE_ARG = " -super_image=%s" +_CMD_LAUNCH_CVD_BOOT_IMAGE_ARG = " -boot_image=%s" + +# In accordance with the number of network interfaces in +# /etc/init.d/cuttlefish-common +_MAX_INSTANCE_ID = 10 + +_INSTANCES_IN_USE_MSG = ("All instances are in use. Try resetting an instance " + "by specifying --local-instance and an id between 1 " + "and %d." % _MAX_INSTANCE_ID) _CONFIRM_RELAUNCH = ("\nCuttlefish AVD[id:%d] is already running. \n" "Enter 'y' to terminate current instance and launch a new " "instance, enter anything else to exit out[y/N]: ") -_LAUNCH_CVD_TIMEOUT_SECS = 120 # default timeout as 120 seconds -_LAUNCH_CVD_TIMEOUT_ERROR = ("Cuttlefish AVD launch timeout, did not complete " - "within %d secs.") -_VIRTUAL_DISK_PATHS = "virtual_disk_paths" + +# The first two fields of this named tuple are image folder and CVD host +# package folder which are essential for local instances. The following fields +# are optional. They are set when the AVD spec requires to mix images. +ArtifactPaths = collections.namedtuple( + "ArtifactPaths", + ["image_dir", "host_bins", "misc_info", "ota_tools_dir", "system_image", + "boot_image"]) class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): @@ -79,9 +131,6 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): avd_spec: AVDSpec object that tells us what we're going to create. no_prompts: Boolean, True to skip all prompts. - Raises: - errors.LaunchCVDFail: Launch AVD failed. - Returns: A Report instance. """ @@ -91,39 +140,131 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): result_report.SetStatus(report.Status.FAIL) return result_report - self.PrintDisclaimer() - local_image_path, host_bins_path = self.GetImageArtifactsPath(avd_spec) + artifact_paths = self.GetImageArtifactsPath(avd_spec) + + try: + ins_id, ins_lock = self._SelectAndLockInstance(avd_spec) + except errors.CreateError as e: + result_report = report.Report(command="create") + result_report.AddError(str(e)) + result_report.SetStatus(report.Status.FAIL) + return result_report + + try: + if not self._CheckRunningCvd(ins_id, no_prompts): + # Mark as in-use so that it won't be auto-selected again. + ins_lock.SetInUse(True) + sys.exit(constants.EXIT_BY_USER) + + result_report = self._CreateInstance(ins_id, artifact_paths, + avd_spec, no_prompts) + # The infrastructure is able to delete the instance only if the + # instance name is reported. This method changes the state to + # in-use after creating the report. + ins_lock.SetInUse(True) + return result_report + finally: + ins_lock.Unlock() + + @staticmethod + def _SelectAndLockInstance(avd_spec): + """Select an id and lock the instance. + + Args: + avd_spec: AVDSpec for the device. + + Returns: + The instance id and the LocalInstanceLock that is locked by this + process. + + Raises: + errors.CreateError if fails to select or lock the instance. + """ + if avd_spec.local_instance_id: + ins_id = avd_spec.local_instance_id + ins_lock = instance.GetLocalInstanceLock(ins_id) + if ins_lock.Lock(): + return ins_id, ins_lock + raise errors.CreateError("Instance %d is locked by another " + "process." % ins_id) + + for ins_id in range(1, _MAX_INSTANCE_ID + 1): + ins_lock = instance.GetLocalInstanceLock(ins_id) + if ins_lock.LockIfNotInUse(timeout_secs=0): + logger.info("Selected instance id: %d", ins_id) + return ins_id, ins_lock + raise errors.CreateError(_INSTANCES_IN_USE_MSG) + + #pylint: disable=too-many-locals + def _CreateInstance(self, local_instance_id, artifact_paths, avd_spec, + no_prompts): + """Create a CVD instance. + + Args: + local_instance_id: Integer of instance id. + artifact_paths: ArtifactPaths object. + avd_spec: AVDSpec for the instance. + no_prompts: Boolean, True to skip all prompts. + + Returns: + A Report instance. + """ + webrtc_port = self.GetWebrtcSigServerPort(local_instance_id) + if avd_spec.connect_webrtc: + utils.ReleasePort(webrtc_port) - launch_cvd_path = os.path.join(host_bins_path, "bin", + cvd_home_dir = instance.GetLocalInstanceHomeDir(local_instance_id) + create_common.PrepareLocalInstanceDir(cvd_home_dir, avd_spec) + super_image_path = None + if artifact_paths.system_image: + super_image_path = self._MixSuperImage(cvd_home_dir, + artifact_paths) + runtime_dir = instance.GetLocalInstanceRuntimeDir(local_instance_id) + # TODO(b/168171781): cvd_status of list/delete via the symbolic. + self.PrepareLocalCvdToolsLink(cvd_home_dir, artifact_paths.host_bins) + launch_cvd_path = os.path.join(artifact_paths.host_bins, "bin", constants.CMD_LAUNCH_CVD) + hw_property = None + if avd_spec.hw_customize: + hw_property = avd_spec.hw_property cmd = self.PrepareLaunchCVDCmd(launch_cvd_path, - avd_spec.hw_property, + hw_property, avd_spec.connect_adb, - local_image_path, - avd_spec.local_instance_id) + artifact_paths.image_dir, + runtime_dir, + avd_spec.connect_webrtc, + avd_spec.connect_vnc, + super_image_path, + artifact_paths.boot_image, + avd_spec.launch_args, + avd_spec.flavor) result_report = report.Report(command="create") - instance_name = instance.GetLocalInstanceName( - avd_spec.local_instance_id) + instance_name = instance.GetLocalInstanceName(local_instance_id) try: - self.CheckLaunchCVD( - cmd, host_bins_path, avd_spec.local_instance_id, local_image_path, - no_prompts, avd_spec.boot_timeout_secs or _LAUNCH_CVD_TIMEOUT_SECS) + self._LaunchCvd(cmd, local_instance_id, artifact_paths.host_bins, + cvd_home_dir, (avd_spec.boot_timeout_secs or + constants.DEFAULT_CF_BOOT_TIMEOUT)) except errors.LaunchCVDFail as launch_error: + err_msg = ("Cannot create cuttlefish instance: %s\n" + "For more detail: %s/launcher.log" % + (launch_error, runtime_dir)) result_report.SetStatus(report.Status.BOOT_FAIL) result_report.AddDeviceBootFailure( - instance_name, constants.LOCALHOST, None, None, - error=str(launch_error)) + instance_name, constants.LOCALHOST, None, None, error=err_msg) return result_report - active_ins = list_instance.GetActiveCVD(avd_spec.local_instance_id) + active_ins = list_instance.GetActiveCVD(local_instance_id) if active_ins: result_report.SetStatus(report.Status.SUCCESS) result_report.AddDevice(instance_name, constants.LOCALHOST, - active_ins.adb_port, active_ins.vnc_port) + active_ins.adb_port, active_ins.vnc_port, + webrtc_port) # Launch vnc client if we're auto-connecting. if avd_spec.connect_vnc: utils.LaunchVNCFromReport(result_report, avd_spec, no_prompts) + if avd_spec.connect_webrtc: + utils.LaunchBrowserFromReport(result_report) if avd_spec.unlock_screen: AdbTools(active_ins.adb_port).AutoUnlockScreen() else: @@ -132,10 +273,21 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): result_report.SetStatus(report.Status.BOOT_FAIL) result_report.AddDeviceBootFailure( instance_name, constants.LOCALHOST, None, None, error=err_msg) - return result_report @staticmethod + def GetWebrtcSigServerPort(instance_id): + """Get the port of the signaling server. + + Args: + instance_id: Integer of instance id. + + Returns: + Integer of signaling server port. + """ + return constants.WEBRTC_LOCAL_PORT + instance_id - 1 + + @staticmethod def _FindCvdHostBinaries(search_paths): """Return the directory that contains CVD host binaries.""" for search_path in search_paths: @@ -143,16 +295,66 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): constants.CMD_LAUNCH_CVD)): return search_path - host_out_dir = os.environ.get(constants.ENV_ANDROID_HOST_OUT) - if (host_out_dir and - os.path.isfile(os.path.join(host_out_dir, "bin", - constants.CMD_LAUNCH_CVD))): - return host_out_dir + for env_host_out in [constants.ENV_ANDROID_SOONG_HOST_OUT, + constants.ENV_ANDROID_HOST_OUT]: + host_out_dir = os.environ.get(env_host_out) + if (host_out_dir and + os.path.isfile(os.path.join(host_out_dir, "bin", + constants.CMD_LAUNCH_CVD))): + return host_out_dir raise errors.GetCvdLocalHostPackageError( "CVD host binaries are not found. Please run `make hosttar`, or " "set --local-tool to an extracted CVD host package.") + @staticmethod + def _FindMiscInfo(image_dir): + """Find misc info in build output dir or extracted target files. + + Args: + image_dir: The directory to search for misc info. + + Returns: + image_dir if the directory structure looks like an output directory + in build environment. + image_dir/META if it looks like extracted target files. + + Raises: + errors.CheckPathError if this method cannot find misc info. + """ + misc_info_path = os.path.join(image_dir, _MISC_INFO_FILE_NAME) + if os.path.isfile(misc_info_path): + return misc_info_path + misc_info_path = os.path.join(image_dir, _TARGET_FILES_META_DIR_NAME, + _MISC_INFO_FILE_NAME) + if os.path.isfile(misc_info_path): + return misc_info_path + raise errors.CheckPathError( + "Cannot find %s in %s." % (_MISC_INFO_FILE_NAME, image_dir)) + + @staticmethod + def _FindImageDir(image_dir): + """Find images in build output dir or extracted target files. + + Args: + image_dir: The directory to search for images. + + Returns: + image_dir if the directory structure looks like an output directory + in build environment. + image_dir/IMAGES if it looks like extracted target files. + + Raises: + errors.GetLocalImageError if this method cannot find images. + """ + if glob.glob(os.path.join(image_dir, "*.img")): + return image_dir + subdir = os.path.join(image_dir, _TARGET_FILES_IMAGES_DIR_NAME) + if glob.glob(os.path.join(subdir, "*.img")): + return subdir + raise errors.GetLocalImageError( + "Cannot find images in %s." % image_dir) + def GetImageArtifactsPath(self, avd_spec): """Get image artifacts path. @@ -165,14 +367,65 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): avd_spec: AVDSpec object that tells us what we're going to create. Returns: - Tuple of (local image file, host bins package) paths. + ArtifactPaths object consisting of image directory and host bins + package. + + Raises: + errors.GetCvdLocalHostPackageError, errors.GetLocalImageError, or + errors.CheckPathError if any artifact is not found. """ - return (avd_spec.local_image_dir, - self._FindCvdHostBinaries(avd_spec.local_tool_dirs)) + image_dir = os.path.abspath(avd_spec.local_image_dir) + host_bins_path = self._FindCvdHostBinaries(avd_spec.local_tool_dirs) + + if avd_spec.local_system_image: + misc_info_path = self._FindMiscInfo(image_dir) + image_dir = self._FindImageDir(image_dir) + ota_tools_dir = os.path.abspath( + ota_tools.FindOtaTools(avd_spec.local_tool_dirs)) + system_image_path = create_common.FindLocalImage( + avd_spec.local_system_image, _SYSTEM_IMAGE_NAME_PATTERN) + else: + misc_info_path = None + ota_tools_dir = None + system_image_path = None + + if avd_spec.local_kernel_image: + boot_image_path = create_common.FindLocalImage( + avd_spec.local_kernel_image, _BOOT_IMAGE_NAME_PATTERN) + else: + boot_image_path = None + + return ArtifactPaths(image_dir, host_bins_path, + misc_info=misc_info_path, + ota_tools_dir=ota_tools_dir, + system_image=system_image_path, + boot_image=boot_image_path) + + @staticmethod + def _MixSuperImage(output_dir, artifact_paths): + """Mix cuttlefish images and a system image into a super image. + + Args: + output_dir: The path to the output directory. + artifact_paths: ArtifactPaths object. + + Returns: + The path to the super image in output_dir. + """ + ota = ota_tools.OtaTools(artifact_paths.ota_tools_dir) + super_image_path = os.path.join(output_dir, _MIXED_SUPER_IMAGE_NAME) + ota.BuildSuperImage( + super_image_path, artifact_paths.misc_info, + lambda partition: ota_tools.GetImageForPartition( + partition, artifact_paths.image_dir, + system=artifact_paths.system_image)) + return super_image_path @staticmethod def PrepareLaunchCVDCmd(launch_cvd_path, hw_property, connect_adb, - system_image_dir, local_instance_id): + image_dir, runtime_dir, connect_webrtc, + connect_vnc, super_image_path, boot_image_path, + launch_args, flavor): """Prepare launch_cvd command. Create the launch_cvd commands with all the required args and add @@ -181,49 +434,84 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): Args: launch_cvd_path: String of launch_cvd path. hw_property: dict object of hw property. - system_image_dir: String of local images path. + image_dir: String of local images path. connect_adb: Boolean flag that enables adb_connector. - local_instance_id: Integer of instance id. + runtime_dir: String of runtime directory path. + connect_webrtc: Boolean of connect_webrtc. + connect_vnc: Boolean of connect_vnc. + super_image_path: String of non-default super image path. + boot_image_path: String of non-default boot image path. + launch_args: String of launch args. + flavor: String of flavor name. Returns: String, launch_cvd cmd. """ - instance_dir = instance.GetLocalInstanceRuntimeDir(local_instance_id) launch_cvd_w_args = launch_cvd_path + _CMD_LAUNCH_CVD_ARGS % ( - hw_property["cpu"], hw_property["x_res"], hw_property["y_res"], - hw_property["dpi"], hw_property["memory"], - ("true" if connect_adb else "false"), system_image_dir, - instance_dir) - if constants.HW_ALIAS_DISK in hw_property: - launch_cvd_w_args = (launch_cvd_w_args + _CMD_LAUNCH_CVD_DISK_ARGS % - hw_property[constants.HW_ALIAS_DISK]) + flavor, ("true" if connect_adb else "false"), image_dir, runtime_dir) + if hw_property: + launch_cvd_w_args = launch_cvd_w_args + _CMD_LAUNCH_CVD_HW_ARGS % ( + hw_property["cpu"], hw_property["x_res"], hw_property["y_res"], + hw_property["dpi"], hw_property["memory"]) + if constants.HW_ALIAS_DISK in hw_property: + launch_cvd_w_args = (launch_cvd_w_args + _CMD_LAUNCH_CVD_DISK_ARGS % + hw_property[constants.HW_ALIAS_DISK]) + if connect_webrtc: + launch_cvd_w_args = launch_cvd_w_args + _CMD_LAUNCH_CVD_WEBRTC_ARGS + + if connect_vnc: + launch_cvd_w_args = launch_cvd_w_args + _CMD_LAUNCH_CVD_VNC_ARG + + if super_image_path: + launch_cvd_w_args = (launch_cvd_w_args + + _CMD_LAUNCH_CVD_SUPER_IMAGE_ARG % + super_image_path) + + if boot_image_path: + launch_cvd_w_args = (launch_cvd_w_args + + _CMD_LAUNCH_CVD_BOOT_IMAGE_ARG % + boot_image_path) + + if launch_args: + launch_cvd_w_args = launch_cvd_w_args + " " + launch_args launch_cmd = utils.AddUserGroupsToCmd(launch_cvd_w_args, constants.LIST_CF_USER_GROUPS) logger.debug("launch_cvd cmd:\n %s", launch_cmd) return launch_cmd - def CheckLaunchCVD(self, cmd, host_bins_path, local_instance_id, - local_image_path, no_prompts=False, - timeout_secs=_LAUNCH_CVD_TIMEOUT_SECS): - """Execute launch_cvd command and wait for boot up completed. + @staticmethod + def PrepareLocalCvdToolsLink(cvd_home_dir, host_bins_path): + """Create symbolic link for the cvd tools directory. - 1. Check if the provided image files are in use by any launch_cvd process. - 2. Check if launch_cvd with the same instance id is running. - 3. Launch local AVD. + local instance's cvd tools could be generated in /out after local build + or be generated in the download image folder. It creates a symbolic + link then only check cvd_status using known link for both cases. Args: - cmd: String, launch_cvd command. + cvd_home_dir: The parent directory of the link host_bins_path: String of host package directory. + + Returns: + String of cvd_tools link path + """ + cvd_tools_link_path = os.path.join(cvd_home_dir, constants.CVD_TOOLS_LINK_NAME) + if os.path.islink(cvd_tools_link_path): + os.unlink(cvd_tools_link_path) + os.symlink(host_bins_path, cvd_tools_link_path) + return cvd_tools_link_path + + @staticmethod + def _CheckRunningCvd(local_instance_id, no_prompts=False): + """Check if launch_cvd with the same instance id is running. + + Args: local_instance_id: Integer of instance id. - local_image_path: String of local image directory. no_prompts: Boolean, True to skip all prompts. - timeout_secs: Integer, the number of seconds to wait for the AVD to boot up. + + Returns: + Whether the user wants to continue. """ - # launch_cvd assumes host bins are in $ANDROID_HOST_OUT, let's overwrite - # it to wherever we're running launch_cvd since they could be in a - # different dir (e.g. downloaded image). - os.environ[constants.ENV_ANDROID_HOST_OUT] = host_bins_path # Check if the instance with same id is running. existing_ins = list_instance.GetActiveCVD(local_instance_id) if existing_ins: @@ -231,24 +519,13 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): local_instance_id): existing_ins.Delete() else: - sys.exit(constants.EXIT_BY_USER) - else: - # Image files can't be shared among instances, so check if any running - # launch_cvd process is using this path. - occupied_ins_id = self.IsLocalImageOccupied(local_image_path) - if occupied_ins_id: - utils.PrintColorString( - "The image path[%s] is already used by current running AVD" - "[id:%d]\nPlease choose another path to launch local " - "instance." % (local_image_path, occupied_ins_id), - utils.TextColors.FAIL) - sys.exit(constants.EXIT_BY_USER) - - self._LaunchCvd(cmd, local_instance_id, timeout=timeout_secs) + return False + return True @staticmethod @utils.TimeExecute(function_description="Waiting for AVD(s) to boot up") - def _LaunchCvd(cmd, local_instance_id, timeout=None): + def _LaunchCvd(cmd, local_instance_id, host_bins_path, cvd_home_dir, + timeout): """Execute Launch CVD. Kick off the launch_cvd command and log the output. @@ -256,61 +533,27 @@ class LocalImageLocalInstance(base_avd_create.BaseAVDCreate): Args: cmd: String, launch_cvd command. local_instance_id: Integer of instance id. + host_bins_path: String of host package directory. + cvd_home_dir: String, the home directory for the instance. timeout: Integer, the number of seconds to wait for the AVD to boot up. Raises: - errors.LaunchCVDFail when any CalledProcessError. + errors.LaunchCVDFail if launch_cvd times out or returns non-zero. """ - # Delete the cvd home/runtime temp if exist. The runtime folder is - # under the cvd home dir, so we only delete them from home dir. - cvd_home_dir = instance.GetLocalInstanceHomeDir(local_instance_id) - cvd_runtime_dir = instance.GetLocalInstanceRuntimeDir(local_instance_id) - shutil.rmtree(cvd_home_dir, ignore_errors=True) - os.makedirs(cvd_runtime_dir) - cvd_env = os.environ.copy() + # launch_cvd assumes host bins are in $ANDROID_HOST_OUT. + cvd_env[constants.ENV_ANDROID_SOONG_HOST_OUT] = host_bins_path + cvd_env[constants.ENV_ANDROID_HOST_OUT] = host_bins_path cvd_env[constants.ENV_CVD_HOME] = cvd_home_dir cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str(local_instance_id) # Check the result of launch_cvd command. # An exit code of 0 is equivalent to VIRTUAL_DEVICE_BOOT_COMPLETED - process = subprocess.Popen(cmd, shell=True, stderr=subprocess.STDOUT, - env=cvd_env) - if timeout: - timer = threading.Timer(timeout, process.kill) - timer.start() - process.wait() - if timeout: - timer.cancel() - if process.returncode == 0: - return - raise errors.LaunchCVDFail( - "Can't launch cuttlefish AVD. Return code:%s. \nFor more detail: " - "%s/launcher.log" % (str(process.returncode), cvd_runtime_dir)) - - @staticmethod - def PrintDisclaimer(): - """Print Disclaimer.""" - utils.PrintColorString( - "(Disclaimer: Local cuttlefish instance is not a fully supported\n" - "runtime configuration, fixing breakages is on a best effort SLO.)\n", - utils.TextColors.WARNING) - - @staticmethod - def IsLocalImageOccupied(local_image_dir): - """Check if the given image path is being used by a running CVD process. - - Args: - local_image_dir: String, path of local image. - - Return: - Integer of instance id which using the same image path. - """ - # TODO(149602560): Remove occupied image checking after after cf disk - # overlay is stable - for cf_runtime_config_path in instance.GetAllLocalInstanceConfigs(): - ins = instance.LocalInstance(cf_runtime_config_path) - if ins.CvdStatus(): - for disk_path in ins.virtual_disk_paths: - if local_image_dir in disk_path: - return ins.instance_id - return None + try: + subprocess.check_call(cmd, shell=True, stderr=subprocess.STDOUT, + env=cvd_env, timeout=timeout) + except subprocess.TimeoutExpired as e: + raise errors.LaunchCVDFail("Device did not boot within %d secs." % + timeout) from e + except subprocess.CalledProcessError as e: + raise errors.LaunchCVDFail("launch_cvd returned %s." % + e.returncode) from e diff --git a/create/local_image_local_instance_test.py b/create/local_image_local_instance_test.py index d1a0acca..4a3ec933 100644 --- a/create/local_image_local_instance_test.py +++ b/create/local_image_local_instance_test.py @@ -16,10 +16,11 @@ """Tests for LocalImageLocalInstance.""" import os -import shutil import subprocess +import tempfile import unittest -import mock + +from unittest import mock from acloud import errors from acloud.create import local_image_local_instance @@ -35,52 +36,152 @@ class LocalImageLocalInstanceTest(driver_test_lib.BaseDriverTest): LAUNCH_CVD_CMD_WITH_DISK = """sg group1 <<EOF sg group2 -launch_cvd -daemon -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -run_adb_connector=true -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -blank_data_image_mb fake -data_policy always_create +launch_cvd -daemon -config=phone -run_adb_connector=true -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,enable_sandbox,config -report_anonymous_usage_stats=y -enable_sandbox=false -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -blank_data_image_mb fake -data_policy always_create -start_vnc_server=true EOF""" LAUNCH_CVD_CMD_NO_DISK = """sg group1 <<EOF sg group2 -launch_cvd -daemon -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -run_adb_connector=true -system_image_dir fake_image_dir -instance_dir fake_cvd_dir +launch_cvd -daemon -config=phone -run_adb_connector=true -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,enable_sandbox,config -report_anonymous_usage_stats=y -enable_sandbox=false -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -start_vnc_server=true +EOF""" + + LAUNCH_CVD_CMD_NO_DISK_WITH_GPU = """sg group1 <<EOF +sg group2 +launch_cvd -daemon -config=phone -run_adb_connector=true -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,enable_sandbox,config -report_anonymous_usage_stats=y -enable_sandbox=false -cpus fake -x_res fake -y_res fake -dpi fake -memory_mb fake -start_vnc_server=true +EOF""" + + LAUNCH_CVD_CMD_WITH_WEBRTC = """sg group1 <<EOF +sg group2 +launch_cvd -daemon -config=auto -run_adb_connector=true -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,enable_sandbox,config -report_anonymous_usage_stats=y -enable_sandbox=false -guest_enforce_security=false -vm_manager=crosvm -start_webrtc=true -webrtc_public_ip=0.0.0.0 +EOF""" + + LAUNCH_CVD_CMD_WITH_MIXED_IMAGES = """sg group1 <<EOF +sg group2 +launch_cvd -daemon -config=phone -run_adb_connector=true -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,enable_sandbox,config -report_anonymous_usage_stats=y -enable_sandbox=false -start_vnc_server=true -super_image=fake_super_image -boot_image=fake_boot_image +EOF""" + + LAUNCH_CVD_CMD_WITH_ARGS = """sg group1 <<EOF +sg group2 +launch_cvd -daemon -config=phone -run_adb_connector=true -system_image_dir fake_image_dir -instance_dir fake_cvd_dir -undefok=report_anonymous_usage_stats,enable_sandbox,config -report_anonymous_usage_stats=y -enable_sandbox=false -start_vnc_server=true -setupwizard_mode=REQUIRED EOF""" _EXPECTED_DEVICES_IN_REPORT = [ { "instance_name": "local-instance-1", - "ip": "127.0.0.1:6520", + "ip": "0.0.0.0:6520", "adb_port": 6520, - "vnc_port": 6444 + "vnc_port": 6444, + "webrtc_port": 8443 } ] _EXPECTED_DEVICES_IN_FAILED_REPORT = [ { "instance_name": "local-instance-1", - "ip": "127.0.0.1" + "ip": "0.0.0.0" } ] def setUp(self): """Initialize new LocalImageLocalInstance.""" - super(LocalImageLocalInstanceTest, self).setUp() + super().setUp() self.local_image_local_instance = local_image_local_instance.LocalImageLocalInstance() # pylint: disable=protected-access @mock.patch("acloud.create.local_image_local_instance.utils") @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, - "PrepareLaunchCVDCmd") - @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, "GetImageArtifactsPath") @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, - "CheckLaunchCVD") - def testCreateAVD(self, mock_check_launch_cvd, mock_get_image, - _mock_prepare, mock_utils): - """Test the report returned by _CreateAVD.""" + "_SelectAndLockInstance") + @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, + "_CheckRunningCvd") + @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, + "_CreateInstance") + def testCreateAVD(self, mock_create, mock_check_running_cvd, + mock_lock_instance, mock_get_image, mock_utils): + """Test _CreateAVD.""" mock_utils.IsSupportedPlatform.return_value = True - mock_get_image.return_value = ("/image/path", "/host/bin/path") - mock_avd_spec = mock.Mock(connect_adb=False, unlock_screen=False) + mock_get_image.return_value = local_image_local_instance.ArtifactPaths( + "/image/path", "/host/bin/path", None, None, None, None) + mock_check_running_cvd.return_value = True + mock_avd_spec = mock.Mock() + mock_lock = mock.Mock() + mock_lock.Unlock.return_value = False + mock_lock_instance.return_value = (1, mock_lock) + + # Success + mock_create.return_value = mock.Mock() + self.local_image_local_instance._CreateAVD( + mock_avd_spec, no_prompts=True) + mock_lock_instance.assert_called_once() + mock_lock.SetInUse.assert_called_once_with(True) + mock_lock.Unlock.assert_called_once() + + mock_lock_instance.reset_mock() + mock_lock.SetInUse.reset_mock() + mock_lock.Unlock.reset_mock() + + # Failure with no report + mock_create.side_effect = ValueError("unit test") + with self.assertRaises(ValueError): + self.local_image_local_instance._CreateAVD( + mock_avd_spec, no_prompts=True) + mock_lock_instance.assert_called_once() + mock_lock.SetInUse.assert_not_called() + mock_lock.Unlock.assert_called_once() + + # Failure with report + mock_lock_instance.side_effect = errors.CreateError("unit test") + report = self.local_image_local_instance._CreateAVD( + mock_avd_spec, no_prompts=True) + self.assertEqual(report.errors, ["unit test"]) + + def testSelectAndLockInstance(self): + """test _SelectAndLockInstance.""" + mock_avd_spec = mock.Mock(local_instance_id=0) + mock_lock = mock.Mock() + mock_lock.Lock.return_value = True + mock_lock.LockIfNotInUse.side_effect = (False, True) + self.Patch(instance, "GetLocalInstanceLock", + return_value=mock_lock) + + ins_id, _ = self.local_image_local_instance._SelectAndLockInstance( + mock_avd_spec) + self.assertEqual(2, ins_id) + mock_lock.Lock.assert_not_called() + self.assertEqual(2, mock_lock.LockIfNotInUse.call_count) + + mock_lock.LockIfNotInUse.reset_mock() + + mock_avd_spec.local_instance_id = 1 + ins_id, _ = self.local_image_local_instance._SelectAndLockInstance( + mock_avd_spec) + self.assertEqual(1, ins_id) + mock_lock.Lock.assert_called_once() + mock_lock.LockIfNotInUse.assert_not_called() + + @mock.patch("acloud.create.local_image_local_instance.utils") + @mock.patch("acloud.create.local_image_local_instance.ota_tools") + @mock.patch("acloud.create.local_image_local_instance.create_common") + @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, + "_LaunchCvd") + @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, + "PrepareLaunchCVDCmd") + @mock.patch.object(instance, "GetLocalInstanceRuntimeDir") + @mock.patch.object(instance, "GetLocalInstanceHomeDir") + def testCreateInstance(self, mock_home_dir, _mock_runtime_dir, + _mock_prepare_cmd, mock_launch_cvd, + _mock_create_common, mock_ota_tools, _mock_utils): + """Test the report returned by _CreateInstance.""" self.Patch(instance, "GetLocalInstanceName", return_value="local-instance-1") - local_ins = mock.MagicMock( + mock_home_dir.return_value = "/local-instance-1" + artifact_paths = local_image_local_instance.ArtifactPaths( + "/image/path", "/host/bin/path", "/misc/info/path", + "/ota/tools/dir", "/system/image/path", "/boot/image/path") + mock_ota_tools_object = mock.Mock() + mock_ota_tools.OtaTools.return_value = mock_ota_tools_object + mock_avd_spec = mock.Mock(unlock_screen=False) + local_ins = mock.Mock( adb_port=6520, vnc_port=6444 ) @@ -89,22 +190,27 @@ EOF""" return_value=local_ins) self.Patch(list_instance, "GetActiveCVD", return_value=local_ins) + self.Patch(os, "symlink") # Success - report = self.local_image_local_instance._CreateAVD( - mock_avd_spec, no_prompts=True) + report = self.local_image_local_instance._CreateInstance( + 1, artifact_paths, mock_avd_spec, no_prompts=True) self.assertEqual(report.data.get("devices"), self._EXPECTED_DEVICES_IN_REPORT) + mock_ota_tools.OtaTools.assert_called_with("/ota/tools/dir") + mock_ota_tools_object.BuildSuperImage.assert_called_with( + "/local-instance-1/mixed_super.img", "/misc/info/path", mock.ANY) + # Failure - mock_check_launch_cvd.side_effect = errors.LaunchCVDFail("timeout") + mock_launch_cvd.side_effect = errors.LaunchCVDFail("unit test") - report = self.local_image_local_instance._CreateAVD( - mock_avd_spec, no_prompts=True) + report = self.local_image_local_instance._CreateInstance( + 1, artifact_paths, mock_avd_spec, no_prompts=True) self.assertEqual(report.data.get("devices_failing_boot"), self._EXPECTED_DEVICES_IN_FAILED_REPORT) - self.assertEqual(report.errors, ["timeout"]) + self.assertIn("unit test", report.errors[0]) # pylint: disable=protected-access @mock.patch("acloud.create.local_image_local_instance.os.path.isfile") @@ -114,7 +220,8 @@ EOF""" mock_isfile.return_value = None with mock.patch.dict("acloud.internal.lib.ota_tools.os.environ", - {"ANDROID_HOST_OUT": cvd_host_dir}, clear=True): + {"ANDROID_HOST_OUT": cvd_host_dir, + "ANDROID_SOONG_HOST_OUT": cvd_host_dir}, clear=True): with self.assertRaises(errors.GetCvdLocalHostPackageError): self.local_image_local_instance._FindCvdHostBinaries( [cvd_host_dir]) @@ -123,7 +230,8 @@ EOF""" lambda path: path == "/unit/test/bin/launch_cvd") with mock.patch.dict("acloud.internal.lib.ota_tools.os.environ", - {"ANDROID_HOST_OUT": cvd_host_dir}, clear=True): + {"ANDROID_HOST_OUT": cvd_host_dir, + "ANDROID_SOONG_HOST_OUT": cvd_host_dir}, clear=True): path = self.local_image_local_instance._FindCvdHostBinaries([]) self.assertEqual(path, cvd_host_dir) @@ -133,99 +241,226 @@ EOF""" [cvd_host_dir]) self.assertEqual(path, cvd_host_dir) - # pylint: disable=protected-access - @mock.patch.object(instance, "GetLocalInstanceRuntimeDir") + @staticmethod + def _CreateEmptyFile(path): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w"): + pass + + @mock.patch("acloud.create.local_image_local_instance.ota_tools") + def testGetImageArtifactsPath(self, mock_ota_tools): + """Test GetImageArtifactsPath without system image dir.""" + with tempfile.TemporaryDirectory() as temp_dir: + image_dir = "/unit/test" + cvd_dir = os.path.join(temp_dir, "cvd-host_package") + self._CreateEmptyFile(os.path.join(cvd_dir, "bin", "launch_cvd")) + + mock_avd_spec = mock.Mock( + local_image_dir=image_dir, + local_kernel_image=None, + local_system_image=None, + local_tool_dirs=[cvd_dir]) + + paths = self.local_image_local_instance.GetImageArtifactsPath( + mock_avd_spec) + + mock_ota_tools.FindOtaTools.assert_not_called() + self.assertEqual(paths, (image_dir, cvd_dir, None, None, None, None)) + + @mock.patch("acloud.create.local_image_local_instance.ota_tools") + def testGetImageFromBuildEnvironment(self, mock_ota_tools): + """Test GetImageArtifactsPath with files in build environment.""" + with tempfile.TemporaryDirectory() as temp_dir: + image_dir = os.path.join(temp_dir, "image") + cvd_dir = os.path.join(temp_dir, "cvd-host_package") + mock_ota_tools.FindOtaTools.return_value = cvd_dir + extra_image_dir = os.path.join(temp_dir, "extra_image") + system_image_path = os.path.join(extra_image_dir, "system.img") + boot_image_path = os.path.join(extra_image_dir, "boot.img") + misc_info_path = os.path.join(image_dir, "misc_info.txt") + self._CreateEmptyFile(os.path.join(image_dir, "vbmeta.img")) + self._CreateEmptyFile(os.path.join(cvd_dir, "bin", "launch_cvd")) + self._CreateEmptyFile(system_image_path) + self._CreateEmptyFile(boot_image_path) + self._CreateEmptyFile(os.path.join(extra_image_dir, + "boot-debug.img")) + self._CreateEmptyFile(misc_info_path) + + mock_avd_spec = mock.Mock( + local_image_dir=image_dir, + local_kernel_image=extra_image_dir, + local_system_image=extra_image_dir, + local_tool_dirs=[]) + + with mock.patch.dict("acloud.create.local_image_local_instance." + "os.environ", + {"ANDROID_SOONG_HOST_OUT": cvd_dir}, + clear=True): + paths = self.local_image_local_instance.GetImageArtifactsPath( + mock_avd_spec) + + mock_ota_tools.FindOtaTools.assert_called_once() + self.assertEqual(paths, + (image_dir, cvd_dir, misc_info_path, cvd_dir, + system_image_path, boot_image_path)) + + @mock.patch("acloud.create.local_image_local_instance.ota_tools") + def testGetImageFromTargetFiles(self, mock_ota_tools): + """Test GetImageArtifactsPath with extracted target files.""" + ota_tools_dir = "/mock_ota_tools" + mock_ota_tools.FindOtaTools.return_value = ota_tools_dir + + with tempfile.TemporaryDirectory() as temp_dir: + image_dir = os.path.join(temp_dir, "image") + cvd_dir = os.path.join(temp_dir, "cvd-host_package") + system_image_path = os.path.join(temp_dir, "system", "test.img") + misc_info_path = os.path.join(image_dir, "META", "misc_info.txt") + boot_image_path = os.path.join(temp_dir, "boot", "test.img") + self._CreateEmptyFile(os.path.join(image_dir, "IMAGES", + "vbmeta.img")) + self._CreateEmptyFile(os.path.join(cvd_dir, "bin", "launch_cvd")) + self._CreateEmptyFile(system_image_path) + self._CreateEmptyFile(misc_info_path) + self._CreateEmptyFile(boot_image_path) + + mock_avd_spec = mock.Mock( + local_image_dir=image_dir, + local_kernel_image=boot_image_path, + local_system_image=system_image_path, + local_tool_dirs=[ota_tools_dir, cvd_dir]) + + paths = self.local_image_local_instance.GetImageArtifactsPath( + mock_avd_spec) + + mock_ota_tools.FindOtaTools.assert_called_once() + self.assertEqual(paths, + (os.path.join(image_dir, "IMAGES"), cvd_dir, + misc_info_path, ota_tools_dir, system_image_path, + boot_image_path)) + @mock.patch.object(utils, "CheckUserInGroups") - def testPrepareLaunchCVDCmd(self, mock_usergroups, mock_cvd_dir): + def testPrepareLaunchCVDCmd(self, mock_usergroups): """test PrepareLaunchCVDCmd.""" mock_usergroups.return_value = False - mock_cvd_dir.return_value = "fake_cvd_dir" hw_property = {"cpu": "fake", "x_res": "fake", "y_res": "fake", "dpi":"fake", "memory": "fake", "disk": "fake"} constants.LIST_CF_USER_GROUPS = ["group1", "group2"] launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( constants.CMD_LAUNCH_CVD, hw_property, True, "fake_image_dir", - "fake_cvd_dir") + "fake_cvd_dir", False, True, None, None, None, "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_DISK) # "disk" doesn't exist in hw_property. hw_property = {"cpu": "fake", "x_res": "fake", "y_res": "fake", - "dpi":"fake", "memory": "fake"} + "dpi": "fake", "memory": "fake"} launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( constants.CMD_LAUNCH_CVD, hw_property, True, "fake_image_dir", - "fake_cvd_dir") + "fake_cvd_dir", False, True, None, None, None, "phone") self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_NO_DISK) - @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, - "_LaunchCvd") + # "gpu" is enabled with "default" + launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( + constants.CMD_LAUNCH_CVD, hw_property, True, "fake_image_dir", + "fake_cvd_dir", False, True, None, None, None, "phone") + self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_NO_DISK_WITH_GPU) + + # Following test with hw_property is None. + launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( + constants.CMD_LAUNCH_CVD, None, True, "fake_image_dir", + "fake_cvd_dir", True, False, None, None, None, "auto") + self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_WEBRTC) + + launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( + constants.CMD_LAUNCH_CVD, None, True, "fake_image_dir", + "fake_cvd_dir", False, True, "fake_super_image", "fake_boot_image", + None, "phone") + self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_MIXED_IMAGES) + + # Add args into launch command with "-setupwizard_mode=REQUIRED" + launch_cmd = self.local_image_local_instance.PrepareLaunchCVDCmd( + constants.CMD_LAUNCH_CVD, None, True, "fake_image_dir", + "fake_cvd_dir", False, True, None, None, + "-setupwizard_mode=REQUIRED", "phone") + self.assertEqual(launch_cmd, self.LAUNCH_CVD_CMD_WITH_ARGS) + @mock.patch.object(utils, "GetUserAnswerYes") @mock.patch.object(list_instance, "GetActiveCVD") - @mock.patch.object(local_image_local_instance.LocalImageLocalInstance, - "IsLocalImageOccupied") - def testCheckLaunchCVD(self, mock_image_occupied, mock_cvd_running, - mock_get_answer, - mock_launch_cvd): - """test CheckLaunchCVD.""" - launch_cvd_cmd = "fake_launch_cvd" - host_bins_path = "fake_host_path" + def testCheckRunningCvd(self, mock_cvd_running, mock_get_answer): + """test _CheckRunningCvd.""" local_instance_id = 3 - local_image_path = "fake_image_path" - # Test if image is in use. - mock_cvd_running.return_value = False - mock_image_occupied.return_value = True - with self.assertRaises(SystemExit): - self.local_image_local_instance.CheckLaunchCVD(launch_cvd_cmd, - host_bins_path, - local_instance_id, - local_image_path) - # Test if launch_cvd is running. - mock_image_occupied.return_value = False + # Test that launch_cvd is running. mock_cvd_running.return_value = True mock_get_answer.return_value = False - with self.assertRaises(SystemExit): - self.local_image_local_instance.CheckLaunchCVD(launch_cvd_cmd, - host_bins_path, - local_instance_id, - local_image_path) - - # Test if there's no using image and no conflict launch_cvd process. - mock_image_occupied.return_value = False + answer = self.local_image_local_instance._CheckRunningCvd( + local_instance_id) + self.assertFalse(answer) + + # Test that launch_cvd is not running. mock_cvd_running.return_value = False - self.local_image_local_instance.CheckLaunchCVD(launch_cvd_cmd, - host_bins_path, - local_instance_id, - local_image_path) - mock_launch_cvd.assert_called_once_with( - "fake_launch_cvd", 3, timeout=local_image_local_instance._LAUNCH_CVD_TIMEOUT_SECS) + answer = self.local_image_local_instance._CheckRunningCvd( + local_instance_id) + self.assertTrue(answer) # pylint: disable=protected-access + @mock.patch("acloud.create.local_image_local_instance.subprocess." + "check_call") @mock.patch.dict("os.environ", clear=True) - def testLaunchCVD(self): - """test _LaunchCvd should call subprocess.Popen with the specific env""" + def testLaunchCVD(self, mock_check_call): + """test _LaunchCvd should call subprocess.check_call with the env.""" local_instance_id = 3 launch_cvd_cmd = "launch_cvd" + host_bins_path = "host_bins_path" + cvd_home_dir = "fake_home" + timeout = 100 cvd_env = {} - cvd_env[constants.ENV_CVD_HOME] = "fake_home" - cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str( - local_instance_id) - process = mock.MagicMock() - process.wait.return_value = True - process.returncode = 0 - self.Patch(subprocess, "Popen", return_value=process) - self.Patch(instance, "GetLocalInstanceHomeDir", - return_value="fake_home") - self.Patch(os, "makedirs") - self.Patch(shutil, "rmtree") + cvd_env[constants.ENV_CVD_HOME] = cvd_home_dir + cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str(local_instance_id) + cvd_env[constants.ENV_ANDROID_SOONG_HOST_OUT] = host_bins_path + cvd_env[constants.ENV_ANDROID_HOST_OUT] = host_bins_path self.local_image_local_instance._LaunchCvd(launch_cvd_cmd, - local_instance_id) - # pylint: disable=no-member - subprocess.Popen.assert_called_once_with(launch_cvd_cmd, - shell=True, - stderr=subprocess.STDOUT, - env=cvd_env) + local_instance_id, + host_bins_path, + cvd_home_dir, + timeout) + + mock_check_call.assert_called_once_with(launch_cvd_cmd, + shell=True, + stderr=subprocess.STDOUT, + env=cvd_env, + timeout=timeout) + + @mock.patch("acloud.create.local_image_local_instance.subprocess." + "check_call") + def testLaunchCVDTimeout(self, mock_check_call): + """test _LaunchCvd with subprocess errors.""" + mock_check_call.side_effect = subprocess.TimeoutExpired( + cmd="launch_cvd", timeout=100) + with self.assertRaises(errors.LaunchCVDFail): + self.local_image_local_instance._LaunchCvd("launch_cvd", + 3, + "host_bins_path", + "cvd_home_dir", + 100) + + mock_check_call.side_effect = subprocess.CalledProcessError( + cmd="launch_cvd", returncode=1) + with self.assertRaises(errors.LaunchCVDFail): + self.local_image_local_instance._LaunchCvd("launch_cvd", + 3, + "host_bins_path", + "cvd_home_dir", + 100) + + def testGetWebrtcSigServerPort(self): + """test GetWebrtcSigServerPort.""" + instance_id = 3 + expected_port = 8445 + self.assertEqual( + self.local_image_local_instance.GetWebrtcSigServerPort(instance_id), + expected_port) if __name__ == "__main__": diff --git a/create/local_image_remote_host.py b/create/local_image_remote_host.py index b8741360..b93d78de 100644 --- a/create/local_image_remote_host.py +++ b/create/local_image_remote_host.py @@ -53,7 +53,8 @@ class LocalImageRemoteHost(base_avd_create.BaseAVDCreate): avd_type=constants.TYPE_CF, boot_timeout_secs=avd_spec.boot_timeout_secs, unlock_screen=avd_spec.unlock_screen, - wait_for_boot=False) + wait_for_boot=False, + connect_webrtc=avd_spec.connect_webrtc) # Launch vnc client if we're auto-connecting. if avd_spec.connect_vnc: utils.LaunchVNCFromReport(report, avd_spec, no_prompts) diff --git a/create/local_image_remote_instance.py b/create/local_image_remote_instance.py index 8177f716..811ebcfa 100644 --- a/create/local_image_remote_instance.py +++ b/create/local_image_remote_instance.py @@ -24,6 +24,7 @@ from acloud.internal import constants from acloud.internal.lib import utils from acloud.public.actions import common_operations from acloud.public.actions import remote_instance_cf_device_factory +from acloud.public.actions import remote_instance_fvp_device_factory class LocalImageRemoteInstance(base_avd_create.BaseAVDCreate): @@ -41,18 +42,28 @@ class LocalImageRemoteInstance(base_avd_create.BaseAVDCreate): Returns: A Report instance. """ - device_factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( - avd_spec, - avd_spec.local_image_artifact, - create_common.GetCvdHostPackage()) + if avd_spec.avd_type == constants.TYPE_CF: + device_factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( + avd_spec, + avd_spec.local_image_artifact, + create_common.GetCvdHostPackage()) + command = "create_cf" + elif avd_spec.avd_type == constants.TYPE_FVP: + device_factory = remote_instance_fvp_device_factory.RemoteInstanceDeviceFactory( + avd_spec) + command = "create_fvp" + report = common_operations.CreateDevices( - "create_cf", avd_spec.cfg, device_factory, avd_spec.num, + command, avd_spec.cfg, device_factory, + avd_spec.num, report_internal_ip=avd_spec.report_internal_ip, autoconnect=avd_spec.autoconnect, - avd_type=constants.TYPE_CF, + avd_type=avd_spec.avd_type, boot_timeout_secs=avd_spec.boot_timeout_secs, unlock_screen=avd_spec.unlock_screen, - wait_for_boot=False) + wait_for_boot=False, + connect_webrtc=avd_spec.connect_webrtc, + client_adb_port=avd_spec.client_adb_port) # Launch vnc client if we're auto-connecting. if avd_spec.connect_vnc: utils.LaunchVNCFromReport(report, avd_spec, no_prompts) diff --git a/create/remote_image_local_instance.py b/create/remote_image_local_instance.py index 9f979073..644b962a 100644 --- a/create/remote_image_local_instance.py +++ b/create/remote_image_local_instance.py @@ -20,12 +20,14 @@ remote image. """ import logging import os +import subprocess import sys from acloud import errors -from acloud.create import create_common from acloud.create import local_image_local_instance from acloud.internal import constants +from acloud.internal.lib import android_build_client +from acloud.internal.lib import auth from acloud.internal.lib import utils from acloud.setup import setup_common @@ -38,6 +40,7 @@ _CONFIRM_DOWNLOAD_DIR = ("Download dir %(download_dir)s does not have enough " "space (available space %(available_space)sGB, " "require %(required_space)sGB).\nPlease enter " "alternate path or 'q' to exit: ") +_HOME_FOLDER = os.path.expanduser("~") # The downloaded image artifacts will take up ~8G: # $du -lh --time $ANDROID_PRODUCT_OUT/aosp_cf_x86_phone-img-eng.XXX.zip # 422M @@ -51,37 +54,61 @@ _REQUIRED_SPACE = 10 def DownloadAndProcessImageFiles(avd_spec): """Download the CF image artifacts and process them. - It will download two artifacts and process them in this function. One is - cvd_host_package.tar.gz, the other is rom image zip. If the build_id is - "1234" and build_target is "aosp_cf_x86_phone-userdebug", - the image zip name is "aosp_cf_x86_phone-img-1234.zip". + To download rom images, Acloud would download the tool fetch_cvd that can + help process mixed build images. Args: avd_spec: AVDSpec object that tells us what we're going to create. Returns: extract_path: String, path to image folder. + + Raises: + errors.GetRemoteImageError: Fails to download rom images. """ cfg = avd_spec.cfg build_id = avd_spec.remote_image[constants.BUILD_ID] + build_branch = avd_spec.remote_image[constants.BUILD_BRANCH] build_target = avd_spec.remote_image[constants.BUILD_TARGET] extract_path = os.path.join( avd_spec.image_download_dir, constants.TEMP_ARTIFACTS_FOLDER, - build_id) + build_id + build_target) logger.debug("Extract path: %s", extract_path) # TODO(b/117189191): If extract folder exists, check if the files are # already downloaded and skip this step if they are. if not os.path.exists(extract_path): os.makedirs(extract_path) - remote_image = "%s-img-%s.zip" % (build_target.split('-')[0], - build_id) - artifacts = [constants.CVD_HOST_PACKAGE, remote_image] - for artifact in artifacts: - create_common.DownloadRemoteArtifact( - cfg, build_target, build_id, artifact, extract_path, decompress=True) + build_api = ( + android_build_client.AndroidBuildClient(auth.CreateCredentials(cfg))) + + # Download rom images via fetch_cvd + fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) + build_api.DownloadFetchcvd(fetch_cvd, cfg.fetch_cvd_version) + fetch_cvd_build_args = build_api.GetFetchBuildArgs( + build_id, build_branch, build_target, + avd_spec.system_build_info.get(constants.BUILD_ID), + avd_spec.system_build_info.get(constants.BUILD_BRANCH), + avd_spec.system_build_info.get(constants.BUILD_TARGET), + avd_spec.kernel_build_info.get(constants.BUILD_ID), + avd_spec.kernel_build_info.get(constants.BUILD_BRANCH), + avd_spec.kernel_build_info.get(constants.BUILD_TARGET), + avd_spec.bootloader_build_info.get(constants.BUILD_ID), + avd_spec.bootloader_build_info.get(constants.BUILD_BRANCH), + avd_spec.bootloader_build_info.get(constants.BUILD_TARGET)) + creds_cache_file = os.path.join(_HOME_FOLDER, cfg.creds_cache_file) + fetch_cvd_cert_arg = build_api.GetFetchCertArg(creds_cache_file) + fetch_cvd_args = [fetch_cvd, "-directory=%s" % extract_path, + fetch_cvd_cert_arg] + fetch_cvd_args.extend(fetch_cvd_build_args) + logger.debug("Download images command: %s", fetch_cvd_args) + try: + subprocess.check_call(fetch_cvd_args) + except subprocess.CalledProcessError as e: + raise errors.GetRemoteImageError("Fails to download images: %s" % e) + return extract_path @@ -140,7 +167,7 @@ class RemoteImageLocalInstance(local_image_local_instance.LocalImageLocalInstanc errors.NoCuttlefishCommonInstalled: cuttlefish-common doesn't install. Returns: - Tuple of (local image file, host bins package) paths. + local_image_local_instance.ArtifactPaths object. """ if not setup_common.PackageInstalled("cuttlefish-common"): raise errors.NoCuttlefishCommonInstalled( @@ -157,4 +184,7 @@ class RemoteImageLocalInstance(local_image_local_instance.LocalImageLocalInstanc raise errors.GetCvdLocalHostPackageError( "No launch_cvd found. Please check downloaded artifacts dir: %s" % image_dir) - return image_dir, image_dir + # This method does not set the optional fields because launch_cvd loads + # the paths from the fetcher config in image_dir. + return local_image_local_instance.ArtifactPaths( + image_dir, image_dir, None, None, None, None) diff --git a/create/remote_image_local_instance_test.py b/create/remote_image_local_instance_test.py index b6e99050..7c5adc77 100644 --- a/create/remote_image_local_instance_test.py +++ b/create/remote_image_local_instance_test.py @@ -16,10 +16,11 @@ import unittest from collections import namedtuple import os -import mock +import subprocess + +from unittest import mock from acloud import errors -from acloud.create import create_common from acloud.create import remote_image_local_instance from acloud.internal.lib import android_build_client from acloud.internal.lib import auth @@ -34,7 +35,7 @@ class RemoteImageLocalInstanceTest(driver_test_lib.BaseDriverTest): def setUp(self): """Initialize remote_image_local_instance.""" - super(RemoteImageLocalInstanceTest, self).setUp() + super().setUp() self.build_client = mock.MagicMock() self.Patch( android_build_client, @@ -43,12 +44,14 @@ class RemoteImageLocalInstanceTest(driver_test_lib.BaseDriverTest): self.Patch(auth, "CreateCredentials", return_value=mock.MagicMock()) self.RemoteImageLocalInstance = remote_image_local_instance.RemoteImageLocalInstance() self._fake_remote_image = {"build_target" : "aosp_cf_x86_phone-userdebug", - "build_id": "1234"} + "build_id": "1234", + "branch": "aosp_master"} self._extract_path = "/tmp/acloud_image_artifacts/1234" @mock.patch.object(remote_image_local_instance, "DownloadAndProcessImageFiles") def testGetImageArtifactsPath(self, mock_proc): """Test get image artifacts path.""" + mock_proc.return_value = "/unit/test" avd_spec = mock.MagicMock() # raise errors.NoCuttlefishCommonInstalled self.Patch(setup_common, "PackageInstalled", return_value=False) @@ -61,33 +64,25 @@ class RemoteImageLocalInstanceTest(driver_test_lib.BaseDriverTest): self.Patch(remote_image_local_instance, "ConfirmDownloadRemoteImageDir", return_value="/tmp") self.Patch(os.path, "exists", return_value=True) - self.RemoteImageLocalInstance.GetImageArtifactsPath(avd_spec) + paths = self.RemoteImageLocalInstance.GetImageArtifactsPath(avd_spec) mock_proc.assert_called_once_with(avd_spec) + self.assertEqual(paths.image_dir, "/unit/test") + self.assertEqual(paths.host_bins, "/unit/test") - @mock.patch.object(create_common, "DownloadRemoteArtifact") - def testDownloadAndProcessImageFiles(self, mock_download): + def testDownloadAndProcessImageFiles(self): """Test process remote cuttlefish image.""" avd_spec = mock.MagicMock() avd_spec.cfg = mock.MagicMock() + avd_spec.cfg.creds_cache_file = "cache.file" avd_spec.remote_image = self._fake_remote_image avd_spec.image_download_dir = "/tmp" self.Patch(os.path, "exists", return_value=False) self.Patch(os, "makedirs") + self.Patch(subprocess, "check_call") remote_image_local_instance.DownloadAndProcessImageFiles(avd_spec) - build_id = "1234" - build_target = "aosp_cf_x86_phone-userdebug" - checkfile1 = "aosp_cf_x86_phone-img-1234.zip" - checkfile2 = "cvd-host_package.tar.gz" - - # To validate DownloadArtifact runs twice. - self.assertEqual(mock_download.call_count, 2) - - # To validate DownloadArtifact arguments correct. - mock_download.assert_has_calls([ - mock.call(avd_spec.cfg, build_target, build_id, checkfile1, - self._extract_path, decompress=True), - mock.call(avd_spec.cfg, build_target, build_id, checkfile2, - self._extract_path, decompress=True)], any_order=True) + + self.assertEqual(self.build_client.GetFetchBuildArgs.call_count, 1) + self.assertEqual(self.build_client.GetFetchCertArg.call_count, 1) def testConfirmDownloadRemoteImageDir(self): """Test confirm download remote image dir""" diff --git a/create/remote_image_remote_host.py b/create/remote_image_remote_host.py index 7ef67c4d..1004fffe 100644 --- a/create/remote_image_remote_host.py +++ b/create/remote_image_remote_host.py @@ -53,7 +53,8 @@ class RemoteImageRemoteHost(base_avd_create.BaseAVDCreate): avd_type=constants.TYPE_CF, boot_timeout_secs=avd_spec.boot_timeout_secs, unlock_screen=avd_spec.unlock_screen, - wait_for_boot=False) + wait_for_boot=False, + connect_webrtc=avd_spec.connect_webrtc) # Launch vnc client if we're auto-connecting. if avd_spec.connect_vnc: utils.LaunchVNCFromReport(report, avd_spec, no_prompts) diff --git a/create/remote_image_remote_instance.py b/create/remote_image_remote_instance.py index 633d32ec..b56c411a 100644 --- a/create/remote_image_remote_instance.py +++ b/create/remote_image_remote_instance.py @@ -18,11 +18,23 @@ r"""RemoteImageRemoteInstance class. Create class that is responsible for creating a remote instance AVD with a remote image. """ + +import logging +import time + from acloud.create import base_avd_create +from acloud.internal import constants +from acloud.internal.lib import engprod_client from acloud.internal.lib import utils from acloud.public.actions import common_operations from acloud.public.actions import remote_instance_cf_device_factory -from acloud.internal import constants +from acloud.public import report + + +logger = logging.getLogger(__name__) +_DEVICE = "device" +_DEVICE_KEY_MAPPING = {"serverUrl": "ip", "sessionId": "instance_name"} +_LAUNCH_CVD_TIME = "launch_cvd_time" class RemoteImageRemoteInstance(base_avd_create.BaseAVDCreate): @@ -40,20 +52,71 @@ class RemoteImageRemoteInstance(base_avd_create.BaseAVDCreate): Returns: A Report instance. """ + if avd_spec.oxygen: + return self._LeaseOxygenAVD(avd_spec) device_factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( avd_spec) - report = common_operations.CreateDevices( + create_report = common_operations.CreateDevices( "create_cf", avd_spec.cfg, device_factory, avd_spec.num, report_internal_ip=avd_spec.report_internal_ip, autoconnect=avd_spec.autoconnect, avd_type=constants.TYPE_CF, boot_timeout_secs=avd_spec.boot_timeout_secs, unlock_screen=avd_spec.unlock_screen, - wait_for_boot=False) + wait_for_boot=False, + connect_webrtc=avd_spec.connect_webrtc, + client_adb_port=avd_spec.client_adb_port) # Launch vnc client if we're auto-connecting. if avd_spec.connect_vnc: - utils.LaunchVNCFromReport(report, avd_spec, no_prompts) + utils.LaunchVNCFromReport(create_report, avd_spec, no_prompts) if avd_spec.connect_webrtc: - utils.LaunchBrowserFromReport(report) + utils.LaunchBrowserFromReport(create_report) + + return create_report + + def _LeaseOxygenAVD(self, avd_spec): + """Lease the AVD from the AVD pool. - return report + Args: + avd_spec: AVDSpec object that tells us what we're going to create. + + Returns: + A Report instance. + """ + timestart = time.time() + response = engprod_client.EngProdClient.LeaseDevice( + avd_spec.remote_image[constants.BUILD_TARGET], + avd_spec.remote_image[constants.BUILD_ID], + avd_spec.cfg.api_key, + avd_spec.cfg.api_url) + execution_time = round(time.time() - timestart, 2) + reporter = report.Report(command="create_cf") + if _DEVICE in response: + reporter.SetStatus(report.Status.SUCCESS) + device_data = response[_DEVICE] + device_data[_LAUNCH_CVD_TIME] = execution_time + self._ReplaceDeviceDataKeys(device_data) + reporter.UpdateData(response) + else: + reporter.SetStatus(report.Status.FAIL) + reporter.AddError(response.get("errorMessage")) + + return reporter + + @staticmethod + def _ReplaceDeviceDataKeys(device_data): + """Replace keys of device data from oxygen response. + + To keep the device data using the same keys in Acloud report. Before + writing data to report, it needs to update the keys. + + Values: + device_data: Dict of device data. e.g. {'sessionId': 'b01ead68', + 'serverUrl': '10.1.1.1'} + """ + for key, val in _DEVICE_KEY_MAPPING.items(): + if key in device_data: + device_data[val] = device_data[key] + del device_data[key] + else: + logger.debug("There is no '%s' data in response.", key) diff --git a/create/remote_image_remote_instance_test.py b/create/remote_image_remote_instance_test.py new file mode 100644 index 00000000..ba8d8107 --- /dev/null +++ b/create/remote_image_remote_instance_test.py @@ -0,0 +1,86 @@ +# Copyright 2021 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for RemoteImageRemoteInstance.""" + +import unittest + +from unittest import mock + +from acloud.create import remote_image_remote_instance +from acloud.internal import constants +from acloud.internal.lib import driver_test_lib +from acloud.internal.lib import engprod_client +from acloud.public import report +from acloud.public.actions import common_operations +from acloud.public.actions import remote_instance_cf_device_factory + + +class RemoteImageRemoteInstanceTest(driver_test_lib.BaseDriverTest): + """Test RemoteImageRemoteInstance method.""" + + def setUp(self): + """Initialize new RemoteImageRemoteInstance.""" + super().setUp() + self.remote_image_remote_instance = remote_image_remote_instance.RemoteImageRemoteInstance() + + # pylint: disable=protected-access + @mock.patch.object(remote_image_remote_instance.RemoteImageRemoteInstance, + "_LeaseOxygenAVD") + @mock.patch.object(common_operations, "CreateDevices") + @mock.patch.object(remote_instance_cf_device_factory, + "RemoteInstanceDeviceFactory") + def testCreateAVD(self, mock_factory, mock_create_device, mock_lease): + """test CreateAVD.""" + avd_spec = mock.Mock() + avd_spec.oxygen = False + self.remote_image_remote_instance._CreateAVD( + avd_spec, no_prompts=True) + mock_factory.assert_called_once() + mock_create_device.assert_called_once() + + avd_spec.oxygen = True + self.remote_image_remote_instance._CreateAVD( + avd_spec, no_prompts=True) + mock_lease.assert_called_once() + + def testLeaseOxygenAVD(self): + """test LeaseOxygenAVD.""" + avd_spec = mock.Mock() + avd_spec.oxygen = True + avd_spec.remote_image = {constants.BUILD_TARGET: "fake_target", + constants.BUILD_ID: "fake_id"} + response_success = {"device": {"sessionId": "fake_device", + "serverUrl": "10.1.1.1"}} + response_fail = {"errorMessage": "Lease device fail."} + self.Patch(engprod_client.EngProdClient, "LeaseDevice", + side_effect=[response_success, response_fail]) + expected_status = report.Status.SUCCESS + reporter = self.remote_image_remote_instance._LeaseOxygenAVD(avd_spec) + self.assertEqual(reporter.status, expected_status) + + expected_status = report.Status.FAIL + reporter = self.remote_image_remote_instance._LeaseOxygenAVD(avd_spec) + self.assertEqual(reporter.status, expected_status) + + + def testReplaceDeviceDataKeys(self): + """test ReplaceDeviceDataKeys.""" + device_data = {"sessionId": "fake_device", "serverUrl": "10.1.1.1"} + expected_result = {"instance_name": "fake_device", "ip": "10.1.1.1"} + self.remote_image_remote_instance._ReplaceDeviceDataKeys(device_data) + self.assertEqual(device_data, expected_result) + + +if __name__ == '__main__': + unittest.main() diff --git a/delete/delete.py b/delete/delete.py index 02ee4841..3a7faff0 100644 --- a/delete/delete.py +++ b/delete/delete.py @@ -26,7 +26,6 @@ import subprocess from acloud import errors from acloud.internal import constants from acloud.internal.lib import auth -from acloud.internal.lib import adb_tools from acloud.internal.lib import cvd_compute_client_multi_stage from acloud.internal.lib import utils from acloud.internal.lib import ssh as ssh_object @@ -52,12 +51,8 @@ def DeleteInstances(cfg, instances_to_delete): instances_to_delete: List of list.Instance() object. Returns: - Report instance if there are instances to delete, None otherwise. + Report object. """ - if not instances_to_delete: - print("No instances to delete") - return None - delete_report = report.Report(command="delete") remote_instance_list = [] for instance in instances_to_delete: @@ -136,7 +131,15 @@ def DeleteLocalCuttlefishInstance(instance, delete_report): Returns: delete_report. """ + ins_lock = instance.GetLock() + if not ins_lock.Lock(): + delete_report.AddError("%s is locked by another process." % + instance.name) + delete_report.SetStatus(report.Status.FAIL) + return delete_report + try: + ins_lock.SetInUse(False) instance.Delete() delete_report.SetStatus(report.Status.SUCCESS) device_driver.AddDeletionResultToReport( @@ -146,6 +149,8 @@ def DeleteLocalCuttlefishInstance(instance, delete_report): except subprocess.CalledProcessError as e: delete_report.AddError(str(e)) delete_report.SetStatus(report.Status.FAIL) + finally: + ins_lock.Unlock() return delete_report @@ -162,22 +167,58 @@ def DeleteLocalGoldfishInstance(instance, delete_report): Returns: delete_report. """ - adb = adb_tools.AdbTools(adb_port=instance.adb_port, - device_serial=instance.device_serial) - if adb.EmuCommand("kill") == 0: - delete_report.SetStatus(report.Status.SUCCESS) - device_driver.AddDeletionResultToReport( - delete_report, [instance.name], failed=[], - error_msgs=[], - resource_name="instance") - else: - delete_report.AddError("Cannot kill %s." % instance.device_serial) + lock = instance.GetLock() + if not lock.Lock(): + delete_report.AddError("%s is locked by another process." % + instance.name) delete_report.SetStatus(report.Status.FAIL) + return delete_report + + try: + lock.SetInUse(False) + if instance.adb.EmuCommand("kill") == 0: + delete_report.SetStatus(report.Status.SUCCESS) + device_driver.AddDeletionResultToReport( + delete_report, [instance.name], failed=[], + error_msgs=[], + resource_name="instance") + else: + delete_report.AddError("Cannot kill %s." % instance.device_serial) + delete_report.SetStatus(report.Status.FAIL) + finally: + lock.Unlock() - instance.DeleteCreationTimestamp(ignore_errors=True) return delete_report +def ResetLocalInstanceLockByName(name, delete_report): + """Set the lock state of a local instance to be not in use. + + Args: + name: The instance name. + delete_report: Report object. + """ + ins_lock = list_instances.GetLocalInstanceLockByName(name) + if not ins_lock: + delete_report.AddError("%s is not a valid local instance name." % name) + delete_report.SetStatus(report.Status.FAIL) + return + + if not ins_lock.Lock(): + delete_report.AddError("%s is locked by another process." % name) + delete_report.SetStatus(report.Status.FAIL) + return + + try: + ins_lock.SetInUse(False) + delete_report.SetStatus(report.Status.SUCCESS) + device_driver.AddDeletionResultToReport( + delete_report, [name], failed=[], error_msgs=[], + resource_name="instance") + finally: + ins_lock.Unlock() + + def CleanUpRemoteHost(cfg, remote_host, host_user, host_ssh_private_key_path=None): """Clean up the remote host. @@ -227,18 +268,22 @@ def DeleteInstanceByNames(cfg, instances): A Report instance. """ delete_report = report.Report(command="delete") - local_instances = [ - ins for ins in instances if ins.startswith(_LOCAL_INSTANCE_PREFIX) - ] - remote_instances = list(set(instances) - set(local_instances)) - if local_instances: - utils.PrintColorString("Deleting local instances") - delete_report = DeleteInstances(cfg, list_instances.FilterInstancesByNames( - list_instances.GetLocalInstances(), local_instances)) - if remote_instances: - delete_report = DeleteRemoteInstances(cfg, - remote_instances, - delete_report) + local_names = set(name for name in instances if + name.startswith(_LOCAL_INSTANCE_PREFIX)) + remote_names = list(set(instances) - set(local_names)) + if local_names: + active_instances = list_instances.GetLocalInstancesByNames(local_names) + inactive_names = local_names.difference(ins.name for ins in + active_instances) + if active_instances: + utils.PrintColorString("Deleting local instances") + delete_report = DeleteInstances(cfg, active_instances) + if inactive_names: + utils.PrintColorString("Unlocking local instances") + for name in inactive_names: + ResetLocalInstanceLockByName(name, delete_report) + if remote_names: + delete_report = DeleteRemoteInstances(cfg, remote_names, delete_report) return delete_report @@ -276,4 +321,6 @@ def Run(args): # user didn't specify instances in args. instances = list_instances.ChooseInstancesFromList(instances) + if not instances: + utils.PrintColorString("No instances to delete") return DeleteInstances(cfg, instances) diff --git a/delete/delete_test.py b/delete/delete_test.py index 829ab7b6..1d7f97fd 100644 --- a/delete/delete_test.py +++ b/delete/delete_test.py @@ -13,8 +13,10 @@ # limitations under the License. """Tests for delete.""" +import subprocess import unittest -import mock + +from unittest import mock from acloud.delete import delete from acloud.internal.lib import driver_test_lib @@ -26,13 +28,14 @@ from acloud.public import report class DeleteTest(driver_test_lib.BaseDriverTest): """Test delete functions.""" - @mock.patch("subprocess.check_call") - def testDeleteLocalCuttlefishInstance(self, mock_subprocess): + def testDeleteLocalCuttlefishInstanceSuccess(self): """Test DeleteLocalCuttlefishInstance.""" - mock_subprocess.return_value = True instance_object = mock.MagicMock() - instance_object.instance_dir = "fake_instance_dir" instance_object.name = "local-instance" + mock_lock = mock.Mock() + mock_lock.Lock.return_value = True + instance_object.GetLock.return_value = mock_lock + delete_report = report.Report(command="delete") delete.DeleteLocalCuttlefishInstance(instance_object, delete_report) self.assertEqual(delete_report.data, { @@ -44,25 +47,44 @@ class DeleteTest(driver_test_lib.BaseDriverTest): ], }) self.assertEqual(delete_report.status, "SUCCESS") + mock_lock.SetInUse.assert_called_once_with(False) + mock_lock.Unlock.assert_called_once() - @mock.patch("acloud.delete.delete.adb_tools.AdbTools") - def testDeleteLocalGoldfishInstanceSuccess(self, mock_adb_tools): + def testDeleteLocalCuttlefishInstanceFailure(self): + """Test DeleteLocalCuttlefishInstance with command failure.""" + instance_object = mock.MagicMock() + instance_object.name = "local-instance" + instance_object.Delete.side_effect = subprocess.CalledProcessError( + 1, "cmd") + mock_lock = mock.Mock() + mock_lock.Lock.return_value = True + instance_object.GetLock.return_value = mock_lock + + delete_report = report.Report(command="delete") + delete.DeleteLocalCuttlefishInstance(instance_object, delete_report) + + self.assertEqual(delete_report.status, "FAIL") + mock_lock.SetInUse.assert_called_once_with(False) + mock_lock.Unlock.assert_called_once() + + def testDeleteLocalGoldfishInstanceSuccess(self): """Test DeleteLocalGoldfishInstance.""" - mock_instance = mock.Mock(adb_port=5555, + mock_adb_tools = mock.Mock() + mock_adb_tools.EmuCommand.return_value = 0 + mock_instance = mock.Mock(adb=mock_adb_tools, + adb_port=5555, device_serial="serial", instance_dir="/unit/test") # name is a positional argument of Mock(). mock_instance.name = "unittest" - - mock_adb_tools_obj = mock.Mock() - mock_adb_tools.return_value = mock_adb_tools_obj - mock_adb_tools_obj.EmuCommand.return_value = 0 + mock_lock = mock.Mock() + mock_lock.Lock.return_value = True + mock_instance.GetLock.return_value = mock_lock delete_report = report.Report(command="delete") delete.DeleteLocalGoldfishInstance(mock_instance, delete_report) - mock_adb_tools_obj.EmuCommand.assert_called_with("kill") - mock_instance.DeleteCreationTimestamp.assert_called() + mock_adb_tools.EmuCommand.assert_called_with("kill") self.assertEqual(delete_report.data, { "deleted": [ { @@ -72,40 +94,79 @@ class DeleteTest(driver_test_lib.BaseDriverTest): ], }) self.assertEqual(delete_report.status, "SUCCESS") + mock_lock.SetInUse.assert_called_once_with(False) + mock_lock.Unlock.assert_called_once() - @mock.patch("acloud.delete.delete.adb_tools.AdbTools") - def testDeleteLocalGoldfishInstanceFailure(self, mock_adb_tools): + def testDeleteLocalGoldfishInstanceFailure(self): """Test DeleteLocalGoldfishInstance with adb command failure.""" - mock_instance = mock.Mock(adb_port=5555, + mock_adb_tools = mock.Mock() + mock_adb_tools.EmuCommand.return_value = 1 + mock_instance = mock.Mock(adb=mock_adb_tools, + adb_port=5555, device_serial="serial", instance_dir="/unit/test") # name is a positional argument of Mock(). mock_instance.name = "unittest" - - mock_adb_tools_obj = mock.Mock() - mock_adb_tools.return_value = mock_adb_tools_obj - mock_adb_tools_obj.EmuCommand.return_value = 1 + mock_lock = mock.Mock() + mock_lock.Lock.return_value = True + mock_instance.GetLock.return_value = mock_lock delete_report = report.Report(command="delete") delete.DeleteLocalGoldfishInstance(mock_instance, delete_report) - mock_adb_tools_obj.EmuCommand.assert_called_with("kill") - mock_instance.DeleteCreationTimestamp.assert_called() + mock_adb_tools.EmuCommand.assert_called_with("kill") + self.assertTrue(len(delete_report.errors) > 0) + self.assertEqual(delete_report.status, "FAIL") + mock_lock.SetInUse.assert_called_once_with(False) + mock_lock.Unlock.assert_called_once() + + def testResetLocalInstanceLockByName(self): + """test ResetLocalInstanceLockByName.""" + mock_lock = mock.Mock() + mock_lock.Lock.return_value = True + self.Patch(list_instances, "GetLocalInstanceLockByName", + return_value=mock_lock) + delete_report = report.Report(command="delete") + delete.ResetLocalInstanceLockByName("unittest", delete_report) + + self.assertEqual(delete_report.data, { + "deleted": [ + { + "type": "instance", + "name": "unittest", + }, + ], + }) + mock_lock.Lock.assert_called_once() + mock_lock.SetInUse.assert_called_once_with(False) + mock_lock.Unlock.assert_called_once() + + def testResetLocalInstanceLockByNameFailure(self): + """test ResetLocalInstanceLockByName with an invalid name.""" + self.Patch(list_instances, "GetLocalInstanceLockByName", + return_value=None) + delete_report = report.Report(command="delete") + delete.ResetLocalInstanceLockByName("unittest", delete_report) + self.assertTrue(len(delete_report.errors) > 0) self.assertEqual(delete_report.status, "FAIL") @mock.patch.object(delete, "DeleteInstances", return_value="") + @mock.patch.object(delete, "ResetLocalInstanceLockByName") @mock.patch.object(delete, "DeleteRemoteInstances", return_value="") def testDeleteInstanceByNames(self, mock_delete_remote_ins, - mock_delete_local_ins): + mock_reset_lock, mock_delete_local_ins): """test DeleteInstanceByNames.""" cfg = mock.Mock() # Test delete local instances. instances = ["local-instance-1", "local-instance-2"] - self.Patch(list_instances, "FilterInstancesByNames", return_value="") - self.Patch(list_instances, "GetLocalInstances", return_value=[]) + mock_local_ins = mock.Mock() + mock_local_ins.name = "local-instance-1" + self.Patch(list_instances, "GetLocalInstancesByNames", + return_value=[mock_local_ins]) delete.DeleteInstanceByNames(cfg, instances) - mock_delete_local_ins.assert_called() + mock_delete_local_ins.assert_called_with(cfg, [mock_local_ins]) + mock_reset_lock.assert_called_with("local-instance-2", mock.ANY) # Test delete remote instances. instances = ["ins-id1-cf-x86-phone-userdebug", @@ -41,7 +41,7 @@ class HttpError(DriverError): def __init__(self, code, message): self.code = code - super(HttpError, self).__init__(message) + super().__init__(message) @staticmethod def CreateFromHttpError(http_error): @@ -83,6 +83,10 @@ class DeviceBootError(DriverError): """To catch device boot errors.""" +class DownloadArtifactError(DriverError): + """To catch download artifact errors.""" + + class NoSubnetwork(DriverError): """When there is no subnetwork for the GCE.""" @@ -131,6 +135,10 @@ class NotSupportedPlatformError(SetupError): """Error related to user using a not supported os.""" +class NotSupportedFieldName(SetupError): + """Unsupported field name for user config.""" + + class CreateError(Exception): """Base Create cmd exception.""" @@ -143,6 +151,10 @@ class CheckPathError(CreateError): """Path does not exist.""" +class CheckGCEZonesQuotaError(CreateError): + """There is no zone have enough quota.""" + + class UnsupportedInstanceImageType(CreateError): """Unsupported create action for given instance/image type.""" @@ -179,6 +191,10 @@ class GetLocalImageError(CreateError): """Can't find the local image.""" +class GetRemoteImageError(CreateError): + """An error to download the remote image.""" + + class GetCvdLocalHostPackageError(CreateError): """Can't find the lost host package.""" diff --git a/gen_version.sh b/gen_version.sh new file mode 100755 index 00000000..2f49dcdc --- /dev/null +++ b/gen_version.sh @@ -0,0 +1,9 @@ +#!/bin/bash +OUTFILE="$1" +BUILD_NUMBER_FROM_FILE=${OUT_DIR}/build_number.txt +if test -f "$BUILD_NUMBER_FROM_FILE"; then + cp ${BUILD_NUMBER_FROM_FILE} ${OUTFILE} +else + DATETIME=$(TZ='UTC' date +'%Y.%m.%d') + echo ${DATETIME}_local_build > ${OUTFILE} +fi diff --git a/internal/constants.py b/internal/constants.py index c5302444..af7e711a 100755 --- a/internal/constants.py +++ b/internal/constants.py @@ -30,14 +30,18 @@ LOGCAT_SERIAL_PORT = 2 # Remote image parameters BUILD_TARGET = "build_target" -BUILD_BRANCH = "build_branch" +BUILD_BRANCH = "branch" BUILD_ID = "build_id" +# Special value of local image parameters +FIND_IN_BUILD_ENV = "" + # AVD types TYPE_CHEEPS = "cheeps" TYPE_CF = "cuttlefish" TYPE_GCE = "gce" TYPE_GF = "goldfish" +TYPE_FVP = "fvp" # Image types IMAGE_SRC_REMOTE = "remote_image" @@ -57,6 +61,15 @@ INSTANCE_TYPE_REMOTE = "remote" INSTANCE_TYPE_LOCAL = "local" INSTANCE_TYPE_HOST = "host" +# CF_AVD_BUILD_TARGET_MAPPING +CF_X86_PATTERN = "cf_x86" +CF_ARM_PATTERN = "cf_arm" +CF_AVD_BUILD_TARGET_PATTERN_MAPPING = { + INSTANCE_TYPE_REMOTE: CF_X86_PATTERN, + INSTANCE_TYPE_LOCAL: CF_X86_PATTERN, + INSTANCE_TYPE_HOST: CF_ARM_PATTERN, +} + # Flavor types FLAVOR_PHONE = "phone" FLAVOR_AUTO = "auto" @@ -65,9 +78,10 @@ FLAVOR_TV = "tv" FLAVOR_IOT = "iot" FLAVOR_TABLET = "tablet" FLAVOR_TABLET_3G = "tablet_3g" +FLAVOR_FOLDABLE = "foldable" ALL_FLAVORS = [ FLAVOR_PHONE, FLAVOR_AUTO, FLAVOR_WEAR, FLAVOR_TV, FLAVOR_IOT, - FLAVOR_TABLET, FLAVOR_TABLET_3G + FLAVOR_TABLET, FLAVOR_TABLET_3G, FLAVOR_FOLDABLE ] # HW Property @@ -99,6 +113,8 @@ INSTANCE_NAME = "instance_name" GCE_USER = "vsoc-01" VNC_PORT = "vnc_port" ADB_PORT = "adb_port" +WEBRTC_PORT = "webrtc_port" +DEVICE_SERIAL = "device_serial" # For cuttlefish remote instances CF_ADB_PORT = 6520 CF_VNC_PORT = 6444 @@ -111,6 +127,10 @@ GCE_VNC_PORT = 6444 # For goldfish remote instances GF_ADB_PORT = 5555 GF_VNC_PORT = 6444 +# For FVP remote instances (no VNC support) +FVP_ADB_PORT = 5555 +# Maximum port number +MAX_PORT = 65535 COMMAND_PS = ["ps", "aux"] CMD_LAUNCH_CVD = "launch_cvd" @@ -119,17 +139,23 @@ CMD_STOP_CVD = "stop_cvd" CMD_RUN_CVD = "run_cvd" ENV_ANDROID_BUILD_TOP = "ANDROID_BUILD_TOP" ENV_ANDROID_EMULATOR_PREBUILTS = "ANDROID_EMULATOR_PREBUILTS" +# TODO(b/172535794): Remove the deprecated "ANDROID_HOST_OUT" by 2021Q4. ENV_ANDROID_HOST_OUT = "ANDROID_HOST_OUT" ENV_ANDROID_PRODUCT_OUT = "ANDROID_PRODUCT_OUT" +ENV_ANDROID_SOONG_HOST_OUT = "ANDROID_SOONG_HOST_OUT" ENV_ANDROID_TMP = "ANDROID_TMP" ENV_BUILD_TARGET = "TARGET_PRODUCT" -LOCALHOST = "127.0.0.1" +LOCALHOST = "0.0.0.0" LOCALHOST_ADB_SERIAL = LOCALHOST + ":%d" +REMOTE_INSTANCE_ADB_SERIAL = "127.0.0.1:%s" SSH_BIN = "ssh" SCP_BIN = "scp" ADB_BIN = "adb" +# Default timeout, the unit is seconds. +DEFAULT_SSH_TIMEOUT = 300 +DEFAULT_CF_BOOT_TIMEOUT = 450 LABEL_CREATE_BY = "created_by" @@ -148,7 +174,6 @@ INS_KEY_AVD_FLAVOR = "flavor" INS_KEY_IS_LOCAL = "remote" INS_KEY_ZONE = "zone" INS_STATUS_RUNNING = "RUNNING" -LOCAL_INS_NAME = "local-instance" ENV_CUTTLEFISH_CONFIG_FILE = "CUTTLEFISH_CONFIG_FILE" ENV_CUTTLEFISH_INSTANCE = "CUTTLEFISH_INSTANCE" ENV_CVD_HOME = "HOME" @@ -156,6 +181,8 @@ CUTTLEFISH_CONFIG_FILE = "cuttlefish_config.json" TEMP_ARTIFACTS_FOLDER = "acloud_image_artifacts" CVD_HOST_PACKAGE = "cvd-host_package.tar.gz" +# cvd tools symbolic link name of local instance. +CVD_TOOLS_LINK_NAME = "host_bins" TOOL_NAME = "acloud" # Exit code in metrics EXIT_SUCCESS = 0 @@ -166,3 +193,26 @@ EXIT_BY_ERROR = -99 # For reuse gce instance SELECT_ONE_GCE_INSTANCE = "select_one_gce_instance" + +# Webrtc +WEBRTC_LOCAL_PORT = 8443 +WEBRTC_LOCAL_HOST = "localhost" + +# Remote Log +REMOTE_LOG_FOLDER = "/home/%s/cuttlefish_runtime" % GCE_USER + +# Cheeps specific stuff. +CHEEPS_BETTY_IMAGE = "betty_image" + +# Key name in report +ERROR_LOG_FOLDER = "error_log_folder" + +# Stages for create progress +STAGE_INIT = 0 +STAGE_GCE = 1 +STAGE_SSH_CONNECT = 2 +STAGE_ARTIFACT = 3 +STAGE_BOOT_UP = 4 + +# The name of download image tool. +FETCH_CVD = "fetch_cvd" diff --git a/internal/lib/adb_tools.py b/internal/lib/adb_tools.py index 434bbd82..4862ba91 100644 --- a/internal/lib/adb_tools.py +++ b/internal/lib/adb_tools.py @@ -40,7 +40,7 @@ _WAIT_ADB_RETRY_BACKOFF_FACTOR = 1.5 _WAIT_ADB_SLEEP_MULTIPLIER = 2 -class AdbTools(object): +class AdbTools: """Adb tools. Attributes: @@ -54,6 +54,8 @@ class AdbTools(object): _device_information: Dict, will be added to adb information include usb, product model, device and transport_id """ + _adb_command = None + def __init__(self, adb_port=None, device_serial=""): """Initialize. @@ -61,7 +63,6 @@ class AdbTools(object): adb_port: String of adb port number. device_serial: String, adb device's serial number. """ - self._adb_command = "" self._adb_port = adb_port self._device_address = "" self._device_serial = "" @@ -83,14 +84,17 @@ class AdbTools(object): self._device_serial = (device_serial if device_serial else self._device_address) - def _CheckAdb(self): + @classmethod + def _CheckAdb(cls): """Find adb bin path. Raises: errors.NoExecuteCmd: Can't find the execute adb bin. """ - self._adb_command = utils.FindExecutable(constants.ADB_BIN) - if not self._adb_command: + if cls._adb_command: + return + cls._adb_command = utils.FindExecutable(constants.ADB_BIN) + if not cls._adb_command: raise errors.NoExecuteCmd("Can't find the adb command.") def GetAdbConnectionStatus(self): @@ -145,7 +149,7 @@ class AdbTools(object): "transport_id":None} """ adb_cmd = [self._adb_command, _ADB_DEVICE, _ADB_STATUS_DEVICE_ARGS] - device_info = subprocess.check_output(adb_cmd) + device_info = utils.CheckOutput(adb_cmd) self._device_information = { attribute: None for attribute in _DEVICE_ATTRIBUTES} @@ -156,6 +160,22 @@ class AdbTools(object): attribute: match.group(attribute) if match.group(attribute) else None for attribute in _DEVICE_ATTRIBUTES} + @classmethod + def GetDeviceSerials(cls): + """Get the serial numbers of connected devices.""" + cls._CheckAdb() + adb_cmd = [cls._adb_command, _ADB_DEVICE] + device_info = utils.CheckOutput(adb_cmd) + serials = [] + # Skip the first line which is "List of devices attached". Each of the + # following lines consists of the serial number, a tab character, and + # the state. The last line is empty. + for line in device_info.splitlines()[1:]: + serial_state = line.split() + if len(serial_state) > 1: + serials.append(serial_state[0]) + return serials + def IsAdbConnectionAlive(self): """Check devices connect alive. diff --git a/internal/lib/adb_tools_test.py b/internal/lib/adb_tools_test.py index cac26d06..2015ab94 100644 --- a/internal/lib/adb_tools_test.py +++ b/internal/lib/adb_tools_test.py @@ -15,23 +15,32 @@ import subprocess import unittest -import mock + +from unittest import mock +from six import b from acloud import errors from acloud.internal.lib import adb_tools from acloud.internal.lib import driver_test_lib -from acloud.internal.lib import utils class AdbToolsTest(driver_test_lib.BaseDriverTest): """Test adb functions.""" - DEVICE_ALIVE = ("List of devices attached\n" - "127.0.0.1:48451 device product:aosp_cf_x86_phone " - "model:Cuttlefish_x86_phone device:vsoc_x86 " - "transport_id:98") - DEVICE_OFFLINE = ("List of devices attached\n" - "127.0.0.1:48451 offline") - DEVICE_NONE = ("List of devices attached") + DEVICE_ALIVE = b("List of devices attached\n" + "127.0.0.1:48451 device product:aosp_cf_x86_phone " + "model:Cuttlefish_x86_phone device:vsoc_x86 " + "transport_id:98") + DEVICE_OFFLINE = b("List of devices attached\n" + "127.0.0.1:48451 offline") + DEVICE_STATE_ONLY = b("List of devices attached\n" + "127.0.0.1:48451\toffline\n" + "emulator-5554\tdevice\n") + DEVICE_NONE = b("List of devices attached") + + def setUp(self): + """Patch the path to adb.""" + super(AdbToolsTest, self).setUp() + self.Patch(adb_tools.AdbTools, "_adb_command", "path/adb") # pylint: disable=no-member def testGetAdbConnectionStatus(self): @@ -89,6 +98,13 @@ class AdbToolsTest(driver_test_lib.BaseDriverTest): adb_cmd = adb_tools.AdbTools(fake_adb_port) self.assertEqual(adb_cmd.device_information, dict_none) + def testGetDeviceSerials(self): + """Test parsing the output of adb devices.""" + self.Patch(subprocess, "check_output", + return_value=self.DEVICE_STATE_ONLY) + serials = adb_tools.AdbTools.GetDeviceSerials() + self.assertEqual(serials, ["127.0.0.1:48451", "emulator-5554"]) + # pylint: disable=no-member,protected-access def testConnectAdb(self): """Test connect adb.""" @@ -152,7 +168,6 @@ class AdbToolsTest(driver_test_lib.BaseDriverTest): """Test emu command.""" fake_adb_port = "48451" fake_device_serial = "fake_device_serial" - self.Patch(utils, "FindExecutable", return_value="path/adb") self.Patch(subprocess, "check_output", return_value=self.DEVICE_NONE) mock_popen_obj = mock.Mock(returncode=1) diff --git a/internal/lib/android_build_client.py b/internal/lib/android_build_client.py index 2f463475..847ed1ed 100644 --- a/internal/lib/android_build_client.py +++ b/internal/lib/android_build_client.py @@ -18,12 +18,17 @@ import collections import io +import json import logging +import os +import ssl +import stat import apiclient from acloud import errors from acloud.internal.lib import base_cloud_client +from acloud.internal.lib import utils logger = logging.getLogger(__name__) @@ -35,6 +40,7 @@ BuildInfo = collections.namedtuple("BuildInfo", [ "build_id", # The build id string "build_target", # The build target string "release_build_id"]) # The release build id string +_DEFAULT_BRANCH = "aosp-master" class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): @@ -57,6 +63,11 @@ class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): ONE_RESULT = 1 BUILD_SUCCESSFUL = True LATEST = "latest" + # FETCH_CVD variables. + FETCHER_NAME = "fetch_cvd" + FETCHER_BUILD_TARGET = "aosp_cf_x86_phone-userdebug" + MAX_RETRY = 3 + RETRY_SLEEP_SECS = 3 # Message constant COPY_TO_MSG = ("build artifact (target: %s, build_id: %s, " @@ -100,6 +111,155 @@ class AndroidBuildClient(base_cloud_client.BaseCloudApiClient): logger.error("Downloading artifact failed: %s", str(e)) raise errors.DriverError(str(e)) + def DownloadFetchcvd(self, local_dest, fetch_cvd_version): + """Get fetch_cvd from Android Build. + + Args: + local_dest: A local path where the artifact should be stored. + e.g. "/tmp/fetch_cvd" + fetch_cvd_version: String of fetch_cvd version. + """ + utils.RetryExceptionType( + exception_types=ssl.SSLError, + max_retries=self.MAX_RETRY, + functor=self.DownloadArtifact, + sleep_multiplier=self.RETRY_SLEEP_SECS, + retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR, + build_target=self.FETCHER_BUILD_TARGET, + build_id=fetch_cvd_version, + resource_id=self.FETCHER_NAME, + local_dest=local_dest, + attempt_id=self.LATEST) + fetch_cvd_stat = os.stat(local_dest) + os.chmod(local_dest, fetch_cvd_stat.st_mode | stat.S_IEXEC) + + @staticmethod + def ProcessBuild(build_id=None, branch=None, build_target=None): + """Create a Cuttlefish fetch_cvd build string. + + Args: + build_id: A specific build number to load from. Takes precedence over `branch`. + branch: A manifest-branch at which to get the latest build. + build_target: A particular device to load at the desired build. + + Returns: + A string, used in the fetch_cvd cmd or None if all args are None. + """ + if not build_target: + return build_id or branch + + if build_target and not branch: + branch = _DEFAULT_BRANCH + return (build_id or branch) + "/" + build_target + + # pylint: disable=too-many-locals + def GetFetchBuildArgs(self, build_id, branch, build_target, system_build_id, + system_branch, system_build_target, kernel_build_id, + kernel_branch, kernel_build_target, bootloader_build_id, + bootloader_branch, bootloader_build_target): + """Get args from build information for fetch_cvd. + + Args: + build_id: String of build id, e.g. "2263051", "P2804227" + branch: String of branch name, e.g. "aosp-master" + build_target: String of target name. + e.g. "aosp_cf_x86_phone-userdebug" + system_build_id: String of the system image build id. + system_branch: String of the system image branch name. + system_build_target: String of the system image target name, + e.g. "cf_x86_phone-userdebug" + kernel_build_id: String of the kernel image build id. + kernel_branch: String of the kernel image branch name. + kernel_build_target: String of the kernel image target name, + bootloader_build_id: String of the bootloader build id. + bootloader_branch: String of the bootloader branch name. + bootloader_build_target: String of the bootloader target name. + + Returns: + List of string args for fetch_cvd. + """ + fetch_cvd_args = [] + + default_build = self.ProcessBuild(build_id, branch, build_target) + if default_build: + fetch_cvd_args.append("-default_build=" + default_build) + system_build = self.ProcessBuild( + system_build_id, system_branch, system_build_target) + if system_build: + fetch_cvd_args.append("-system_build=" + system_build) + bootloader_build = self.ProcessBuild(bootloader_build_id, + bootloader_branch, + bootloader_build_target) + if bootloader_build: + fetch_cvd_args.append("-bootloader_build=%s" % bootloader_build) + kernel_build = self.GetKernelBuild(kernel_build_id, + kernel_branch, + kernel_build_target) + if kernel_build: + fetch_cvd_args.append("-kernel_build=" + kernel_build) + + return fetch_cvd_args + + @staticmethod + # pylint: disable=broad-except + def GetFetchCertArg(certification_file): + """Get cert arg from certification file for fetch_cvd. + + Parse the certification file to get access token of the latest + credential data and pass it to fetch_cvd command. + Example of certification file: + { + "data": [ + { + "credential": { + "_class": "OAuth2Credentials", + "_module": "oauth2client.client", + "access_token": "token_strings", + "client_id": "179485041932", + } + }] + } + + + Args: + certification_file: String of certification file path. + + Returns: + String of certificate arg for fetch_cvd. If there is no + certification file, return empty string for aosp branch. + """ + cert_arg = "" + + try: + with open(certification_file) as cert_file: + auth_token = json.load(cert_file).get("data")[-1].get( + "credential").get("access_token") + if auth_token: + cert_arg = "-credential_source=%s" % auth_token + except Exception as e: + utils.PrintColorString( + "Fail to open the certification file(%s): %s" % + (certification_file, e), utils.TextColors.WARNING) + return cert_arg + + def GetKernelBuild(self, kernel_build_id, kernel_branch, kernel_build_target): + """Get kernel build args for fetch_cvd. + + Args: + kernel_branch: Kernel branch name, e.g. "kernel-common-android-4.14" + kernel_build_id: Kernel build id, a string, e.g. "223051", "P280427" + kernel_build_target: String, Kernel build target name. + + Returns: + String of kernel build args for fetch_cvd. + If no kernel build then return None. + """ + # kernel_target have default value "kernel". If user provide kernel_build_id + # or kernel_branch, then start to process kernel image. + if kernel_build_id or kernel_branch: + return self.ProcessBuild(kernel_build_id, kernel_branch, kernel_build_target) + return None + def CopyTo(self, build_target, build_id, diff --git a/internal/lib/android_build_client_test.py b/internal/lib/android_build_client_test.py index e2c49d20..0aeeb448 100644 --- a/internal/lib/android_build_client_test.py +++ b/internal/lib/android_build_client_test.py @@ -20,7 +20,9 @@ import io import time import unittest -import mock + +from unittest import mock +import six import apiclient @@ -163,6 +165,89 @@ class AndroidBuildClientTest(driver_test_lib.BaseDriverTest): successful=self.client.BUILD_SUCCESSFUL) self.assertEqual(build_id, build_info.get("builds")[0].get("buildId")) + def testGetFetchBuildArgs(self): + """Test GetFetchBuildArgs.""" + build_id = "1234" + build_branch = "base_branch" + build_target = "base_target" + system_build_id = "2345" + system_build_branch = "system_branch" + system_build_target = "system_target" + kernel_build_id = "3456" + kernel_build_branch = "kernel_branch" + kernel_build_target = "kernel_target" + + # Test base image. + expected_args = ["-default_build=1234/base_target"] + self.assertEqual( + expected_args, + self.client.GetFetchBuildArgs( + build_id, build_branch, build_target, None, None, None, None, + None, None, None, None, None)) + + # Test base image with system image. + expected_args = ["-default_build=1234/base_target", + "-system_build=2345/system_target"] + self.assertEqual( + expected_args, + self.client.GetFetchBuildArgs( + build_id, build_branch, build_target, system_build_id, + system_build_branch, system_build_target, None, None, None, + None, None, None)) + + # Test base image with kernel image. + expected_args = ["-default_build=1234/base_target", + "-kernel_build=3456/kernel_target"] + self.assertEqual( + expected_args, + self.client.GetFetchBuildArgs( + build_id, build_branch, build_target, None, None, None, + kernel_build_id, kernel_build_branch, kernel_build_target, + None, None, None)) + + def testGetFetchCertArg(self): + """Test GetFetchCertArg.""" + cert_file_path = "fake_path" + certification = ( + "{" + " \"data\": [" + " {" + " \"credential\": {" + " \"access_token\": \"fake_token\"" + " }" + " }" + " ]" + "}" + ) + expected_arg = "-credential_source=fake_token" + self.Patch(six.moves.builtins, "open", mock.mock_open(read_data=certification)) + cert_arg = self.client.GetFetchCertArg(cert_file_path) + self.assertEqual(expected_arg, cert_arg) + + def testProcessBuild(self): + """Test creating "cuttlefish build" strings.""" + self.assertEqual( + self.client.ProcessBuild( + build_id="123", branch="abc", build_target="def"), "123/def") + self.assertEqual( + self.client.ProcessBuild( + build_id=None, branch="abc", build_target="def"), "abc/def") + self.assertEqual( + self.client.ProcessBuild( + build_id="123", branch=None, build_target="def"), "123/def") + self.assertEqual( + self.client.ProcessBuild( + build_id="123", branch="abc", build_target=None), "123") + self.assertEqual( + self.client.ProcessBuild( + build_id=None, branch="abc", build_target=None), "abc") + self.assertEqual( + self.client.ProcessBuild( + build_id="123", branch=None, build_target=None), "123") + self.assertEqual( + self.client.ProcessBuild( + build_id=None, branch=None, build_target=None), None) + if __name__ == "__main__": unittest.main() diff --git a/internal/lib/android_compute_client.py b/internal/lib/android_compute_client.py index 1bc69bcf..fac23a7d 100755 --- a/internal/lib/android_compute_client.py +++ b/internal/lib/android_compute_client.py @@ -41,9 +41,12 @@ from acloud import errors from acloud.internal import constants from acloud.internal.lib import gcompute_client from acloud.internal.lib import utils +from acloud.public import config logger = logging.getLogger(__name__) +_ZONE = "zone" +_VERSION = "version" class AndroidComputeClient(gcompute_client.ComputeClient): @@ -52,9 +55,7 @@ class AndroidComputeClient(gcompute_client.ComputeClient): DATA_DISK_NAME_FMT = "data-{instance}" BOOT_COMPLETED_MSG = "VIRTUAL_DEVICE_BOOT_COMPLETED" BOOT_STARTED_MSG = "VIRTUAL_DEVICE_BOOT_STARTED" - BOOT_TIMEOUT_SECS = 5 * 60 # 5 mins, usually it should take ~2 mins BOOT_CHECK_INTERVAL_SECS = 10 - OPERATION_TIMEOUT_SECS = 20 * 60 # Override parent value, 20 mins NAME_LENGTH_LIMIT = 63 @@ -81,6 +82,23 @@ class AndroidComputeClient(gcompute_client.ComputeClient): self._launch_args = acloud_config.launch_args self._instance_name_pattern = acloud_config.instance_name_pattern self._AddPerInstanceSshkey() + self._dict_report = {_ZONE: self._zone, + _VERSION: config.GetVersion()} + + # TODO(147047953): New args to contorl zone metrics check. + def _VerifyZoneByQuota(self): + """Verify the zone must have enough quota to create instance. + + Returns: + Boolean, True if zone have enough quota to create instance. + + Raises: + errors.CheckGCEZonesQuotaError: the zone doesn't have enough quota. + """ + if self.EnoughMetricsInZone(self._zone): + return True + raise errors.CheckGCEZonesQuotaError( + "There is no enough quota in zone: %s" % self._zone) def _AddPerInstanceSshkey(self): """Add per-instance ssh key. @@ -352,12 +370,12 @@ class AndroidComputeClient(gcompute_client.ComputeClient): boot_timeout_secs: Integer, the maximum time in seconds used to wait for the AVD to boot. """ - boot_timeout_secs = boot_timeout_secs or self.BOOT_TIMEOUT_SECS + boot_timeout_secs = boot_timeout_secs or constants.DEFAULT_CF_BOOT_TIMEOUT logger.info("Waiting for instance to boot up %s for %s secs", instance, boot_timeout_secs) timeout_exception = errors.DeviceBootTimeoutError( "Device %s did not finish on boot within timeout (%s secs)" % - (instance, boot_timeout_secs)), + (instance, boot_timeout_secs)) utils.PollAndWait( func=self.CheckBoot, expected_return=True, @@ -397,20 +415,16 @@ class AndroidComputeClient(gcompute_client.ComputeClient): return super(AndroidComputeClient, self).GetSerialPortOutput( instance, zone or self._zone, port) - def GetInstanceNamesByIPs(self, ips, zone=None): - """Get Instance names by IPs. - - This function will go through all instances, which - could be slow if there are too many instances. However, currently - GCE doesn't support search for instance by IP. + def ExtendReportData(self, key, value): + """Extend the report data. Args: - ips: A set of IPs. - zone: String, representing zone name, e.g. "us-central1-f" - - Returns: - A dictionary where key is ip and value is instance name or None - if instance is not found for the given IP. + key: string of key name. + value: string of data value. """ - return super(AndroidComputeClient, self).GetInstanceNamesByIPs( - ips, zone or self._zone) + self._dict_report.update({key: value}) + + @property + def dict_report(self): + """Return dict_report""" + return self._dict_report diff --git a/internal/lib/android_compute_client_test.py b/internal/lib/android_compute_client_test.py index 0c30e8b9..56c6041e 100644 --- a/internal/lib/android_compute_client_test.py +++ b/internal/lib/android_compute_client_test.py @@ -15,7 +15,8 @@ # limitations under the License. """Tests for android_compute_client.""" import unittest -import mock + +from unittest import mock from acloud import errors from acloud.internal.lib import android_compute_client diff --git a/internal/lib/auth.py b/internal/lib/auth.py index 50668784..ad03d6b1 100644 --- a/internal/lib/auth.py +++ b/internal/lib/auth.py @@ -32,7 +32,7 @@ service account* | oauth2 + private key non-google-owned service account can not access Android Build API. Only local build artifact can be used. -* Google-owned service account, if used, needs to be whitelisted by +* Google-owned service account, if used, needs to be allowed by Android Build team so that acloud can access build api. """ @@ -86,14 +86,20 @@ def _CreateOauthServiceAccountCreds(email, private_key_path, scopes): " error message: %s" % (private_key_path, str(e))) return credentials + # pylint: disable=invalid-name -def _CreateOauthServiceAccountCredsWithJsonKey(json_private_key_path, scopes): +def _CreateOauthServiceAccountCredsWithJsonKey(json_private_key_path, scopes, + creds_cache_file, user_agent): """Create credentials with a normal service account from json key file. Args: json_private_key_path: Path to the service account json key file. scopes: string, multiple scopes should be saperated by space. Api scopes to request for the oauth token. + creds_cache_file: String, file name for the credential cache. + e.g. .acloud_oauth2.dat + Will be created at home folder. + user_agent: String, the user agent for the credential, e.g. "acloud" Returns: An oauth2client.OAuth2Credentials instance. @@ -102,17 +108,23 @@ def _CreateOauthServiceAccountCredsWithJsonKey(json_private_key_path, scopes): errors.AuthenticationError: if failed to authenticate. """ try: - return ( - oauth2_service_account.ServiceAccountCredentials - .from_json_keyfile_name( - json_private_key_path, scopes=scopes)) + credentials = oauth2_service_account.ServiceAccountCredentials.from_json_keyfile_name( + json_private_key_path, scopes=scopes) + storage = multistore_file.get_credential_storage( + filename=os.path.abspath(creds_cache_file), + client_id=credentials.client_id, + user_agent=user_agent, + scope=scopes) + credentials.set_store(storage) except EnvironmentError as e: raise errors.AuthenticationError( "Could not authenticate using json private key file (%s) " " error message: %s" % (json_private_key_path, str(e))) + return credentials + -class RunFlowFlags(object): +class RunFlowFlags(): """Flags for oauth2client.tools.run_flow.""" def __init__(self, browser_auth): @@ -173,6 +185,8 @@ def _CreateOauthUserCreds(creds_cache_file, client_id, client_secret, scope=scopes) credentials = storage.get() if credentials is not None: + if not credentials.access_token_expired and not credentials.invalid: + return credentials try: credentials.refresh(httplib2.Http()) except oauth2_client.AccessTokenRefreshError: @@ -197,18 +211,24 @@ def CreateCredentials(acloud_config, scopes=_ALL_SCOPES): Returns: An oauth2client.OAuth2Credentials instance. """ + if os.path.isabs(acloud_config.creds_cache_file): + creds_cache_file = acloud_config.creds_cache_file + else: + creds_cache_file = os.path.join(HOME_FOLDER, + acloud_config.creds_cache_file) + if acloud_config.service_account_json_private_key_path: return _CreateOauthServiceAccountCredsWithJsonKey( acloud_config.service_account_json_private_key_path, - scopes=scopes) - elif acloud_config.service_account_private_key_path: + scopes=scopes, + creds_cache_file=creds_cache_file, + user_agent=acloud_config.user_agent) + if acloud_config.service_account_private_key_path: return _CreateOauthServiceAccountCreds( acloud_config.service_account_name, acloud_config.service_account_private_key_path, scopes=scopes) - creds_cache_file = os.path.join(HOME_FOLDER, - acloud_config.creds_cache_file) return _CreateOauthUserCreds( creds_cache_file=creds_cache_file, client_id=acloud_config.client_id, diff --git a/internal/lib/base_cloud_client.py b/internal/lib/base_cloud_client.py index cf9ee062..6e4400c5 100755 --- a/internal/lib/base_cloud_client.py +++ b/internal/lib/base_cloud_client.py @@ -17,18 +17,17 @@ BasicCloudApiCliend does basic setup for a cloud API. """ -import httplib import logging import socket import ssl import six +from six.moves import http_client # pylint: disable=import-error +import httplib2 from apiclient import errors as gerrors from apiclient.discovery import build -import apiclient.http -import httplib2 from oauth2client import client from acloud import errors @@ -38,7 +37,7 @@ from acloud.internal.lib import utils logger = logging.getLogger(__name__) -class BaseCloudApiClient(object): +class BaseCloudApiClient(): """A class that does basic setup for a cloud API.""" # To be overriden by subclasses. @@ -58,7 +57,7 @@ class BaseCloudApiClient(object): 502, # Bad Gateway 503, # Service Unavailable ] - RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error, + RETRIABLE_ERRORS = (http_client.HTTPException, httplib2.HttpLib2Error, socket.error, ssl.SSLError) RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, ) @@ -246,7 +245,7 @@ class BaseCloudApiClient(object): def _CallBack(request_id, response, exception): results[request_id] = (response, self._TranslateError(exception)) - batch = apiclient.http.BatchHttpRequest() + batch = self._service.new_batch_http_request() for request_id, request in six.iteritems(requests): batch.add( request=request, callback=_CallBack, request_id=request_id) diff --git a/internal/lib/base_cloud_client_test.py b/internal/lib/base_cloud_client_test.py index de74cd8b..fc75358d 100644 --- a/internal/lib/base_cloud_client_test.py +++ b/internal/lib/base_cloud_client_test.py @@ -19,9 +19,8 @@ import time import unittest -import mock -import apiclient +from unittest import mock from acloud import errors from acloud.internal.lib import base_cloud_client @@ -59,7 +58,8 @@ class BaseCloudApiClientTest(driver_test_lib.BaseDriverTest): return_value=mock.MagicMock()) return base_cloud_client.BaseCloudApiClient(mock.MagicMock()) - def _SetupBatchHttpRequestMock(self, rid_to_responses, rid_to_exceptions): + def _SetupBatchHttpRequestMock(self, rid_to_responses, rid_to_exceptions, + client): """Setup BatchHttpRequest mock.""" rid_to_exceptions = rid_to_exceptions or {} @@ -86,10 +86,8 @@ class BaseCloudApiClientTest(driver_test_lib.BaseDriverTest): mock_batch.execute = _Execute return mock_batch - self.Patch( - apiclient.http, - "BatchHttpRequest", - side_effect=_CreatMockBatchHttpRequest) + self.Patch(client.service, "new_batch_http_request", + side_effect=_CreatMockBatchHttpRequest) def testBatchExecute(self): """Test BatchExecute.""" @@ -103,7 +101,7 @@ class BaseCloudApiClientTest(driver_test_lib.BaseDriverTest): error_2 = FakeError("fake retriable error.") responses = {"r1": response, "r2": None, "r3": None} exceptions = {"r1": None, "r2": error_1, "r3": error_2} - self._SetupBatchHttpRequestMock(responses, exceptions) + self._SetupBatchHttpRequestMock(responses, exceptions, client) results = client.BatchExecute( requests, other_retriable_errors=(FakeError, )) expected_results = { diff --git a/internal/lib/cheeps_compute_client.py b/internal/lib/cheeps_compute_client.py index a84c9b54..31f7dfb5 100644 --- a/internal/lib/cheeps_compute_client.py +++ b/internal/lib/cheeps_compute_client.py @@ -88,11 +88,9 @@ class CheepsComputeClient(android_compute_client.AndroidComputeClient): metadata["user"] = avd_spec.username metadata["password"] = avd_spec.password - if avd_spec.remote_image[constants.BUILD_ID]: - metadata['android_build_id'] = avd_spec.remote_image[constants.BUILD_ID] - - if avd_spec.remote_image[constants.BUILD_TARGET]: - metadata['android_build_target'] = avd_spec.remote_image[constants.BUILD_TARGET] + metadata["android_build_id"] = avd_spec.remote_image[constants.BUILD_ID] + metadata["android_build_target"] = avd_spec.remote_image[constants.BUILD_TARGET] + metadata["betty_image"] = avd_spec.remote_image[constants.CHEEPS_BETTY_IMAGE] gcompute_client.ComputeClient.CreateInstance( self, diff --git a/internal/lib/cheeps_compute_client_test.py b/internal/lib/cheeps_compute_client_test.py index 309aefb5..73ded3e8 100644 --- a/internal/lib/cheeps_compute_client_test.py +++ b/internal/lib/cheeps_compute_client_test.py @@ -16,7 +16,8 @@ """Tests for acloud.internal.lib.cheeps_compute_client.""" import unittest -import mock + +from unittest import mock from acloud.internal import constants from acloud.internal.lib import cheeps_compute_client @@ -43,6 +44,7 @@ class CheepsComputeClientTest(driver_test_lib.BaseDriverTest): Y_RES = 1280 USER = "test_user" PASSWORD = "test_password" + CHEEPS_BETTY_IMAGE = 'abcasdf' def _GetFakeConfig(self): """Create a fake configuration object. @@ -85,6 +87,7 @@ class CheepsComputeClientTest(driver_test_lib.BaseDriverTest): 'android_build_id': self.ANDROID_BUILD_ID, 'android_build_target': self.ANDROID_BUILD_TARGET, 'avd_type': "cheeps", + 'betty_image': self.CHEEPS_BETTY_IMAGE, 'cvd_01_dpi': str(self.DPI), 'cvd_01_x_res': str(self.X_RES), 'cvd_01_y_res': str(self.Y_RES), @@ -106,6 +109,54 @@ class CheepsComputeClientTest(driver_test_lib.BaseDriverTest): avd_spec.remote_image = { constants.BUILD_ID: self.ANDROID_BUILD_ID, constants.BUILD_TARGET: self.ANDROID_BUILD_TARGET, + constants.CHEEPS_BETTY_IMAGE: self.CHEEPS_BETTY_IMAGE, + } + + self.cheeps_compute_client.CreateInstance( + self.INSTANCE, + self.IMAGE, + self.IMAGE_PROJECT, + avd_spec) + # pylint: disable=no-member + gcompute_client.ComputeClient.CreateInstance.assert_called_with( + self.cheeps_compute_client, + instance=self.INSTANCE, + image_name=self.IMAGE, + image_project=self.IMAGE_PROJECT, + disk_args=None, + metadata=expected_metadata, + machine_type=self.MACHINE_TYPE, + network=self.NETWORK, + zone=self.ZONE) + + def testCreateInstanceMissingParams(self): + """Test CreateInstance with optional avd_spec parameters missing.""" + expected_metadata = { + 'android_build_id': self.ANDROID_BUILD_ID, + 'android_build_target': self.ANDROID_BUILD_TARGET, + 'avd_type': "cheeps", + 'betty_image': None, + 'cvd_01_dpi': str(self.DPI), + 'cvd_01_x_res': str(self.X_RES), + 'cvd_01_y_res': str(self.Y_RES), + 'display': "%sx%s (%s)"%( + str(self.X_RES), + str(self.Y_RES), + str(self.DPI)), + } + expected_metadata.update(self.METADATA) + + + avd_spec = mock.MagicMock() + avd_spec.hw_property = {constants.HW_X_RES: str(self.X_RES), + constants.HW_Y_RES: str(self.Y_RES), + constants.HW_ALIAS_DPI: str(self.DPI)} + avd_spec.username = None + avd_spec.password = None + avd_spec.remote_image = { + constants.BUILD_ID: self.ANDROID_BUILD_ID, + constants.BUILD_TARGET: self.ANDROID_BUILD_TARGET, + constants.CHEEPS_BETTY_IMAGE: None, } self.cheeps_compute_client.CreateInstance( diff --git a/internal/lib/cvd_compute_client_multi_stage.py b/internal/lib/cvd_compute_client_multi_stage.py index 2c31b25c..cc86a21e 100644 --- a/internal/lib/cvd_compute_client_multi_stage.py +++ b/internal/lib/cvd_compute_client_multi_stage.py @@ -37,7 +37,6 @@ Android build, and start Android within the host instance. import logging import os -import stat import subprocess import tempfile import time @@ -54,8 +53,11 @@ from acloud.pull import pull logger = logging.getLogger(__name__) +_CONFIG_ARG = "-config" _DECOMPRESS_KERNEL_ARG = "-decompress_kernel=true" -_GPU_ARG = "-gpu_mode=drm_virgl" +_AGREEMENT_PROMPT_ARG = "-report_anonymous_usage_stats=y" +_UNDEFOK_ARG = "-undefok=report_anonymous_usage_stats,config" +_NUM_AVDS_ARG = "-num_instances=%(num_AVD)s" _DEFAULT_BRANCH = "aosp-master" _FETCHER_BUILD_TARGET = "aosp_cf_x86_phone-userdebug" _FETCHER_NAME = "fetch_cvd" @@ -66,33 +68,21 @@ _LAUNCH_CVD = "launch_cvd_time" # WebRTC args for launching AVD _GUEST_ENFORCE_SECURITY_FALSE = "--guest_enforce_security=false" _START_WEBRTC = "--start_webrtc" +_WEBRTC_ID = "--webrtc_device_id=%(instance)s" _VM_MANAGER = "--vm_manager=crosvm" _WEBRTC_ARGS = [_GUEST_ENFORCE_SECURITY_FALSE, _START_WEBRTC, _VM_MANAGER] -_WEBRTC_PUBLIC_IP = "--webrtc_public_ip=%s" - - -def _ProcessBuild(build_id=None, branch=None, build_target=None): - """Create a Cuttlefish fetch_cvd build string. - - Args: - build_id: A specific build number to load from. Takes precedence over `branch`. - branch: A manifest-branch at which to get the latest build. - build_target: A particular device to load at the desired build. - - Returns: - A string, used in the fetch_cvd cmd or None if all args are None. - """ - if not build_target: - return build_id or branch - elif build_target and not branch: - branch = _DEFAULT_BRANCH - return (build_id or branch) + "/" + build_target +_VNC_ARGS = ["--start_vnc_server=true"] +_NO_RETRY = 0 +# Launch cvd command for acloud report +_LAUNCH_CVD_COMMAND = "launch_cvd_command" class CvdComputeClient(android_compute_client.AndroidComputeClient): """Client that manages Android Virtual Device.""" DATA_POLICY_CREATE_IF_MISSING = "create_if_missing" + # Data policy to customize disk size. + DATA_POLICY_ALWAYS_CREATE = "always_create" def __init__(self, acloud_config, @@ -114,7 +104,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): external ip. gpu: String, GPU to attach to the device. """ - super(CvdComputeClient, self).__init__(acloud_config, oauth2_credentials) + super().__init__(acloud_config, oauth2_credentials) self._fetch_cvd_version = acloud_config.fetch_cvd_version self._build_api = ( @@ -130,6 +120,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): self._ssh = None self._ip = None self._user = constants.GCE_USER + self._stage = constants.STAGE_INIT self._execution_time = {_FETCH_ARTIFACT: 0, _GCE_CREATE: 0, _LAUNCH_CVD: 0} def InitRemoteHost(self, ssh, ip, user): @@ -144,6 +135,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): remote host, e.g. "external:140.110.20.1, internal:10.0.0.1" user: String of user log in to the instance. """ + self.SetStage(constants.STAGE_SSH_CONNECT) self._ssh = ssh self._ip = ip self._user = user @@ -151,6 +143,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): self.StopCvd() self.CleanUp() + # TODO(171376263): Refactor CreateInstance() args with avd_spec. # pylint: disable=arguments-differ,too-many-locals,broad-except def CreateInstance(self, instance, image_name, image_project, build_target=None, branch=None, build_id=None, @@ -158,7 +151,8 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): kernel_build_target=None, blank_data_disk_size_gb=None, avd_spec=None, extra_scopes=None, system_build_target=None, system_branch=None, - system_build_id=None): + system_build_id=None, bootloader_build_target=None, + bootloader_branch=None, bootloader_build_id=None): """Create/Reuse a single configured cuttlefish device. 1. Prepare GCE instance. @@ -180,10 +174,13 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): blank_data_disk_size_gb: Size of the blank data disk in GB. avd_spec: An AVDSpec instance. extra_scopes: A list of extra scopes to be passed to the instance. - system_build_target: Target name for the system image, - e.g. "cf_x86_phone-userdebug" - system_branch: A String, branch name for the system image. - system_build_id: A string, build id for the system image. + system_build_target: String of the system image target name, + e.g. "cf_x86_phone-userdebug" + system_branch: String of the system image branch name. + system_build_id: String of the system image build id. + bootloader_build_target: String of the bootloader target name. + bootloader_branch: String of the bootloader branch name. + bootloader_build_id: String of the bootloader build id. Returns: A string, representing instance name. @@ -195,9 +192,15 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): int(self.GetImage(image_name, image_project)["diskSizeGb"]) + blank_data_disk_size_gb) + # Record the build info into metadata. + self._RecordBuildInfo(avd_spec, build_id, build_target, + system_build_id, system_build_target, + kernel_build_id, kernel_build_target) + if avd_spec and avd_spec.instance_name_to_reuse: self._ip = self._ReusingGceInstance(avd_spec) else: + self._VerifyZoneByQuota() self._ip = self._CreateGceInstance(instance, image_name, image_project, extra_scopes, boot_disk_size_gb, avd_spec) @@ -207,6 +210,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): extra_args_ssh_tunnel=self._extra_args_ssh_tunnel, report_internal_ip=self._report_internal_ip) try: + self.SetStage(constants.STAGE_SSH_CONNECT) self._ssh.WaitForSsh(timeout=self._ins_timeout_secs) if avd_spec: if avd_spec.instance_name_to_reuse: @@ -219,13 +223,10 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): self.FetchBuild(build_id, branch, build_target, system_build_id, system_branch, system_build_target, kernel_build_id, - kernel_branch, kernel_build_target) - kernel_build = self.GetKernelBuild(kernel_build_id, - kernel_branch, - kernel_build_target) + kernel_branch, kernel_build_target, bootloader_build_id, + bootloader_branch, bootloader_build_target) self.LaunchCvd(instance, blank_data_disk_size_gb=blank_data_disk_size_gb, - kernel_build=kernel_build, boot_timeout_secs=self._boot_timeout_secs) return instance @@ -233,21 +234,58 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): self._all_failures[instance] = e return instance + def _RecordBuildInfo(self, avd_spec, build_id, build_target, + system_build_id, system_build_target, + kernel_build_id, kernel_build_target): + """Rocord the build information into metadata. + + The build information includes build id and build target of base image, + system image, and kernel image. + + Args: + avd_spec: An AVDSpec instance. + build_id: String, build id for the base image. + build_target: String, target name for the base image, + e.g. "cf_x86_phone-userdebug" + system_build_id: String, build id for the system image. + system_build_target: String, system build target name, + e.g. "cf_x86_phone-userdebug" + kernel_build_id: String, kernel build id, e.g. "223051", "P280427" + kernel_build_target: String, kernel build target name. + """ + if avd_spec and avd_spec.image_source == constants.IMAGE_SRC_REMOTE: + build_id = avd_spec.remote_image.get(constants.BUILD_ID) + build_target = avd_spec.remote_image.get(constants.BUILD_TARGET) + system_build_id = avd_spec.system_build_info.get(constants.BUILD_ID) + system_build_target = avd_spec.system_build_info.get(constants.BUILD_TARGET) + kernel_build_id = avd_spec.kernel_build_info.get(constants.BUILD_ID) + kernel_build_target = avd_spec.kernel_build_info.get(constants.BUILD_TARGET) + if build_id and build_target: + self._metadata.update({"build_id": build_id}) + self._metadata.update({"build_target": build_target}) + if system_build_id and system_build_target: + self._metadata.update({"system_build_id": system_build_id}) + self._metadata.update({"system_build_target": system_build_target}) + if kernel_build_id and kernel_build_target: + self._metadata.update({"kernel_build_id": kernel_build_id}) + self._metadata.update({"kernel_build_target": kernel_build_target}) + + # pylint: disable=too-many-branches def _GetLaunchCvdArgs(self, avd_spec=None, blank_data_disk_size_gb=None, - kernel_build=None, decompress_kernel=None): + decompress_kernel=None, instance=None): """Get launch_cvd args. Args: avd_spec: An AVDSpec instance. blank_data_disk_size_gb: Size of the blank data disk in GB. - kernel_build: String, kernel build info. decompress_kernel: Boolean, if true decompress the kernel. + instance: String, instance name. Returns: String, args of launch_cvd. """ launch_cvd_args = [] - if blank_data_disk_size_gb > 0: + if blank_data_disk_size_gb and blank_data_disk_size_gb > 0: # Policy 'create_if_missing' would create a blank userdata disk if # missing. If already exist, reuse the disk. launch_cvd_args.append( @@ -255,65 +293,64 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): launch_cvd_args.append( "-blank_data_image_mb=%d" % (blank_data_disk_size_gb * 1024)) if avd_spec: - launch_cvd_args.append( - "-x_res=" + avd_spec.hw_property[constants.HW_X_RES]) - launch_cvd_args.append( - "-y_res=" + avd_spec.hw_property[constants.HW_Y_RES]) - launch_cvd_args.append( - "-dpi=" + avd_spec.hw_property[constants.HW_ALIAS_DPI]) - if constants.HW_ALIAS_DISK in avd_spec.hw_property: + launch_cvd_args.append("-config=%s" % avd_spec.flavor) + if avd_spec.hw_customize or not self._ArgSupportInLaunchCVD(_CONFIG_ARG): launch_cvd_args.append( - "-data_policy=" + self.DATA_POLICY_CREATE_IF_MISSING) + "-x_res=" + avd_spec.hw_property[constants.HW_X_RES]) launch_cvd_args.append( - "-blank_data_image_mb=" - + avd_spec.hw_property[constants.HW_ALIAS_DISK]) - if constants.HW_ALIAS_CPUS in avd_spec.hw_property: + "-y_res=" + avd_spec.hw_property[constants.HW_Y_RES]) launch_cvd_args.append( - "-cpus=%s" % avd_spec.hw_property[constants.HW_ALIAS_CPUS]) - if constants.HW_ALIAS_MEMORY in avd_spec.hw_property: - launch_cvd_args.append( - "-memory_mb=%s" % avd_spec.hw_property[constants.HW_ALIAS_MEMORY]) + "-dpi=" + avd_spec.hw_property[constants.HW_ALIAS_DPI]) + if constants.HW_ALIAS_DISK in avd_spec.hw_property: + launch_cvd_args.append( + "-data_policy=" + self.DATA_POLICY_ALWAYS_CREATE) + launch_cvd_args.append( + "-blank_data_image_mb=" + + avd_spec.hw_property[constants.HW_ALIAS_DISK]) + if constants.HW_ALIAS_CPUS in avd_spec.hw_property: + launch_cvd_args.append( + "-cpus=%s" % avd_spec.hw_property[constants.HW_ALIAS_CPUS]) + if constants.HW_ALIAS_MEMORY in avd_spec.hw_property: + launch_cvd_args.append( + "-memory_mb=%s" % avd_spec.hw_property[constants.HW_ALIAS_MEMORY]) if avd_spec.connect_webrtc: - launch_cvd_args.append(_WEBRTC_PUBLIC_IP % self._ip.external) launch_cvd_args.extend(_WEBRTC_ARGS) + launch_cvd_args.append(_WEBRTC_ID % {"instance": instance}) + if avd_spec.connect_vnc: + launch_cvd_args.extend(_VNC_ARGS) + if avd_spec.num_avds_per_instance > 1: + launch_cvd_args.append( + _NUM_AVDS_ARG % {"num_AVD": avd_spec.num_avds_per_instance}) + if avd_spec.launch_args: + launch_cvd_args.append(avd_spec.launch_args) else: resolution = self._resolution.split("x") launch_cvd_args.append("-x_res=" + resolution[0]) launch_cvd_args.append("-y_res=" + resolution[1]) launch_cvd_args.append("-dpi=" + resolution[3]) - if kernel_build: - launch_cvd_args.append("-kernel_path=kernel") - - if self._launch_args: + if not avd_spec and self._launch_args: launch_cvd_args.append(self._launch_args) if decompress_kernel: launch_cvd_args.append(_DECOMPRESS_KERNEL_ARG) - if self._gpu: - launch_cvd_args.append(_GPU_ARG) - + launch_cvd_args.append(_UNDEFOK_ARG) + launch_cvd_args.append(_AGREEMENT_PROMPT_ARG) return launch_cvd_args - @staticmethod - def GetKernelBuild(kernel_build_id, kernel_branch, kernel_build_target): - """Get kernel build args for fetch_cvd. + def _ArgSupportInLaunchCVD(self, arg): + """Check if the arg is supported in launch_cvd. Args: - kernel_branch: Kernel branch name, e.g. "kernel-common-android-4.14" - kernel_build_id: Kernel build id, a string, e.g. "223051", "P280427" - kernel_build_target: String, Kernel build target name. + arg: String of the arg. e.g. "-config". Returns: - String of kernel build args for fetch_cvd. - If no kernel build then return None. + True if this arg is supported. Otherwise False. """ - # kernel_target have default value "kernel". If user provide kernel_build_id - # or kernel_branch, then start to process kernel image. - if kernel_build_id or kernel_branch: - return _ProcessBuild(kernel_build_id, kernel_branch, kernel_build_target) - return None + if arg in self._ssh.GetCmdOutput("./bin/launch_cvd --help"): + return True + return False def StopCvd(self): """Stop CVD. @@ -344,7 +381,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): @utils.TimeExecute(function_description="Launching AVD(s) and waiting for boot up", result_evaluator=utils.BootEvaluator) def LaunchCvd(self, instance, avd_spec=None, - blank_data_disk_size_gb=None, kernel_build=None, + blank_data_disk_size_gb=None, decompress_kernel=None, boot_timeout_secs=None): """Launch CVD. @@ -356,7 +393,6 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): instance: String, instance name. avd_spec: An AVDSpec instance. blank_data_disk_size_gb: Size of the blank data disk in GB. - kernel_build: String, kernel build info. decompress_kernel: Boolean, if true decompress the kernel. boot_timeout_secs: Integer, the maximum time to wait for the command to respond. @@ -365,16 +401,18 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): dict of faliures, return this dict for BootEvaluator to handle LaunchCvd success or fail messages. """ + self.SetStage(constants.STAGE_BOOT_UP) timestart = time.time() error_msg = "" launch_cvd_args = self._GetLaunchCvdArgs(avd_spec, blank_data_disk_size_gb, - kernel_build, - decompress_kernel) - boot_timeout_secs = boot_timeout_secs or self.BOOT_TIMEOUT_SECS + decompress_kernel, + instance) + boot_timeout_secs = boot_timeout_secs or constants.DEFAULT_CF_BOOT_TIMEOUT ssh_command = "./bin/launch_cvd -daemon " + " ".join(launch_cvd_args) try: - self._ssh.Run(ssh_command, boot_timeout_secs) + self.ExtendReportData(_LAUNCH_CVD_COMMAND, ssh_command) + self._ssh.Run(ssh_command, boot_timeout_secs, retry=_NO_RETRY) except (subprocess.CalledProcessError, errors.DeviceConnectionError) as e: # TODO(b/140475060): Distinguish the error is command return error # or timeout error. @@ -398,8 +436,9 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): instance: String, instance name. """ log_files = pull.GetAllLogFilePaths(self._ssh) - download_folder = pull.GetDownloadLogFolder(instance) - pull.PullLogs(self._ssh, log_files, download_folder) + error_log_folder = pull.GetDownloadLogFolder(instance) + pull.PullLogs(self._ssh, log_files, error_log_folder) + self.ExtendReportData(constants.ERROR_LOG_FOLDER, error_log_folder) @utils.TimeExecute(function_description="Reusing GCE instance") def _ReusingGceInstance(self, avd_spec): @@ -436,6 +475,7 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): Returns: ssh.IP object, that stores internal and external ip of the instance. """ + self.SetStage(constants.STAGE_GCE) timestart = time.time() metadata = self._metadata.copy() @@ -446,6 +486,9 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): avd_spec.hw_property[constants.HW_X_RES], avd_spec.hw_property[constants.HW_Y_RES], avd_spec.hw_property[constants.HW_ALIAS_DPI])) + if avd_spec.gce_metadata: + for key, value in avd_spec.gce_metadata.items(): + metadata[key] = value disk_args = self._GetDiskArgs( instance, image_name, image_project, boot_disk_size_gb) @@ -479,18 +522,10 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): is on the instance, future commands can use it to download relevant Cuttlefish files from the Build API on the instance itself. """ - # TODO(schuffelen): Support fetch_cvd_version="latest" when there is - # stronger automated testing on it. + self.SetStage(constants.STAGE_ARTIFACT) download_dir = tempfile.mkdtemp() download_target = os.path.join(download_dir, _FETCHER_NAME) - self._build_api.DownloadArtifact( - build_target=_FETCHER_BUILD_TARGET, - build_id=self._fetch_cvd_version, - resource_id=_FETCHER_NAME, - local_dest=download_target, - attempt_id="latest") - fetch_cvd_stat = os.stat(download_target) - os.chmod(download_target, fetch_cvd_stat.st_mode | stat.S_IEXEC) + self._build_api.DownloadFetchcvd(download_target, self._fetch_cvd_version) self._ssh.ScpPushFile(src_file=download_target, dst_file=_FETCHER_NAME) os.remove(download_target) os.rmdir(download_dir) @@ -498,28 +533,40 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): @utils.TimeExecute(function_description="Downloading build on instance") def FetchBuild(self, build_id, branch, build_target, system_build_id, system_branch, system_build_target, kernel_build_id, - kernel_branch, kernel_build_target): + kernel_branch, kernel_build_target, bootloader_build_id, + bootloader_branch, bootloader_build_target): """Execute fetch_cvd on the remote instance to get Cuttlefish runtime files. Args: - fetch_args: String of arguments to pass to fetch_cvd. + build_id: String of build id, e.g. "2263051", "P2804227" + branch: String of branch name, e.g. "aosp-master" + build_target: String of target name. + e.g. "aosp_cf_x86_phone-userdebug" + system_build_id: String of the system image build id. + system_branch: String of the system image branch name. + system_build_target: String of the system image target name, + e.g. "cf_x86_phone-userdebug" + kernel_build_id: String of the kernel image build id. + kernel_branch: String of the kernel image branch name. + kernel_build_target: String of the kernel image target name, + bootloader_build_id: String of the bootloader build id. + bootloader_branch: String of the bootloader branch name. + bootloader_build_target: String of the bootloader target name. + + Returns: + List of string args for fetch_cvd. """ timestart = time.time() fetch_cvd_args = ["-credential_source=gce"] - - default_build = _ProcessBuild(build_id, branch, build_target) - if default_build: - fetch_cvd_args.append("-default_build=" + default_build) - system_build = _ProcessBuild(system_build_id, system_branch, system_build_target) - if system_build: - fetch_cvd_args.append("-system_build=" + system_build) - kernel_build = self.GetKernelBuild(kernel_build_id, - kernel_branch, - kernel_build_target) - if kernel_build: - fetch_cvd_args.append("-kernel_build=" + kernel_build) - - self._ssh.Run("./fetch_cvd " + " ".join(fetch_cvd_args)) + fetch_cvd_build_args = self._build_api.GetFetchBuildArgs( + build_id, branch, build_target, system_build_id, system_branch, + system_build_target, kernel_build_id, kernel_branch, + kernel_build_target, bootloader_build_id, bootloader_branch, + bootloader_build_target) + fetch_cvd_args.extend(fetch_cvd_build_args) + + self._ssh.Run("./fetch_cvd " + " ".join(fetch_cvd_args), + timeout=constants.DEFAULT_SSH_TIMEOUT) self._execution_time[_FETCH_ARTIFACT] = round(time.time() - timestart, 2) def GetInstanceIP(self, instance=None): @@ -539,6 +586,41 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): return gcompute_client.ComputeClient.GetInstanceIP( self, instance=instance, zone=self._zone) + def GetHostImageName(self, stable_image_name, image_family, image_project): + """Get host image name. + + Args: + stable_image_name: String of stable host image name. + image_family: String of image family. + image_project: String of image project. + + Returns: + String of stable host image name. + + Raises: + errors.ConfigError: There is no host image name in config file. + """ + if stable_image_name: + return stable_image_name + + if image_family: + image_name = gcompute_client.ComputeClient.GetImageFromFamily( + self, image_family, image_project)["name"] + logger.debug("Get the host image name from image family: %s", image_name) + return image_name + + raise errors.ConfigError( + "Please specify 'stable_host_image_name' or 'stable_host_image_family'" + " in config.") + + def SetStage(self, stage): + """Set stage to know the create progress. + + Args: + stage: Integer, the stage would like STAGE_INIT, STAGE_GCE. + """ + self._stage = stage + @property def all_failures(self): """Return all_failures""" @@ -548,3 +630,13 @@ class CvdComputeClient(android_compute_client.AndroidComputeClient): def execution_time(self): """Return execution_time""" return self._execution_time + + @property + def stage(self): + """Return stage""" + return self._stage + + @property + def build_api(self): + """Return build_api""" + return self._build_api diff --git a/internal/lib/cvd_compute_client_multi_stage_test.py b/internal/lib/cvd_compute_client_multi_stage_test.py index 98ce7e43..08875e4c 100644 --- a/internal/lib/cvd_compute_client_multi_stage_test.py +++ b/internal/lib/cvd_compute_client_multi_stage_test.py @@ -20,7 +20,8 @@ import glob import os import subprocess import unittest -import mock + +from unittest import mock from acloud.create import avd_spec from acloud.internal import constants @@ -32,8 +33,6 @@ from acloud.internal.lib import utils from acloud.internal.lib.ssh import Ssh from acloud.list import list as list_instances -from acloud.internal.lib.cvd_compute_client_multi_stage import _ProcessBuild - class CvdComputeClientTest(driver_test_lib.BaseDriverTest): """Test CvdComputeClient.""" @@ -82,8 +81,13 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): def setUp(self): """Set up the test.""" - super(CvdComputeClientTest, self).setUp() + super().setUp() self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "InitResourceHandle") + self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "_VerifyZoneByQuota", + return_value=True) + self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, + "_ArgSupportInLaunchCVD", + return_value=True) self.Patch(android_build_client.AndroidBuildClient, "InitResourceHandle") self.Patch(android_build_client.AndroidBuildClient, "DownloadArtifact") self.Patch(list_instances, "GetInstancesFromInstanceNames", return_value=mock.MagicMock()) @@ -94,45 +98,43 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): self.cvd_compute_client_multi_stage = cvd_compute_client_multi_stage.CvdComputeClient( self._GetFakeConfig(), mock.MagicMock(), gpu=self.GPU) self.args = mock.MagicMock() - self.args.local_image = None + self.args.local_image = constants.FIND_IN_BUILD_ENV + self.args.local_system_image = None self.args.config_file = "" self.args.avd_type = constants.TYPE_CF self.args.flavor = "phone" self.args.adb_port = None - self.args.hw_property = "cpu:2,resolution:1080x1920,dpi:240,memory:4g" + self.args.hw_property = "cpu:2,resolution:1080x1920,dpi:240,memory:4g,disk:10g" + self.args.num_avds_per_instance = 2 + self.args.remote_host = False + self.args.launch_args = self.LAUNCH_ARGS # pylint: disable=protected-access - @mock.patch.object(utils, "GetBuildEnvironmentVariable", return_value="fake_env") + @mock.patch.object(utils, "GetBuildEnvironmentVariable", return_value="fake_env_cf_x86") @mock.patch.object(glob, "glob", return_value=["fake.img"]) def testGetLaunchCvdArgs(self, _mock_check_img, _mock_env): """test GetLaunchCvdArgs.""" # test GetLaunchCvdArgs with avd_spec fake_avd_spec = avd_spec.AVDSpec(self.args) - expeted_args = ['-x_res=1080', '-y_res=1920', '-dpi=240', '-cpus=2', - '-memory_mb=4096', '--setupwizard_mode=REQUIRED', - '-gpu_mode=drm_virgl'] + expeted_args = ["-config=phone", "-x_res=1080", "-y_res=1920", "-dpi=240", + "-data_policy=always_create", "-blank_data_image_mb=10240", + "-cpus=2", "-memory_mb=4096", "-num_instances=2", + "--setupwizard_mode=REQUIRED", + "-undefok=report_anonymous_usage_stats,config", + "-report_anonymous_usage_stats=y"] launch_cvd_args = self.cvd_compute_client_multi_stage._GetLaunchCvdArgs(fake_avd_spec) self.assertEqual(launch_cvd_args, expeted_args) # test GetLaunchCvdArgs without avd_spec - expeted_args = ['-x_res=720', '-y_res=1280', '-dpi=160', - '--setupwizard_mode=REQUIRED', '-gpu_mode=drm_virgl'] + expeted_args = ["-x_res=720", "-y_res=1280", "-dpi=160", + "--setupwizard_mode=REQUIRED", + "-undefok=report_anonymous_usage_stats,config", + "-report_anonymous_usage_stats=y"] launch_cvd_args = self.cvd_compute_client_multi_stage._GetLaunchCvdArgs( avd_spec=None) self.assertEqual(launch_cvd_args, expeted_args) - # pylint: disable=protected-access - def testProcessBuild(self): - """Test creating "cuttlefish build" strings.""" - self.assertEqual(_ProcessBuild(build_id="123", branch="abc", build_target="def"), "123/def") - self.assertEqual(_ProcessBuild(build_id=None, branch="abc", build_target="def"), "abc/def") - self.assertEqual(_ProcessBuild(build_id="123", branch=None, build_target="def"), "123/def") - self.assertEqual(_ProcessBuild(build_id="123", branch="abc", build_target=None), "123") - self.assertEqual(_ProcessBuild(build_id=None, branch="abc", build_target=None), "abc") - self.assertEqual(_ProcessBuild(build_id="123", branch=None, build_target=None), "123") - self.assertEqual(_ProcessBuild(build_id=None, branch=None, build_target=None), None) - - @mock.patch.object(utils, "GetBuildEnvironmentVariable", return_value="fake_env") + @mock.patch.object(utils, "GetBuildEnvironmentVariable", return_value="fake_env_cf_x86") @mock.patch.object(glob, "glob", return_value=["fake.img"]) @mock.patch.object(gcompute_client.ComputeClient, "CompareMachineSize", return_value=1) @@ -157,10 +159,12 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): created_subprocess = mock.MagicMock() created_subprocess.stdout = mock.MagicMock() - created_subprocess.stdout.readline = mock.MagicMock(return_value='') + created_subprocess.stdout.readline = mock.MagicMock(return_value=b"") created_subprocess.poll = mock.MagicMock(return_value=0) created_subprocess.returncode = 0 created_subprocess.communicate = mock.MagicMock(return_value=('', '')) + self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, + "_RecordBuildInfo") self.Patch(subprocess, "Popen", return_value=created_subprocess) self.Patch(subprocess, "check_call") self.Patch(os, "chmod") @@ -220,6 +224,57 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): gpu=self.GPU, tags=None) + def testRecordBuildInfo(self): + """Test RecordBuildInfo""" + build_id = "build_id" + build_target = "build_target" + system_build_id = "system_id" + system_build_target = "system_target" + kernel_build_id = "kernel_id" + kernel_build_target = "kernel_target" + fake_avd_spec = mock.MagicMock() + fake_avd_spec.image_source = constants.IMAGE_SRC_REMOTE + fake_avd_spec.remote_image = {constants.BUILD_ID: build_id, + constants.BUILD_TARGET: build_target} + fake_avd_spec.system_build_info = {constants.BUILD_ID: system_build_id, + constants.BUILD_TARGET: system_build_target} + fake_avd_spec.kernel_build_info = {constants.BUILD_ID: kernel_build_id, + constants.BUILD_TARGET: kernel_build_target} + expected_metadata = dict() + expected_metadata.update(self.METADATA) + expected_metadata.update({"build_id": build_id}) + expected_metadata.update({"build_target": build_target}) + expected_metadata.update({"system_build_id": system_build_id}) + expected_metadata.update({"system_build_target": system_build_target}) + expected_metadata.update({"kernel_build_id": kernel_build_id}) + expected_metadata.update({"kernel_build_target": kernel_build_target}) + + # Test record metadata with avd_spec for acloud create + self.cvd_compute_client_multi_stage._RecordBuildInfo( + fake_avd_spec, build_id=None, build_target=None, system_build_id=None, + system_build_target=None, kernel_build_id=None, kernel_build_target=None) + self.assertEqual(self.cvd_compute_client_multi_stage._metadata, expected_metadata) + + # Test record metadata with build info for acloud create_cf + self.cvd_compute_client_multi_stage._RecordBuildInfo( + None, build_id, build_target, system_build_id, system_build_target, + kernel_build_id, kernel_build_target) + self.assertEqual(self.cvd_compute_client_multi_stage._metadata, expected_metadata) + + def testSetStage(self): + """Test SetStage""" + device_stage = "fake_stage" + self.cvd_compute_client_multi_stage.SetStage(device_stage) + self.assertEqual(self.cvd_compute_client_multi_stage.stage, + device_stage) + + def testArgSupportInLaunchCVD(self): + """Test ArgSupportInLaunchCVD""" + self.Patch(Ssh, "GetCmdOutput", return_value="-config (Config)") + self.assertTrue( + self.cvd_compute_client_multi_stage._ArgSupportInLaunchCVD( + "-config")) + if __name__ == "__main__": unittest.main() diff --git a/internal/lib/cvd_compute_client_test.py b/internal/lib/cvd_compute_client_test.py index dfd237e1..d9809b1d 100644 --- a/internal/lib/cvd_compute_client_test.py +++ b/internal/lib/cvd_compute_client_test.py @@ -18,7 +18,8 @@ import glob import unittest -import mock + +from unittest import mock from acloud.create import avd_spec from acloud.internal import constants @@ -75,14 +76,14 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): def setUp(self): """Set up the test.""" - super(CvdComputeClientTest, self).setUp() + super().setUp() self.Patch(cvd_compute_client.CvdComputeClient, "InitResourceHandle") self.Patch(list_instances, "ChooseOneRemoteInstance", return_value=mock.MagicMock()) self.Patch(list_instances, "GetInstancesFromInstanceNames", return_value=mock.MagicMock()) self.cvd_compute_client = cvd_compute_client.CvdComputeClient( self._GetFakeConfig(), mock.MagicMock()) - @mock.patch.object(utils, "GetBuildEnvironmentVariable", return_value="fake_env") + @mock.patch.object(utils, "GetBuildEnvironmentVariable", return_value="fake_cf_x86") @mock.patch.object(glob, "glob", return_value=["fake.img"]) @mock.patch.object(gcompute_client.ComputeClient, "CompareMachineSize", return_value=1) @@ -147,11 +148,14 @@ class CvdComputeClientTest(driver_test_lib.BaseDriverTest): local_image_metadata = dict(expected_metadata_local_image) args = mock.MagicMock() mock_check_img.return_value = True - args.local_image = None + args.local_image = constants.FIND_IN_BUILD_ENV + args.local_system_image = None args.config_file = "" args.avd_type = constants.TYPE_CF args.flavor = "phone" args.adb_port = None + args.remote_host = False + args.launch_args = None fake_avd_spec = avd_spec.AVDSpec(args) fake_avd_spec.hw_property[constants.HW_X_RES] = str(self.X_RES) fake_avd_spec.hw_property[constants.HW_Y_RES] = str(self.Y_RES) diff --git a/internal/lib/cvd_runtime_config.py b/internal/lib/cvd_runtime_config.py index fd5bba68..968a48d3 100644 --- a/internal/lib/cvd_runtime_config.py +++ b/internal/lib/cvd_runtime_config.py @@ -19,7 +19,7 @@ import re from acloud import errors -_CFG_KEY_ADB_CONNECTOR_BINARY = "adb_connector_binary" +_CFG_KEY_CROSVM_BINARY = "crosvm_binary" _CFG_KEY_X_RES = "x_res" _CFG_KEY_Y_RES = "y_res" _CFG_KEY_DPI = "dpi" @@ -29,6 +29,7 @@ _CFG_KEY_ADB_IP_PORT = "adb_ip_and_port" _CFG_KEY_INSTANCE_DIR = "instance_dir" _CFG_KEY_VNC_PORT = "vnc_server_port" _CFG_KEY_ADB_PORT = "host_port" +_CFG_KEY_ENABLE_WEBRTC = "enable_webrtc" # TODO(148648620): Check instance_home_[id] for backward compatible. _RE_LOCAL_INSTANCE_ID = re.compile(r".+(?:local-instance-|instance_home_)" r"(?P<ins_id>\d+).+") @@ -46,10 +47,10 @@ def _GetIdFromInstanceDirStr(instance_dir): match = _RE_LOCAL_INSTANCE_ID.match(instance_dir) if match: return match.group("ins_id") - else: - # To support the device which is not created by acloud. - if os.path.expanduser("~") in instance_dir: - return "1" + + # To support the device which is not created by acloud. + if os.path.expanduser("~") in instance_dir: + return "1" return None @@ -66,7 +67,7 @@ class CvdRuntimeConfig(object): [ "/path-to-image" ], - "adb_ip_and_port" : "127.0.0.1:6520", + "adb_ip_and_port" : "0.0.0.0:6520", "instance_dir" : "/path-to-instance-dir", } @@ -79,7 +80,7 @@ class CvdRuntimeConfig(object): { "1" : { - "adb_ip_and_port" : "127.0.0.1:6520", + "adb_ip_and_port" : "0.0.0.0:6520", "instance_dir" : "/path-to-instance-dir", "virtual_disk_paths" : [ @@ -89,18 +90,31 @@ class CvdRuntimeConfig(object): } } + If the avd enable the webrtc, the config will be as below: + { + "enable_webrtc" : true, + "vnc_server_binary" : "/home/vsoc-01/bin/vnc_server", + "webrtc_assets_dir" : "/home/vsoc-01/usr/share/webrtc/assets", + "webrtc_binary" : "/home/vsoc-01/bin/webRTC", + "webrtc_certs_dir" : "/home/vsoc-01/usr/share/webrtc/certs", + "webrtc_enable_adb_websocket" : false, + "webrtc_public_ip" : "0.0.0.0", + } + """ - def __init__(self, config_path): + def __init__(self, config_path=None, raw_data=None): self._config_path = config_path - self._config_dict = self._GetCuttlefishRuntimeConfig(config_path) - self._instance_id = _GetIdFromInstanceDirStr(self._config_path) + self._instance_id = "1" if raw_data else _GetIdFromInstanceDirStr( + config_path) + self._config_dict = self._GetCuttlefishRuntimeConfig(config_path, + raw_data) self._x_res = self._config_dict.get(_CFG_KEY_X_RES) self._y_res = self._config_dict.get(_CFG_KEY_Y_RES) self._dpi = self._config_dict.get(_CFG_KEY_DPI) - adb_connector = self._config_dict.get(_CFG_KEY_ADB_CONNECTOR_BINARY) - self._cvd_tools_path = (os.path.dirname(adb_connector) - if adb_connector else None) + crosvm_bin = self._config_dict.get(_CFG_KEY_CROSVM_BINARY) + self._cvd_tools_path = (os.path.dirname(crosvm_bin) + if crosvm_bin else None) # Below properties will be collected inside of instance id node if there # are more than one instance. @@ -110,6 +124,7 @@ class CvdRuntimeConfig(object): self._adb_ip_port = self._config_dict.get(_CFG_KEY_ADB_IP_PORT) self._virtual_disk_paths = self._config_dict.get( _CFG_KEY_VIRTUAL_DISK_PATHS) + self._enable_webrtc = self._config_dict.get(_CFG_KEY_ENABLE_WEBRTC) if not self._instance_dir: ins_cfg = self._config_dict.get(_CFG_KEY_INSTANCES) ins_dict = ins_cfg.get(self._instance_id) @@ -124,11 +139,12 @@ class CvdRuntimeConfig(object): self._virtual_disk_paths = ins_dict.get(_CFG_KEY_VIRTUAL_DISK_PATHS) @staticmethod - def _GetCuttlefishRuntimeConfig(runtime_cf_config_path): + def _GetCuttlefishRuntimeConfig(runtime_cf_config_path, raw_data=None): """Get and parse cuttlefish_config.json. Args: runtime_cf_config_path: String, path of the cvd runtime config. + raw_data: String, data of the cvd runtime config. Returns: A dictionary that parsed from cuttlefish runtime config. @@ -136,6 +152,16 @@ class CvdRuntimeConfig(object): Raises: errors.ConfigError: if file not found or config load failed. """ + if raw_data: + # if remote instance couldn't fetch the config will return message such as + # 'cat: .../cuttlefish_config.json: No such file or directory'. + # Add this condition to prevent from JSONDecodeError. + try: + return json.loads(raw_data) + except ValueError as e: + raise errors.ConfigError( + "An exception happened when loading the raw_data of the " + "cvd runtime config:\n%s" % str(e)) if not os.path.exists(runtime_cf_config_path): raise errors.ConfigError( "file does not exist: %s" % runtime_cf_config_path) @@ -196,3 +222,8 @@ class CvdRuntimeConfig(object): def instance_id(self): """Return _instance_id""" return self._instance_id + + @property + def enable_webrtc(self): + """Return _enable_webrtc""" + return self._enable_webrtc diff --git a/internal/lib/cvd_runtime_config_test.py b/internal/lib/cvd_runtime_config_test.py index f540a889..a7cf8d23 100644 --- a/internal/lib/cvd_runtime_config_test.py +++ b/internal/lib/cvd_runtime_config_test.py @@ -16,7 +16,9 @@ """Tests for cvd_runtime_config class.""" import os -import mock +import unittest + +from unittest import mock import six from acloud.internal.lib import cvd_runtime_config as cf_cfg @@ -31,7 +33,7 @@ class CvdRuntimeconfigTest(driver_test_lib.BaseDriverTest): "x_res" : 720, "y_res" : 1280, "instances": { - "1":{ + "2":{ "adb_ip_and_port": "127.0.0.1:6520", "host_port": 6520, "instance_dir": "/path-to-instance-dir", @@ -41,7 +43,33 @@ class CvdRuntimeconfigTest(driver_test_lib.BaseDriverTest): } """ - # pylint: disable=protected-access + CF_RUNTIME_CONFIG_WEBRTC = """ +{"x_display" : ":20", + "x_res" : 720, + "y_res" : 1280, + "dpi" : 320, + "instances" : { + "1":{ + "adb_ip_and_port": "127.0.0.1:6520", + "host_port": 6520, + "instance_dir": "/path-to-instance-dir", + "vnc_server_port": 6444, + "virtual_disk_paths": ["/path-to-image"] + } + }, + "enable_webrtc" : true, + "vnc_server_binary" : "/home/vsoc-01/bin/vnc_server", + "crosvm_binary" : "/home/vsoc-01/bin/crosvm", + "webrtc_assets_dir" : "/home/vsoc-01/usr/share/webrtc/assets", + "webrtc_binary" : "/home/vsoc-01/bin/webRTC", + "webrtc_certs_dir" : "/home/vsoc-01/usr/share/webrtc/certs", + "webrtc_enable_adb_websocket" : false, + "webrtc_public_ip" : "127.0.0.1" +} +""" + + + # pylint: disable=protected-access, no-member def testGetCuttlefishRuntimeConfig(self): """Test GetCuttlefishRuntimeConfig.""" # Should raise error when file does not exist. @@ -52,7 +80,7 @@ class CvdRuntimeconfigTest(driver_test_lib.BaseDriverTest): u'x_res': 720, u'x_display': u':20', u'instances': - {u'1': + {u'2': {u'adb_ip_and_port': u'127.0.0.1:6520', u'host_port': 6520, u'instance_dir': u'/path-to-instance-dir', @@ -60,7 +88,51 @@ class CvdRuntimeconfigTest(driver_test_lib.BaseDriverTest): }, } mock_open = mock.mock_open(read_data=self.CF_RUNTIME_CONFIG) - cf_cfg_path = "/fake-path/local-instance-1/fake.config" + cf_cfg_path = "/fake-path/local-instance-2/fake.config" with mock.patch.object(six.moves.builtins, "open", mock_open): - self.assertEqual(expected_dict, - cf_cfg.CvdRuntimeConfig(cf_cfg_path)._config_dict) + fake_cvd_runtime_config = cf_cfg.CvdRuntimeConfig(cf_cfg_path) + self.assertEqual(fake_cvd_runtime_config._config_dict, expected_dict) + self.assertEqual(fake_cvd_runtime_config.enable_webrtc, None) + self.assertEqual(fake_cvd_runtime_config.config_path, + "/fake-path/local-instance-2/fake.config") + self.assertEqual(fake_cvd_runtime_config.instance_id, "2") + + # Test read runtime config from raw_data and webrtc AVD. + self.Patch(cf_cfg, "_GetIdFromInstanceDirStr") + fake_cvd_runtime_config_webrtc = cf_cfg.CvdRuntimeConfig( + raw_data=self.CF_RUNTIME_CONFIG_WEBRTC) + cf_cfg._GetIdFromInstanceDirStr.assert_not_called() + self.assertEqual(fake_cvd_runtime_config_webrtc.config_path, None) + self.assertEqual(fake_cvd_runtime_config_webrtc.instance_id, "1") + self.assertEqual(fake_cvd_runtime_config_webrtc.enable_webrtc, True) + self.assertEqual(fake_cvd_runtime_config_webrtc.x_res, 720) + self.assertEqual(fake_cvd_runtime_config_webrtc.y_res, 1280) + self.assertEqual(fake_cvd_runtime_config_webrtc.dpi, 320) + self.assertEqual(fake_cvd_runtime_config_webrtc.adb_ip_port, "127.0.0.1:6520") + self.assertEqual(fake_cvd_runtime_config_webrtc.instance_dir, "/path-to-instance-dir") + self.assertEqual(fake_cvd_runtime_config_webrtc.vnc_port, 6444) + self.assertEqual(fake_cvd_runtime_config_webrtc.adb_port, 6520) + self.assertEqual(fake_cvd_runtime_config_webrtc.virtual_disk_paths, ['/path-to-image']) + self.assertEqual(fake_cvd_runtime_config_webrtc.cvd_tools_path, "/home/vsoc-01/bin") + + +class CvdRuntimeconfigFunctionTest(driver_test_lib.BaseDriverTest): + """Test CvdRuntimeconfigFunctionTest class.""" + + # pylint: disable=protected-access + def testGetIdFromInstanceDirStr(self): + """Test GetIdFromInstanceDirStr.""" + fake_instance_dir = "/path-to-instance-dir" + self.assertEqual(cf_cfg._GetIdFromInstanceDirStr(fake_instance_dir), None) + + fake_instance_dir = "/fake-path/local-instance-1/" + self.assertEqual(cf_cfg._GetIdFromInstanceDirStr(fake_instance_dir), "1") + + fake_home_path = "/home/fake_user/" + self.Patch(os.path, 'expanduser', return_value=fake_home_path) + fake_instance_dir = "/home/fake_user/local-instance/" + self.assertEqual(cf_cfg._GetIdFromInstanceDirStr(fake_instance_dir), "1") + + +if __name__ == "__main__": + unittest.main() diff --git a/internal/lib/driver_test_lib.py b/internal/lib/driver_test_lib.py index a9f212c7..339a8fcd 100644 --- a/internal/lib/driver_test_lib.py +++ b/internal/lib/driver_test_lib.py @@ -15,7 +15,8 @@ # limitations under the License. """Driver test library.""" import unittest -import mock + +from unittest import mock class BaseDriverTest(unittest.TestCase): diff --git a/internal/lib/engprod_client.py b/internal/lib/engprod_client.py new file mode 100644 index 00000000..26043543 --- /dev/null +++ b/internal/lib/engprod_client.py @@ -0,0 +1,47 @@ +# Copyright 2021 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A client that talks to EngProd APIs.""" + +import json +import subprocess + +from urllib.parse import urljoin + + +class EngProdClient(): + """Client that manages EngProd api.""" + + @staticmethod + def LeaseDevice(build_target, build_id, api_key, api_url): + """Lease one cuttlefish device. + + Args: + build_target: Target name, e.g. "aosp_cf_x86_phone-userdebug" + build_id: Build ID, a string, e.g. "2263051", "P2804227" + api_key: String of api key. + api_url: String of api url. + + Returns: + The response of curl command. + """ + request_data = "{\"target\": \"%s\", \"build_id\": \"%s\"}" % ( + build_target, build_id) + lease_url = urljoin(api_url, "lease?key=%s" % api_key) + response = subprocess.check_output([ + "curl", "--request", "POST", lease_url, "-H", + "Accept: application/json", "-H", "Content-Type: application/json", + "-d", request_data + ]) + return json.loads(response) diff --git a/internal/lib/gcompute_client.py b/internal/lib/gcompute_client.py index 3c0b2951..a9bb8a90 100755 --- a/internal/lib/gcompute_client.py +++ b/internal/lib/gcompute_client.py @@ -51,6 +51,17 @@ _SSH_KEYS_NAME = "sshKeys" _ITEMS = "items" _METADATA = "metadata" _ZONE_RE = re.compile(r"^zones/(?P<zone>.+)") +# Quota metrics +_METRIC_CPUS = "CPUS" +_METRIC_DISKS_GB = "DISKS_TOTAL_GB" +_METRIC_USE_ADDRESSES = "IN_USE_ADDRESSES" +_METRICS = [_METRIC_CPUS, _METRIC_DISKS_GB, _METRIC_USE_ADDRESSES] +_USAGE = "usage" +_LIMIT = "limit" +# The minimum requirement to create an instance. +_REQUIRE_METRICS = {_METRIC_CPUS: 8, + _METRIC_DISKS_GB: 1000, + _METRIC_USE_ADDRESSES: 1} BASE_DISK_ARGS = { "type": "PERSISTENT", @@ -196,11 +207,75 @@ class ComputeClient(base_cloud_client.BaseCloudApiClient): """Get project information. Returns: - A project resource in json. + A project resource in json. """ api = self.service.projects().get(project=self._project) return self.Execute(api) + def GetRegionInfo(self): + """Get region information that includes all quotas limit. + + The region info example: + {"items": + [{"status": "UP", + "name": "asia-east1", + "quotas": + [{"usage": 92, "metric": "CPUS", "limit": 100}, + {"usage": 640, "metric": "DISKS_TOTAL_GB", "limit": 10240}, + ...]]} + } + + Returns: + A region resource in json. + """ + api = self.service.regions().list(project=self._project) + return self.Execute(api) + + @staticmethod + def GetMetricQuota(regions_info, zone, metric): + """Get CPU quota limit in specific zone and project. + + Args: + regions_info: Dict, regions resource in json. + zone: String, name of zone. + metric: String, name of metric, e.g. "CPUS". + + Returns: + A dict of quota information. Such as + {"usage": 100, "metric": "CPUS", "limit": 200} + """ + for region_info in regions_info["items"]: + if region_info["name"] in zone: + for quota in region_info["quotas"]: + if quota["metric"] == metric: + return quota + logger.info("Can't get %s quota info from zone(%s)", metric, zone) + return None + + def EnoughMetricsInZone(self, zone): + """Check the zone have enough metrics to create instance. + + The metrics include CPUS and DISKS. + + Args: + zone: String, name of zone. + + Returns: + Boolean. True if zone have enough quota. + """ + regions_info = self.GetRegionInfo() + for metric in _METRICS: + quota = self.GetMetricQuota(regions_info, zone, metric) + if not quota: + logger.debug( + "Can't query the metric(%s) in zone(%s)", metric, zone) + return False + if quota[_LIMIT] - quota[_USAGE] < _REQUIRE_METRICS[metric]: + logger.debug( + "The metric(%s) is over limit in zone(%s)", metric, zone) + return False + return True + def GetDisk(self, disk_name, zone): """Get disk information. @@ -470,6 +545,21 @@ class ComputeClient(base_cloud_client.BaseCloudApiClient): project=image_project or self._project, image=image_name) return self.Execute(api) + def GetImageFromFamily(self, image_family, image_project=None): + """Get image information from image_family. + + Args: + image_family: String of image family. + image_project: String of image project. + + Returns: + An image resource in json. + https://cloud.google.com/compute/docs/reference/latest/images#resource + """ + api = self.service.images().getFromFamily( + project=image_project or self._project, family=image_family) + return self.Execute(api) + def DeleteImage(self, image_name): """Delete an image. @@ -960,7 +1050,7 @@ class ComputeClient(base_cloud_client.BaseCloudApiClient): # Initialize return values failed = [] error_msgs = [] - for resource_name, (_, error) in results.iteritems(): + for resource_name, (_, error) in six.iteritems(results): if error is not None: failed.append(resource_name) error_msgs.append(str(error)) @@ -1185,6 +1275,7 @@ class ComputeClient(base_cloud_client.BaseCloudApiClient): "email": "default", "scopes": scopes, }], + "enableVtpm": True, } if tags: @@ -1201,7 +1292,7 @@ class ComputeClient(base_cloud_client.BaseCloudApiClient): metadata_list = [{ _METADATA_KEY: key, _METADATA_KEY_VALUE: val - } for key, val in metadata.iteritems()] + } for key, val in six.iteritems(metadata)] body[_METADATA] = {_ITEMS: metadata_list} logger.info("Creating instance: project %s, zone %s, body:%s", self._project, zone, body) diff --git a/internal/lib/gcompute_client_test.py b/internal/lib/gcompute_client_test.py index d46a2363..aab1c3a0 100644 --- a/internal/lib/gcompute_client_test.py +++ b/internal/lib/gcompute_client_test.py @@ -20,12 +20,11 @@ import copy import os import unittest -import mock + +from unittest import mock import six # pylint: disable=import-error -import apiclient.http - from acloud import errors from acloud.internal import constants from acloud.internal.lib import driver_test_lib @@ -49,6 +48,7 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): IMAGE = "fake-image" IMAGE_URL = "http://fake-image-url" IMAGE_OTHER = "fake-image-other" + DISK = "fake-disk" MACHINE_TYPE = "fake-machine-type" MACHINE_TYPE_URL = "http://fake-machine-type-url" METADATA = ("metadata_key", "metadata_value") @@ -405,7 +405,9 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): mock_batch = mock.MagicMock() mock_batch.add = _Add mock_batch.execute = _Execute - self.Patch(apiclient.http, "BatchHttpRequest", return_value=mock_batch) + self.Patch(self.compute_client._service, + "new_batch_http_request", + return_value=mock_batch) @mock.patch.object(gcompute_client.ComputeClient, "WaitOnOperation") def testDeleteImages(self, mock_wait): @@ -601,6 +603,7 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): "value": self.METADATA[1]}], }, "labels":{constants.LABEL_CREATE_BY: "fake_user"}, + "enableVtpm": True, } self.compute_client.CreateInstance( @@ -678,6 +681,7 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): "value": self.METADATA[1]}], }, "labels":{'created_by': "fake_user"}, + "enableVtpm": True, } self.compute_client.CreateInstance( @@ -752,6 +756,7 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): }], }, "labels":{'created_by': "fake_user"}, + "enableVtpm": True, } self.compute_client.CreateInstance( @@ -1461,6 +1466,70 @@ class ComputeClientTest(driver_test_lib.BaseDriverTest): side_effect=error) self.assertFalse(self.compute_client.CheckAccess()) + def testEnoughMetricsInZone(self): + """Test EnoughMetricsInZone.""" + region_info_enough_quota = { + "items": [{ + "name": "asia-east1", + "quotas": [{ + "usage": 50, + "metric": "CPUS", + "limit": 100 + }, { + "usage": 640, + "metric": "DISKS_TOTAL_GB", + "limit": 10240 + }, { + "usage": 20, + "metric": "IN_USE_ADDRESSES", + "limit": 100 + }] + }] + } + self.Patch( + gcompute_client.ComputeClient, "GetRegionInfo", + return_value=region_info_enough_quota) + self.assertTrue(self.compute_client.EnoughMetricsInZone("asia-east1-b")) + self.assertFalse(self.compute_client.EnoughMetricsInZone("fake_zone")) + + region_info_not_enough_quota = { + "items": [{ + "name": "asia-east1", + "quotas": [{ + "usage": 100, + "metric": "CPUS", + "limit": 100 + }, { + "usage": 640, + "metric": "DISKS_TOTAL_GB", + "limit": 10240 + }, { + "usage": 20, + "metric": "IN_USE_ADDRESSES", + "limit": 100 + }] + }] + } + self.Patch( + gcompute_client.ComputeClient, "GetRegionInfo", + return_value=region_info_not_enough_quota) + self.assertFalse(self.compute_client.EnoughMetricsInZone("asia-east1-b")) + + def testGetDisk(self): + """Test GetDisk.""" + resource_mock = mock.MagicMock() + mock_api = mock.MagicMock() + self.compute_client._service.disks = mock.MagicMock( + return_value=resource_mock) + resource_mock.get = mock.MagicMock(return_value=mock_api) + mock_api.execute = mock.MagicMock(return_value={"name": self.DISK}) + result = self.compute_client.GetDisk(self.DISK, self.ZONE) + self.assertEqual(result, {"name": self.DISK}) + resource_mock.get.assert_called_with(project=PROJECT, + zone=self.ZONE, + disk=self.DISK) + self.assertTrue(self.compute_client.CheckDiskExists(self.DISK, self.ZONE)) + if __name__ == "__main__": unittest.main() diff --git a/internal/lib/goldfish_compute_client.py b/internal/lib/goldfish_compute_client.py index d9d1d206..33de884b 100644 --- a/internal/lib/goldfish_compute_client.py +++ b/internal/lib/goldfish_compute_client.py @@ -107,15 +107,16 @@ class GoldfishComputeClient(android_compute_client.AndroidComputeClient): instance: String Raises: - Raises an errors.DeviceBootError exception if a failure is detected. + errors.DownloadArtifactError: If it fails to download artifact. + errors.DeviceBootError: If it fails to boot up. """ if self.BOOT_FAILED_MSG in serial_out: if self.EMULATOR_FETCH_FAILED_MSG in serial_out: - raise errors.DeviceBootError( + raise errors.DownloadArtifactError( "Failed to download emulator build. Re-run with a newer build." ) if self.ANDROID_FETCH_FAILED_MSG in serial_out: - raise errors.DeviceBootError( + raise errors.DownloadArtifactError( "Failed to download system image build. Re-run with a newer build." ) if self.BOOT_TIMEOUT_MSG in serial_out: @@ -124,6 +125,17 @@ class GoldfishComputeClient(android_compute_client.AndroidComputeClient): @staticmethod def GetKernelBuildArtifact(target): + """Get kernel build artifact name. + + Args: + target: String, kernel target name. + + Returns: + String of artifact name. + + Raises: + errors.DeviceBootError: If it fails to get artifact name. + """ if target == "kernel": return "bzImage" if target == "kernel_x86_64": @@ -151,7 +163,8 @@ class GoldfishComputeClient(android_compute_client.AndroidComputeClient): gpu=None, avd_spec=None, extra_scopes=None, - tags=None): + tags=None, + launch_args=None): """Create a goldfish instance given a stable host image and a build id. Args: @@ -174,7 +187,9 @@ class GoldfishComputeClient(android_compute_client.AndroidComputeClient): extra_scopes: A list of extra scopes to be passed to the instance. tags: A list of tags to associate with the instance. e.g. ["http-server", "https-server"] + launch_args: String of args for launch command. """ + self._VerifyZoneByQuota() self._CheckMachineSize() # Add space for possible data partition. @@ -205,6 +220,8 @@ class GoldfishComputeClient(android_compute_client.AndroidComputeClient): metadata[ "cvd_01_fetch_emulator_bid"] = "{branch}/{build_id}".format( branch=emulator_branch, build_id=emulator_build_id) + if launch_args: + metadata["launch_args"] = launch_args metadata["cvd_01_launch"] = "1" # Update metadata by avd_spec diff --git a/internal/lib/goldfish_compute_client_test.py b/internal/lib/goldfish_compute_client_test.py index cfee21fe..bdfc119f 100644 --- a/internal/lib/goldfish_compute_client_test.py +++ b/internal/lib/goldfish_compute_client_test.py @@ -15,7 +15,8 @@ # limitations under the License. """Tests for acloud.internal.lib.goldfish_compute_client.""" import unittest -import mock + +from unittest import mock from acloud.internal.lib import driver_test_lib from acloud.internal.lib import gcompute_client @@ -50,6 +51,7 @@ class GoldfishComputeClientTest(driver_test_lib.BaseDriverTest): GPU = "nvidia-tesla-k80" EXTRA_SCOPES = "scope1" TAGS = ['http-server'] + LAUNCH_ARGS = "fake-args" def _GetFakeConfig(self): """Create a fake configuration object. @@ -67,11 +69,12 @@ class GoldfishComputeClientTest(driver_test_lib.BaseDriverTest): fake_cfg.metadata_variable = self.METADATA fake_cfg.extra_data_disk_size_gb = self.EXTRA_DATA_DISK_SIZE_GB fake_cfg.extra_scopes = self.EXTRA_SCOPES + fake_cfg.launch_args = self.LAUNCH_ARGS return fake_cfg def setUp(self): """Set up the test.""" - super(GoldfishComputeClientTest, self).setUp() + super().setUp() self.Patch(goldfish_compute_client.GoldfishComputeClient, "InitResourceHandle") self.goldfish_compute_client = ( @@ -92,6 +95,9 @@ class GoldfishComputeClientTest(driver_test_lib.BaseDriverTest): return_value=[{ "fake_arg": "fake_value" }]) + self.Patch(goldfish_compute_client.GoldfishComputeClient, + "_VerifyZoneByQuota", + return_value=True) @mock.patch("getpass.getuser", return_value="fake_user") def testCreateInstance(self, _mock_user): @@ -118,6 +124,7 @@ class GoldfishComputeClientTest(driver_test_lib.BaseDriverTest): "cvd_01_dpi": str(self.DPI), "cvd_01_x_res": str(self.X_RES), "cvd_01_y_res": str(self.Y_RES), + "launch_args" : self.LAUNCH_ARGS, } expected_metadata.update(self.METADATA) expected_disk_args = [{"fake_arg": "fake_value"}] @@ -131,7 +138,8 @@ class GoldfishComputeClientTest(driver_test_lib.BaseDriverTest): self.EMULATOR_BRANCH, self.EMULATOR_BUILD_ID, self.EXTRA_DATA_DISK_SIZE_GB, self.GPU, extra_scopes=self.EXTRA_SCOPES, - tags=self.TAGS) + tags=self.TAGS, + launch_args=self.LAUNCH_ARGS) # pylint: disable=no-member gcompute_client.ComputeClient.CreateInstance.assert_called_with( diff --git a/internal/lib/gstorage_client_test.py b/internal/lib/gstorage_client_test.py index e2426e1e..600d20e4 100644 --- a/internal/lib/gstorage_client_test.py +++ b/internal/lib/gstorage_client_test.py @@ -4,7 +4,8 @@ import io import time import unittest -import mock + +from unittest import mock import apiclient diff --git a/internal/lib/local_instance_lock.py b/internal/lib/local_instance_lock.py new file mode 100644 index 00000000..725eef79 --- /dev/null +++ b/internal/lib/local_instance_lock.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# +# Copyright 2020 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LocalInstanceLock class.""" + +import errno +import fcntl +import logging +import os + +from acloud import errors +from acloud.internal.lib import utils + + +logger = logging.getLogger(__name__) + +_LOCK_FILE_SIZE = 1 +# An empty file is equivalent to NOT_IN_USE. +_IN_USE_STATE = b"I" +_NOT_IN_USE_STATE = b"N" + +_DEFAULT_TIMEOUT_SECS = 5 + + +class LocalInstanceLock: + """The class that controls a lock file for a local instance. + + Acloud acquires the lock file of a local instance before it creates, + deletes, or queries it. The lock prevents multiple acloud processes from + accessing an instance simultaneously. + + The lock file records whether the instance is in use. Acloud checks the + state when it needs an unused id to create a new instance. + + Attributes: + _file_path: The path to the lock file. + _file_desc: The file descriptor of the file. It is set to None when + this object does not hold the lock. + """ + + def __init__(self, file_path): + self._file_path = file_path + self._file_desc = None + + def _Flock(self, timeout_secs): + """Call fcntl.flock with timeout. + + Args: + timeout_secs: An integer or a float, the timeout for acquiring the + lock file. 0 indicates non-block. + + Returns: + True if the file is locked successfully. False if timeout. + + Raises: + OSError: if any file operation fails. + """ + try: + if timeout_secs > 0: + wrapper = utils.TimeoutException(timeout_secs) + wrapper(fcntl.flock)(self._file_desc, fcntl.LOCK_EX) + else: + fcntl.flock(self._file_desc, fcntl.LOCK_EX | fcntl.LOCK_NB) + except errors.FunctionTimeoutError as e: + logger.debug("Cannot lock %s within %s seconds", + self._file_path, timeout_secs) + return False + except (OSError, IOError) as e: + # flock raises IOError in python2; OSError in python3. + if e.errno in (errno.EACCES, errno.EAGAIN): + logger.debug("Cannot lock %s", self._file_path) + return False + raise + return True + + def Lock(self, timeout_secs=_DEFAULT_TIMEOUT_SECS): + """Acquire the lock file. + + Args: + timeout_secs: An integer or a float, the timeout for acquiring the + lock file. 0 indicates non-block. + + Returns: + True if the file is locked successfully. False if timeout. + + Raises: + OSError: if any file operation fails. + """ + if self._file_desc is not None: + raise OSError("%s has been locked." % self._file_path) + parent_dir = os.path.dirname(self._file_path) + if not os.path.exists(parent_dir): + os.makedirs(parent_dir) + successful = False + self._file_desc = os.open(self._file_path, os.O_CREAT | os.O_RDWR, + 0o666) + try: + successful = self._Flock(timeout_secs) + finally: + if not successful: + os.close(self._file_desc) + self._file_desc = None + return successful + + def _CheckFileDescriptor(self): + """Raise an error if the file is not opened or locked.""" + if self._file_desc is None: + raise RuntimeError("%s has not been locked." % self._file_path) + + def SetInUse(self, in_use): + """Write the instance state to the file. + + Args: + in_use: A boolean, whether to set the instance to be in use. + + Raises: + OSError: if any file operation fails. + """ + self._CheckFileDescriptor() + os.lseek(self._file_desc, 0, os.SEEK_SET) + state = _IN_USE_STATE if in_use else _NOT_IN_USE_STATE + if os.write(self._file_desc, state) != _LOCK_FILE_SIZE: + raise OSError("Cannot write " + self._file_path) + + def Unlock(self): + """Unlock the file. + + Raises: + OSError: if any file operation fails. + """ + self._CheckFileDescriptor() + fcntl.flock(self._file_desc, fcntl.LOCK_UN) + os.close(self._file_desc) + self._file_desc = None + + def LockIfNotInUse(self, timeout_secs=_DEFAULT_TIMEOUT_SECS): + """Lock the file if the instance is not in use. + + Returns: + True if the file is locked successfully. + False if timeout or the instance is in use. + + Raises: + OSError: if any file operation fails. + """ + if not self.Lock(timeout_secs): + return False + in_use = True + try: + in_use = os.read(self._file_desc, _LOCK_FILE_SIZE) == _IN_USE_STATE + finally: + if in_use: + self.Unlock() + return not in_use diff --git a/internal/lib/local_instance_lock_test.py b/internal/lib/local_instance_lock_test.py new file mode 100644 index 00000000..047171f0 --- /dev/null +++ b/internal/lib/local_instance_lock_test.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# +# Copyright 2020 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for LocalInstanceLock.""" + +import fcntl +import os +import shutil +import tempfile +import unittest + +from unittest import mock + +from acloud import errors +from acloud.internal.lib import local_instance_lock + + +class LocalInstanceLockTest(unittest.TestCase): + """Test LocalInstanceLock methods.""" + + def setUp(self): + self._temp_dir = tempfile.mkdtemp() + self._lock_path = os.path.join(self._temp_dir, "temp.lock") + self._lock = local_instance_lock.LocalInstanceLock( + self._lock_path) + + def tearDown(self): + shutil.rmtree(self._temp_dir, ignore_errors=True) + + def testLock(self): + """Test the method calls that don't raise errors.""" + self.assertTrue(self._lock.LockIfNotInUse()) + self.assertTrue(os.path.isfile(self._lock_path)) + self._lock.Unlock() + + self.assertTrue(self._lock.LockIfNotInUse(timeout_secs=0)) + self._lock.SetInUse(True) + self._lock.Unlock() + + self.assertFalse(self._lock.LockIfNotInUse()) + + self.assertTrue(self._lock.Lock()) + self._lock.SetInUse(False) + self._lock.Unlock() + + self.assertTrue(self._lock.Lock(timeout_secs=0)) + self._lock.Unlock() + + def testOperationsWithoutLock(self): + """Test raising errors when the file is not locked.""" + self.assertRaises(RuntimeError, self._lock.Unlock) + self.assertRaises(RuntimeError, self._lock.SetInUse, True) + self.assertRaises(RuntimeError, self._lock.SetInUse, False) + + def testNonBlockingLock(self): + """Test failing to lock in non-blocking mode.""" + lock = local_instance_lock.LocalInstanceLock(self._lock_path) + self.assertTrue(lock.Lock(timeout_secs=0)) + try: + self.assertFalse(self._lock.Lock(timeout_secs=0)) + self.assertFalse(self._lock.LockIfNotInUse(timeout_secs=0)) + finally: + lock.Unlock() + + @mock.patch("acloud.internal.lib.local_instance_lock." + "utils.TimeoutException") + def testLockWithTimeout(self, mock_timeout_exception): + """Test failing to lock due to timeout.""" + mock_wrapped_flock = mock.Mock(side_effect=errors.FunctionTimeoutError) + mock_wrapper = mock.Mock(return_value=mock_wrapped_flock) + mock_timeout_exception.return_value = mock_wrapper + + self.assertFalse(self._lock.Lock(timeout_secs=1)) + + mock_wrapper.assert_called_once_with(fcntl.flock) + mock_wrapped_flock.assert_called_once_with(mock.ANY, fcntl.LOCK_EX) + mock_wrapper.reset_mock() + mock_wrapped_flock.reset_mock() + + self.assertFalse(self._lock.LockIfNotInUse(timeout_secs=1)) + + mock_wrapper.assert_called_once_with(fcntl.flock) + mock_wrapped_flock.assert_called_once_with(mock.ANY, fcntl.LOCK_EX) + + +if __name__ == "__main__": + unittest.main() diff --git a/internal/lib/ota_tools.py b/internal/lib/ota_tools.py index 25ee4f91..6906be4f 100644 --- a/internal/lib/ota_tools.py +++ b/internal/lib/ota_tools.py @@ -15,10 +15,12 @@ import logging import os -import stat import subprocess import tempfile +from six import b + + from acloud import errors from acloud.internal import constants from acloud.internal.lib import utils @@ -58,18 +60,46 @@ def FindOtaTools(search_paths): if os.path.isfile(os.path.join(search_path, _BIN_DIR_NAME, _BUILD_SUPER_IMAGE)): return search_path - - host_out_dir = os.environ.get(constants.ENV_ANDROID_HOST_OUT) - if (host_out_dir and - os.path.isfile(os.path.join(host_out_dir, _BIN_DIR_NAME, - _BUILD_SUPER_IMAGE))): - return host_out_dir + for env_host_out in [constants.ENV_ANDROID_SOONG_HOST_OUT, + constants.ENV_ANDROID_HOST_OUT]: + host_out_dir = os.environ.get(env_host_out) + if (host_out_dir and + os.path.isfile(os.path.join(host_out_dir, _BIN_DIR_NAME, + _BUILD_SUPER_IMAGE))): + return host_out_dir raise errors.CheckPathError(_MISSING_OTA_TOOLS_MSG % {"tool_name": "OTA tool directory"}) -class OtaTools(object): +def GetImageForPartition(partition_name, image_dir, **image_paths): + """Map a partition name to an image path. + + This function is used with BuildSuperImage or MkCombinedImg to mix + image_dir and image_paths into the output file. + + Args: + partition_name: String, e.g., "system", "product", and "vendor". + image_dir: String, the directory to search for the images that are not + given in image_paths. + image_paths: Pairs of partition names and image paths. + + Returns: + The image path if the partition is in image_paths. + Otherwise, this function returns the path under image_dir. + + Raises + errors.GetLocalImageError if the image does not exist. + """ + image_path = (image_paths.get(partition_name) or + os.path.join(image_dir, partition_name + ".img")) + if not os.path.isfile(image_path): + raise errors.GetLocalImageError( + "Cannot find image for partition %s" % partition_name) + return image_path + + +class OtaTools: """The class that executes OTA tool commands.""" def __init__(self, ota_tools_dir): @@ -91,9 +121,7 @@ class OtaTools(object): if not os.path.isfile(path): raise errors.NoExecuteCmd(_MISSING_OTA_TOOLS_MSG % {"tool_name": name}) - mode = os.stat(path).st_mode - os.chmod(path, mode | (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | - stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)) + utils.SetExecutable(path) return path @staticmethod @@ -116,6 +144,14 @@ class OtaTools(object): popen_args["stdin"] = subprocess.PIPE popen_args["stdout"] = subprocess.PIPE popen_args["stderr"] = subprocess.PIPE + + # Some OTA tools are Python scripts in different versions. The + # PYTHONPATH for acloud may be incompatible with the tools. + if "env" not in popen_args and "PYTHONPATH" in os.environ: + popen_env = os.environ.copy() + del popen_env["PYTHONPATH"] + popen_args["env"] = popen_env + proc = subprocess.Popen(command, **popen_args) stdout, stderr = proc.communicate() logger.info("%s stdout: %s", command[0], stdout) @@ -164,18 +200,18 @@ class OtaTools(object): if split_line[0] == "dynamic_partition_list": partition_names = split_line[1].split() elif split_line[0] == "lpmake": - output_file.write("lpmake=%s\n" % lpmake_path) + output_file.write(b("lpmake=%s\n" % lpmake_path)) continue elif split_line[0].endswith("_image"): continue - output_file.write(line) + output_file.write(b(line)) if not partition_names: logger.w("No dynamic partition list in misc info.") for partition_name in partition_names: - output_file.write("%s_image=%s\n" % - (partition_name, get_image(partition_name))) + output_file.write(b("%s_image=%s\n" % + (partition_name, get_image(partition_name)))) @utils.TimeExecute(function_description="Build super image") @utils.TimeoutException(_BUILD_SUPER_IMAGE_TIMEOUT_SECS) @@ -196,8 +232,8 @@ class OtaTools(object): try: with open(misc_info_path, "r") as misc_info: with tempfile.NamedTemporaryFile( - prefix="misc_info_", suffix=".txt", - delete=False) as new_misc_info: + prefix="misc_info_", suffix=".txt", + delete=False) as new_misc_info: new_misc_info_path = new_misc_info.name self._RewriteMiscInfo(new_misc_info, misc_info, lpmake, get_image) @@ -246,11 +282,11 @@ class OtaTools(object): for line in input_file: split_line = line.split() if len(split_line) == 3: - output_file.write("%s %s %s\n" % (get_image(split_line[1]), - split_line[1], - split_line[2])) + output_file.write(b("%s %s %s\n" % (get_image(split_line[1]), + split_line[1], + split_line[2]))) else: - output_file.write(line) + output_file.write(b(line)) @utils.TimeExecute(function_description="Make combined image") @utils.TimeoutException(_MK_COMBINED_IMG_TIMEOUT_SECS) @@ -272,8 +308,8 @@ class OtaTools(object): try: with open(system_qemu_config_path, "r") as config: with tempfile.NamedTemporaryFile( - prefix="system-qemu-config_", suffix=".txt", - delete=False) as new_config: + prefix="system-qemu-config_", suffix=".txt", + delete=False) as new_config: new_config_path = new_config.name self._RewriteSystemQemuConfig(new_config, config, get_image) diff --git a/internal/lib/ota_tools_test.py b/internal/lib/ota_tools_test.py index 3f0363dc..97dc4bcc 100644 --- a/internal/lib/ota_tools_test.py +++ b/internal/lib/ota_tools_test.py @@ -17,7 +17,8 @@ import os import shutil import tempfile import unittest -import mock + +from unittest import mock from acloud import errors from acloud.internal.lib import ota_tools @@ -55,7 +56,7 @@ def _GetImage(name): return "/path/to/" + name + ".img" -class CapturedFile(object): +class CapturedFile: """Capture intermediate files created by OtaTools.""" def __init__(self): @@ -125,7 +126,8 @@ class OtaToolsTest(unittest.TestCase): # CVD host package contains lpmake but not all tools. self._CreateBinary("lpmake") with mock.patch.dict("acloud.internal.lib.ota_tools.os.environ", - {"ANDROID_HOST_OUT": self._temp_dir}, clear=True): + {"ANDROID_HOST_OUT": self._temp_dir, + "ANDROID_SOONG_HOST_OUT": self._temp_dir}, clear=True): with self.assertRaises(errors.CheckPathError): ota_tools.FindOtaTools([self._temp_dir]) @@ -138,9 +140,35 @@ class OtaToolsTest(unittest.TestCase): # ANDROID_HOST_OUT contains OTA tools in build environment. with mock.patch.dict("acloud.internal.lib.ota_tools.os.environ", - {"ANDROID_HOST_OUT": self._temp_dir}, clear=True): + {"ANDROID_HOST_OUT": self._temp_dir, + "ANDROID_SOONG_HOST_OUT": self._temp_dir}, clear=True): self.assertEqual(ota_tools.FindOtaTools([]), self._temp_dir) + def testGetImageForPartition(self): + """Test GetImageForPartition.""" + image_dir = os.path.join(self._temp_dir, "images") + vendor_path = os.path.join(image_dir, "vendor.img") + override_system_path = os.path.join(self._temp_dir, "system.img") + self._CreateFile(vendor_path, "") + self._CreateFile(os.path.join(image_dir, "system.img"), "") + self._CreateFile(override_system_path, "") + + returned_path = ota_tools.GetImageForPartition( + "system", image_dir, system=override_system_path) + self.assertEqual(returned_path, override_system_path) + + returned_path = ota_tools.GetImageForPartition( + "vendor", image_dir, system=override_system_path) + self.assertEqual(returned_path, vendor_path) + + with self.assertRaises(errors.GetLocalImageError): + ota_tools.GetImageForPartition("not_exist", image_dir) + + with self.assertRaises(errors.GetLocalImageError): + ota_tools.GetImageForPartition( + "system", image_dir, + system=os.path.join(self._temp_dir, "not_exist")) + # pylint: disable=broad-except def _TestBuildSuperImage(self, mock_popen, mock_popen_object, expected_error=None): @@ -204,7 +232,9 @@ class OtaToolsTest(unittest.TestCase): mock_popen.return_value = self._MockPopen(return_value=0) - self._ota.MakeDisabledVbmetaImage("/unit/test") + with mock.patch.dict("acloud.internal.lib.ota_tools.os.environ", + {"PYTHONPATH": "/unit/test"}, clear=True): + self._ota.MakeDisabledVbmetaImage("/unit/test") expected_cmd = ( avbtool, "make_vbmeta_image", @@ -215,6 +245,7 @@ class OtaToolsTest(unittest.TestCase): mock_popen.assert_called_once() self.assertEqual(mock_popen.call_args[0][0], expected_cmd) + self.assertFalse(mock_popen.call_args[1]["env"]) # pylint: disable=broad-except def _TestMkCombinedImg(self, mock_popen, mock_popen_object, diff --git a/internal/lib/ssh.py b/internal/lib/ssh.py index 5411e671..0f93218a 100755 --- a/internal/lib/ssh.py +++ b/internal/lib/ssh.py @@ -30,7 +30,7 @@ _SSH_CMD = ("-i %(rsa_key_file)s " _SSH_IDENTITY = "-l %(login_user)s %(ip_addr)s" _SSH_CMD_MAX_RETRY = 5 _SSH_CMD_RETRY_SLEEP = 3 -_WAIT_FOR_SSH_MAX_TIMEOUT = 60 +_CONNECTION_TIMEOUT = 10 def _SshCallWait(cmd, timeout=None): @@ -109,7 +109,8 @@ def _SshLogOutput(cmd, timeout=None, show_output=False): cmd = "exec " + cmd logger.info("Running command \"%s\"", cmd) process = subprocess.Popen(cmd, shell=True, stdin=None, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True) if timeout: # TODO: if process is killed, out error message to log. timer = threading.Timer(timeout, process.kill) @@ -130,7 +131,8 @@ def _SshLogOutput(cmd, timeout=None, show_output=False): raise subprocess.CalledProcessError(process.returncode, cmd) -def ShellCmdWithRetry(cmd, timeout=None, show_output=False): +def ShellCmdWithRetry(cmd, timeout=None, show_output=False, + retry=_SSH_CMD_MAX_RETRY): """Runs a shell command on remote device. If the network is unstable and causes SSH connect fail, it will retry. When @@ -142,14 +144,15 @@ def ShellCmdWithRetry(cmd, timeout=None, show_output=False): cmd: String of the full SSH command to run, including the SSH binary and its arguments. timeout: Optional integer, number of seconds to give. show_output: Boolean, True to show command output in screen. + retry: Integer, the retry times. Raises: errors.DeviceConnectionError: For any non-zero return code of remote_cmd. """ utils.RetryExceptionType( - exception_types=errors.DeviceConnectionError, - max_retries=_SSH_CMD_MAX_RETRY, + exception_types=(errors.DeviceConnectionError, subprocess.CalledProcessError), + max_retries=retry, functor=_SshLogOutput, sleep_multiplier=_SSH_CMD_RETRY_SLEEP, retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR, @@ -188,7 +191,8 @@ class Ssh(object): self._ssh_private_key_path = ssh_private_key_path self._extra_args_ssh_tunnel = extra_args_ssh_tunnel - def Run(self, target_command, timeout=None, show_output=False): + def Run(self, target_command, timeout=None, show_output=False, + retry=_SSH_CMD_MAX_RETRY): """Run a shell command over SSH on a remote instance. Example: @@ -203,10 +207,12 @@ class Ssh(object): target_command: String, text of command to run on the remote instance. timeout: Integer, the maximum time to wait for the command to respond. show_output: Boolean, True to show command output in screen. + retry: Integer, the retry times. """ ShellCmdWithRetry(self.GetBaseCmd(constants.SSH_BIN) + " " + target_command, timeout, - show_output) + show_output, + retry) def GetBaseCmd(self, execute_bin): """Get a base command over SSH on a remote instance. @@ -240,6 +246,23 @@ class Ssh(object): raise errors.UnknownType("Don't support the execute bin %s." % execute_bin) + def GetCmdOutput(self, cmd): + """Runs a single SSH command and get its output. + + Args: + cmd: String, text of command to run on the remote instance. + + Returns: + String of the command output. + """ + ssh_cmd = "exec " + self.GetBaseCmd(constants.SSH_BIN) + " " + cmd + logger.info("Running command \"%s\"", ssh_cmd) + process = subprocess.Popen(ssh_cmd, shell=True, stdin=None, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True) + stdout, _ = process.communicate() + return stdout + def CheckSshConnection(self, timeout): """Run remote 'uptime' ssh command to check ssh connection. @@ -258,27 +281,27 @@ class Ssh(object): "Ssh isn't ready in the remote instance.") @utils.TimeExecute(function_description="Waiting for SSH server") - def WaitForSsh(self, timeout=None, sleep_for_retry=_SSH_CMD_RETRY_SLEEP, - max_retry=_SSH_CMD_MAX_RETRY): + def WaitForSsh(self, timeout=None, max_retry=_SSH_CMD_MAX_RETRY): """Wait until the remote instance is ready to accept commands over SSH. Args: timeout: Integer, the maximum time in seconds to wait for the command to respond. - sleep_for_retry: Integer, the sleep time in seconds for retry. max_retry: Integer, the maximum number of retry. Raises: errors.DeviceConnectionError: Ssh isn't ready in the remote instance. """ - timeout_one_round = timeout / max_retry if timeout else None + ssh_timeout = timeout or constants.DEFAULT_SSH_TIMEOUT + sleep_multiplier = ssh_timeout / sum(range(max_retry + 1)) + logger.debug("Retry with interval time: %s secs", str(sleep_multiplier)) utils.RetryExceptionType( exception_types=errors.DeviceConnectionError, max_retries=max_retry, functor=self.CheckSshConnection, - sleep_multiplier=sleep_for_retry, + sleep_multiplier=sleep_multiplier, retry_backoff_factor=utils.DEFAULT_RETRY_BACKOFF_FACTOR, - timeout=timeout_one_round or _WAIT_FOR_SSH_MAX_TIMEOUT) + timeout=_CONNECTION_TIMEOUT) def ScpPushFile(self, src_file, dst_file): """Scp push file to remote. diff --git a/internal/lib/ssh_test.py b/internal/lib/ssh_test.py index 2e010043..f6268fe9 100644 --- a/internal/lib/ssh_test.py +++ b/internal/lib/ssh_test.py @@ -18,7 +18,10 @@ import subprocess import unittest -import mock +import threading +import time + +from unittest import mock from acloud import errors from acloud.internal import constants @@ -40,7 +43,7 @@ class SshTest(driver_test_lib.BaseDriverTest): super(SshTest, self).setUp() self.created_subprocess = mock.MagicMock() self.created_subprocess.stdout = mock.MagicMock() - self.created_subprocess.stdout.readline = mock.MagicMock(return_value='') + self.created_subprocess.stdout.readline = mock.MagicMock(return_value=b"") self.created_subprocess.poll = mock.MagicMock(return_value=0) self.created_subprocess.returncode = 0 self.created_subprocess.communicate = mock.MagicMock(return_value= @@ -48,6 +51,7 @@ class SshTest(driver_test_lib.BaseDriverTest): def testSSHExecuteWithRetry(self): """test SSHExecuteWithRetry method.""" + self.Patch(time, "sleep") self.Patch(subprocess, "Popen", side_effect=subprocess.CalledProcessError( None, "ssh command fail.")) @@ -88,7 +92,8 @@ class SshTest(driver_test_lib.BaseDriverTest): shell=True, stderr=-2, stdin=None, - stdout=-1) + stdout=-1, + universal_newlines=True) def testSshRunCmdwithExtraArgs(self): """test ssh rum command with extra command.""" @@ -106,7 +111,8 @@ class SshTest(driver_test_lib.BaseDriverTest): shell=True, stderr=-2, stdin=None, - stdout=-1) + stdout=-1, + universal_newlines=True) def testScpPullFileCmd(self): """Test scp pull file command.""" @@ -119,7 +125,8 @@ class SshTest(driver_test_lib.BaseDriverTest): shell=True, stderr=-2, stdin=None, - stdout=-1) + stdout=-1, + universal_newlines=True) def testScpPullFileCmdwithExtraArgs(self): """Test scp pull file command.""" @@ -137,7 +144,8 @@ class SshTest(driver_test_lib.BaseDriverTest): shell=True, stderr=-2, stdin=None, - stdout=-1) + stdout=-1, + universal_newlines=True) def testScpPushFileCmd(self): """Test scp push file command.""" @@ -150,7 +158,8 @@ class SshTest(driver_test_lib.BaseDriverTest): shell=True, stderr=-2, stdin=None, - stdout=-1) + stdout=-1, + universal_newlines=True) def testScpPushFileCmdwithExtraArgs(self): """Test scp pull file command.""" @@ -168,7 +177,8 @@ class SshTest(driver_test_lib.BaseDriverTest): shell=True, stderr=-2, stdin=None, - stdout=-1) + stdout=-1, + universal_newlines=True) # pylint: disable=protected-access def testIPAddress(self): @@ -205,9 +215,58 @@ class SshTest(driver_test_lib.BaseDriverTest): self.assertRaises(errors.DeviceConnectionError, ssh_object.WaitForSsh, timeout=1, - sleep_for_retry=1, max_retry=1) + def testSshCallWait(self): + """Test SshCallWait.""" + self.Patch(subprocess, "Popen", return_value=self.created_subprocess) + self.Patch(threading, "Timer") + fake_cmd = "fake command" + ssh._SshCallWait(fake_cmd) + threading.Timer.assert_not_called() + + def testSshCallWaitTimeout(self): + """Test SshCallWait with timeout.""" + self.Patch(subprocess, "Popen", return_value=self.created_subprocess) + self.Patch(threading, "Timer") + fake_cmd = "fake command" + fake_timeout = 30 + ssh._SshCallWait(fake_cmd, fake_timeout) + threading.Timer.assert_called_once() + + def testSshCall(self): + """Test _SshCall.""" + self.Patch(subprocess, "Popen", return_value=self.created_subprocess) + self.Patch(threading, "Timer") + fake_cmd = "fake command" + ssh._SshCall(fake_cmd) + threading.Timer.assert_not_called() + + def testSshCallTimeout(self): + """Test SshCallWait with timeout.""" + self.Patch(subprocess, "Popen", return_value=self.created_subprocess) + self.Patch(threading, "Timer") + fake_cmd = "fake command" + fake_timeout = 30 + ssh._SshCall(fake_cmd, fake_timeout) + threading.Timer.assert_called_once() + + def testSshLogOutput(self): + """Test _SshCall.""" + self.Patch(subprocess, "Popen", return_value=self.created_subprocess) + self.Patch(threading, "Timer") + fake_cmd = "fake command" + ssh._SshLogOutput(fake_cmd) + threading.Timer.assert_not_called() + + def testSshLogOutputTimeout(self): + """Test SshCallWait with timeout.""" + self.Patch(subprocess, "Popen", return_value=self.created_subprocess) + self.Patch(threading, "Timer") + fake_cmd = "fake command" + fake_timeout = 30 + ssh._SshLogOutput(fake_cmd, fake_timeout) + threading.Timer.assert_called_once() if __name__ == "__main__": unittest.main() diff --git a/internal/lib/utils.py b/internal/lib/utils.py index a60c6c3f..819aba4c 100755 --- a/internal/lib/utils.py +++ b/internal/lib/utils.py @@ -15,7 +15,6 @@ # pylint: disable=too-many-lines from __future__ import print_function -from distutils.spawn import find_executable import base64 import binascii import collections @@ -30,6 +29,7 @@ import shutil import signal import struct import socket +import stat import subprocess import sys import tarfile @@ -57,11 +57,18 @@ GET_BUILD_VAR_CMD = ["build/soong/soong_ui.bash", "--dumpvar-mode"] DEFAULT_RETRY_BACKOFF_FACTOR = 1 DEFAULT_SLEEP_MULTIPLIER = 0 -_SSH_TUNNEL_ARGS = ("-i %(rsa_key_file)s -o UserKnownHostsFile=/dev/null " - "-o StrictHostKeyChecking=no " - "-L %(vnc_port)d:127.0.0.1:%(target_vnc_port)d " - "-L %(adb_port)d:127.0.0.1:%(target_adb_port)d " - "-N -f -l %(ssh_user)s %(ip_addr)s") +_SSH_TUNNEL_ARGS = ( + "-i %(rsa_key_file)s -o UserKnownHostsFile=/dev/null " + "-o StrictHostKeyChecking=no " + "%(port_mapping)s" + "-N -f -l %(ssh_user)s %(ip_addr)s") +PORT_MAPPING = "-L %(local_port)d:127.0.0.1:%(target_port)d " +_RELEASE_PORT_CMD = "kill $(lsof -t -i :%d)" +_WEBRTC_TARGET_PORT = 8443 +WEBRTC_PORTS_MAPPING = [{"local": constants.WEBRTC_LOCAL_PORT, + "target": _WEBRTC_TARGET_PORT}, + {"local": 15550, "target": 15550}, + {"local": 15551, "target": 15551}] _ADB_CONNECT_ARGS = "connect 127.0.0.1:%(adb_port)d" # Store the ports that vnc/adb are forwarded to, both are integers. ForwardedPorts = collections.namedtuple("ForwardedPorts", [constants.VNC_PORT, @@ -74,7 +81,8 @@ AVD_PORT_DICT = { constants.TYPE_GF: ForwardedPorts(constants.GF_VNC_PORT, constants.GF_ADB_PORT), constants.TYPE_CHEEPS: ForwardedPorts(constants.CHEEPS_VNC_PORT, - constants.CHEEPS_ADB_PORT) + constants.CHEEPS_ADB_PORT), + constants.TYPE_FVP: ForwardedPorts(None, constants.FVP_ADB_PORT), } _VNC_BIN = "ssvnc" @@ -88,8 +96,7 @@ _DEFAULT_DISPLAY_SCALE = 1.0 _DIST_DIR = "DIST_DIR" # For webrtc -_WEBRTC_URL = "https://" -_WEBRTC_PORT = "8443" +_WEBRTC_URL = "https://%(webrtc_ip)s:%(webrtc_port)d" _CONFIRM_CONTINUE = ("In order to display the screen to the AVD, we'll need to " "install a vnc client (ssvnc). \nWould you like acloud to " @@ -98,12 +105,12 @@ _CONFIRM_CONTINUE = ("In order to display the screen to the AVD, we'll need to " _EvaluatedResult = collections.namedtuple("EvaluatedResult", ["is_result_ok", "result_message"]) # dict of supported system and their distributions. -_SUPPORTED_SYSTEMS_AND_DISTS = {"Linux": ["Ubuntu", "Debian"]} +_SUPPORTED_SYSTEMS_AND_DISTS = {"Linux": ["Ubuntu", "ubuntu", "Debian", "debian"]} _DEFAULT_TIMEOUT_ERR = "Function did not complete within %d secs." _SSVNC_VIEWER_PATTERN = "vnc://127.0.0.1:%(vnc_port)d" -class TempDir(object): +class TempDir: """A context manager that ceates a temporary directory. Attributes: @@ -275,11 +282,10 @@ def PollAndWait(func, expected_return, timeout_exception, timeout_secs, return_value = func(*args, **kwargs) if return_value == expected_return: return - elif time.time() - start > timeout_secs: + if time.time() - start > timeout_secs: raise timeout_exception - else: - if sleep_interval_secs > 0: - time.sleep(sleep_interval_secs) + if sleep_interval_secs > 0: + time.sleep(sleep_interval_secs) def GenerateUniqueName(prefix=None, suffix=None): @@ -349,7 +355,7 @@ def CreateSshKeyPairIfNotExist(private_key_path, public_key_path): if private_key_exist: cmd = SSH_KEYGEN_PUB_CMD + ["-f", private_key_path] with open(public_key_path, 'w') as outfile: - stream_content = subprocess.check_output(cmd) + stream_content = CheckOutput(cmd) outfile.write( stream_content.rstrip('\n') + " " + getpass.getuser()) logger.info( @@ -407,7 +413,7 @@ def VerifyRsaPubKey(rsa): key_type, data, _ = elements try: - binary_data = base64.decodestring(data) + binary_data = base64.decodebytes(six.b(data)) # number of bytes of int type int_length = 4 # binary_data is like "7ssh-key..." in a binary format. @@ -417,7 +423,7 @@ def VerifyRsaPubKey(rsa): # We will verify that the rsa conforms to this format. # ">I" in the following line means "big-endian unsigned integer". type_length = struct.unpack(">I", binary_data[:int_length])[0] - if binary_data[int_length:int_length + type_length] != key_type: + if binary_data[int_length:int_length + type_length] != six.b(key_type): raise errors.DriverError("rsa key is invalid: %s" % rsa) except (struct.error, binascii.Error) as e: raise errors.DriverError( @@ -448,7 +454,7 @@ def Decompress(sourcefile, dest=None): "for zip or tar.gz.") -# pylint: disable=old-style-class,no-init +# pylint: disable=no-init class TextColors: """A class that defines common color ANSI code.""" @@ -513,7 +519,7 @@ def GetUserAnswerYes(question): return answer.lower() in constants.USER_ANSWER_YES -class BatchHttpRequestExecutor(object): +class BatchHttpRequestExecutor: """A helper class that executes requests in batch with retry. This executor executes http requests in a batch and retry @@ -681,7 +687,7 @@ def BootEvaluator(boot_dict): return _EvaluatedResult(is_result_ok=True, result_message=None) -class TimeExecute(object): +class TimeExecute: """Count the function execute time.""" def __init__(self, function_description=None, print_before_call=True, @@ -802,6 +808,55 @@ def _ExecuteCommand(cmd, args): subprocess.check_call(command, stderr=dev_null, stdout=dev_null) +def ReleasePort(port): + """Release local port. + + Args: + port: Integer of local port number. + """ + try: + with open(os.devnull, "w") as dev_null: + subprocess.check_call(_RELEASE_PORT_CMD % port, + stderr=dev_null, stdout=dev_null, shell=True) + except subprocess.CalledProcessError: + logger.debug("The port %d is available.", constants.WEBRTC_LOCAL_PORT) + + +def EstablishWebRTCSshTunnel(ip_addr, rsa_key_file, ssh_user, + extra_args_ssh_tunnel=None): + """Create ssh tunnels for webrtc. + + # TODO(151418177): Before fully supporting webrtc feature, we establish one + # WebRTC tunnel at a time. so always delete the previous connection before + # establishing new one. + + Args: + ip_addr: String, use to build the adb & vnc tunnel between local + and remote instance. + rsa_key_file: String, Private key file path to use when creating + the ssh tunnels. + ssh_user: String of user login into the instance. + extra_args_ssh_tunnel: String, extra args for ssh tunnel connection. + """ + ReleasePort(constants.WEBRTC_LOCAL_PORT) + try: + port_mapping = [PORT_MAPPING % { + "local_port":port["local"], + "target_port":port["target"]} for port in WEBRTC_PORTS_MAPPING] + ssh_tunnel_args = _SSH_TUNNEL_ARGS % { + "rsa_key_file": rsa_key_file, + "ssh_user": ssh_user, + "ip_addr": ip_addr, + "port_mapping":" ".join(port_mapping)} + ssh_tunnel_args_list = shlex.split(ssh_tunnel_args) + if extra_args_ssh_tunnel is not None: + ssh_tunnel_args_list.extend(shlex.split(extra_args_ssh_tunnel)) + _ExecuteCommand(constants.SSH_BIN, ssh_tunnel_args_list) + except subprocess.CalledProcessError as e: + PrintColorString("\n%s\nFailed to create ssh tunnels, retry with '#acloud " + "reconnect'." % e, TextColors.FAIL) + + # TODO(147337696): create ssh tunnels tear down as adb and vnc. # pylint: disable=too-many-locals def AutoConnect(ip_addr, rsa_key_file, target_vnc_port, target_adb_port, @@ -823,19 +878,24 @@ def AutoConnect(ip_addr, rsa_key_file, target_vnc_port, target_adb_port, NamedTuple of (vnc_port, adb_port) SSHTUNNEL of the connect, both are integers. """ - local_free_vnc_port = PickFreePort() local_adb_port = client_adb_port or PickFreePort() + port_mapping = [PORT_MAPPING % { + "local_port":local_adb_port, + "target_port":target_adb_port}] + local_free_vnc_port = None + if target_vnc_port: + local_free_vnc_port = PickFreePort() + port_mapping += [PORT_MAPPING % { + "local_port":local_free_vnc_port, + "target_port":target_vnc_port}] try: ssh_tunnel_args = _SSH_TUNNEL_ARGS % { "rsa_key_file": rsa_key_file, - "vnc_port": local_free_vnc_port, - "adb_port": local_adb_port, - "target_vnc_port": target_vnc_port, - "target_adb_port": target_adb_port, + "port_mapping": " ".join(port_mapping), "ssh_user": ssh_user, "ip_addr": ip_addr} ssh_tunnel_args_list = shlex.split(ssh_tunnel_args) - if extra_args_ssh_tunnel: + if extra_args_ssh_tunnel is not None: ssh_tunnel_args_list.extend(shlex.split(extra_args_ssh_tunnel)) _ExecuteCommand(constants.SSH_BIN, ssh_tunnel_args_list) except subprocess.CalledProcessError as e: @@ -912,6 +972,7 @@ def LaunchVNCFromReport(report, avd_spec, no_prompts=False): PrintColorString("No VNC port specified, skipping VNC startup.", TextColors.FAIL) + def LaunchBrowserFromReport(report): """Open browser when autoconnect to webrtc according to the instances report. @@ -925,18 +986,28 @@ def LaunchBrowserFromReport(report): for device in report.data.get("devices", []): if device.get("ip"): - webrtc_link = "%s%s:%s" % (_WEBRTC_URL, device.get("ip"), - _WEBRTC_PORT) - if os.environ.get(_ENV_DISPLAY, None): - webbrowser.open_new_tab(webrtc_link) - else: - PrintColorString("Remote terminal can't support launch webbrowser.", - TextColors.FAIL) - PrintColorString("Open %s to remotely control AVD on the " - "browser." % webrtc_link) - else: - PrintColorString("Auto-launch devices webrtc in browser failed!", - TextColors.FAIL) + LaunchBrowser(constants.WEBRTC_LOCAL_HOST, + device.get(constants.WEBRTC_PORT, + constants.WEBRTC_LOCAL_PORT)) + + +def LaunchBrowser(ip_addr, port): + """Launch browser to connect the webrtc AVD. + + Args: + ip_addr: String, use to connect to webrtc AVD on the instance. + port: Integer, port number. + """ + webrtc_link = _WEBRTC_URL % { + "webrtc_ip": ip_addr, + "webrtc_port": port} + if os.environ.get(_ENV_DISPLAY, None): + webbrowser.open_new_tab(webrtc_link) + else: + PrintColorString("Remote terminal can't support launch webbrowser.", + TextColors.FAIL) + PrintColorString("WebRTC AVD URL: %s "% webrtc_link) + def LaunchVncClient(port, avd_width=None, avd_height=None, no_prompts=False): """Launch ssvnc. @@ -951,7 +1022,9 @@ def LaunchVncClient(port, avd_width=None, avd_height=None, no_prompts=False): os.environ[_ENV_DISPLAY] except KeyError: PrintColorString("Remote terminal can't support VNC. " - "Skipping VNC startup.", TextColors.FAIL) + "Skipping VNC startup. " + "VNC server is listening at 127.0.0.1:{}.".format(port), + TextColors.FAIL) return if IsSupportedPlatform() and not FindExecutable(_VNC_BIN): @@ -959,7 +1032,7 @@ def LaunchVncClient(port, avd_width=None, avd_height=None, no_prompts=False): try: PrintColorString("Installing ssvnc vnc client... ", end="") sys.stdout.flush() - subprocess.check_output(_CMD_INSTALL_SSVNC, shell=True) + CheckOutput(_CMD_INSTALL_SSVNC, shell=True) PrintColorString("Done", TextColors.OKGREEN) except subprocess.CalledProcessError as cpe: PrintColorString("Failed to install ssvnc: %s" % @@ -995,17 +1068,23 @@ def PrintDeviceSummary(report): PrintColorString("\n") PrintColorString("Device summary:") for device in report.data.get("devices", []): - adb_serial = "(None)" - adb_port = device.get("adb_port") - if adb_port: - adb_serial = constants.LOCALHOST_ADB_SERIAL % adb_port + adb_serial = device.get(constants.DEVICE_SERIAL) + if not adb_serial: + adb_port = device.get("adb_port") + if adb_port: + adb_serial = constants.LOCALHOST_ADB_SERIAL % adb_port + else: + adb_serial = "(None)" + instance_name = device.get("instance_name") instance_ip = device.get("ip") instance_details = "" if not instance_name else "(%s[%s])" % ( instance_name, instance_ip) PrintColorString(" - device serial: %s %s" % (adb_serial, instance_details)) - PrintColorString(" export ANDROID_SERIAL=%s" % adb_serial) + PrintColorString("\n") + PrintColorString("Note: To ensure Tradefed use this AVD, please run:") + PrintColorString("\texport ANDROID_SERIAL=%s" % adb_serial) # TODO(b/117245508): Help user to delete instance if it got created. if report.errors: @@ -1013,6 +1092,7 @@ def PrintDeviceSummary(report): PrintColorString("Fail in:\n%s\n" % error_msg, TextColors.FAIL) +# pylint: disable=import-outside-toplevel def CalculateVNCScreenRatio(avd_width, avd_height): """calculate the vnc screen scale ratio to fit into user's monitor. @@ -1026,7 +1106,13 @@ def CalculateVNCScreenRatio(avd_width, avd_height): import Tkinter # Some python interpreters may not be configured for Tk, just return default scale ratio. except ImportError: - return _DEFAULT_DISPLAY_SCALE + try: + import tkinter as Tkinter + except ImportError: + PrintColorString( + "no module named tkinter, vnc display scale were not be fit." + "please run 'sudo apt-get install python3-tk' to install it.") + return _DEFAULT_DISPLAY_SCALE root = Tkinter.Tk() margin = 100 # leave some space on user's monitor. screen_height = root.winfo_screenheight() - margin @@ -1117,18 +1203,23 @@ def CheckUserInGroups(group_name_list): True if current user is in all the groups. """ logger.info("Checking if user is in following groups: %s", group_name_list) - current_groups = [grp.getgrgid(g).gr_name for g in os.getgroups()] - all_groups_present = True + all_groups = [g.gr_name for g in grp.getgrall()] for group in group_name_list: - if group not in current_groups: - all_groups_present = False - logger.info("missing group: %s", group) - return all_groups_present + if group not in all_groups: + logger.info("This group doesn't exist: %s", group) + return False + if getpass.getuser() not in grp.getgrnam(group).gr_mem: + logger.info("Current user isn't in this group: %s", group) + return False + return True def IsSupportedPlatform(print_warning=False): """Check if user's os is the supported platform. + platform.version() return such as '#1 SMP Debian 5.6.14-1rodete2...' + and use to judge supported or not. + Args: print_warning: Boolean, print the unsupported warning if True. @@ -1136,17 +1227,19 @@ def IsSupportedPlatform(print_warning=False): Boolean, True if user is using supported platform. """ system = platform.system() - # TODO(b/143197659): linux_distribution() deprecated in python 3. To fix it - # try to use another package "import distro". - dist = platform.linux_distribution()[0] - platform_supported = (system in _SUPPORTED_SYSTEMS_AND_DISTS and - dist in _SUPPORTED_SYSTEMS_AND_DISTS[system]) - - logger.info("supported system and dists: %s", + # TODO(b/161085678): After python3 fully migrated, then use distro to fix. + platform_supported = False + if system in _SUPPORTED_SYSTEMS_AND_DISTS: + for dist in _SUPPORTED_SYSTEMS_AND_DISTS[system]: + if dist in platform.version(): + platform_supported = True + break + + logger.info("Updated supported system and dists: %s", _SUPPORTED_SYSTEMS_AND_DISTS) platform_supported_msg = ("%s[%s] %s supported platform" % (system, - dist, + platform.version(), "is a" if platform_supported else "is not a")) if print_warning and not platform_supported: PrintColorString(platform_supported_msg, TextColors.WARNING) @@ -1164,7 +1257,7 @@ def GetDistDir(): dist_cmd = GET_BUILD_VAR_CMD[:] dist_cmd.append(_DIST_DIR) try: - dist_dir = subprocess.check_output(dist_cmd, cwd=android_build_top) + dist_dir = CheckOutput(dist_cmd, cwd=android_build_top) except subprocess.CalledProcessError: return None return os.path.join(android_build_top, dist_dir.strip()) @@ -1235,7 +1328,7 @@ def GetBuildEnvironmentVariable(variable_name): ) -# pylint: disable=no-member +# pylint: disable=no-member,import-outside-toplevel def FindExecutable(filename): """A compatibility function to find execution file path. @@ -1245,7 +1338,11 @@ def FindExecutable(filename): Returns: String: execution file path. """ - return find_executable(filename) if six.PY2 else shutil.which(filename) + try: + from distutils.spawn import find_executable + return find_executable(filename) + except ImportError: + return shutil.which(filename) def GetDictItems(namedtuple_object): @@ -1270,3 +1367,47 @@ def CleanupSSVncviewer(vnc_port): """ ssvnc_viewer_pattern = _SSVNC_VIEWER_PATTERN % {"vnc_port":vnc_port} CleanupProcess(ssvnc_viewer_pattern) + + +def CheckOutput(cmd, **kwargs): + """Call subprocess.check_output to get output. + + The subprocess.check_output return type is "bytes" in python 3, we have + to convert bytes as string with .decode() in advance. + + Args: + cmd: String of command. + **kwargs: dictionary of keyword based args to pass to func. + + Return: + String to command output. + """ + return subprocess.check_output(cmd, **kwargs).decode() + + +def SetExecutable(path): + """Grant the persmission to execute a file. + + Args: + path: String, the file path. + + Raises: + OSError if any file operation fails. + """ + mode = os.stat(path).st_mode + os.chmod(path, mode | (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | + stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)) + + +def SetDirectoryTreeExecutable(dir_path): + """Grant the permission to execute all files in a directory. + + Args: + dir_path: String, the directory path. + + Raises: + OSError if any file operation fails. + """ + for parent_dir, _, file_names in os.walk(dir_path): + for name in file_names: + SetExecutable(os.path.join(parent_dir, name)) diff --git a/internal/lib/utils_test.py b/internal/lib/utils_test.py index 169a36d7..45bb1777 100644 --- a/internal/lib/utils_test.py +++ b/internal/lib/utils_test.py @@ -15,6 +15,7 @@ # limitations under the License. """Tests for acloud.internal.lib.utils.""" +import collections import errno import getpass import grp @@ -23,9 +24,11 @@ import shutil import subprocess import tempfile import time +import webbrowser import unittest -import mock + +from unittest import mock import six from acloud import errors @@ -33,6 +36,12 @@ from acloud.internal.lib import driver_test_lib from acloud.internal.lib import utils +GroupInfo = collections.namedtuple("GroupInfo", [ + "gr_name", + "gr_passwd", + "gr_gid", + "gr_mem"]) + # Tkinter may not be supported so mock it out. try: import Tkinter @@ -40,7 +49,7 @@ except ImportError: Tkinter = mock.Mock() -class FakeTkinter(object): +class FakeTkinter: """Fake implementation of Tkinter.Tk()""" def __init__(self, width=None, height=None): @@ -300,28 +309,23 @@ class UtilsTest(driver_test_lib.BaseDriverTest): avd_w = 1080 self.assertEqual(utils.CalculateVNCScreenRatio(avd_w, avd_h), 0.6) - # pylint: disable=protected-access def testCheckUserInGroups(self): """Test CheckUserInGroups.""" - self.Patch(os, "getgroups", return_value=[1, 2, 3]) - gr1 = mock.MagicMock() - gr1.gr_name = "fake_gr_1" - gr2 = mock.MagicMock() - gr2.gr_name = "fake_gr_2" - gr3 = mock.MagicMock() - gr3.gr_name = "fake_gr_3" - self.Patch(grp, "getgrgid", side_effect=[gr1, gr2, gr3]) - - # User in all required groups should return true. - self.assertTrue( - utils.CheckUserInGroups( - ["fake_gr_1", "fake_gr_2"])) - - # User not in all required groups should return False. - self.Patch(grp, "getgrgid", side_effect=[gr1, gr2, gr3]) - self.assertFalse( - utils.CheckUserInGroups( - ["fake_gr_1", "fake_gr_4"])) + self.Patch(getpass, "getuser", return_value="user_0") + self.Patch(grp, "getgrall", return_value=[ + GroupInfo("fake_group1", "passwd_1", 0, ["user_1", "user_2"]), + GroupInfo("fake_group2", "passwd_2", 1, ["user_1", "user_2"])]) + self.Patch(grp, "getgrnam", return_value=GroupInfo( + "fake_group1", "passwd_1", 0, ["user_1", "user_2"])) + # Test Group name doesn't exist. + self.assertFalse(utils.CheckUserInGroups(["Non_exist_group"])) + + # Test User isn't in group. + self.assertFalse(utils.CheckUserInGroups(["fake_group1"])) + + # Test User is in group. + self.Patch(getpass, "getuser", return_value="user_1") + self.assertTrue(utils.CheckUserInGroups(["fake_group1"])) @mock.patch.object(utils, "CheckUserInGroups") def testAddUserGroupsToCmd(self, mock_user_group): @@ -382,7 +386,7 @@ class UtilsTest(driver_test_lib.BaseDriverTest): # pylint: disable=protected-access,no-member def testExtraArgsSSHTunnel(self): - """Tesg extra args will be the same with expanded args.""" + """Test extra args will be the same with expanded args.""" fake_ip_addr = "1.1.1.1" fake_rsa_key_file = "/tmp/rsa_file" fake_target_vnc_port = 8888 @@ -403,14 +407,52 @@ class UtilsTest(driver_test_lib.BaseDriverTest): args_list = ["-i", "/tmp/rsa_file", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", - "-L", "12345:127.0.0.1:8888", "-L", "12345:127.0.0.1:9999", + "-L", "12345:127.0.0.1:8888", "-N", "-f", "-l", "fake_user", "1.1.1.1", "-o", "command=shell %s %h", "-o", "command1=ls -la"] first_call_args = utils._ExecuteCommand.call_args_list[0][0] self.assertEqual(first_call_args[1], args_list) + # pylint: disable=protected-access,no-member + def testEstablishWebRTCSshTunnel(self): + """Test establish WebRTC ssh tunnel.""" + fake_ip_addr = "1.1.1.1" + fake_rsa_key_file = "/tmp/rsa_file" + ssh_user = "fake_user" + self.Patch(utils, "ReleasePort") + self.Patch(utils, "_ExecuteCommand") + self.Patch(subprocess, "check_call", return_value=True) + extra_args_ssh_tunnel = "-o command='shell %s %h' -o command1='ls -la'" + utils.EstablishWebRTCSshTunnel( + ip_addr=fake_ip_addr, rsa_key_file=fake_rsa_key_file, + ssh_user=ssh_user, extra_args_ssh_tunnel=None) + args_list = ["-i", "/tmp/rsa_file", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "StrictHostKeyChecking=no", + "-L", "8443:127.0.0.1:8443", + "-L", "15550:127.0.0.1:15550", + "-L", "15551:127.0.0.1:15551", + "-N", "-f", "-l", "fake_user", "1.1.1.1"] + first_call_args = utils._ExecuteCommand.call_args_list[0][0] + self.assertEqual(first_call_args[1], args_list) + + extra_args_ssh_tunnel = "-o command='shell %s %h'" + utils.EstablishWebRTCSshTunnel( + ip_addr=fake_ip_addr, rsa_key_file=fake_rsa_key_file, + ssh_user=ssh_user, extra_args_ssh_tunnel=extra_args_ssh_tunnel) + args_list_with_extra_args = ["-i", "/tmp/rsa_file", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "StrictHostKeyChecking=no", + "-L", "8443:127.0.0.1:8443", + "-L", "15550:127.0.0.1:15550", + "-L", "15551:127.0.0.1:15551", + "-N", "-f", "-l", "fake_user", "1.1.1.1", + "-o", "command=shell %s %h"] + first_call_args = utils._ExecuteCommand.call_args_list[1][0] + self.assertEqual(first_call_args[1], args_list_with_extra_args) + # pylint: disable=protected-access, no-member def testCleanupSSVncviwer(self): """test cleanup ssvnc viewer.""" @@ -427,6 +469,51 @@ class UtilsTest(driver_test_lib.BaseDriverTest): utils.CleanupSSVncviewer(fake_vnc_port) subprocess.check_call.assert_not_called() + def testLaunchBrowserFromReport(self): + """test launch browser from report.""" + self.Patch(webbrowser, "open_new_tab") + fake_report = mock.MagicMock(data={}) + + # test remote instance + self.Patch(os.environ, "get", return_value=True) + fake_report.data = { + "devices": [{"instance_name": "remote_cf_instance_name", + "ip": "192.168.1.1",},],} + + utils.LaunchBrowserFromReport(fake_report) + webbrowser.open_new_tab.assert_called_once_with("https://localhost:8443") + webbrowser.open_new_tab.call_count = 0 + + # test local instance + fake_report.data = { + "devices": [{"instance_name": "local-instance1", + "ip": "127.0.0.1:6250",},],} + utils.LaunchBrowserFromReport(fake_report) + webbrowser.open_new_tab.assert_called_once_with("https://localhost:8443") + webbrowser.open_new_tab.call_count = 0 + + # verify terminal can't support launch webbrowser. + self.Patch(os.environ, "get", return_value=False) + utils.LaunchBrowserFromReport(fake_report) + self.assertEqual(webbrowser.open_new_tab.call_count, 0) + + def testSetExecutable(self): + """test setting a file to be executable.""" + with tempfile.NamedTemporaryFile(delete=True) as temp_file: + utils.SetExecutable(temp_file.name) + self.assertEqual(os.stat(temp_file.name).st_mode & 0o777, 0o755) + + def testSetDirectoryTreeExecutable(self): + """test setting a file in a directory to be executable.""" + with tempfile.TemporaryDirectory() as temp_dir: + subdir = os.path.join(temp_dir, "subdir") + file_path = os.path.join(subdir, "file") + os.makedirs(subdir) + with open(file_path, "w"): + pass + utils.SetDirectoryTreeExecutable(temp_dir) + self.assertEqual(os.stat(file_path).st_mode & 0o777, 0o755) + if __name__ == "__main__": unittest.main() diff --git a/internal/proto/user_config.proto b/internal/proto/user_config.proto index 1f7688a7..0dec7717 100755 --- a/internal/proto/user_config.proto +++ b/internal/proto/user_config.proto @@ -62,49 +62,60 @@ message UserConfig { // [CVD only] The name of the stable host image released by Cloud Android team optional string stable_host_image_name = 16; + // [CVD only] The name of the host image family released by Cloud Android team + optional string stable_host_image_family = 17; // [CVD only] The project that the stable host image is released to - optional string stable_host_image_project = 17; + optional string stable_host_image_project = 18; // [GOLDFISH only] The name of the stable host image released by Android // Emulator (emu-dev) team - optional string stable_goldfish_host_image_name = 18; + optional string stable_goldfish_host_image_name = 19; // [GOLDFISH only] The project that the stable goldfish host image is // released to (emu-dev-cts) - optional string stable_goldfish_host_image_project = 19; + optional string stable_goldfish_host_image_project = 20; // Account information for accessing Cloud API // This is the new way to provide service account auth. - optional string service_account_json_private_key_path = 20; + optional string service_account_json_private_key_path = 21; // Desired hw_property - optional string hw_property = 21; + optional string hw_property = 22; // [CHEEPS only] The name of the stable host image released by the ARC // (arc-eng) team - optional string stable_cheeps_host_image_name = 22; + optional string stable_cheeps_host_image_name = 23; // [CHEEPS only] The project that the stable host image is released to - optional string stable_cheeps_host_image_project = 23; + optional string stable_cheeps_host_image_project = 24; // [CVD only] It will get passed into the launch_cvd command if not empty. // In version 0.7.2 and later. - optional string launch_args = 24; + optional string launch_args = 25; // The pattern of the instance name, e.g. ins-{uuid}-{build_id}-{build_target} // the parts in {} will be automatically replaced with the actual value if // you specify them in the pattern, uuid will be automatically generated. - optional string instance_name_pattern = 25; + optional string instance_name_pattern = 26; // List of scopes that will be given to the instance // https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#changeserviceaccountandscopes - repeated string extra_scopes = 26; + repeated string extra_scopes = 27; // Provide some additional parameters to build the ssh tunnel. - optional string extra_args_ssh_tunnel = 27; + optional string extra_args_ssh_tunnel = 28; // [CVD only] Version of fetch_cvd to use. - optional string fetch_cvd_version = 28; + optional string fetch_cvd_version = 29; // [CVD only] Enable multi stage function. - optional bool enable_multi_stage = 29; + optional bool enable_multi_stage = 30; + + // [CHEEPS only] The name of the L1 betty image (used with Cheeps controller) + optional string betty_image = 31; + + // [Oxygen only] The OAuth Credentials of API key. + optional string api_key = 32; + + // [Oxygen only] The API service url. + optional string api_url = 33; } diff --git a/list/instance.py b/list/instance.py index fb98960a..30473907 100644 --- a/list/instance.py +++ b/list/instance.py @@ -38,10 +38,12 @@ import tempfile import dateutil.parser import dateutil.tz +from acloud.create import local_image_local_instance from acloud.internal import constants from acloud.internal.lib import cvd_runtime_config from acloud.internal.lib import utils from acloud.internal.lib.adb_tools import AdbTools +from acloud.internal.lib.local_instance_lock import LocalInstanceLock logger = logging.getLogger(__name__) @@ -49,7 +51,11 @@ logger = logging.getLogger(__name__) _ACLOUD_CVD_TEMP = os.path.join(tempfile.gettempdir(), "acloud_cvd_temp") _CVD_RUNTIME_FOLDER_NAME = "cuttlefish_runtime" _CVD_STATUS_BIN = "cvd_status" +_LOCAL_INSTANCE_NAME_FORMAT = "local-instance-%(id)d" +_LOCAL_INSTANCE_NAME_PATTERN = re.compile(r"^local-instance-(?P<id>\d+)$") +_ACLOUDWEB_INSTANCE_START_STRING = "cf-" _MSG_UNABLE_TO_CALCULATE = "Unable to calculate" +_NO_ANDROID_ENV = "android source not available" _RE_GROUP_ADB = "local_adb_port" _RE_GROUP_VNC = "local_vnc_port" _RE_SSH_TUNNEL_PATTERN = (r"((.*\s*-L\s)(?P<%s>\d+):127.0.0.1:%s)" @@ -75,8 +81,11 @@ def GetDefaultCuttlefishConfig(): Return: String, path of cf runtime config. """ - return os.path.join(os.path.expanduser("~"), _CVD_RUNTIME_FOLDER_NAME, - constants.CUTTLEFISH_CONFIG_FILE) + cfg_path = os.path.join(os.path.expanduser("~"), _CVD_RUNTIME_FOLDER_NAME, + constants.CUTTLEFISH_CONFIG_FILE) + if os.path.isfile(cfg_path): + return cfg_path + return None def GetLocalInstanceName(local_instance_id): @@ -88,7 +97,23 @@ def GetLocalInstanceName(local_instance_id): Return: String, the instance name. """ - return "%s-%d" % (constants.LOCAL_INS_NAME, local_instance_id) + return _LOCAL_INSTANCE_NAME_FORMAT % {"id": local_instance_id} + + +def GetLocalInstanceIdByName(name): + """Get local cuttlefish instance id by name. + + Args: + name: String of instance name. + + Return: + The instance id as an integer if the name is in valid format. + None if the name does not represent a local cuttlefish instance. + """ + match = _LOCAL_INSTANCE_NAME_PATTERN.match(name) + if match: + return int(match.group("id")) + return None def GetLocalInstanceConfig(local_instance_id): @@ -108,27 +133,27 @@ def GetLocalInstanceConfig(local_instance_id): def GetAllLocalInstanceConfigs(): - """Get the list of instance config. + """Get all cuttlefish runtime configs from the known locations. Return: - List of instance config path. + List of tuples. Each tuple consists of an instance id and a config + path. """ - cfg_list = [] + id_cfg_pairs = [] # Check if any instance config is under home folder. cfg_path = GetDefaultCuttlefishConfig() - if os.path.isfile(cfg_path): - cfg_list.append(cfg_path) + if cfg_path: + id_cfg_pairs.append((1, cfg_path)) # Check if any instance config is under acloud cvd temp folder. if os.path.exists(_ACLOUD_CVD_TEMP): for ins_name in os.listdir(_ACLOUD_CVD_TEMP): - cfg_path = os.path.join(_ACLOUD_CVD_TEMP, - ins_name, - _CVD_RUNTIME_FOLDER_NAME, - constants.CUTTLEFISH_CONFIG_FILE) - if os.path.isfile(cfg_path): - cfg_list.append(cfg_path) - return cfg_list + ins_id = GetLocalInstanceIdByName(ins_name) + if ins_id is not None: + cfg_path = GetLocalInstanceConfig(ins_id) + if cfg_path: + id_cfg_pairs.append((ins_id, cfg_path)) + return id_cfg_pairs def GetLocalInstanceHomeDir(local_instance_id): @@ -144,6 +169,20 @@ def GetLocalInstanceHomeDir(local_instance_id): GetLocalInstanceName(local_instance_id)) +def GetLocalInstanceLock(local_instance_id): + """Get local instance lock. + + Args: + local_instance_id: Integer of instance id. + + Returns: + LocalInstanceLock object. + """ + file_path = os.path.join(_ACLOUD_CVD_TEMP, + GetLocalInstanceName(local_instance_id) + ".lock") + return LocalInstanceLock(file_path) + + def GetLocalInstanceRuntimeDir(local_instance_id): """Get instance runtime dir @@ -186,6 +225,7 @@ def _GetElapsedTime(start_time): return _MSG_UNABLE_TO_CALCULATE +# pylint: disable=useless-object-inheritance class Instance(object): """Class to store data of instance.""" @@ -193,7 +233,8 @@ class Instance(object): def __init__(self, name, fullname, display, ip, status=None, adb_port=None, vnc_port=None, ssh_tunnel_is_connected=None, createtime=None, elapsed_time=None, avd_type=None, avd_flavor=None, - is_local=False, device_information=None, zone=None): + is_local=False, device_information=None, zone=None, + webrtc_port=None): self._name = name self._fullname = fullname self._status = status @@ -201,6 +242,7 @@ class Instance(object): self._ip = ip self._adb_port = adb_port # adb port which is forwarding to remote self._vnc_port = vnc_port # vnc port which is forwarding to remote + self._webrtc_port = webrtc_port # True if ssh tunnel is still connected self._ssh_tunnel_is_connected = ssh_tunnel_is_connected self._createtime = createtime @@ -227,6 +269,7 @@ class Instance(object): representation.append("%s display: %s" % (_INDENT, self._display)) representation.append("%s vnc: 127.0.0.1:%s" % (_INDENT, self._vnc_port)) representation.append("%s zone: %s" % (_INDENT, self._zone)) + representation.append("%s webrtc port: %s" % (_INDENT, self._webrtc_port)) if self._adb_port and self._device_information: representation.append("%s adb serial: 127.0.0.1:%s" % @@ -244,6 +287,16 @@ class Instance(object): return "\n".join(representation) + def AdbConnected(self): + """Check AVD adb connected. + + Returns: + Boolean, True when adb status of AVD is connected. + """ + if self._adb_port and self._device_information: + return True + return False + @property def name(self): """Return the instance name.""" @@ -305,6 +358,11 @@ class Instance(object): return self._vnc_port @property + def webrtc_port(self): + """Return webrtc_port.""" + return self._webrtc_port + + @property def zone(self): """Return zone.""" return self._zone @@ -330,27 +388,29 @@ class LocalInstance(Instance): # cuttlefish_config.json so far. name = GetLocalInstanceName(self._local_instance_id) fullname = (_FULL_NAME_STRING % - {"device_serial": "127.0.0.1:%s" % self._cf_runtime_cfg.adb_port, + {"device_serial": "0.0.0.0:%s" % self._cf_runtime_cfg.adb_port, "instance_name": name, "elapsed_time": None}) adb_device = AdbTools(self._cf_runtime_cfg.adb_port) + webrtc_port = local_image_local_instance.LocalImageLocalInstance.GetWebrtcSigServerPort( + self._local_instance_id) device_information = None if adb_device.IsAdbConnected(): device_information = adb_device.device_information - super(LocalInstance, self).__init__( - name=name, fullname=fullname, display=display, ip="127.0.0.1", + super().__init__( + name=name, fullname=fullname, display=display, ip="0.0.0.0", status=constants.INS_STATUS_RUNNING, adb_port=self._cf_runtime_cfg.adb_port, vnc_port=self._cf_runtime_cfg.vnc_port, createtime=None, elapsed_time=None, avd_type=constants.TYPE_CF, is_local=True, device_information=device_information, - zone=_LOCAL_ZONE) + zone=_LOCAL_ZONE, webrtc_port=webrtc_port) def Summary(self): """Return the string that this class is holding.""" instance_home = "%s instance home: %s" % (_INDENT, self._instance_dir) - return "%s\n%s" % (super(LocalInstance, self).Summary(), instance_home) + return "%s\n%s" % (super().Summary(), instance_home) def CvdStatus(self): """check if local instance is active. @@ -360,6 +420,10 @@ class LocalInstance(Instance): Returns True if instance is active. """ + if not self._cf_runtime_cfg.cvd_tools_path: + logger.debug("No cvd tools path found from config:%s", + self._cf_runtime_cfg.config_path) + return False cvd_env = os.environ.copy() cvd_env[constants.ENV_CUTTLEFISH_CONFIG_FILE] = self._cf_runtime_cfg.config_path cvd_env[constants.ENV_CVD_HOME] = GetLocalInstanceHomeDir(self._local_instance_id) @@ -367,6 +431,20 @@ class LocalInstance(Instance): try: cvd_status_cmd = os.path.join(self._cf_runtime_cfg.cvd_tools_path, _CVD_STATUS_BIN) + # TODO(b/150575261): Change the cvd home and cvd artifact path to + # another place instead of /tmp to prevent from the file not + # found exception. + if not os.path.exists(cvd_status_cmd): + logger.warning("Cvd tools path doesn't exist:%s", cvd_status_cmd) + for env_host_out in [constants.ENV_ANDROID_SOONG_HOST_OUT, + constants.ENV_ANDROID_HOST_OUT]: + if os.environ.get(env_host_out, _NO_ANDROID_ENV) in cvd_status_cmd: + logger.warning( + "Can't find the cvd_status tool (Try lunching a " + "cuttlefish target like aosp_cf_x86_phone-userdebug " + "and running 'make hosttar' before list/delete local " + "instances)") + return False logger.debug("Running cmd[%s] to check cvd status.", cvd_status_cmd) process = subprocess.Popen(cvd_status_cmd, stdin=None, @@ -397,26 +475,29 @@ class LocalInstance(Instance): stop_cvd_cmd = os.path.join(self.cf_runtime_cfg.cvd_tools_path, constants.CMD_STOP_CVD) logger.debug("Running cmd[%s] to delete local cvd", stop_cvd_cmd) - with open(os.devnull, "w") as dev_null: - cvd_env = os.environ.copy() - if self.instance_dir: - cvd_env[constants.ENV_CUTTLEFISH_CONFIG_FILE] = self._cf_runtime_cfg.config_path - cvd_env[constants.ENV_CVD_HOME] = GetLocalInstanceHomeDir( - self._local_instance_id) - cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str(self._local_instance_id) - else: - logger.error("instance_dir is null!! instance[%d] might not be" - " deleted", self._local_instance_id) - subprocess.check_call( - utils.AddUserGroupsToCmd(stop_cvd_cmd, - constants.LIST_CF_USER_GROUPS), - stderr=dev_null, stdout=dev_null, shell=True, env=cvd_env) + cvd_env = os.environ.copy() + if self.instance_dir: + cvd_env[constants.ENV_CUTTLEFISH_CONFIG_FILE] = self._cf_runtime_cfg.config_path + cvd_env[constants.ENV_CVD_HOME] = GetLocalInstanceHomeDir( + self._local_instance_id) + cvd_env[constants.ENV_CUTTLEFISH_INSTANCE] = str(self._local_instance_id) + else: + logger.error("instance_dir is null!! instance[%d] might not be" + " deleted", self._local_instance_id) + subprocess.check_call( + utils.AddUserGroupsToCmd(stop_cvd_cmd, + constants.LIST_CF_USER_GROUPS), + stderr=subprocess.STDOUT, shell=True, env=cvd_env) adb_cmd = AdbTools(self.adb_port) # When relaunch a local instance, we need to pass in retry=True to make # sure adb device is completely gone since it will use the same adb port adb_cmd.DisconnectAdb(retry=True) + def GetLock(self): + """Return the LocalInstanceLock for this object.""" + return GetLocalInstanceLock(self._local_instance_id) + @property def instance_dir(self): """Return _instance_dir.""" @@ -439,14 +520,23 @@ class LocalInstance(Instance): class LocalGoldfishInstance(Instance): - """Class to store data of local goldfish instance.""" + """Class to store data of local goldfish instance. + + A goldfish instance binds to a console port and an adb port. The console + port is for `adb emu` to send emulator-specific commands. The adb port is + for `adb connect` to start a TCP connection. By convention, the console + port is an even number, and the adb port is the console port + 1. The first + instance uses port 5554 and 5555, the second instance uses 5556 and 5557, + and so on. + """ _INSTANCE_NAME_PATTERN = re.compile( r"^local-goldfish-instance-(?P<id>\d+)$") - _CREATION_TIMESTAMP_FILE_NAME = "creation_timestamp.txt" _INSTANCE_NAME_FORMAT = "local-goldfish-instance-%(id)s" _EMULATOR_DEFAULT_CONSOLE_PORT = 5554 - _GF_ADB_DEVICE_SERIAL = "emulator-%(console_port)s" + _DEFAULT_ADB_LOCAL_TRANSPORT_MAX_PORT = 5585 + _DEVICE_SERIAL_FORMAT = "emulator-%(console_port)s" + _DEVICE_SERIAL_PATTERN = re.compile(r"^emulator-(?P<console_port>\d+)$") def __init__(self, local_instance_id, avd_flavor=None, create_time=None, x_res=None, y_res=None, dpi=None): @@ -461,8 +551,9 @@ class LocalGoldfishInstance(Instance): dpi: Integer of dpi. """ self._id = local_instance_id - # By convention, adb port is console port + 1. adb_port = self.console_port + 1 + self._adb = AdbTools(adb_port=adb_port, + device_serial=self.device_serial) name = self._INSTANCE_NAME_FORMAT % {"id": local_instance_id} @@ -478,11 +569,10 @@ class LocalGoldfishInstance(Instance): else: display = "unknown" - adb = AdbTools(adb_port) - device_information = (adb.device_information if - adb.device_information else None) + device_information = (self._adb.device_information if + self._adb.device_information else None) - super(LocalGoldfishInstance, self).__init__( + super().__init__( name=name, fullname=fullname, display=display, ip="127.0.0.1", status=None, adb_port=adb_port, avd_type=constants.TYPE_GF, createtime=create_time, elapsed_time=elapsed_time, @@ -495,15 +585,20 @@ class LocalGoldfishInstance(Instance): return os.path.join(tempfile.gettempdir(), "acloud_gf_temp") @property + def adb(self): + """Return the AdbTools to send emulator commands to this instance.""" + return self._adb + + @property def console_port(self): - """Return the console port as an integer""" + """Return the console port as an integer.""" # Emulator requires the console port to be an even number. return self._EMULATOR_DEFAULT_CONSOLE_PORT + (self._id - 1) * 2 @property def device_serial(self): """Return the serial number that contains the console port.""" - return self._GF_ADB_DEVICE_SERIAL % {"console_port": self.console_port} + return self._DEVICE_SERIAL_FORMAT % {"console_port": self.console_port} @property def instance_dir(self): @@ -511,53 +606,61 @@ class LocalGoldfishInstance(Instance): return os.path.join(self._GetInstanceDirRoot(), self._INSTANCE_NAME_FORMAT % {"id": self._id}) - @property - def creation_timestamp_path(self): - """Return the file path containing the creation timestamp.""" - return os.path.join(self.instance_dir, - self._CREATION_TIMESTAMP_FILE_NAME) - - def WriteCreationTimestamp(self): - """Write creation timestamp to file.""" - with open(self.creation_timestamp_path, "w") as timestamp_file: - timestamp_file.write(str(_GetCurrentLocalTime())) - - def DeleteCreationTimestamp(self, ignore_errors): - """Delete the creation timestamp file. + @classmethod + def GetIdByName(cls, name): + """Get id by name. Args: - ignore_errors: Boolean, whether to ignore the errors. + name: String of instance name. - Raises: - OSError if fails to delete the file. + Return: + The instance id as an integer if the name is in valid format. + None if the name does not represent a local goldfish instance. """ - try: - os.remove(self.creation_timestamp_path) - except OSError as e: - if not ignore_errors: - raise - logger.warning("Can't delete creation timestamp: %s", e) + match = cls._INSTANCE_NAME_PATTERN.match(name) + if match: + return int(match.group("id")) + return None @classmethod - def GetExistingInstances(cls): - """Get a list of instances that have creation timestamp files.""" - instance_root = cls._GetInstanceDirRoot() - if not os.path.isdir(instance_root): - return [] + def GetLockById(cls, instance_id): + """Get LocalInstanceLock by id.""" + lock_path = os.path.join( + cls._GetInstanceDirRoot(), + (cls._INSTANCE_NAME_FORMAT % {"id": instance_id}) + ".lock") + return LocalInstanceLock(lock_path) + def GetLock(self): + """Return the LocalInstanceLock for this object.""" + return self.GetLockById(self._id) + + @classmethod + def GetExistingInstances(cls): + """Get the list of instances that adb can send emu commands to.""" instances = [] - for name in os.listdir(instance_root): - match = cls._INSTANCE_NAME_PATTERN.match(name) - timestamp_path = os.path.join(instance_root, name, - cls._CREATION_TIMESTAMP_FILE_NAME) - if match and os.path.isfile(timestamp_path): - instance_id = int(match.group("id")) - with open(timestamp_path, "r") as timestamp_file: - timestamp = timestamp_file.read().strip() - instances.append(LocalGoldfishInstance(instance_id, - create_time=timestamp)) + for serial in AdbTools.GetDeviceSerials(): + match = cls._DEVICE_SERIAL_PATTERN.match(serial) + if not match: + continue + port = int(match.group("console_port")) + instance_id = (port - cls._EMULATOR_DEFAULT_CONSOLE_PORT) // 2 + 1 + instances.append(LocalGoldfishInstance(instance_id)) return instances + @classmethod + def GetMaxNumberOfInstances(cls): + """Get number of emulators that adb can detect.""" + max_port = os.environ.get("ADB_LOCAL_TRANSPORT_MAX_PORT", + cls._DEFAULT_ADB_LOCAL_TRANSPORT_MAX_PORT) + try: + max_port = int(max_port) + except ValueError: + max_port = cls._DEFAULT_ADB_LOCAL_TRANSPORT_MAX_PORT + if (max_port < cls._EMULATOR_DEFAULT_CONSOLE_PORT or + max_port > constants.MAX_PORT): + max_port = cls._DEFAULT_ADB_LOCAL_TRANSPORT_MAX_PORT + return (max_port + 1 - cls._EMULATOR_DEFAULT_CONSOLE_PORT) // 2 + class RemoteInstance(Instance): """Class to store data of remote instance.""" @@ -609,6 +712,9 @@ class RemoteInstance(Instance): avd_type = value elif key == constants.INS_KEY_AVD_FLAVOR: avd_flavor = value + # TODO(176884236): Insert avd information into metadata of instance. + if not avd_type and name.startswith(_ACLOUDWEB_INSTANCE_START_STRING): + avd_type = constants.TYPE_CF # Find ssl tunnel info. adb_port = None @@ -640,14 +746,15 @@ class RemoteInstance(Instance): "instance_name": name, "elapsed_time": elapsed_time}) - super(RemoteInstance, self).__init__( + super().__init__( name=name, fullname=fullname, display=display, ip=ip, status=status, adb_port=adb_port, vnc_port=vnc_port, ssh_tunnel_is_connected=ssh_tunnel_is_connected, createtime=create_time, elapsed_time=elapsed_time, avd_type=avd_type, avd_flavor=avd_flavor, is_local=False, device_information=device_information, - zone=zone) + zone=zone, + webrtc_port=constants.WEBRTC_LOCAL_PORT) @staticmethod def _GetZoneName(zone_info): @@ -687,12 +794,14 @@ class RemoteInstance(Instance): default_vnc_port = utils.AVD_PORT_DICT[avd_type].vnc_port default_adb_port = utils.AVD_PORT_DICT[avd_type].adb_port + # TODO(165888525): Align the SSH tunnel for the order of adb port and + # vnc port. re_pattern = re.compile(_RE_SSH_TUNNEL_PATTERN % - (_RE_GROUP_VNC, default_vnc_port, - _RE_GROUP_ADB, default_adb_port, ip)) + (_RE_GROUP_ADB, default_adb_port, + _RE_GROUP_VNC, default_vnc_port, ip)) adb_port = None vnc_port = None - process_output = subprocess.check_output(constants.COMMAND_PS) + process_output = utils.CheckOutput(constants.COMMAND_PS) for line in process_output.splitlines(): match = re_pattern.match(line) if match: diff --git a/list/instance_test.py b/list/instance_test.py index a2c79224..de734100 100644 --- a/list/instance_test.py +++ b/list/instance_test.py @@ -18,9 +18,10 @@ import collections import datetime import subprocess - import unittest -import mock + +from unittest import mock +from six import b # pylint: disable=import-error import dateutil.parser @@ -29,20 +30,19 @@ import dateutil.tz from acloud.internal import constants from acloud.internal.lib import cvd_runtime_config from acloud.internal.lib import driver_test_lib +from acloud.internal.lib import utils from acloud.internal.lib.adb_tools import AdbTools from acloud.list import instance class InstanceTest(driver_test_lib.BaseDriverTest): """Test instance.""" - PS_SSH_TUNNEL = ("/fake_ps_1 --fake arg \n" - "/fake_ps_2 --fake arg \n" - "/usr/bin/ssh -i ~/.ssh/acloud_rsa " - "-o UserKnownHostsFile=/dev/null " - "-o StrictHostKeyChecking=no -L 12345:127.0.0.1:6444 " - "-L 54321:127.0.0.1:6520 -N -f -l user 1.1.1.1") - PS_LAUNCH_CVD = ("Sat Nov 10 21:55:10 2018 /fake_path/bin/run_cvd ") - PS_RUNTIME_CF_CONFIG = {"x_res": "1080", "y_res": "1920", "dpi": "480"} + PS_SSH_TUNNEL = b("/fake_ps_1 --fake arg \n" + "/fake_ps_2 --fake arg \n" + "/usr/bin/ssh -i ~/.ssh/acloud_rsa " + "-o UserKnownHostsFile=/dev/null " + "-o StrictHostKeyChecking=no -L 54321:127.0.0.1:6520 " + "-L 12345:127.0.0.1:6444 -N -f -l user 1.1.1.1") GCE_INSTANCE = { constants.INS_KEY_NAME: "fake_ins_name", constants.INS_KEY_CREATETIME: "fake_create_time", @@ -58,11 +58,10 @@ class InstanceTest(driver_test_lib.BaseDriverTest): "value":"fake_flavor"}]} } - # pylint: disable=protected-access - def testCreateLocalInstance(self): - """"Test get local instance info from launch_cvd process.""" - self.Patch(subprocess, "check_output", return_value=self.PS_LAUNCH_CVD) - cf_config = mock.MagicMock( + @staticmethod + def _MockCvdRuntimeConfig(): + """Create a mock CvdRuntimeConfig.""" + return mock.MagicMock( instance_id=2, x_res=1080, y_res=1920, @@ -71,21 +70,57 @@ class InstanceTest(driver_test_lib.BaseDriverTest): adb_port=6521, vnc_port=6445, adb_ip_port="127.0.0.1:6521", + cvd_tools_path="fake_cvd_tools_path", + config_path="fake_config_path", ) + + @mock.patch("acloud.list.instance.AdbTools") + def testCreateLocalInstance(self, mock_adb_tools): + """Test getting local instance info from cvd runtime config.""" + mock_adb_tools_object = mock.Mock(device_information={}) + mock_adb_tools_object.IsAdbConnected.return_value = True + mock_adb_tools.return_value = mock_adb_tools_object self.Patch(cvd_runtime_config, "CvdRuntimeConfig", - return_value=cf_config) - local_instance = instance.LocalInstance(cf_config) + return_value=self._MockCvdRuntimeConfig()) + local_instance = instance.LocalInstance("fake_config_path") - self.assertEqual(constants.LOCAL_INS_NAME + "-2", local_instance.name) + self.assertEqual("local-instance-2", local_instance.name) self.assertEqual(True, local_instance.islocal) self.assertEqual("1080x1920 (480)", local_instance.display) - expected_full_name = ("device serial: 127.0.0.1:%s (%s) elapsed time: %s" + expected_full_name = ("device serial: 0.0.0.0:%s (%s) elapsed time: %s" % ("6521", - constants.LOCAL_INS_NAME + "-2", + "local-instance-2", "None")) self.assertEqual(expected_full_name, local_instance.fullname) self.assertEqual(6521, local_instance.adb_port) self.assertEqual(6445, local_instance.vnc_port) + self.assertEqual(8444, local_instance.webrtc_port) + + @mock.patch("acloud.list.instance.AdbTools") + def testDeleteLocalInstance(self, mock_adb_tools): + """Test executing stop_cvd command.""" + self.Patch(cvd_runtime_config, "CvdRuntimeConfig", + return_value=self._MockCvdRuntimeConfig()) + mock_adb_tools_object = mock.Mock(device_information={}) + mock_adb_tools_object.IsAdbConnected.return_value = True + mock_adb_tools.return_value = mock_adb_tools_object + self.Patch(utils, "AddUserGroupsToCmd", + side_effect=lambda cmd, groups: cmd) + mock_check_call = self.Patch(subprocess, "check_call") + + local_instance = instance.LocalInstance("fake_config_path") + with mock.patch.dict("acloud.list.instance.os.environ", clear=True): + local_instance.Delete() + + expected_env = { + 'CUTTLEFISH_INSTANCE': '2', + 'HOME': '/tmp/acloud_cvd_temp/local-instance-2', + 'CUTTLEFISH_CONFIG_FILE': 'fake_config_path', + } + mock_check_call.assert_called_with( + 'fake_cvd_tools_path/stop_cvd', stderr=subprocess.STDOUT, + shell=True, env=expected_env) + mock_adb_tools_object.DisconnectAdb.assert_called() @mock.patch("acloud.list.instance.tempfile") @mock.patch("acloud.list.instance.AdbTools") @@ -105,59 +140,35 @@ class InstanceTest(driver_test_lib.BaseDriverTest): self.assertEqual(inst.instance_dir, "/unit/test/acloud_gf_temp/local-goldfish-instance-1") - @mock.patch("acloud.list.instance.open", - mock.mock_open(read_data="test createtime")) - @mock.patch("acloud.list.instance.os.path.isfile") - @mock.patch("acloud.list.instance.os.listdir") - @mock.patch("acloud.list.instance.os.path.isdir") - @mock.patch("acloud.list.instance.tempfile") @mock.patch("acloud.list.instance.AdbTools") - @mock.patch("acloud.list.instance._GetElapsedTime") - def testGetLocalGoldfishInstances(self, mock_get_elapsed_time, - mock_adb_tools, mock_tempfile, - mock_isdir, mock_listdir, mock_isfile): + def testGetLocalGoldfishInstances(self, mock_adb_tools): """Test LocalGoldfishInstance.GetExistingInstances.""" - mock_get_elapsed_time.return_value = datetime.timedelta(hours=10) - mock_adb_tools.return_value = mock.Mock(device_information={}) - mock_tempfile.gettempdir.return_value = "/unit/test" - acloud_gf_temp_path = "/unit/test/acloud_gf_temp" - subdir_names = ( - "local-goldfish-instance-1", - "local-goldfish-instance-2", - "local-goldfish-instance-3") - timestamp_paths = ( - "/unit/test/acloud_gf_temp/local-goldfish-instance-1/" - "creation_timestamp.txt", - "/unit/test/acloud_gf_temp/local-goldfish-instance-2/" - "creation_timestamp.txt", - "/unit/test/acloud_gf_temp/local-goldfish-instance-3/" - "creation_timestamp.txt") - mock_isdir.side_effect = lambda path: path == acloud_gf_temp_path - mock_listdir.side_effect = lambda path: ( - subdir_names if path == acloud_gf_temp_path else []) - mock_isfile.side_effect = lambda path: ( - path in (timestamp_paths[0], timestamp_paths[2])) + mock_adb_tools.GetDeviceSerials.return_value = [ + "127.0.0.1:6520", "emulator-5554", "ABCD", "emulator-5558"] instances = instance.LocalGoldfishInstance.GetExistingInstances() - mock_isdir.assert_called_with(acloud_gf_temp_path) - mock_listdir.assert_called_with(acloud_gf_temp_path) - for timestamp_path in timestamp_paths: - mock_isfile.assert_any_call(timestamp_path) self.assertEqual(len(instances), 2) self.assertEqual(instances[0].console_port, 5554) - self.assertEqual(instances[0].createtime, "test createtime") - self.assertEqual(instances[0].fullname, - "device serial: emulator-5554 " - "(local-goldfish-instance-1) " - "elapsed time: 10:00:00") + self.assertEqual(instances[0].name, "local-goldfish-instance-1") self.assertEqual(instances[1].console_port, 5558) - self.assertEqual(instances[1].createtime, "test createtime") - self.assertEqual(instances[1].fullname, - "device serial: emulator-5558 " - "(local-goldfish-instance-3) " - "elapsed time: 10:00:00") + self.assertEqual(instances[1].name, "local-goldfish-instance-3") + + def testGetMaxNumberOfGoldfishInstances(self): + """Test LocalGoldfishInstance.GetMaxNumberOfInstances.""" + mock_environ = {} + with mock.patch.dict("acloud.list.instance.os.environ", + mock_environ, clear=True): + num = instance.LocalGoldfishInstance.GetMaxNumberOfInstances() + self.assertEqual(num, 16) + mock_environ["ADB_LOCAL_TRANSPORT_MAX_PORT"] = "5565" + with mock.patch.dict("acloud.list.instance.os.environ", + mock_environ, clear=True): + num = instance.LocalGoldfishInstance.GetMaxNumberOfInstances() + self.assertEqual(num, 6) + + # pylint: disable=protected-access def testGetElapsedTime(self): """Test _GetElapsedTime""" # Instance time can't parse @@ -270,6 +281,7 @@ class InstanceTest(driver_test_lib.BaseDriverTest): " display: None\n " " vnc: 127.0.0.1:654321\n " " zone: fake_zone\n " + " webrtc port: 8443\n " " adb serial: 127.0.0.1:123456\n " " product: None\n " " model: None\n " @@ -293,6 +305,7 @@ class InstanceTest(driver_test_lib.BaseDriverTest): " display: None\n " " vnc: 127.0.0.1:None\n " " zone: fake_zone\n " + " webrtc port: 8443\n " " adb serial: disconnected") self.assertEqual(remote_instance.Summary(), result_summary) diff --git a/list/list.py b/list/list.py index febd6f39..6ccbac1a 100644 --- a/list/list.py +++ b/list/list.py @@ -49,6 +49,20 @@ def _ProcessInstances(instance_list): return [instance.RemoteInstance(gce_instance) for gce_instance in instance_list] +def _SortInstancesForDisplay(instances): + """Sort the instances by connected first and then by age. + + Args: + instances: List of instance.Instance() + + Returns: + List of instance.Instance() after sorted. + """ + instances.sort(key=lambda ins: ins.createtime, reverse=True) + instances.sort(key=lambda ins: ins.AdbConnected(), reverse=True) + return instances + + def PrintInstancesDetails(instance_list, verbose=False): """Display instances information. @@ -110,31 +124,47 @@ def GetRemoteInstances(cfg): logger.debug("Instance list from: (filter: %s\n%s):", filter_item, all_instances) - return _ProcessInstances(all_instances) + return _SortInstancesForDisplay(_ProcessInstances(all_instances)) -def _GetLocalCuttlefishInstances(): +def _GetLocalCuttlefishInstances(id_cfg_pairs): """Look for local cuttelfish instances. Gather local instances information from cuttlefish runtime config. + Args: + id_cfg_pairs: List of tuples. Each tuple consists of an instance id and + a config path. + Returns: instance_list: List of local instances. """ local_instance_list = [] - for cf_runtime_config_path in instance.GetAllLocalInstanceConfigs(): - ins = instance.LocalInstance(cf_runtime_config_path) - if ins.CvdStatus(): - local_instance_list.append(ins) - else: - logger.info("cvd runtime config found but instance is not active:%s" - , cf_runtime_config_path) + for ins_id, cfg_path in id_cfg_pairs: + ins_lock = instance.GetLocalInstanceLock(ins_id) + if not ins_lock.Lock(): + logger.warning("Cuttlefish Instance %d is locked by another " + "process.", ins_id) + continue + try: + if not os.path.isfile(cfg_path): + continue + ins = instance.LocalInstance(cfg_path) + if ins.CvdStatus(): + local_instance_list.append(ins) + else: + logger.info("Cvd runtime config is found at %s but instance " + "%d is not active.", cfg_path, ins_id) + finally: + ins_lock.Unlock() return local_instance_list def GetActiveCVD(local_instance_id): """Check if the local AVD with specific instance id is running + This function does not lock the instance. + Args: local_instance_id: Integer of instance id. @@ -147,7 +177,7 @@ def GetActiveCVD(local_instance_id): if ins.CvdStatus(): return ins cfg_path = instance.GetDefaultCuttlefishConfig() - if local_instance_id == 1 and os.path.isfile(cfg_path): + if local_instance_id == 1 and cfg_path: ins = instance.LocalInstance(cfg_path) if ins.CvdStatus(): return ins @@ -164,7 +194,8 @@ def GetLocalInstances(): if not utils.IsSupportedPlatform(): return [] - return (_GetLocalCuttlefishInstances() + + id_cfg_pairs = instance.GetAllLocalInstanceConfigs() + return (_GetLocalCuttlefishInstances(id_cfg_pairs) + instance.LocalGoldfishInstance.GetExistingInstances()) @@ -244,7 +275,7 @@ def ChooseOneRemoteInstance(cfg): return instances_list[0] -def FilterInstancesByNames(instances, names): +def _FilterInstancesByNames(instances, names): """Find instances by names. Args: @@ -272,6 +303,57 @@ def FilterInstancesByNames(instances, names): return found_instances +def GetLocalInstanceLockByName(name): + """Get the lock of a local cuttelfish or goldfish instance. + + Args: + name: The instance name. + + Returns: + LocalInstanceLock object. None if the name is invalid. + """ + cf_id = instance.GetLocalInstanceIdByName(name) + if cf_id is not None: + return instance.GetLocalInstanceLock(cf_id) + + gf_id = instance.LocalGoldfishInstance.GetIdByName(name) + if gf_id is not None: + return instance.LocalGoldfishInstance.GetLockById(gf_id) + + return None + + +def GetLocalInstancesByNames(names): + """Get local cuttlefish and goldfish instances by names. + + This method does not raise an error if it cannot find all instances. + + Args: + names: Collection of instance names. + + Returns: + List consisting of LocalInstance and LocalGoldfishInstance objects. + """ + id_cfg_pairs = [] + for name in names: + ins_id = instance.GetLocalInstanceIdByName(name) + if ins_id is None: + continue + cfg_path = instance.GetLocalInstanceConfig(ins_id) + if cfg_path: + id_cfg_pairs.append((ins_id, cfg_path)) + if ins_id == 1: + cfg_path = instance.GetDefaultCuttlefishConfig() + if cfg_path: + id_cfg_pairs.append((ins_id, cfg_path)) + + gf_instances = [ins for ins in + instance.LocalGoldfishInstance.GetExistingInstances() + if ins.name in names] + + return _GetLocalCuttlefishInstances(id_cfg_pairs) + gf_instances + + def GetInstancesFromInstanceNames(cfg, instance_names): """Get instances from instance names. @@ -287,7 +369,9 @@ def GetInstancesFromInstanceNames(cfg, instance_names): Raises: errors.NoInstancesFound: No instances found. """ - return FilterInstancesByNames(GetInstances(cfg), instance_names) + return _FilterInstancesByNames( + GetLocalInstancesByNames(instance_names) + GetRemoteInstances(cfg), + instance_names) def FilterInstancesByAdbPort(instances, adb_port): diff --git a/list/list_test.py b/list/list_test.py index a4b466c0..b9077a64 100644 --- a/list/list_test.py +++ b/list/list_test.py @@ -15,7 +15,7 @@ import unittest -import mock +from unittest import mock from acloud import errors from acloud.internal.lib import cvd_runtime_config @@ -25,7 +25,7 @@ from acloud.list import list as list_instance from acloud.list import instance -class InstanceObject(object): +class InstanceObject: """Mock to store data of instance.""" def __init__(self, name): @@ -44,8 +44,10 @@ class ListTest(driver_test_lib.BaseDriverTest): alive_instance2 = InstanceObject("alive_instance2") alive_local_instance = InstanceObject("alive_local_instance") - instance_alive = [alive_instance1, alive_instance2, alive_local_instance] - self.Patch(list_instance, "GetInstances", return_value=instance_alive) + self.Patch(list_instance, "GetLocalInstancesByNames", + return_value=[alive_local_instance]) + self.Patch(list_instance, "GetRemoteInstances", + return_value=[alive_instance1, alive_instance2]) instances_list = list_instance.GetInstancesFromInstanceNames(cfg, instance_names) instances_name_in_list = [instance_object.name for instance_object in instances_list] self.assertEqual(instances_name_in_list.sort(), instance_names.sort()) @@ -58,7 +60,7 @@ class ListTest(driver_test_lib.BaseDriverTest): # test get instance from instance name error with invalid input. instance_names = ["miss2_local_instance", "alive_instance1"] miss_instance_names = ["miss2_local_instance"] - self.assertRaisesRegexp( + self.assertRaisesRegex( errors.NoInstancesFound, "Did not find the following instances: %s" % ' '.join(miss_instance_names), list_instance.GetInstancesFromInstanceNames, @@ -88,6 +90,30 @@ class ListTest(driver_test_lib.BaseDriverTest): expected_instance = "cf_instance2" self.assertEqual(list_instance.ChooseOneRemoteInstance(cfg), expected_instance) + def testGetLocalInstancesByNames(self): + """test GetLocalInstancesByNames.""" + self.Patch( + instance, "GetLocalInstanceIdByName", + side_effect=lambda name: 1 if name == "local-instance-1" else None) + self.Patch(instance, "GetLocalInstanceConfig", + return_value="path1") + self.Patch(instance, "GetDefaultCuttlefishConfig", + return_value="path2") + mock_cf_ins = mock.Mock() + mock_cf_ins.name = "local-instance-1" + mock_get_cf = self.Patch(list_instance, + "_GetLocalCuttlefishInstances", + return_value=[mock_cf_ins]) + mock_gf_ins = mock.Mock() + mock_gf_ins.name = "local-goldfish-instance-1" + self.Patch(instance.LocalGoldfishInstance, "GetExistingInstances", + return_value=[mock_gf_ins]) + + ins_list = list_instance.GetLocalInstancesByNames([ + mock_cf_ins.name, "local-instance-6", mock_gf_ins.name]) + self.assertEqual([mock_cf_ins, mock_gf_ins], ins_list) + mock_get_cf.assert_called_with([(1, "path1"), (1, "path2")]) + # pylint: disable=attribute-defined-outside-init def testFilterInstancesByAdbPort(self): """test FilterInstancesByAdbPort.""" @@ -107,22 +133,42 @@ class ListTest(driver_test_lib.BaseDriverTest): def testGetLocalCuttlefishInstances(self): """test _GetLocalCuttlefishInstances.""" # Test getting two instance case - self.Patch(instance, "GetAllLocalInstanceConfigs", - return_value=["fake_path1", "fake_path2"]) - self.Patch(instance, "GetLocalInstanceRuntimeDir") + id_cfg_pairs = [(1, "fake_path1"), (2, "fake_path2")] + mock_isfile = self.Patch(list_instance.os.path, "isfile", + return_value=True) + + mock_lock = mock.Mock() + mock_lock.Lock.return_value = True + self.Patch(instance, "GetLocalInstanceLock", return_value=mock_lock) local_ins = mock.MagicMock() local_ins.CvdStatus.return_value = True self.Patch(instance, "LocalInstance", return_value=local_ins) - ins_list = list_instance._GetLocalCuttlefishInstances() + ins_list = list_instance._GetLocalCuttlefishInstances(id_cfg_pairs) self.assertEqual(2, len(ins_list)) + mock_isfile.assert_called() + local_ins.CvdStatus.assert_called() + self.assertEqual(2, mock_lock.Lock.call_count) + self.assertEqual(2, mock_lock.Unlock.call_count) + + local_ins.CvdStatus.reset_mock() + mock_lock.Lock.reset_mock() + mock_lock.Lock.return_value = False + mock_lock.Unlock.reset_mock() + ins_list = list_instance._GetLocalCuttlefishInstances(id_cfg_pairs) + self.assertEqual(0, len(ins_list)) + local_ins.CvdStatus.assert_not_called() + self.assertEqual(2, mock_lock.Lock.call_count) + mock_lock.Unlock.assert_not_called() - local_ins = mock.MagicMock() + mock_lock.Lock.reset_mock() + mock_lock.Lock.return_value = True local_ins.CvdStatus.return_value = False - self.Patch(instance, "LocalInstance", return_value=local_ins) - ins_list = list_instance._GetLocalCuttlefishInstances() + ins_list = list_instance._GetLocalCuttlefishInstances(id_cfg_pairs) self.assertEqual(0, len(ins_list)) + self.assertEqual(2, mock_lock.Lock.call_count) + self.assertEqual(2, mock_lock.Unlock.call_count) # pylint: disable=no-member def testPrintInstancesDetails(self): diff --git a/powerwash/__init__.py b/powerwash/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/powerwash/__init__.py diff --git a/powerwash/powerwash.py b/powerwash/powerwash.py new file mode 100644 index 00000000..05ab4e58 --- /dev/null +++ b/powerwash/powerwash.py @@ -0,0 +1,90 @@ +# Copyright 2020 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r"""Powerwash entry point. + +This command will powerwash the AVD from a remote instance. +""" + +import logging +import subprocess + +from acloud import errors +from acloud.internal import constants +from acloud.internal.lib import utils +from acloud.internal.lib.ssh import Ssh +from acloud.internal.lib.ssh import IP +from acloud.list import list as list_instances +from acloud.public import config +from acloud.public import report + + +logger = logging.getLogger(__name__) + + +def PowerwashFromInstance(cfg, instance, instance_id): + """Powerwash AVD from remote CF instance. + + Args: + cfg: AcloudConfig object. + instance: list.Instance() object. + instance_id: Integer of the instance id. + + Returns: + A Report instance. + """ + ssh = Ssh(ip=IP(ip=instance.ip), + user=constants.GCE_USER, + ssh_private_key_path=cfg.ssh_private_key_path, + extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) + logger.info("Start to powerwash AVD id (%s) from the instance: %s.", + instance_id, instance.name) + PowerwashDevice(ssh, instance_id) + return report.Report(command="powerwash") + + +@utils.TimeExecute(function_description="Waiting for AVD to powerwash") +def PowerwashDevice(ssh, instance_id): + """Powerwash AVD with the instance id. + + Args: + ssh: Ssh object. + instance_id: Integer of the instance id. + """ + ssh_command = "./bin/powerwash_cvd --instance_num=%d" % (instance_id) + try: + ssh.Run(ssh_command) + except (subprocess.CalledProcessError, errors.DeviceConnectionError) as e: + logger.debug(str(e)) + utils.PrintColorString(str(e), utils.TextColors.FAIL) + + +def Run(args): + """Run powerwash. + + After powerwash command executed, tool will return one Report instance. + + Args: + args: Namespace object from argparse.parse_args. + + Returns: + A Report instance. + """ + cfg = config.GetAcloudConfig(args) + if args.instance_name: + instance = list_instances.GetInstancesFromInstanceNames( + cfg, [args.instance_name]) + return PowerwashFromInstance(cfg, instance[0], args.instance_id) + return PowerwashFromInstance(cfg, + list_instances.ChooseOneRemoteInstance(cfg), + args.instance_id) diff --git a/powerwash/powerwash_args.py b/powerwash/powerwash_args.py new file mode 100644 index 00000000..3c5c1c80 --- /dev/null +++ b/powerwash/powerwash_args.py @@ -0,0 +1,59 @@ +# Copyright 2020 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r"""Powerwash args. + +Defines the powerwash arg parser that holds powerwash specific args. +""" +import argparse + + +CMD_POWERWASH = "powerwash" + + +def GetPowerwashArgParser(subparser): + """Return the powerwash arg parser. + + Args: + subparser: argparse.ArgumentParser that is attached to main acloud cmd. + + Returns: + argparse.ArgumentParser with powerwash options defined. + """ + powerwash_parser = subparser.add_parser(CMD_POWERWASH) + powerwash_parser.required = False + powerwash_parser.set_defaults(which=CMD_POWERWASH) + powerwash_group = powerwash_parser.add_mutually_exclusive_group() + powerwash_group.add_argument( + "--instance-name", + dest="instance_name", + type=str, + required=False, + help="The name of the remote instance that need to reset the AVDs.") + # TODO(b/118439885): Old arg formats to support transition, delete when + # transistion is done. + powerwash_group.add_argument( + "--instance_name", + dest="instance_name", + type=str, + required=False, + help=argparse.SUPPRESS) + powerwash_parser.add_argument( + "--instance-id", + dest="instance_id", + type=int, + required=False, + default=1, + help="The instance id of the remote instance that need to be reset.") + + return powerwash_parser diff --git a/public/acloud_kernel/kernel_swapper_test.py b/public/acloud_kernel/kernel_swapper_test.py index dabe4a93..cf4cfb51 100644 --- a/public/acloud_kernel/kernel_swapper_test.py +++ b/public/acloud_kernel/kernel_swapper_test.py @@ -18,7 +18,8 @@ import subprocess import unittest -import mock + +from unittest import mock from acloud.internal.lib import android_compute_client from acloud.internal.lib import auth diff --git a/public/acloud_main.py b/public/acloud_main.py index a4833481..b8d89d5b 100644 --- a/public/acloud_main.py +++ b/public/acloud_main.py @@ -70,8 +70,10 @@ Try $acloud [cmd] --help for further details. from __future__ import print_function import argparse import logging +import os import platform import sys +import sysconfig import traceback # TODO: Remove this once we switch over to embedded launcher. @@ -93,6 +95,12 @@ if (sys.version_info.major == 2 print(" - or -") print(" POSIXLY_CORRECT=1 port -N install python27") sys.exit(1) +# This is a workaround to put '/usr/lib/python3.X' ahead of googleapiclient of +# build system path list to fix python3 issue of http.client(b/144743252) +# that googleapiclient existed http.py conflict with python3 build-in lib. +# Using embedded_launcher(b/135639220) perhaps work whereas it didn't solve yet. +if sys.version_info.major == 3: + sys.path.insert(0, os.path.dirname(sysconfig.get_paths()['purelib'])) # By Default silence root logger's stream handler since 3p lib may initial # root logger no matter what level we're using. The acloud logger behavior will @@ -115,24 +123,41 @@ from acloud.reconnect import reconnect_args from acloud.list import list as list_instances from acloud.list import list_args from acloud.metrics import metrics +from acloud.powerwash import powerwash +from acloud.powerwash import powerwash_args from acloud.public import acloud_common from acloud.public import config +from acloud.public import report from acloud.public.actions import create_cuttlefish_action from acloud.public.actions import create_goldfish_action from acloud.pull import pull from acloud.pull import pull_args +from acloud.restart import restart +from acloud.restart import restart_args from acloud.setup import setup from acloud.setup import setup_args LOGGING_FMT = "%(asctime)s |%(levelname)s| %(module)s:%(lineno)s| %(message)s" ACLOUD_LOGGER = "acloud" +_LOGGER = logging.getLogger(ACLOUD_LOGGER) NO_ERROR_MESSAGE = "" +PROG = "acloud" +_ACLOUD_CONFIG_ERROR = "ACLOUD_CONFIG_ERROR" # Commands CMD_CREATE_CUTTLEFISH = "create_cf" CMD_CREATE_GOLDFISH = "create_gf" +# Config requires fields. +_CREATE_REQUIRE_FIELDS = ["project", "zone", "machine_type"] +_CREATE_CF_REQUIRE_FIELDS = ["resolution"] +# show contact info to user. +_CONTACT_INFO = ("If you have any question or need acloud team support, " + "please feel free to contact us by email at " + "buganizer-system+419709@google.com") +_LOG_INFO = " and attach those log files from %s" + # pylint: disable=too-many-statements def _ParseArgs(args): @@ -151,11 +176,15 @@ def _ParseArgs(args): delete_args.CMD_DELETE, reconnect_args.CMD_RECONNECT, pull_args.CMD_PULL, + restart_args.CMD_RESTART, ]) parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, usage="acloud {" + usage + "} ...") + parser = argparse.ArgumentParser(prog=PROG) + parser.add_argument('--version', action='version', version=( + '%(prog)s ' + config.GetVersion())) subparsers = parser.add_subparsers(metavar="{" + usage + "}") subparser_list = [] @@ -222,6 +251,12 @@ def _ParseArgs(args): # Command "reconnect" subparser_list.append(reconnect_args.GetReconnectArgParser(subparsers)) + # Command "restart" + subparser_list.append(restart_args.GetRestartArgParser(subparsers)) + + # Command "powerwash" + subparser_list.append(powerwash_args.GetPowerwashArgParser(subparsers)) + # Command "pull" subparser_list.append(pull_args.GetPullArgParser(subparsers)) @@ -229,6 +264,10 @@ def _ParseArgs(args): for subparser in subparser_list: acloud_common.AddCommonArguments(subparser) + if not args: + parser.print_help() + sys.exit(constants.EXIT_BY_WRONG_CMD) + return parser.parse_args(args) @@ -247,6 +286,8 @@ def _VerifyArgs(parsed_args): """ if parsed_args.which == create_args.CMD_CREATE: create_args.VerifyArgs(parsed_args) + if parsed_args.which == setup_args.CMD_SETUP: + setup_args.VerifyArgs(parsed_args) if parsed_args.which == CMD_CREATE_CUTTLEFISH: if not parsed_args.build_id and not parsed_args.branch: raise errors.CommandArgError( @@ -275,6 +316,26 @@ def _VerifyArgs(parsed_args): "--serial_log_file must ends with .tar.gz") +def _ParsingConfig(args, cfg): + """Parse config to check if missing any field. + + Args: + args: Namespace object from argparse.parse_args. + cfg: AcloudConfig object. + + Returns: + error message about list of missing config fields. + """ + missing_fields = [] + if args.which == create_args.CMD_CREATE and args.local_instance is None: + missing_fields = cfg.GetMissingFields(_CREATE_REQUIRE_FIELDS) + if args.which == CMD_CREATE_CUTTLEFISH: + missing_fields.extend(cfg.GetMissingFields(_CREATE_CF_REQUIRE_FIELDS)) + if missing_fields: + return "Missing required configuration fields: %s" % missing_fields + return None + + def _SetupLogging(log_file, verbose): """Setup logging. @@ -340,23 +401,25 @@ def main(argv=None): Job status: Integer, 0 if success. None-zero if fails. Stack trace: String of errors. """ - if argv is None: - argv = sys.argv[1:] - args = _ParseArgs(argv) _SetupLogging(args.log_file, args.verbose) _VerifyArgs(args) + _LOGGER.info("Acloud version: %s", config.GetVersion()) cfg = config.GetAcloudConfig(args) + parsing_config_error = _ParsingConfig(args, cfg) # TODO: Move this check into the functions it is actually needed. # Check access. # device_driver.CheckAccess(cfg) - report = None - if args.which == create_args.CMD_CREATE: - report = create.Run(args) + reporter = None + if parsing_config_error: + reporter = report.Report(command=args.which) + reporter.UpdateFailure(parsing_config_error, _ACLOUD_CONFIG_ERROR) + elif args.which == create_args.CMD_CREATE: + reporter = create.Run(args) elif args.which == CMD_CREATE_CUTTLEFISH: - report = create_cuttlefish_action.CreateDevices( + reporter = create_cuttlefish_action.CreateDevices( cfg=cfg, build_target=args.build_target, build_id=args.build_id, @@ -367,6 +430,9 @@ def main(argv=None): system_branch=args.system_branch, system_build_id=args.system_build_id, system_build_target=args.system_build_target, + bootloader_branch=args.bootloader_branch, + bootloader_build_id=args.bootloader_build_id, + bootloader_build_target=args.bootloader_build_target, gpu=args.gpu, num=args.num, serial_log_file=args.serial_log_file, @@ -375,7 +441,7 @@ def main(argv=None): boot_timeout_secs=args.boot_timeout_secs, ins_timeout_secs=args.ins_timeout_secs) elif args.which == CMD_CREATE_GOLDFISH: - report = create_goldfish_action.CreateDevices( + reporter = create_goldfish_action.CreateDevices( cfg=cfg, build_target=args.build_target, build_id=args.build_id, @@ -390,15 +456,20 @@ def main(argv=None): serial_log_file=args.serial_log_file, autoconnect=args.autoconnect, tags=args.tags, - report_internal_ip=args.report_internal_ip) + report_internal_ip=args.report_internal_ip, + boot_timeout_secs=args.boot_timeout_secs) elif args.which == delete_args.CMD_DELETE: - report = delete.Run(args) + reporter = delete.Run(args) elif args.which == list_args.CMD_LIST: list_instances.Run(args) elif args.which == reconnect_args.CMD_RECONNECT: reconnect.Run(args) + elif args.which == restart_args.CMD_RESTART: + reporter = restart.Run(args) + elif args.which == powerwash_args.CMD_POWERWASH: + reporter = powerwash.Run(args) elif args.which == pull_args.CMD_PULL: - report = pull.Run(args) + reporter = pull.Run(args) elif args.which == setup_args.CMD_SETUP: setup.Run(args) else: @@ -406,11 +477,15 @@ def main(argv=None): sys.stderr.write(error_msg) return constants.EXIT_BY_WRONG_CMD, error_msg - if report and args.report_file: - report.Dump(args.report_file) - if report and report.errors: - error_msg = "\n".join(report.errors) - sys.stderr.write("Encountered the following errors:\n%s\n" % error_msg) + if reporter and args.report_file: + reporter.Dump(args.report_file) + if reporter and reporter.errors: + error_msg = "\n".join(reporter.errors) + help_msg = _CONTACT_INFO + if reporter.data.get(constants.ERROR_LOG_FOLDER): + help_msg += _LOG_INFO % reporter.data.get(constants.ERROR_LOG_FOLDER) + sys.stderr.write("Encountered the following errors:\n%s\n\n%s.\n" % + (error_msg, help_msg)) return constants.EXIT_BY_FAIL_REPORT, error_msg return constants.EXIT_SUCCESS, NO_ERROR_MESSAGE diff --git a/public/actions/common_operations.py b/public/actions/common_operations.py index 65c04710..32907c08 100644 --- a/public/actions/common_operations.py +++ b/public/actions/common_operations.py @@ -32,6 +32,20 @@ from acloud.internal.lib.adb_tools import AdbTools logger = logging.getLogger(__name__) +_ACLOUD_BOOT_UP_ERROR = "ACLOUD_BOOT_UP_ERROR" +_ACLOUD_DOWNLOAD_ARTIFACT_ERROR = "ACLOUD_DOWNLOAD_ARTIFACT_ERROR" +_ACLOUD_GENERIC_ERROR = "ACLOUD_GENERIC_ERROR" +_ACLOUD_SSH_CONNECT_ERROR = "ACLOUD_SSH_CONNECT_ERROR" +# Error type of GCE quota error. +_GCE_QUOTA_ERROR = "GCE_QUOTA_ERROR" +_GCE_QUOTA_ERROR_MSG = "Quota exceeded for quota" +_DICT_ERROR_TYPE = { + constants.STAGE_INIT: "ACLOUD_INIT_ERROR", + constants.STAGE_GCE: "ACLOUD_CREATE_GCE_ERROR", + constants.STAGE_SSH_CONNECT: _ACLOUD_SSH_CONNECT_ERROR, + constants.STAGE_ARTIFACT: _ACLOUD_DOWNLOAD_ARTIFACT_ERROR, + constants.STAGE_BOOT_UP: _ACLOUD_BOOT_UP_ERROR, +} def CreateSshKeyPairIfNecessary(cfg): @@ -66,7 +80,7 @@ def CreateSshKeyPairIfNecessary(cfg): "Unexpected error in CreateSshKeyPairIfNecessary") -class DevicePool(object): +class DevicePool: """A class that manages a pool of virtual devices. Attributes: @@ -100,9 +114,11 @@ class DevicePool(object): ip = self._compute_client.GetInstanceIP(instance) time_info = self._compute_client.execution_time if hasattr( self._compute_client, "execution_time") else {} + stage = self._compute_client.stage if hasattr( + self._compute_client, "stage") else 0 self.devices.append( avd.AndroidVirtualDevice(ip=ip, instance_name=instance, - time_info=time_info)) + time_info=time_info, stage=stage)) @utils.TimeExecute(function_description="Waiting for AVD(s) to boot up", result_evaluator=utils.BootEvaluator) @@ -126,6 +142,14 @@ class DevicePool(object): failures[device.instance_name] = e return failures + def UpdateReport(self, reporter): + """Update report from compute client. + + Args: + reporter: Report object. + """ + reporter.UpdateData(self._compute_client.dict_report) + def CollectSerialPortLogs(self, output_file, port=constants.DEFAULT_SERIAL_PORT): """Tar the instance serial logs into specified output_file. @@ -164,12 +188,31 @@ class DevicePool(object): """ return self._devices +def _GetErrorType(error): + """Get proper error type from the exception error. + + Args: + error: errors object. + + Returns: + String of error type. e.g. "ACLOUD_BOOT_UP_ERROR". + """ + if isinstance(error, errors.CheckGCEZonesQuotaError): + return _GCE_QUOTA_ERROR + if isinstance(error, errors.DownloadArtifactError): + return _ACLOUD_DOWNLOAD_ARTIFACT_ERROR + if isinstance(error, errors.DeviceConnectionError): + return _ACLOUD_SSH_CONNECT_ERROR + if _GCE_QUOTA_ERROR_MSG in str(error): + return _GCE_QUOTA_ERROR + return _ACLOUD_GENERIC_ERROR + # pylint: disable=too-many-locals,unused-argument,too-many-branches def CreateDevices(command, cfg, device_factory, num, avd_type, report_internal_ip=False, autoconnect=False, serial_log_file=None, client_adb_port=None, boot_timeout_secs=None, unlock_screen=False, - wait_for_boot=True): + wait_for_boot=True, connect_webrtc=False): """Create a set of devices using the given factory. Main jobs in create devices. @@ -191,6 +234,7 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, unlock_screen: Boolean, whether to unlock screen after invoke vnc client. wait_for_boot: Boolean, True to check serial log include boot up message. + connect_webrtc: Boolean, whether to auto connect webrtc to device. Raises: errors: Create instance fail. @@ -219,6 +263,7 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, device_pool.CollectSerialPortLogs( serial_log_file, port=constants.DEFAULT_SERIAL_PORT) + device_pool.UpdateReport(reporter) # Write result to report. for device in device_pool.devices: ip = (device.ip.internal if report_internal_ip @@ -242,14 +287,27 @@ def CreateDevices(command, cfg, device_factory, num, avd_type, extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) device_dict[constants.VNC_PORT] = forwarded_ports.vnc_port device_dict[constants.ADB_PORT] = forwarded_ports.adb_port + device_dict[constants.DEVICE_SERIAL] = ( + constants.REMOTE_INSTANCE_ADB_SERIAL % + forwarded_ports.adb_port) if unlock_screen: AdbTools(forwarded_ports.adb_port).AutoUnlockScreen() + if connect_webrtc: + utils.EstablishWebRTCSshTunnel( + ip_addr=ip, + rsa_key_file=cfg.ssh_private_key_path, + ssh_user=constants.GCE_USER, + extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) if device.instance_name in failures: + reporter.SetErrorType(_ACLOUD_BOOT_UP_ERROR) + if device.stage: + reporter.SetErrorType(_DICT_ERROR_TYPE[device.stage]) reporter.AddData(key="devices_failing_boot", value=device_dict) reporter.AddError(str(failures[device.instance_name])) else: reporter.AddData(key="devices", value=device_dict) - except errors.DriverError as e: + except (errors.DriverError, errors.CheckGCEZonesQuotaError) as e: + reporter.SetErrorType(_GetErrorType(e)) reporter.AddError(str(e)) reporter.SetStatus(report.Status.FAIL) return reporter diff --git a/public/actions/common_operations_test.py b/public/actions/common_operations_test.py index 1226b4b1..b01ee1e7 100644 --- a/public/actions/common_operations_test.py +++ b/public/actions/common_operations_test.py @@ -18,13 +18,17 @@ from __future__ import absolute_import from __future__ import division +import shlex import unittest -import mock +from unittest import mock + +from acloud import errors from acloud.internal.lib import android_build_client from acloud.internal.lib import android_compute_client from acloud.internal.lib import auth from acloud.internal.lib import driver_test_lib +from acloud.internal.lib import utils from acloud.internal.lib import ssh from acloud.public import report from acloud.public.actions import common_operations @@ -43,7 +47,7 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): # pylint: disable=protected-access def setUp(self): """Set up the test.""" - super(CommonOperationsTest, self).setUp() + super().setUp() self.build_client = mock.MagicMock() self.device_factory = mock.MagicMock() self.Patch( @@ -113,6 +117,33 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "gcs_bucket_build_id": self.BUILD_ID, }]}) + def testCreateDevicesWithAdbPort(self): + """Test Create Devices with adb port for cuttlefish avd type.""" + self.Patch(utils, "_ExecuteCommand") + self.Patch(utils, "PickFreePort", return_value=56789) + self.Patch(shlex, "split", return_value=[]) + cfg = self._CreateCfg() + _report = common_operations.CreateDevices(self.CMD, cfg, + self.device_factory, 1, + "cuttlefish", + autoconnect=True, + client_adb_port=12345) + self.assertEqual(_report.command, self.CMD) + self.assertEqual(_report.status, report.Status.SUCCESS) + self.assertEqual( + _report.data, + {"devices": [{ + "ip": self.IP.external, + "instance_name": self.INSTANCE, + "branch": self.BRANCH, + "build_id": self.BUILD_ID, + "adb_port": 12345, + "device_serial": "127.0.0.1:12345", + "vnc_port": 56789, + "build_target": self.BUILD_TARGET, + "gcs_bucket_build_id": self.BUILD_ID, + }]}) + def testCreateDevicesInternalIP(self): """Test Create Devices and report internal IP.""" cfg = self._CreateCfg() @@ -133,5 +164,33 @@ class CommonOperationsTest(driver_test_lib.BaseDriverTest): "gcs_bucket_build_id": self.BUILD_ID, }]}) + def testGetErrorType(self): + """Test GetErrorType.""" + # Test with CheckGCEZonesQuotaError() + error = errors.CheckGCEZonesQuotaError() + expected_result = common_operations._GCE_QUOTA_ERROR + self.assertEqual(common_operations._GetErrorType(error), expected_result) + + # Test with DownloadArtifactError() + error = errors.DownloadArtifactError() + expected_result = common_operations._ACLOUD_DOWNLOAD_ARTIFACT_ERROR + self.assertEqual(common_operations._GetErrorType(error), expected_result) + + # Test with DeviceConnectionError() + error = errors.DeviceConnectionError() + expected_result = common_operations._ACLOUD_SSH_CONNECT_ERROR + self.assertEqual(common_operations._GetErrorType(error), expected_result) + + # Test with ACLOUD_GENERIC_ERROR + error = errors.DriverError() + expected_result = common_operations._ACLOUD_GENERIC_ERROR + self.assertEqual(common_operations._GetErrorType(error), expected_result) + + # Test with error message about GCE quota issue + error = errors.DriverError("Quota exceeded for quota read group.") + expected_result = common_operations._GCE_QUOTA_ERROR + self.assertEqual(common_operations._GetErrorType(error), expected_result) + + if __name__ == "__main__": unittest.main() diff --git a/public/actions/create_cuttlefish_action.py b/public/actions/create_cuttlefish_action.py index bc1886c9..c8ed5d30 100644 --- a/public/actions/create_cuttlefish_action.py +++ b/public/actions/create_cuttlefish_action.py @@ -55,8 +55,9 @@ class CuttlefishDeviceFactory(base_device_factory.BaseDeviceFactory): kernel_build_id=None, kernel_branch=None, kernel_build_target=None, system_branch=None, system_build_id=None, system_build_target=None, - boot_timeout_secs=None, ins_timeout_secs=None, - report_internal_ip=None, gpu=None): + bootloader_branch=None, bootloader_build_id=None, + bootloader_build_target=None, boot_timeout_secs=None, + ins_timeout_secs=None, report_internal_ip=None, gpu=None): self.credentials = auth.CreateCredentials(cfg) @@ -90,6 +91,8 @@ class CuttlefishDeviceFactory(base_device_factory.BaseDeviceFactory): kernel_branch) self.system_build_info = self._build_client.GetBuildInfo( system_build_target or build_target, system_build_id, system_branch) + self.bootloader_build_info = self._build_client.GetBuildInfo( + bootloader_build_target, bootloader_build_id, bootloader_branch) def GetBuildInfoDict(self): """Get build info dictionary. @@ -108,6 +111,10 @@ class CuttlefishDeviceFactory(base_device_factory.BaseDeviceFactory): {"system_%s" % key: val for key, val in utils.GetDictItems(self.system_build_info) if val} ) + build_info_dict.update( + {"bootloader_%s" % key: val + for key, val in utils.GetDictItems(self.bootloader_build_info) if val} + ) return build_info_dict def GetFailures(self): @@ -161,10 +168,14 @@ class CuttlefishDeviceFactory(base_device_factory.BaseDeviceFactory): remote_system_build_id = self._GetGcsBucketBuildId( self.system_build_info.build_id, self.system_build_info.release_build_id) + host_image_name = self._compute_client.GetHostImageName( + self._cfg.stable_host_image_name, + self._cfg.stable_host_image_family, + self._cfg.stable_host_image_project) # Create an instance from Stable Host Image self._compute_client.CreateInstance( instance=instance, - image_name=self._cfg.stable_host_image_name, + image_name=host_image_name, image_project=self._cfg.stable_host_image_project, build_target=self.build_info.build_target, branch=self.build_info.branch, @@ -176,7 +187,10 @@ class CuttlefishDeviceFactory(base_device_factory.BaseDeviceFactory): extra_scopes=self._extra_scopes, system_build_target=self.system_build_info.build_target, system_branch=self.system_build_info.branch, - system_build_id=remote_system_build_id) + system_build_id=remote_system_build_id, + bootloader_build_target=self.bootloader_build_info.build_target, + bootloader_branch=self.bootloader_build_info.branch, + bootloader_build_id=self.bootloader_build_info.build_id) return instance @@ -192,6 +206,9 @@ def CreateDevices(cfg, system_branch=None, system_build_id=None, system_build_target=None, + bootloader_branch=None, + bootloader_build_id=None, + bootloader_build_target=None, gpu=None, num=1, serial_log_file=None, @@ -212,6 +229,9 @@ def CreateDevices(cfg, system_branch: Branch name to consume the system.img from, a string. system_build_id: System branch build id, a string. system_build_target: System image build target, a string. + bootloader_branch: String of the bootloader branch name. + bootloader_build_id: String of the bootloader build id. + bootloader_build_target: String of the bootloader target name. gpu: String, GPU to attach to the device or None. e.g. "nvidia-tesla-k80" num: Integer, Number of devices to create. serial_log_file: String, A path to a tar file where serial output should @@ -242,14 +262,18 @@ def CreateDevices(cfg, "system_branch: %s, " "system_build_id: %s, " "system_build_target: %s, " + "bootloader_branch: %s, " + "bootloader_build_id: %s, " + "bootloader_build_target: %s, " "gpu: %s" "num: %s, " "serial_log_file: %s, " "autoconnect: %s, " "report_internal_ip: %s", cfg.project, build_target, build_id, branch, kernel_build_id, kernel_branch, kernel_build_target, - system_branch, system_build_id, system_build_target, gpu, num, - serial_log_file, autoconnect, report_internal_ip) + system_branch, system_build_id, system_build_target, bootloader_branch, + bootloader_build_id, bootloader_build_target, gpu, num, serial_log_file, + autoconnect, report_internal_ip) # If multi_stage enable, launch_cvd don't write serial log to instance. So # it doesn't go WaitForBoot function. if cfg.enable_multi_stage: @@ -260,6 +284,9 @@ def CreateDevices(cfg, kernel_build_target=kernel_build_target, system_branch=system_branch, system_build_id=system_build_id, system_build_target=system_build_target, + bootloader_branch=bootloader_branch, + bootloader_build_id=bootloader_build_id, + bootloader_build_target=bootloader_build_target, boot_timeout_secs=boot_timeout_secs, ins_timeout_secs=ins_timeout_secs, report_internal_ip=report_internal_ip, diff --git a/public/actions/create_cuttlefish_action_test.py b/public/actions/create_cuttlefish_action_test.py index f4788c59..cce79631 100644 --- a/public/actions/create_cuttlefish_action_test.py +++ b/public/actions/create_cuttlefish_action_test.py @@ -21,7 +21,8 @@ Tests for acloud.public.actions.create_cuttlefish_action. import uuid import unittest -import mock + +from unittest import mock from acloud.internal.lib import android_build_client from acloud.internal.lib import android_compute_client @@ -39,15 +40,18 @@ class CreateCuttlefishActionTest(driver_test_lib.BaseDriverTest): IP = ssh.IP(external="127.0.0.1", internal="10.0.0.1") INSTANCE = "fake-instance" IMAGE = "fake-image" - BUILD_TARGET = "fake-build-target" + BRANCH = "fake-branch" BUILD_ID = "12345" + BUILD_TARGET = "fake-build-target" KERNEL_BRANCH = "fake-kernel-branch" KERNEL_BUILD_ID = "54321" KERNEL_BUILD_TARGET = "kernel" - BRANCH = "fake-branch" SYSTEM_BRANCH = "fake-system-branch" SYSTEM_BUILD_ID = "23456" SYSTEM_BUILD_TARGET = "fake-system-build-target" + BOOTLOADER_BRANCH = "fake-bootloader-branch" + BOOTLOADER_BUILD_ID = "34567" + BOOTLOADER_BUILD_TARGET = "fake-bootloader-build-target" STABLE_HOST_IMAGE_NAME = "fake-stable-host-image-name" STABLE_HOST_IMAGE_PROJECT = "fake-stable-host-image-project" EXTRA_DATA_DISK_GB = 4 @@ -106,6 +110,7 @@ class CreateCuttlefishActionTest(driver_test_lib.BaseDriverTest): self.compute_client.GetInstanceIP.return_value = self.IP self.compute_client.GenerateImageName.return_value = self.IMAGE self.compute_client.GenerateInstanceName.return_value = self.INSTANCE + self.compute_client.GetHostImageName.return_value = self.STABLE_HOST_IMAGE_NAME # Mock build client method self.build_client.GetBuildInfo.side_effect = [ @@ -116,7 +121,10 @@ class CreateCuttlefishActionTest(driver_test_lib.BaseDriverTest): self.KERNEL_BUILD_TARGET, None), android_build_client.BuildInfo( self.SYSTEM_BRANCH, self.SYSTEM_BUILD_ID, - self.SYSTEM_BUILD_TARGET, None)] + self.SYSTEM_BUILD_TARGET, None), + android_build_client.BuildInfo( + self.BOOTLOADER_BRANCH, self.BOOTLOADER_BUILD_ID, + self.BOOTLOADER_BUILD_TARGET, None)] # Call CreateDevices report = create_cuttlefish_action.CreateDevices( @@ -124,7 +132,10 @@ class CreateCuttlefishActionTest(driver_test_lib.BaseDriverTest): kernel_build_id=self.KERNEL_BUILD_ID, system_build_target=self.SYSTEM_BUILD_TARGET, system_branch=self.SYSTEM_BRANCH, - system_build_id=self.SYSTEM_BUILD_ID) + system_build_id=self.SYSTEM_BUILD_ID, + bootloader_build_target=self.BOOTLOADER_BUILD_TARGET, + bootloader_branch=self.BOOTLOADER_BRANCH, + bootloader_build_id=self.BOOTLOADER_BUILD_ID) # Verify self.compute_client.CreateInstance.assert_called_with( @@ -140,6 +151,9 @@ class CreateCuttlefishActionTest(driver_test_lib.BaseDriverTest): system_branch=self.SYSTEM_BRANCH, system_build_id=self.SYSTEM_BUILD_ID, system_build_target=self.SYSTEM_BUILD_TARGET, + bootloader_branch=self.BOOTLOADER_BRANCH, + bootloader_build_id=self.BOOTLOADER_BUILD_ID, + bootloader_build_target=self.BOOTLOADER_BUILD_TARGET, blank_data_disk_size_gb=self.EXTRA_DATA_DISK_GB, extra_scopes=self.EXTRA_SCOPES) @@ -155,6 +169,9 @@ class CreateCuttlefishActionTest(driver_test_lib.BaseDriverTest): "system_branch": self.SYSTEM_BRANCH, "system_build_id": self.SYSTEM_BUILD_ID, "system_build_target": self.SYSTEM_BUILD_TARGET, + "bootloader_branch": self.BOOTLOADER_BRANCH, + "bootloader_build_id": self.BOOTLOADER_BUILD_ID, + "bootloader_build_target": self.BOOTLOADER_BUILD_TARGET, "instance_name": self.INSTANCE, "ip": self.IP.external, }, diff --git a/public/actions/create_goldfish_action.py b/public/actions/create_goldfish_action.py index 719e91b0..d2800e6c 100644 --- a/public/actions/create_goldfish_action.py +++ b/public/actions/create_goldfish_action.py @@ -166,7 +166,8 @@ class GoldfishDeviceFactory(base_device_factory.BaseDeviceFactory): blank_data_disk_size_gb=self._blank_data_disk_size_gb, avd_spec=self._avd_spec, tags=self._tags, - extra_scopes=self._extra_scopes) + extra_scopes=self._extra_scopes, + launch_args=self._cfg.launch_args) return instance @@ -242,7 +243,8 @@ def CreateDevices(avd_spec=None, autoconnect=False, branch=None, tags=None, - report_internal_ip=False): + report_internal_ip=False, + boot_timeout_secs=None): """Create one or multiple Goldfish devices. Args: @@ -267,12 +269,13 @@ def CreateDevices(avd_spec=None, ["http-server", "https-server"] report_internal_ip: Boolean to report the internal ip instead of external ip. + boot_timeout_secs: Integer, the maximum time in seconds used to + wait for the AVD to boot. Returns: A Report instance. """ client_adb_port = None - boot_timeout_secs = None if avd_spec: cfg = avd_spec.cfg build_target = avd_spec.remote_image[constants.BUILD_TARGET] diff --git a/public/actions/create_goldfish_action_test.py b/public/actions/create_goldfish_action_test.py index d031167a..4e71a934 100644 --- a/public/actions/create_goldfish_action_test.py +++ b/public/actions/create_goldfish_action_test.py @@ -16,7 +16,8 @@ """Tests for acloud.public.actions.create_goldfish_actions.""" import uuid import unittest -import mock + +from unittest import mock from acloud.internal import constants from acloud.internal.lib import android_build_client @@ -48,6 +49,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): GOLDFISH_HOST_IMAGE_PROJECT = "fake-stable-host-image-project" EXTRA_DATA_DISK_GB = 4 EXTRA_SCOPES = None + LAUNCH_ARGS = "fake-args" def setUp(self): """Sets up the test.""" @@ -91,6 +93,7 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): cfg.emulator_build_target = self.EMULATOR_BUILD_TARGET cfg.extra_data_disk_size_gb = self.EXTRA_DATA_DISK_GB cfg.extra_scopes = self.EXTRA_SCOPES + cfg.launch_args = self.LAUNCH_ARGS return cfg def testCreateDevices(self): @@ -145,7 +148,8 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): gpu=self.GPU, avd_spec=none_avd_spec, extra_scopes=self.EXTRA_SCOPES, - tags=None) + tags=None, + launch_args=self.LAUNCH_ARGS) self.assertEqual(report.data, { "devices": [ @@ -201,7 +205,8 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): gpu=self.GPU, avd_spec=self.avd_spec, extra_scopes=self.EXTRA_SCOPES, - tags=None) + tags=None, + launch_args=self.LAUNCH_ARGS) def testCreateDevicesWithoutBuildId(self): """Test CreateDevices when emulator sysimage buildid is not provided.""" @@ -265,7 +270,8 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): gpu=self.GPU, avd_spec=none_avd_spec, extra_scopes=self.EXTRA_SCOPES, - tags=None) + tags=None, + launch_args=self.LAUNCH_ARGS) self.assertEqual(report.data, { "devices": [{ @@ -319,7 +325,8 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): gpu=self.GPU, avd_spec=self.avd_spec, extra_scopes=self.EXTRA_SCOPES, - tags=None) + tags=None, + launch_args=self.LAUNCH_ARGS) #pylint: disable=invalid-name def testCreateDevicesWithoutEmulatorBuildId(self): @@ -376,7 +383,8 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): gpu=self.GPU, avd_spec=none_avd_spec, extra_scopes=self.EXTRA_SCOPES, - tags=None) + tags=None, + launch_args=self.LAUNCH_ARGS) self.assertEqual(report.data, { "devices": [{ @@ -430,7 +438,8 @@ class CreateGoldfishActionTest(driver_test_lib.BaseDriverTest): gpu=self.GPU, avd_spec=self.avd_spec, extra_scopes=self.EXTRA_SCOPES, - tags=None) + tags=None, + launch_args=self.LAUNCH_ARGS) if __name__ == "__main__": diff --git a/public/actions/gce_device_factory.py b/public/actions/gce_device_factory.py new file mode 100644 index 00000000..f3ec2508 --- /dev/null +++ b/public/actions/gce_device_factory.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# +# Copyright 2018 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GCE device factory. + +GCEDeviceFactory provides a base class for AVDs that run on GCE. +""" + +import os + +from acloud.internal import constants +from acloud.internal.lib import auth +from acloud.internal.lib import cvd_compute_client_multi_stage +from acloud.internal.lib import ssh +from acloud.public.actions import base_device_factory + + +class GCEDeviceFactory(base_device_factory.BaseDeviceFactory): + """A base class for AVDs that run on GCE.""" + + _USER_BUILD = "userbuild" + + def __init__(self, avd_spec, local_image_artifact=None): + """Constructs a new remote instance device factory.""" + self._avd_spec = avd_spec + self._cfg = avd_spec.cfg + self._local_image_artifact = local_image_artifact + self._report_internal_ip = avd_spec.report_internal_ip + self.credentials = auth.CreateCredentials(avd_spec.cfg) + # Control compute_client with enable_multi_stage + compute_client = cvd_compute_client_multi_stage.CvdComputeClient( + acloud_config=avd_spec.cfg, + oauth2_credentials=self.credentials, + ins_timeout_secs=avd_spec.ins_timeout_secs, + report_internal_ip=avd_spec.report_internal_ip, + gpu=avd_spec.gpu) + super(GCEDeviceFactory, self).__init__(compute_client) + self._ssh = None + + def _CreateGceInstance(self): + """Create a single configured GCE instance. + + build_target: The format is like "aosp_cf_x86_phone". We only get info + from the user build image file name. If the file name is + not custom format (no "-"), we will use $TARGET_PRODUCT + from environment variable as build_target. + + Returns: + A string, representing instance name. + """ + image_name = os.path.basename( + self._local_image_artifact) if self._local_image_artifact else "" + build_target = (os.environ.get(constants.ENV_BUILD_TARGET) if "-" not + in image_name else image_name.split("-")[0]) + build_id = self._USER_BUILD + if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: + build_id = self._avd_spec.remote_image[constants.BUILD_ID] + build_target = self._avd_spec.remote_image[constants.BUILD_TARGET] + + if self._avd_spec.instance_name_to_reuse: + instance = self._avd_spec.instance_name_to_reuse + else: + instance = self._compute_client.GenerateInstanceName( + build_target=build_target, build_id=build_id) + + host_image_name = self._compute_client.GetHostImageName( + self._cfg.stable_host_image_name, + self._cfg.stable_host_image_family, + self._cfg.stable_host_image_project) + + # Create an instance from Stable Host Image + self._compute_client.CreateInstance( + instance=instance, + image_name=host_image_name, + image_project=self._cfg.stable_host_image_project, + blank_data_disk_size_gb=self._cfg.extra_data_disk_size_gb, + avd_spec=self._avd_spec) + ip = self._compute_client.GetInstanceIP(instance) + self._ssh = ssh.Ssh(ip=ip, + user=constants.GCE_USER, + ssh_private_key_path=self._cfg.ssh_private_key_path, + extra_args_ssh_tunnel=self._cfg.extra_args_ssh_tunnel, + report_internal_ip=self._report_internal_ip) + return instance + + def GetFailures(self): + """Get failures from all devices. + + Returns: + A dictionary that contains all the failures. + The key is the name of the instance that fails to boot, + and the value is an errors.DeviceBootError object. + """ + return self._compute_client.all_failures + + def _SetFailures(self, instance, error_msg): + """Set failures from this device. + + Record the failures for any steps in AVD creation. + + Args: + instance: String of instance name. + error_msg: String of error message. + """ + self._compute_client.all_failures[instance] = error_msg diff --git a/public/actions/remote_instance_cf_device_factory.py b/public/actions/remote_instance_cf_device_factory.py index 8935aecf..c08df622 100644 --- a/public/actions/remote_instance_cf_device_factory.py +++ b/public/actions/remote_instance_cf_device_factory.py @@ -19,24 +19,26 @@ import glob import logging import os import shutil +import subprocess import tempfile from acloud import errors -from acloud.create import create_common from acloud.internal import constants -from acloud.internal.lib import auth -from acloud.internal.lib import cvd_compute_client_multi_stage from acloud.internal.lib import utils from acloud.internal.lib import ssh -from acloud.public.actions import base_device_factory +from acloud.public.actions import gce_device_factory logger = logging.getLogger(__name__) +_ALL_FILES = "*" +# bootloader and kernel are files required to launch AVD. +_BOOTLOADER = "bootloader" +_KERNEL = "kernel" +_ARTIFACT_FILES = ["*.img", _BOOTLOADER, _KERNEL] +_HOME_FOLDER = os.path.expanduser("~") -_USER_BUILD = "userbuild" - -class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): +class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): """A class that can produce a cuttlefish device. Attributes: @@ -52,23 +54,10 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): """ def __init__(self, avd_spec, local_image_artifact=None, cvd_host_package_artifact=None): - """Constructs a new remote instance device factory.""" - self._avd_spec = avd_spec - self._cfg = avd_spec.cfg - self._local_image_artifact = local_image_artifact + super().__init__(avd_spec, local_image_artifact) self._cvd_host_package_artifact = cvd_host_package_artifact - self._report_internal_ip = avd_spec.report_internal_ip - self.credentials = auth.CreateCredentials(avd_spec.cfg) - # Control compute_client with enable_multi_stage - compute_client = cvd_compute_client_multi_stage.CvdComputeClient( - acloud_config=avd_spec.cfg, - oauth2_credentials=self.credentials, - ins_timeout_secs=avd_spec.ins_timeout_secs, - report_internal_ip=avd_spec.report_internal_ip, - gpu=avd_spec.gpu) - super(RemoteInstanceDeviceFactory, self).__init__(compute_client) - self._ssh = None + # pylint: disable=broad-except def CreateInstance(self): """Create a single configured cuttlefish device. @@ -90,7 +79,7 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): instance = self._InitRemotehost() self._ProcessRemoteHostArtifacts() self._LaunchCvd(instance=instance, - decompress_kernel=True, + decompress_kernel=None, boot_timeout_secs=self._avd_spec.boot_timeout_secs) else: instance = self._CreateGceInstance() @@ -101,7 +90,7 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): self._ProcessArtifacts(self._avd_spec.image_source) self._LaunchCvd(instance=instance, boot_timeout_secs=self._avd_spec.boot_timeout_secs) - except errors.DeviceConnectionError as e: + except Exception as e: self._SetFailures(instance, e) return instance @@ -125,7 +114,7 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): self._local_image_artifact) if self._local_image_artifact else "" build_target = (os.environ.get(constants.ENV_BUILD_TARGET) if "-" not in image_name else image_name.split("-")[0]) - build_id = _USER_BUILD + build_id = self._USER_BUILD if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: build_id = self._avd_spec.remote_image[constants.BUILD_ID] @@ -148,25 +137,46 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): def _DownloadArtifacts(self, extract_path): """Download the CF image artifacts and process them. - - Download image from the Android Build system, then decompress it. + - Download images from the Android Build system. - Download cvd host package from the Android Build system. Args: extract_path: String, a path include extracted files. + + Raises: + errors.GetRemoteImageError: Fails to download rom images. """ cfg = self._avd_spec.cfg build_id = self._avd_spec.remote_image[constants.BUILD_ID] + build_branch = self._avd_spec.remote_image[constants.BUILD_BRANCH] build_target = self._avd_spec.remote_image[constants.BUILD_TARGET] - # Image zip - remote_image = "%s-img-%s.zip" % (build_target.split('-')[0], build_id) - create_common.DownloadRemoteArtifact( - cfg, build_target, build_id, remote_image, extract_path, decompress=True) - - # Cvd host package - create_common.DownloadRemoteArtifact( - cfg, build_target, build_id, constants.CVD_HOST_PACKAGE, - extract_path) + # Download images with fetch_cvd + fetch_cvd = os.path.join(extract_path, constants.FETCH_CVD) + self._compute_client.build_api.DownloadFetchcvd(fetch_cvd, + cfg.fetch_cvd_version) + fetch_cvd_build_args = self._compute_client.build_api.GetFetchBuildArgs( + build_id, build_branch, build_target, + self._avd_spec.system_build_info.get(constants.BUILD_ID), + self._avd_spec.system_build_info.get(constants.BUILD_BRANCH), + self._avd_spec.system_build_info.get(constants.BUILD_TARGET), + self._avd_spec.kernel_build_info.get(constants.BUILD_ID), + self._avd_spec.kernel_build_info.get(constants.BUILD_BRANCH), + self._avd_spec.kernel_build_info.get(constants.BUILD_TARGET), + self._avd_spec.bootloader_build_info.get(constants.BUILD_ID), + self._avd_spec.bootloader_build_info.get(constants.BUILD_BRANCH), + self._avd_spec.bootloader_build_info.get(constants.BUILD_TARGET)) + creds_cache_file = os.path.join(_HOME_FOLDER, cfg.creds_cache_file) + fetch_cvd_cert_arg = self._compute_client.build_api.GetFetchCertArg( + creds_cache_file) + fetch_cvd_args = [fetch_cvd, "-directory=%s" % extract_path, + fetch_cvd_cert_arg] + fetch_cvd_args.extend(fetch_cvd_build_args) + logger.debug("Download images command: %s", fetch_cvd_args) + try: + subprocess.check_call(fetch_cvd_args) + except subprocess.CalledProcessError as e: + raise errors.GetRemoteImageError("Fails to download images: %s" % e) def _ProcessRemoteHostArtifacts(self): """Process remote host artifacts. @@ -177,8 +187,9 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): build to local and unzip it then upload to remote host, because there is no permission to fetch build rom on the remote host. """ + self._compute_client.SetStage(constants.STAGE_ARTIFACT) if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL: - self._UploadArtifacts( + self._UploadLocalImageArtifacts( self._local_image_artifact, self._cvd_host_package_artifact, self._avd_spec.local_image_dir) else: @@ -186,10 +197,7 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): artifacts_path = tempfile.mkdtemp() logger.debug("Extracted path of artifacts: %s", artifacts_path) self._DownloadArtifacts(artifacts_path) - self._UploadArtifacts( - None, - os.path.join(artifacts_path, constants.CVD_HOST_PACKAGE), - artifacts_path) + self._UploadRemoteImageArtifacts(artifacts_path) finally: shutil.rmtree(artifacts_path) @@ -206,92 +214,38 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): image_source: String, the type of image source is remote or local. """ if image_source == constants.IMAGE_SRC_LOCAL: - self._UploadArtifacts(self._local_image_artifact, - self._cvd_host_package_artifact, - self._avd_spec.local_image_dir) + self._UploadLocalImageArtifacts(self._local_image_artifact, + self._cvd_host_package_artifact, + self._avd_spec.local_image_dir) elif image_source == constants.IMAGE_SRC_REMOTE: self._compute_client.UpdateFetchCvd() - self._FetchBuild( - self._avd_spec.remote_image[constants.BUILD_ID], - self._avd_spec.remote_image[constants.BUILD_BRANCH], - self._avd_spec.remote_image[constants.BUILD_TARGET], - self._avd_spec.system_build_info[constants.BUILD_ID], - self._avd_spec.system_build_info[constants.BUILD_BRANCH], - self._avd_spec.system_build_info[constants.BUILD_TARGET], - self._avd_spec.kernel_build_info[constants.BUILD_ID], - self._avd_spec.kernel_build_info[constants.BUILD_BRANCH], - self._avd_spec.kernel_build_info[constants.BUILD_TARGET]) - - def _FetchBuild(self, build_id, branch, build_target, system_build_id, - system_branch, system_build_target, kernel_build_id, - kernel_branch, kernel_build_target): + self._FetchBuild(self._avd_spec) + + def _FetchBuild(self, avd_spec): """Download CF artifacts from android build. Args: - build_branch: String, git branch name. e.g. "aosp-master" - build_target: String, the build target, e.g. cf_x86_phone-userdebug - build_id: String, build id, e.g. "2263051", "P2804227" - kernel_branch: Kernel branch name, e.g. "kernel-common-android-4.14" - kernel_build_id: Kernel build id, a string, e.g. "223051", "P280427" - kernel_build_target: String, Kernel build target name. - system_build_target: Target name for the system image, - e.g. "cf_x86_phone-userdebug" - system_branch: A String, branch name for the system image. - system_build_id: A string, build id for the system image. - + avd_spec: AVDSpec object that tells us what we're going to create. """ self._compute_client.FetchBuild( - build_id, branch, build_target, system_build_id, - system_branch, system_build_target, kernel_build_id, - kernel_branch, kernel_build_target) - - def _CreateGceInstance(self): - """Create a single configured cuttlefish device. - - Override method from parent class. - build_target: The format is like "aosp_cf_x86_phone". We only get info - from the user build image file name. If the file name is - not custom format (no "-"), we will use $TARGET_PRODUCT - from environment variable as build_target. - - Returns: - A string, representing instance name. - """ - image_name = os.path.basename( - self._local_image_artifact) if self._local_image_artifact else "" - build_target = (os.environ.get(constants.ENV_BUILD_TARGET) if "-" not - in image_name else image_name.split("-")[0]) - build_id = _USER_BUILD - if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: - build_id = self._avd_spec.remote_image[constants.BUILD_ID] - build_target = self._avd_spec.remote_image[constants.BUILD_TARGET] - - if self._avd_spec.instance_name_to_reuse: - instance = self._avd_spec.instance_name_to_reuse - else: - instance = self._compute_client.GenerateInstanceName( - build_target=build_target, build_id=build_id) - - # Create an instance from Stable Host Image - self._compute_client.CreateInstance( - instance=instance, - image_name=self._cfg.stable_host_image_name, - image_project=self._cfg.stable_host_image_project, - blank_data_disk_size_gb=self._cfg.extra_data_disk_size_gb, - avd_spec=self._avd_spec) - ip = self._compute_client.GetInstanceIP(instance) - self._ssh = ssh.Ssh(ip=ip, - user=constants.GCE_USER, - ssh_private_key_path=self._cfg.ssh_private_key_path, - extra_args_ssh_tunnel=self._cfg.extra_args_ssh_tunnel, - report_internal_ip=self._report_internal_ip) - return instance + avd_spec.remote_image[constants.BUILD_ID], + avd_spec.remote_image[constants.BUILD_BRANCH], + avd_spec.remote_image[constants.BUILD_TARGET], + avd_spec.system_build_info[constants.BUILD_ID], + avd_spec.system_build_info[constants.BUILD_BRANCH], + avd_spec.system_build_info[constants.BUILD_TARGET], + avd_spec.kernel_build_info[constants.BUILD_ID], + avd_spec.kernel_build_info[constants.BUILD_BRANCH], + avd_spec.kernel_build_info[constants.BUILD_TARGET], + avd_spec.bootloader_build_info[constants.BUILD_ID], + avd_spec.bootloader_build_info[constants.BUILD_BRANCH], + avd_spec.bootloader_build_info[constants.BUILD_TARGET]) @utils.TimeExecute(function_description="Processing and uploading local images") - def _UploadArtifacts(self, - local_image_zip, - cvd_host_package_artifact, - images_dir): + def _UploadLocalImageArtifacts(self, + local_image_zip, + cvd_host_package_artifact, + images_dir): """Upload local images and avd local host package to instance. There are two ways to upload local images. @@ -312,8 +266,18 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): self._ssh.Run(remote_cmd) else: # Compress image files for faster upload. - artifact_files = [os.path.basename(image) for image in glob.glob( - os.path.join(images_dir, "*.img"))] + try: + images_path = os.path.join(images_dir, "required_images") + with open(images_path, "r") as images: + artifact_files = images.read().splitlines() + except IOError: + # Older builds may not have a required_images file. In this case + # we fall back to *.img. + artifact_files = [] + for file_name in _ARTIFACT_FILES: + artifact_files.extend( + os.path.basename(image) for image in glob.glob( + os.path.join(images_dir, file_name))) cmd = ("tar -cf - --lzop -S -C {images_dir} {artifact_files} | " "{ssh_cmd} -- tar -xf - --lzop -S".format( images_dir=images_dir, @@ -327,6 +291,26 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): logger.debug("remote_cmd:\n %s", remote_cmd) self._ssh.Run(remote_cmd) + @utils.TimeExecute(function_description="Uploading remote image artifacts") + def _UploadRemoteImageArtifacts(self, images_dir): + """Upload remote image artifacts to instance. + + Args: + images_dir: String, directory of local artifacts downloaded by fetch_cvd. + """ + artifact_files = [ + os.path.basename(image) + for image in glob.glob(os.path.join(images_dir, _ALL_FILES)) + ] + # TODO(b/182259589): Refactor upload image command into a function. + cmd = ("tar -cf - --lzop -S -C {images_dir} {artifact_files} | " + "{ssh_cmd} -- tar -xf - --lzop -S".format( + images_dir=images_dir, + artifact_files=" ".join(artifact_files), + ssh_cmd=self._ssh.GetBaseCmd(constants.SSH_BIN))) + logger.debug("cmd:\n %s", cmd) + ssh.ShellCmdWithRetry(cmd) + def _LaunchCvd(self, instance, decompress_kernel=None, boot_timeout_secs=None): """Launch CVD. @@ -336,42 +320,14 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): boot_timeout_secs: Integer, the maximum time to wait for the command to respond. """ - kernel_build = None # TODO(b/140076771) Support kernel image for local image mode. - if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE: - kernel_build = self._compute_client.GetKernelBuild( - self._avd_spec.kernel_build_info[constants.BUILD_ID], - self._avd_spec.kernel_build_info[constants.BUILD_BRANCH], - self._avd_spec.kernel_build_info[constants.BUILD_TARGET]) self._compute_client.LaunchCvd( instance, self._avd_spec, self._cfg.extra_data_disk_size_gb, - kernel_build, decompress_kernel, boot_timeout_secs) - def GetFailures(self): - """Get failures from all devices. - - Returns: - A dictionary that contains all the failures. - The key is the name of the instance that fails to boot, - and the value is an errors.DeviceBootError object. - """ - return self._compute_client.all_failures - - def _SetFailures(self, instance, error_msg): - """Set failures from this device. - - Record the failures for any steps in AVD creation. - - Args: - instance: String of instance name. - error_msg: String of error message. - """ - self._compute_client.all_failures[instance] = error_msg - def GetBuildInfoDict(self): """Get build info dictionary. @@ -395,4 +351,8 @@ class RemoteInstanceDeviceFactory(base_device_factory.BaseDeviceFactory): {"system_%s" % key: val for key, val in self._avd_spec.system_build_info.items() if val} ) + build_info_dict.update( + {"bootloader_%s" % key: val + for key, val in self._avd_spec.bootloader_build_info.items() if val} + ) return build_info_dict diff --git a/public/actions/remote_instance_cf_device_factory_test.py b/public/actions/remote_instance_cf_device_factory_test.py index e6807236..1e0aa0bd 100644 --- a/public/actions/remote_instance_cf_device_factory_test.py +++ b/public/actions/remote_instance_cf_device_factory_test.py @@ -16,15 +16,17 @@ import glob import os import shutil +import subprocess import tempfile import unittest import uuid -import mock +from unittest import mock + +import six from acloud.create import avd_spec from acloud.internal import constants -from acloud.create import create_common from acloud.internal.lib import android_build_client from acloud.internal.lib import auth from acloud.internal.lib import cvd_compute_client_multi_stage @@ -40,21 +42,21 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): def setUp(self): """Set up the test.""" - super(RemoteInstanceDeviceFactoryTest, self).setUp() + super().setUp() self.Patch(auth, "CreateCredentials", return_value=mock.MagicMock()) self.Patch(android_build_client.AndroidBuildClient, "InitResourceHandle") self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "InitResourceHandle") self.Patch(list_instances, "GetInstancesFromInstanceNames", return_value=mock.MagicMock()) self.Patch(list_instances, "ChooseOneRemoteInstance", return_value=mock.MagicMock()) self.Patch(utils, "GetBuildEnvironmentVariable", - return_value="test_environ") + return_value="test_env_cf_arm") self.Patch(glob, "glob", return_vale=["fake.img"]) # pylint: disable=protected-access @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, "_FetchBuild") @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_UploadArtifacts") + "_UploadLocalImageArtifacts") def testProcessArtifacts(self, mock_upload, mock_download): """test ProcessArtifacts.""" # Test image source type is local. @@ -62,7 +64,9 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.config_file = "" args.avd_type = constants.TYPE_CF args.flavor = "phone" - args.local_image = None + args.local_image = constants.FIND_IN_BUILD_ENV + args.local_system_image = None + args.launch_args = None avd_spec_local_img = avd_spec.AVDSpec(args) fake_image_name = "/fake/aosp_cf_x86_phone-img-eng.username.zip" fake_host_package_name = "/fake/host_package.tar.gz" @@ -74,7 +78,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.assertEqual(mock_upload.call_count, 1) # Test image source type is remote. - args.local_image = "" + args.local_image = None args.build_id = "1234" args.branch = "fake_branch" args.build_target = "fake_target" @@ -100,8 +104,10 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.config_file = "" args.avd_type = constants.TYPE_CF args.flavor = "phone" - args.local_image = None + args.local_image = constants.FIND_IN_BUILD_ENV + args.local_system_image = None args.adb_port = None + args.launch_args = None fake_avd_spec = avd_spec.AVDSpec(args) fake_avd_spec.cfg.enable_multi_stage = True fake_avd_spec._instance_name_to_reuse = None @@ -109,6 +115,8 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_uuid = mock.MagicMock(hex="1234") self.Patch(uuid, "uuid4", return_value=fake_uuid) self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "CreateInstance") + self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, + "GetHostImageName", return_value="fake_image") fake_host_package_name = "/fake/host_package.tar.gz" fake_image_name = "/fake/aosp_cf_x86_phone-img-eng.username.zip" @@ -142,8 +150,10 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.avd_type = constants.TYPE_CF args.flavor = "phone" args.remote_host = "1.1.1.1" - args.local_image = None + args.local_image = constants.FIND_IN_BUILD_ENV + args.local_system_image = None args.adb_port = None + args.launch_args = None fake_avd_spec = avd_spec.AVDSpec(args) fake_avd_spec.cfg.enable_multi_stage = True fake_avd_spec._instance_name_to_reuse = None @@ -174,14 +184,18 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.config_file = "" args.avd_type = constants.TYPE_CF args.flavor = "phone" - args.local_image = None + args.local_image = constants.FIND_IN_BUILD_ENV + args.local_system_image = None args.adb_port = None + args.launch_args = None fake_avd_spec = avd_spec.AVDSpec(args) fake_avd_spec.cfg.enable_multi_stage = True fake_avd_spec._instance_name_to_reuse = "fake-1234-userbuild-fake-target" fake_uuid = mock.MagicMock(hex="1234") self.Patch(uuid, "uuid4", return_value=fake_uuid) self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "CreateInstance") + self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, + "GetHostImageName", return_value="fake_image") fake_host_package_name = "/fake/host_package.tar.gz" fake_image_name = "/fake/aosp_cf_x86_phone-img-eng.username.zip" factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( @@ -200,7 +214,10 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.avd_type = constants.TYPE_CF args.flavor = "phone" args.local_image = "fake_local_image" + args.local_system_image = None args.adb_port = None + args.cheeps_betty_image = None + args.launch_args = None avd_spec_local_image = avd_spec.AVDSpec(args) factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( avd_spec_local_image, @@ -209,7 +226,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.assertEqual(factory.GetBuildInfoDict(), None) # Test image source type is remote. - args.local_image = "" + args.local_image = None args.build_id = "123" args.branch = "fake_branch" args.build_target = "fake_target" @@ -219,6 +236,9 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.kernel_build_id = "345" args.kernel_branch = "kernel_branch" args.kernel_build_target = "kernel_target" + args.bootloader_build_id = "456" + args.bootloader_branch = "bootloader_branch" + args.bootloader_build_target = "bootloader_target" avd_spec_remote_image = avd_spec.AVDSpec(args) factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( avd_spec_remote_image, @@ -226,14 +246,17 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_host_package_name) expected_build_info = { "build_id": "123", - "build_branch": "fake_branch", + "branch": "fake_branch", "build_target": "fake_target", "system_build_id": "234", - "system_build_branch": "sys_branch", + "system_branch": "sys_branch", "system_build_target": "sys_target", "kernel_build_id": "345", - "kernel_build_branch": "kernel_branch", - "kernel_build_target": "kernel_target" + "kernel_branch": "kernel_branch", + "kernel_build_target": "kernel_target", + "bootloader_build_id": "456", + "bootloader_branch": "bootloader_branch", + "bootloader_build_target": "bootloader_target" } self.assertEqual(factory.GetBuildInfoDict(), expected_build_info) @@ -251,7 +274,9 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): args.avd_type = constants.TYPE_CF args.flavor = "phone" args.local_image = "fake_local_image" + args.local_system_image = None args.adb_port = None + args.launch_args = None avd_spec_local_image = avd_spec.AVDSpec(args) factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( avd_spec_local_image, @@ -260,7 +285,9 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): factory._ssh = ssh.Ssh(ip=fake_ip, user=constants.GCE_USER, ssh_private_key_path="/fake/acloud_rea") - factory._UploadArtifacts(fake_image, fake_host_package, fake_local_image_dir) + factory._UploadLocalImageArtifacts(fake_image, + fake_host_package, + fake_local_image_dir) expected_cmd1 = ("/usr/bin/install_zip.sh . < %s" % fake_image) expected_cmd2 = ("tar -x -z -f - < %s" % fake_host_package) mock_ssh_run.assert_has_calls([ @@ -269,10 +296,63 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): # Test local image get from local folder case. fake_image = None - self.Patch(glob, "glob", return_value=["fake.img"]) - factory._UploadArtifacts(fake_image, fake_host_package, fake_local_image_dir) + self.Patch(glob, "glob", side_effect=[["fake.img"], ["bootloader"], ["kernel"]]) + factory._UploadLocalImageArtifacts(fake_image, + fake_host_package, + fake_local_image_dir) expected_cmd = ( - "tar -cf - --lzop -S -C %s fake.img | " + "tar -cf - --lzop -S -C %s fake.img bootloader kernel | " + "%s -- tar -xf - --lzop -S" % + (fake_local_image_dir, factory._ssh.GetBaseCmd(constants.SSH_BIN))) + mock_shell.assert_called_once_with(expected_cmd) + + mock_shell.reset_mock() + required_images = mock.mock_open(read_data=( + "boot.img\n" + "cache.img\n" + "super.img\n" + "userdata.img\n" + "bootloader\n")) + with mock.patch.object(six.moves.builtins, "open", required_images): + factory._UploadLocalImageArtifacts(fake_image, + fake_host_package, + fake_local_image_dir) + expected_cmd = ( + "tar -cf - --lzop -S -C %s boot.img cache.img super.img userdata.img bootloader | " + "%s -- tar -xf - --lzop -S" % + (fake_local_image_dir, factory._ssh.GetBaseCmd(constants.SSH_BIN))) + mock_shell.assert_called_once_with(expected_cmd) + + @mock.patch.object(ssh, "ShellCmdWithRetry") + def testUploadRemoteImageArtifacts(self, mock_shell): + """Test UploadRemoteImageArtifacts.""" + fake_host_package = "/fake/host_package.tar.gz" + fake_image_zip = None + fake_local_image_dir = "/fake_image" + fake_ip = ssh.IP(external="1.1.1.1", internal="10.1.1.1") + args = mock.MagicMock() + # Test local image extract from image zip case. + args.config_file = "" + args.avd_type = constants.TYPE_CF + args.flavor = "phone" + args.local_image = "fake_local_image" + args.local_system_image = None + args.adb_port = None + args.launch_args = None + avd_spec_local_image = avd_spec.AVDSpec(args) + factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( + avd_spec_local_image, + fake_image_zip, + fake_host_package) + factory._ssh = ssh.Ssh(ip=fake_ip, + user=constants.GCE_USER, + ssh_private_key_path="/fake/acloud_rea") + + self.Patch(glob, "glob", return_value=["fake.img", "bootloader", "kernel"]) + factory._UploadRemoteImageArtifacts(fake_local_image_dir) + + expected_cmd = ( + "tar -cf - --lzop -S -C %s fake.img bootloader kernel | " "%s -- tar -xf - --lzop -S" % (fake_local_image_dir, factory._ssh.GetBaseCmd(constants.SSH_BIN))) mock_shell.assert_called_once_with(expected_cmd) @@ -280,7 +360,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, "_InitRemotehost") @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_UploadArtifacts") + "_UploadLocalImageArtifacts") @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, "_LaunchCvd") def testLocalImageRemoteHost(self, mock_launchcvd, mock_upload, mock_init_remote_host): @@ -307,7 +387,7 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, "_CreateGceInstance") @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_UploadArtifacts") + "_UploadLocalImageArtifacts") @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, "_LaunchCvd") def testLocalImageCreateInstance(self, mock_launchcvd, mock_upload, mock_create_gce_instance): @@ -332,11 +412,12 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.assertEqual(mock_launchcvd.call_count, 1) # pylint: disable=no-member - @mock.patch.object(create_common, "DownloadRemoteArtifact") - def testDownloadArtifacts(self, mock_download): + @mock.patch.object(subprocess, "check_call") + def testDownloadArtifacts(self, mock_check_call): """Test process remote cuttlefish image.""" extract_path = "/tmp/1111/" fake_remote_image = {"build_target" : "aosp_cf_x86_phone-userdebug", + "branch" : "aosp-master", "build_id": "1234"} self.Patch( cvd_compute_client_multi_stage, @@ -346,32 +427,24 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): self.Patch(shutil, "rmtree") fake_avd_spec = mock.MagicMock() fake_avd_spec.cfg = mock.MagicMock() + fake_avd_spec.cfg.creds_cache_file = "cache.file" fake_avd_spec.remote_image = fake_remote_image fake_avd_spec.image_download_dir = "/tmp" self.Patch(os.path, "exists", return_value=False) self.Patch(os, "makedirs") factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( fake_avd_spec) + factory._DownloadArtifacts(extract_path) - build_id = "1234" - build_target = "aosp_cf_x86_phone-userdebug" - checkfile1 = "aosp_cf_x86_phone-img-1234.zip" - checkfile2 = "cvd-host_package.tar.gz" - - # To validate DownloadArtifact runs twice. - self.assertEqual(mock_download.call_count, 2) - - # To validate DownloadArtifact arguments correct. - mock_download.assert_has_calls([ - mock.call(fake_avd_spec.cfg, build_target, build_id, checkfile1, - extract_path, decompress=True), - mock.call(fake_avd_spec.cfg, build_target, build_id, checkfile2, - extract_path)], any_order=True) - - @mock.patch.object(create_common, "DownloadRemoteArtifact") + self.assertEqual(mock_check_call.call_count, 1) + @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, - "_UploadArtifacts") - def testProcessRemoteHostArtifacts(self, mock_upload, mock_download): + "_UploadLocalImageArtifacts") + @mock.patch.object(remote_instance_cf_device_factory.RemoteInstanceDeviceFactory, + "_UploadRemoteImageArtifacts") + def testProcessRemoteHostArtifacts(self, + mock_upload_remote_image, + mock_upload_local_image): """Test process remote host artifacts.""" self.Patch( cvd_compute_client_multi_stage, @@ -383,6 +456,8 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_avd_spec.instance_type = constants.INSTANCE_TYPE_HOST fake_avd_spec.image_source = constants.IMAGE_SRC_LOCAL fake_avd_spec._instance_name_to_reuse = None + fake_avd_spec.cfg = mock.MagicMock() + fake_avd_spec.cfg.creds_cache_file = "cache.file" fake_host_package_name = "/fake/host_package.tar.gz" fake_image_name = "" factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( @@ -390,21 +465,20 @@ class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): fake_image_name, fake_host_package_name) factory._ProcessRemoteHostArtifacts() - self.assertEqual(mock_upload.call_count, 1) + self.assertEqual(mock_upload_local_image.call_count, 1) # Test process remote host artifacts with remote images. fake_tmp_folder = "/tmp/1111/" - mock_upload.call_count = 0 self.Patch(tempfile, "mkdtemp", return_value=fake_tmp_folder) self.Patch(shutil, "rmtree") + self.Patch(subprocess, "check_call") fake_avd_spec.instance_type = constants.INSTANCE_TYPE_HOST fake_avd_spec.image_source = constants.IMAGE_SRC_REMOTE fake_avd_spec._instance_name_to_reuse = None factory = remote_instance_cf_device_factory.RemoteInstanceDeviceFactory( fake_avd_spec) factory._ProcessRemoteHostArtifacts() - self.assertEqual(mock_upload.call_count, 1) - self.assertEqual(mock_download.call_count, 2) + self.assertEqual(mock_upload_remote_image.call_count, 1) shutil.rmtree.assert_called_once_with(fake_tmp_folder) diff --git a/public/actions/remote_instance_fvp_device_factory.py b/public/actions/remote_instance_fvp_device_factory.py new file mode 100644 index 00000000..6c6dcbe2 --- /dev/null +++ b/public/actions/remote_instance_fvp_device_factory.py @@ -0,0 +1,95 @@ +# Copyright 2020 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""RemoteInstanceDeviceFactory provides basic interface to create an FVP +device factory.""" + +import os + +from acloud.internal import constants +from acloud.internal.lib import utils +from acloud.internal.lib import ssh +from acloud.public.actions import gce_device_factory + +class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory): + def __init__(self, avd_spec): + super(RemoteInstanceDeviceFactory, self).__init__(avd_spec) + + def CreateInstance(self): + """Start a GCE instance, copy the necessary artifacts to it and then + start FVP. + + Returns: + The instance. + """ + instance = self._CreateGceInstance() + if instance in self.GetFailures(): + return instance + + try: + self._UploadArtifacts() + self._StartFVP() + except Exception as e: + self._SetFailures(instance, e) + + return instance + + @utils.TimeExecute(function_description="Processing and uploading local images") + def _UploadArtifacts(self): + """Copy artifacts to the GCE instance: the local images, the model + itself and support files. + """ + images_dir = self._avd_spec.local_image_dir + images_path = os.path.join(images_dir, "required_images") + with open(images_path, "r") as images: + artifact_files = images.read().splitlines() + ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) + + cmd = ("tar -cf - --lzop -S -C {images_dir} {artifact_files} | " + "{ssh_cmd} -- tar -xf - --lzop -S".format( + images_dir=images_dir, + artifact_files=" ".join(artifact_files), + ssh_cmd=ssh_cmd)) + ssh.ShellCmdWithRetry(cmd) + + model_bin = utils.GetBuildEnvironmentVariable("MODEL_BIN") + cmd = ("tar -cf - --lzop -S -C {model_dir} . | " + "{ssh_cmd} -- tar -xf - --lzop -S".format( + model_dir=os.path.dirname(model_bin), + ssh_cmd=ssh_cmd)) + ssh.ShellCmdWithRetry(cmd) + + self._ssh.ScpPushFile( + src_file="device/generic/goldfish/fvpbase/run_model_only", + dst_file="run_model_only") + + cmd = "{ssh_cmd} -- mkdir -p lib64".format(ssh_cmd=ssh_cmd) + ssh.ShellCmdWithRetry(cmd) + host_out = utils.GetBuildEnvironmentVariable("ANDROID_HOST_OUT") + self._ssh.ScpPushFile( + src_file="%s/lib64/bind_to_localhost.so" % host_out, + dst_file="lib64/bind_to_localhost.so") + + @utils.TimeExecute(function_description="Starting FVP") + def _StartFVP(self): + """Start the model on the GCE instance.""" + ssh_cmd = self._ssh.GetBaseCmd(constants.SSH_BIN) + model_bin = utils.GetBuildEnvironmentVariable("MODEL_BIN") + + cmd = ("{ssh_cmd} -- sh -c \"'ANDROID_HOST_OUT=. " + "ANDROID_PRODUCT_OUT=. MODEL_BIN=./{model_basename} " + "./run_model_only > /dev/null 2> /dev/null &'\"".format( + ssh_cmd=ssh_cmd, + model_basename=os.path.basename(model_bin))) + ssh.ShellCmdWithRetry(cmd) diff --git a/public/actions/remote_instance_fvp_device_factory_test.py b/public/actions/remote_instance_fvp_device_factory_test.py new file mode 100644 index 00000000..74330733 --- /dev/null +++ b/public/actions/remote_instance_fvp_device_factory_test.py @@ -0,0 +1,112 @@ +# Copyright 2019 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for remote_instance_cf_device_factory.""" + +import glob +import os +import unittest + +from unittest import mock + +import six + +from acloud.create import avd_spec +from acloud.internal import constants +from acloud.internal.lib import android_build_client +from acloud.internal.lib import auth +from acloud.internal.lib import cvd_compute_client_multi_stage +from acloud.internal.lib import driver_test_lib +from acloud.internal.lib import ssh +from acloud.list import list as list_instances +from acloud.public.actions import remote_instance_fvp_device_factory + + +class RemoteInstanceDeviceFactoryTest(driver_test_lib.BaseDriverTest): + """Test RemoteInstanceDeviceFactory.""" + + def setUp(self): + super().setUp() + self.Patch(auth, "CreateCredentials", return_value=mock.MagicMock()) + self.Patch(android_build_client.AndroidBuildClient, "InitResourceHandle") + self.Patch(cvd_compute_client_multi_stage.CvdComputeClient, "InitResourceHandle") + self.Patch(list_instances, "GetInstancesFromInstanceNames", return_value=mock.MagicMock()) + self.Patch(list_instances, "ChooseOneRemoteInstance", return_value=mock.MagicMock()) + self.Patch(glob, "glob", return_vale=["fake.img"]) + + # pylint: disable=protected-access + @staticmethod + @mock.patch.object( + remote_instance_fvp_device_factory.RemoteInstanceDeviceFactory, + "_CreateGceInstance") + @mock.patch.object(ssh, "ShellCmdWithRetry") + @mock.patch.dict(os.environ, { + constants.ENV_BUILD_TARGET:'fvp', + "ANDROID_HOST_OUT":'/path/to/host/out', + "ANDROID_PRODUCT_OUT":'/path/to/product/out', + "MODEL_BIN":'/path/to/model/FVP_Base_RevC-2xAEMv8A', + }) + def testCreateInstance(mock_shell, mock_create_gce): + """Test CreateInstance.""" + fake_ip = ssh.IP(external="1.1.1.1", internal="10.1.1.1") + args = mock.MagicMock() + # Test local image extract from image zip case. + args.config_file = "" + args.avd_type = constants.TYPE_FVP + args.flavor = "phone" + args.local_image = "fake_local_image" + args.local_system_image = None + args.adb_port = None + args.launch_args = None + avd_spec_local_image = avd_spec.AVDSpec(args) + factory = remote_instance_fvp_device_factory.RemoteInstanceDeviceFactory( + avd_spec_local_image) + factory._ssh = ssh.Ssh(ip=fake_ip, + user=constants.GCE_USER, + ssh_private_key_path="/fake/acloud_rea") + mock_open = mock.mock_open(read_data = ( + "bl1.bin\n" + "boot.img\n" + "fip.bin\n" + "system-qemu.img\n" + "userdata.img\n")) + with mock.patch.object(six.moves.builtins, "open", mock_open): + factory.CreateInstance() + + mock_create_gce.assert_called_once() + + expected_cmds = [ + ("tar -cf - --lzop -S -C /path/to/product/out bl1.bin boot.img " + "fip.bin system-qemu.img userdata.img | " + "%s -- tar -xf - --lzop -S" % + factory._ssh.GetBaseCmd(constants.SSH_BIN)), + ("tar -cf - --lzop -S -C /path/to/model . | " + "%s -- tar -xf - --lzop -S" % + factory._ssh.GetBaseCmd(constants.SSH_BIN)), + ("%s device/generic/goldfish/fvpbase/run_model_only " + "vsoc-01@1.1.1.1:run_model_only" % + factory._ssh.GetBaseCmd(constants.SCP_BIN)), + ("%s -- mkdir -p lib64" % + factory._ssh.GetBaseCmd(constants.SSH_BIN)), + ("%s /path/to/host/out/lib64/bind_to_localhost.so " + "vsoc-01@1.1.1.1:lib64/bind_to_localhost.so" % + factory._ssh.GetBaseCmd(constants.SCP_BIN)), + ("%s -- sh -c \"'ANDROID_HOST_OUT=. ANDROID_PRODUCT_OUT=. " + "MODEL_BIN=./FVP_Base_RevC-2xAEMv8A " + "./run_model_only > /dev/null 2> /dev/null &'\"" % + factory._ssh.GetBaseCmd(constants.SSH_BIN)), + ] + mock_shell.assert_has_calls([mock.call(cmd) for cmd in expected_cmds]) + +if __name__ == "__main__": + unittest.main() diff --git a/public/avd.py b/public/avd.py index bb38e2a5..4ac1bb31 100755 --- a/public/avd.py +++ b/public/avd.py @@ -38,7 +38,7 @@ logger = logging.getLogger(__name__) class AndroidVirtualDevice(object): """Represent an Android device.""" - def __init__(self, instance_name, ip=None, time_info=None): + def __init__(self, instance_name, ip=None, time_info=None, stage=None): """Initialize. Args: @@ -46,6 +46,7 @@ class AndroidVirtualDevice(object): ip: namedtuple (internal, external) that holds IP address of the gce instance, e.g. "external:140.110.20.1, internal:10.0.0.1" time_info: Dict of time cost information, e.g. {"launch_cvd": 5} + stage: Integer of AVD in which stage, e.g. STAGE_GCE, STAGE_BOOT_UP """ self._ip = ip self._instance_name = instance_name @@ -72,6 +73,7 @@ class AndroidVirtualDevice(object): # "system_build_target": "cf_x86_phone-userdebug", # "system_gcs_bucket_build_id": "12345"} self._build_info = {} + self._stage = stage @property def ip(self): @@ -100,6 +102,11 @@ class AndroidVirtualDevice(object): """Getter of _time_info.""" return self._time_info + @property + def stage(self): + """Getter of _stage.""" + return self._stage + @build_info.setter def build_info(self, value): self._build_info = value diff --git a/public/config.py b/public/config.py index d538230a..e7eb8e14 100755 --- a/public/config.py +++ b/public/config.py @@ -63,6 +63,28 @@ logger = logging.getLogger(__name__) _CONFIG_DATA_PATH = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data") _DEFAULT_CONFIG_FILE = "acloud.config" +_DEFAULT_HW_PROPERTY = "cpu:4,resolution:720x1280,dpi:320,memory:4g" + +# VERSION +_VERSION_FILE = "VERSION" +_UNKNOWN = "UNKNOWN" +_NUM_INSTANCES_ARG = "-num_instances" + + +def GetVersion(): + """Print the version of acloud. + + The VERSION file is built into the acloud binary. The version file path is + under "public/data". + + Returns: + String of the acloud version. + """ + version_file_path = os.path.join(_CONFIG_DATA_PATH, _VERSION_FILE) + if os.path.exists(version_file_path): + with open(version_file_path) as version_file: + return version_file.read() + return _UNKNOWN def GetDefaultConfigFile(): @@ -89,7 +111,7 @@ def GetAcloudConfig(args): return cfg -class AcloudConfig(object): +class AcloudConfig(): """A class that holds all configurations for acloud.""" REQUIRED_FIELD = [ @@ -179,6 +201,7 @@ class AcloudConfig(object): self.orientation = usr_cfg.orientation self.resolution = usr_cfg.resolution + self.stable_host_image_family = usr_cfg.stable_host_image_family self.stable_host_image_name = ( usr_cfg.stable_host_image_name or internal_cfg.default_usr_cfg.stable_host_image_name) @@ -201,6 +224,7 @@ class AcloudConfig(object): self.stable_cheeps_host_image_project = ( usr_cfg.stable_cheeps_host_image_project or internal_cfg.default_usr_cfg.stable_cheeps_host_image_project) + self.betty_image = usr_cfg.betty_image self.extra_args_ssh_tunnel = usr_cfg.extra_args_ssh_tunnel @@ -208,6 +232,8 @@ class AcloudConfig(object): self.hw_property = usr_cfg.hw_property self.launch_args = usr_cfg.launch_args + self.api_key = usr_cfg.api_key + self.api_url = usr_cfg.api_url self.instance_name_pattern = ( usr_cfg.instance_name_pattern or internal_cfg.default_usr_cfg.instance_name_pattern) @@ -224,6 +250,7 @@ class AcloudConfig(object): # Verify validity of configurations. self.Verify() + # pylint: disable=too-many-branches def OverrideWithArgs(self, parsed_args): """Override configuration values with args passed in from cmd line. @@ -244,30 +271,47 @@ class AcloudConfig(object): parsed_args.service_account_json_private_key_path) if parsed_args.which == "create_gf" and parsed_args.base_image: self.stable_goldfish_host_image_name = parsed_args.base_image - if parsed_args.which == create_args.CMD_CREATE and not self.hw_property: - flavor = parsed_args.flavor or constants.FLAVOR_PHONE - self.hw_property = self.common_hw_property_map.get(flavor, "") if parsed_args.which in [create_args.CMD_CREATE, "create_cf"]: if parsed_args.network: self.network = parsed_args.network if parsed_args.multi_stage_launch is not None: self.enable_multi_stage = parsed_args.multi_stage_launch - - def OverrideHwPropertyWithFlavor(self, flavor): - """Override hw configuration values with flavor name. - - HwProperty will be overrided according to the change of flavor. - If flavor is None, set hw configuration with phone(default flavor). + if parsed_args.which in [create_args.CMD_CREATE, "create_cf", "create_gf"]: + if parsed_args.zone: + self.zone = parsed_args.zone + if (parsed_args.which == "create_cf" and + parsed_args.num_avds_per_instance > 1): + scrubbed_args = [arg for arg in self.launch_args.split() + if _NUM_INSTANCES_ARG not in arg] + scrubbed_args.append("%s=%d" % (_NUM_INSTANCES_ARG, + parsed_args.num_avds_per_instance)) + + self.launch_args = " ".join(scrubbed_args) + + def GetDefaultHwProperty(self, flavor, instance_type=None): + """Get default hw configuration values. + + HwProperty will be overrided according to the change of flavor and + instance type. The format of key is flavor or instance_type-flavor. + e.g: 'phone' or 'local-phone'. + If the giving key is not found, get hw configuration with a default + phone property. Args: - flavor: string of flavor name. + flavor: String of flavor name. + instance_type: String of instance type. + + Returns: + String of device hardware property, it would be like + "cpu:4,resolution:720x1280,dpi:320,memory:4g". """ - self.hw_property = self.common_hw_property_map.get( - flavor, constants.FLAVOR_PHONE) + hw_key = ("%s-%s" % (instance_type, flavor) + if instance_type == constants.INSTANCE_TYPE_LOCAL else flavor) + return self.common_hw_property_map.get(hw_key, _DEFAULT_HW_PROPERTY) def Verify(self): """Verify configuration fields.""" - missing = [f for f in self.REQUIRED_FIELD if not getattr(self, f)] + missing = self.GetMissingFields(self.REQUIRED_FIELD) if missing: raise errors.ConfigError( "Missing required configuration fields: %s" % missing) @@ -278,12 +322,23 @@ class AcloudConfig(object): "invalid value: %d" % (self.precreated_data_image_map.keys(), self.extra_data_disk_size_gb)) + def GetMissingFields(self, fields): + """Get missing required fields. + + Args: + fields: List of field names. + + Returns: + List of missing field names. + """ + return [f for f in fields if not getattr(self, f)] + def SupportRemoteInstance(self): """Return True if gcp project is provided in config.""" - return True if self.project else False + return bool(self.project) -class AcloudConfigManager(object): +class AcloudConfigManager(): """A class that loads configurations.""" _DEFAULT_INTERNAL_CONFIG_PATH = os.path.join(_CONFIG_DATA_PATH, diff --git a/public/config_test.py b/public/config_test.py index 510fb13d..0a1f3f84 100644 --- a/public/config_test.py +++ b/public/config_test.py @@ -18,7 +18,8 @@ import unittest import os import tempfile -import mock + +from unittest import mock import six @@ -54,6 +55,7 @@ metadata_variable { hw_property: "cpu:3,resolution:1080x1920,dpi:480,memory:4g,disk:10g" extra_scopes: "scope1" extra_scopes: "scope2" +betty_image: "fake_betty_image" """ INTERNAL_CONFIG = """ @@ -108,10 +110,23 @@ common_hw_property_map { key: "phone" value: "cpu:2,resolution:1080x1920,dpi:420,memory:4g,disk:8g" } + +common_hw_property_map { + key: "auto" + value: "cpu:4,resolution:1280x800,dpi:160,memory:4g" +} """ def setUp(self): self.config_file = mock.MagicMock() + # initial config with test config. + self.config_file.read.return_value = self.INTERNAL_CONFIG + internal_cfg = config.AcloudConfigManager.LoadConfigFromProtocolBuffer( + self.config_file, internal_config_pb2.InternalConfig) + self.config_file.read.return_value = self.USER_CONFIG + usr_cfg = config.AcloudConfigManager.LoadConfigFromProtocolBuffer( + self.config_file, user_config_pb2.UserConfig) + self.cfg = config.AcloudConfig(usr_cfg, internal_cfg) # pylint: disable=no-member def testLoadUserConfig(self): @@ -143,6 +158,7 @@ common_hw_property_map { "cpu:3,resolution:1080x1920,dpi:480,memory:4g," "disk:10g") self.assertEqual(cfg.extra_scopes, ["scope1", "scope2"]) + self.assertEqual(cfg.betty_image, "fake_betty_image") # pylint: disable=protected-access @mock.patch("os.makedirs") @@ -251,7 +267,8 @@ common_hw_property_map { # hw property self.assertEqual( {key: val for key, val in six.iteritems(cfg.common_hw_property_map)}, - {"phone": "cpu:2,resolution:1080x1920,dpi:420,memory:4g,disk:8g"}) + {"phone": "cpu:2,resolution:1080x1920,dpi:420,memory:4g,disk:8g", + "auto": "cpu:4,resolution:1280x800,dpi:160,memory:4g"}) def testLoadConfigFails(self): """Test loading a bad file.""" @@ -260,33 +277,33 @@ common_hw_property_map { config.AcloudConfigManager.LoadConfigFromProtocolBuffer( self.config_file, internal_config_pb2.InternalConfig) - def testOverrideWithHWProperty(self): - """Test override hw property by flavor type.""" - # initial config with test config. - self.config_file.read.return_value = self.INTERNAL_CONFIG - internal_cfg = config.AcloudConfigManager.LoadConfigFromProtocolBuffer( - self.config_file, internal_config_pb2.InternalConfig) - self.config_file.read.return_value = self.USER_CONFIG - usr_cfg = config.AcloudConfigManager.LoadConfigFromProtocolBuffer( - self.config_file, user_config_pb2.UserConfig) - cfg = config.AcloudConfig(usr_cfg, internal_cfg) - - # test override with an exist flavor. - cfg.hw_property = None + def testOverrideWithArgs(self): + """Test OverrideWithArgs.""" + # test override zone. + self.cfg.zone = "us-central1-f" args = mock.MagicMock() - args.flavor = "phone" args.which = "create" - cfg.OverrideWithArgs(args) - self.assertEqual(cfg.hw_property, - "cpu:2,resolution:1080x1920,dpi:420,memory:4g,disk:8g") + args.flavor = "phone" + args.zone = "us-central1-b" + self.cfg.OverrideWithArgs(args) + self.assertEqual(self.cfg.zone, "us-central1-b") - # test override with a nonexistent flavor. - cfg.hw_property = None - args = mock.MagicMock() - args.flavor = "non-exist-flavor" - args.which = "create" - cfg.OverrideWithArgs(args) - self.assertEqual(cfg.hw_property, "") + def testGetDefaultHwProperty(self): + """Test GetDefaultHwProperty.""" + # test with "phone" flavor + expected = "cpu:2,resolution:1080x1920,dpi:420,memory:4g,disk:8g" + self.assertEqual(expected, self.cfg.GetDefaultHwProperty("phone")) + + # test with "auto" flavor + expected = "cpu:4,resolution:1280x800,dpi:160,memory:4g" + self.assertEqual(expected, self.cfg.GetDefaultHwProperty("auto")) + + def testGetMissingFields(self): + """Test GetMissingFields.""" + fields = ["project", "zone", "hw_property"] + self.cfg.hw_property = "" + expected = ["hw_property"] + self.assertEqual(expected, self.cfg.GetMissingFields(fields)) if __name__ == "__main__": diff --git a/public/data/default.config b/public/data/default.config index 8ef13221..ebce64de 100644 --- a/public/data/default.config +++ b/public/data/default.config @@ -18,7 +18,7 @@ default_usr_cfg { network: "default" extra_data_disk_size_gb: 0 instance_name_pattern: "ins-{uuid}-{build_id}-{build_target}" - fetch_cvd_version: "6170097" + fetch_cvd_version: "6904202" metadata_variable { key: "camera_front" @@ -54,29 +54,60 @@ default_usr_cfg { # Below are common HW properties, the values also could be referred in the # AVD manager of android sdk. # https://developer.android.com/studio/run/managing-avds +# Cuttlefish config reference: google/cuttlefish/shared/config +common_hw_property_map { + key: "local-phone" + value: "cpu:4,resolution:720x1280,dpi:320,memory:6g" +} + +common_hw_property_map { + key: "local-auto" + value: "cpu:4,resolution:1280x800,dpi:160,memory:6g" +} + +common_hw_property_map { + key: "local-wear" + value: "cpu:4,resolution:320x320,dpi:240,memory:2g" +} + +common_hw_property_map { + key: "local-tablet" + value: "cpu:4,resolution:2560x1800,dpi:320,memory:6g" +} + +common_hw_property_map { + key: "local-foldable" + value: "cpu:4,resolution:1768x2208,dpi:386,memory:4g" +} + common_hw_property_map { key: "phone" - value: "cpu:2,resolution:720x1280,dpi:320,memory:4g" + value: "cpu:4,resolution:720x1280,dpi:320,memory:4g" } common_hw_property_map { key: "auto" - value: "cpu:2,resolution:1280x800,dpi:160,memory:4g" + value: "cpu:4,resolution:1280x800,dpi:160,memory:4g" } common_hw_property_map { key: "wear" - value: "cpu:2,resolution:320x320,dpi:240,memory:2g" + value: "cpu:4,resolution:320x320,dpi:240,memory:2g" } common_hw_property_map { key: "tablet" - value: "cpu:2,resolution:2560x1800,dpi:320,memory:4g" + value: "cpu:4,resolution:2560x1800,dpi:320,memory:4g" } common_hw_property_map { key: "tv" - value: "cpu:2,resolution:1280x720,dpi:213,memory:2g" + value: "cpu:4,resolution:1920x1080,dpi:213,memory:2g" +} + +common_hw_property_map { + key: "foldable" + value: "cpu:4,resolution:1768x2208,dpi:386,memory:4g" } # Device resolution diff --git a/public/device_driver_test.py b/public/device_driver_test.py index 0e86c501..e3c44f21 100644 --- a/public/device_driver_test.py +++ b/public/device_driver_test.py @@ -19,7 +19,8 @@ import uuid import unittest -import mock + +from unittest import mock from acloud.internal.lib import auth from acloud.internal.lib import android_build_client diff --git a/public/report.py b/public/report.py index dd95c4e4..6afff82d 100755 --- a/public/report.py +++ b/public/report.py @@ -32,6 +32,7 @@ The json format of a report dump looks like: "errors": [ "Can't find instances: ['104.197.110.255']" ], + "error_type": "error_type_1", "status": "FAIL" } @@ -65,7 +66,7 @@ from acloud.internal import constants logger = logging.getLogger(__name__) -class Status(object): +class Status(): """Status of acloud command.""" SUCCESS = "SUCCESS" @@ -97,7 +98,7 @@ class Status(object): return cls.SEVERITY_ORDER[candidate] > cls.SEVERITY_ORDER[reference] -class Report(object): +class Report(): """A class that stores and generates report.""" def __init__(self, command): @@ -109,6 +110,7 @@ class Report(object): self.command = command self.status = Status.UNKNOWN self.errors = [] + self.error_type = "" self.data = {} def AddData(self, key, value): @@ -120,6 +122,14 @@ class Report(object): """ self.data.setdefault(key, []).append(value) + def UpdateData(self, dict_data): + """Update a dict data to the report. + + Args: + dict_data: A dict of report data. + """ + self.data.update(dict_data) + def AddError(self, error): """Add error message. @@ -136,6 +146,14 @@ class Report(object): """ self.errors.extend(errors) + def SetErrorType(self, error_type): + """Set error type. + + Args: + error_type: String of error type. + """ + self.error_type = error_type + def SetStatus(self, status): """Set status. @@ -151,7 +169,7 @@ class Report(object): self.status, status) def AddDevice(self, instance_name, ip_address, adb_port, vnc_port, - key="devices"): + webrtc_port=None, device_serial=None, key="devices"): """Add a record of a device. Args: @@ -159,6 +177,8 @@ class Report(object): ip_address: A string. adb_port: An integer. vnc_port: An integer. + webrtc_port: An integer, the port to display device screen. + device_serial: String of device serial. key: A string, the data entry where the record is added. """ device = {constants.INSTANCE_NAME: instance_name} @@ -168,12 +188,19 @@ class Report(object): else: device[constants.IP] = ip_address + if device_serial: + device[constants.DEVICE_SERIAL] = device_serial + if vnc_port: device[constants.VNC_PORT] = vnc_port + + if webrtc_port: + device[constants.WEBRTC_PORT] = webrtc_port self.AddData(key=key, value=device) def AddDeviceBootFailure(self, instance_name, ip_address, adb_port, - vnc_port, error): + vnc_port, error, device_serial=None, + webrtc_port=None): """Add a record of device boot failure. Args: @@ -182,10 +209,24 @@ class Report(object): adb_port: An integer. vnc_port: An integer. Can be None if the device doesn't support it. error: A string, the error message. + device_serial: String of device serial. + webrtc_port: An integer. """ self.AddDevice(instance_name, ip_address, adb_port, vnc_port, - "devices_failing_boot") + webrtc_port, device_serial, "devices_failing_boot") + self.AddError(error) + + def UpdateFailure(self, error, error_type=None): + """Update the falure information of report. + + Args: + error: String, the error message. + error_type: String, the error type. + """ self.AddError(error) + self.SetStatus(Status.FAIL) + if error_type: + self.SetErrorType(error_type) def Dump(self, report_file): """Dump report content to a file. @@ -198,6 +239,7 @@ class Report(object): command=self.command, status=self.status, errors=self.errors, + error_type=self.error_type, data=self.data) logger.info("Report: %s", json.dumps(result, indent=2, sort_keys=True)) if not report_file: diff --git a/public/report_test.py b/public/report_test.py index d3987c80..de1d7bcf 100644 --- a/public/report_test.py +++ b/public/report_test.py @@ -60,16 +60,50 @@ class ReportTest(unittest.TestCase): test_report.SetStatus(report.Status.FAIL) self.assertEqual(test_report.status, "BOOT_FAIL") + def testSetErrorType(self): + """test SetErrorType.""" + error_type = "GCE_QUOTA_ERROR" + test_report = report.Report("create") + test_report.SetErrorType(error_type) + self.assertEqual(test_report.error_type, error_type) + + def testUpdateFailure(self): + """test UpdateFailure.""" + error_type = "GCE_QUOTA_ERROR" + error_msg = "Reach quota limit." + test_report = report.Report("create") + test_report.UpdateFailure(error_msg, error_type) + self.assertEqual(test_report.status, "FAIL") + self.assertEqual(test_report.errors, [error_msg]) + self.assertEqual(test_report.error_type, error_type) + def testAddDevice(self): """test AddDevice.""" test_report = report.Report("create") - test_report.AddDevice("instance_1", "127.0.0.1", 6520, 6444) + test_report.AddDevice("instance_1", "127.0.0.1", 6520, 6444, 8443) + expected = { + "devices": [{ + "instance_name": "instance_1", + "ip": "127.0.0.1:6520", + "adb_port": 6520, + "vnc_port": 6444, + "webrtc_port": 8443 + }] + } + self.assertEqual(test_report.data, expected) + + # Write report with "device_serial" + test_report = report.Report("create") + device_serial = "emulator-test" + test_report.AddDevice("instance_1", "127.0.0.1", 6520, 6444, + device_serial=device_serial) expected = { "devices": [{ "instance_name": "instance_1", "ip": "127.0.0.1:6520", "adb_port": 6520, - "vnc_port": 6444 + "vnc_port": 6444, + "device_serial": device_serial }] } self.assertEqual(test_report.data, expected) @@ -77,14 +111,16 @@ class ReportTest(unittest.TestCase): def testAddDeviceBootFailure(self): """test AddDeviceBootFailure.""" test_report = report.Report("create") + device_serial = "emulator-test" test_report.AddDeviceBootFailure("instance_1", "127.0.0.1", 6520, 6444, - "some errors") + "some errors", device_serial) expected = { "devices_failing_boot": [{ "instance_name": "instance_1", "ip": "127.0.0.1:6520", "adb_port": 6520, - "vnc_port": 6444 + "vnc_port": 6444, + "device_serial": device_serial }] } self.assertEqual(test_report.data, expected) diff --git a/pull/pull.py b/pull/pull.py index 5da45d5c..7f118da4 100644 --- a/pull/pull.py +++ b/pull/pull.py @@ -34,8 +34,7 @@ from acloud.public import report logger = logging.getLogger(__name__) -_REMOTE_LOG_FOLDER = "/home/%s/cuttlefish_runtime" % constants.GCE_USER -_FIND_LOG_FILE_CMD = "find %s -type f" % _REMOTE_LOG_FOLDER +_FIND_LOG_FILE_CMD = "find -L %s -type f" % constants.REMOTE_LOG_FOLDER # Black list for log files. _KERNEL = "kernel" _IMG_FILE_EXTENSION = ".img" @@ -143,7 +142,7 @@ def SelectLogFileToPull(ssh, file_name=None): """ log_files = GetAllLogFilePaths(ssh) if file_name: - file_path = os.path.join(_REMOTE_LOG_FOLDER, file_name) + file_path = os.path.join(constants.REMOTE_LOG_FOLDER, file_name) if file_path in log_files: return [file_path] raise errors.CheckPathError("Can't find this log file(%s) from remote " @@ -157,7 +156,7 @@ def SelectLogFileToPull(ssh, file_name=None): return utils.GetAnswerFromList(log_files, enable_choose_all=True) raise errors.CheckPathError("Can't find any log file in folder(%s) from " - "remote instance." % _REMOTE_LOG_FOLDER) + "remote instance." % constants.REMOTE_LOG_FOLDER) def GetAllLogFilePaths(ssh): @@ -172,11 +171,11 @@ def GetAllLogFilePaths(ssh): ssh_cmd = [ssh.GetBaseCmd(constants.SSH_BIN), _FIND_LOG_FILE_CMD] log_files = [] try: - files_output = subprocess.check_output(" ".join(ssh_cmd), shell=True) + files_output = utils.CheckOutput(" ".join(ssh_cmd), shell=True) log_files = FilterLogfiles(files_output.splitlines()) except subprocess.CalledProcessError: logger.debug("The folder(%s) that running launch_cvd doesn't exist.", - _REMOTE_LOG_FOLDER) + constants.REMOTE_LOG_FOLDER) return log_files diff --git a/pull/pull_test.py b/pull/pull_test.py index f57c6c46..b5e7e77b 100644 --- a/pull/pull_test.py +++ b/pull/pull_test.py @@ -16,7 +16,8 @@ import unittest import os import tempfile -import mock + +from unittest import mock from acloud import errors from acloud.internal import constants diff --git a/reconnect/reconnect.py b/reconnect/reconnect.py index d354b4e2..e0a3b61f 100644 --- a/reconnect/reconnect.py +++ b/reconnect/reconnect.py @@ -19,21 +19,81 @@ Reconnect will: - restart vnc for remote/local instances """ +import logging +import os import re from acloud import errors from acloud.internal import constants from acloud.internal.lib import auth from acloud.internal.lib import android_compute_client +from acloud.internal.lib import cvd_runtime_config from acloud.internal.lib import utils +from acloud.internal.lib import ssh as ssh_object from acloud.internal.lib.adb_tools import AdbTools from acloud.list import list as list_instance from acloud.public import config from acloud.public import report +logger = logging.getLogger(__name__) + _RE_DISPLAY = re.compile(r"([\d]+)x([\d]+)\s.*") _VNC_STARTED_PATTERN = "ssvnc vnc://127.0.0.1:%(vnc_port)d" +_WEBRTC_PORTS_SEARCH = "".join( + [utils.PORT_MAPPING % {"local_port":port["local"], + "target_port":port["target"]} + for port in utils.WEBRTC_PORTS_MAPPING]) + + +def _IsWebrtcEnable(instance, host_user, host_ssh_private_key_path, + extra_args_ssh_tunnel): + """Check local/remote instance webRTC is enable. + + Args: + instance: Local/Remote Instance object. + host_user: String of user login into the instance. + host_ssh_private_key_path: String of host key for logging in to the + host. + extra_args_ssh_tunnel: String, extra args for ssh tunnel connection. + + Returns: + Boolean: True if cf_runtime_cfg.enable_webrtc is True. + """ + if instance.islocal: + return instance.cf_runtime_cfg.enable_webrtc + ssh = ssh_object.Ssh(ip=ssh_object.IP(ip=instance.ip), user=host_user, + ssh_private_key_path=host_ssh_private_key_path, + extra_args_ssh_tunnel=extra_args_ssh_tunnel) + remote_cuttlefish_config = os.path.join(constants.REMOTE_LOG_FOLDER, + constants.CUTTLEFISH_CONFIG_FILE) + raw_data = ssh.GetCmdOutput("cat " + remote_cuttlefish_config) + try: + cf_runtime_cfg = cvd_runtime_config.CvdRuntimeConfig( + raw_data=raw_data.strip()) + return cf_runtime_cfg.enable_webrtc + except errors.ConfigError: + logger.debug("No cuttlefish config[%s] found!", + remote_cuttlefish_config) + return False + + +def _WebrtcPortOccupied(): + """To decide whether need to release port. + + Remote webrtc instance will create a ssh tunnel which may conflict with + local webrtc instance default port. Searching process cmd in the pattern + of _WEBRTC_PORTS_SEARCH to decide whether to release port. + + Return: + True if need to release port. + """ + process_output = utils.CheckOutput(constants.COMMAND_PS) + for line in process_output.splitlines(): + match = re.search(_WEBRTC_PORTS_SEARCH, line) + if match: + return True + return False def StartVnc(vnc_port, display): @@ -112,6 +172,7 @@ def ReconnectInstance(ssh_private_key_path, adb_cmd = AdbTools(instance.adb_port) vnc_port = instance.vnc_port adb_port = instance.adb_port + webrtc_port = instance.webrtc_port # ssh tunnel is up but device is disconnected on adb if instance.ssh_tunnel_is_connected and not adb_cmd.IsAdbConnectionAlive(): adb_cmd.DisconnectAdb() @@ -128,8 +189,29 @@ def ReconnectInstance(ssh_private_key_path, extra_args_ssh_tunnel=extra_args_ssh_tunnel) vnc_port = forwarded_ports.vnc_port adb_port = forwarded_ports.adb_port - - if vnc_port and connect_vnc: + if _IsWebrtcEnable(instance, + constants.GCE_USER, + ssh_private_key_path, + extra_args_ssh_tunnel): + if instance.islocal: + if _WebrtcPortOccupied(): + raise errors.PortOccupied("\nReconnect to a local webrtc instance " + "is not work because remote webrtc " + "instance has established ssh tunnel " + "which occupied local webrtc instance " + "port. If you want to connect to a " + "local-instance of webrtc. please run " + "'acloud create --local-instance " + "--autoconnect webrtc' directly.") + else: + utils.EstablishWebRTCSshTunnel( + ip_addr=instance.ip, + rsa_key_file=ssh_private_key_path, + ssh_user=constants.GCE_USER, + extra_args_ssh_tunnel=extra_args_ssh_tunnel) + utils.LaunchBrowser(constants.WEBRTC_LOCAL_HOST, + webrtc_port) + elif(vnc_port and connect_vnc): StartVnc(vnc_port, instance.display) device_dict = { @@ -138,6 +220,9 @@ def ReconnectInstance(ssh_private_key_path, constants.VNC_PORT: vnc_port, constants.ADB_PORT: adb_port } + if adb_port and not instance.islocal: + device_dict[constants.DEVICE_SERIAL] = ( + constants.REMOTE_INSTANCE_ADB_SERIAL % adb_port) if vnc_port and adb_port: reconnect_report.AddData(key="devices", value=device_dict) diff --git a/reconnect/reconnect_test.py b/reconnect/reconnect_test.py index 70ea801b..a90df99e 100644 --- a/reconnect/reconnect_test.py +++ b/reconnect/reconnect_test.py @@ -17,7 +17,7 @@ import collections import unittest import subprocess -import mock +from unittest import mock from acloud import errors from acloud.internal import constants @@ -34,12 +34,13 @@ ForwardedPorts = collections.namedtuple("ForwardedPorts", class ReconnectTest(driver_test_lib.BaseDriverTest): """Test reconnect functions.""" - # pylint: disable=no-member + # pylint: disable=no-member, too-many-statements def testReconnectInstance(self): """Test Reconnect Instances.""" - ssh_private_key_path = "/fake/acloud_rea" + ssh_private_key_path = "/fake/acloud_rsa" fake_report = mock.MagicMock() instance_object = mock.MagicMock() + instance_object.name = "fake_name" instance_object.ip = "1.1.1.1" instance_object.islocal = False instance_object.adb_port = "8686" @@ -50,22 +51,32 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): self.Patch(AdbTools, "IsAdbConnected", return_value=False) self.Patch(AdbTools, "IsAdbConnectionAlive", return_value=False) self.Patch(utils, "IsCommandRunning", return_value=False) + self.Patch(reconnect, "_IsWebrtcEnable", return_value=False) + fake_device_dict = { + constants.IP: "1.1.1.1", + constants.INSTANCE_NAME: "fake_name", + constants.VNC_PORT: 6666, + constants.ADB_PORT: "8686", + constants.DEVICE_SERIAL: "127.0.0.1:8686" + } - #test ssh tunnel not connected, remote instance. + # test ssh tunnel not connected, remote instance. instance_object.vnc_port = 6666 instance_object.display = "" utils.AutoConnect.call_count = 0 reconnect.ReconnectInstance(ssh_private_key_path, instance_object, fake_report) utils.AutoConnect.assert_not_called() utils.LaunchVncClient.assert_called_with(6666) + fake_report.AddData.assert_called_with(key="devices", value=fake_device_dict) instance_object.display = "888x777 (99)" utils.AutoConnect.call_count = 0 reconnect.ReconnectInstance(ssh_private_key_path, instance_object, fake_report) utils.AutoConnect.assert_not_called() utils.LaunchVncClient.assert_called_with(6666, "888", "777") + fake_report.AddData.assert_called_with(key="devices", value=fake_device_dict) - #test ssh tunnel connected , remote instance. + # test ssh tunnel connected , remote instance. instance_object.ssh_tunnel_is_connected = False instance_object.display = "" utils.AutoConnect.call_count = 0 @@ -81,6 +92,14 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): ssh_user=constants.GCE_USER, extra_args_ssh_tunnel=extra_args_ssh_tunnel) utils.LaunchVncClient.assert_called_with(11111) + fake_device_dict = { + constants.IP: "1.1.1.1", + constants.INSTANCE_NAME: "fake_name", + constants.VNC_PORT: 11111, + constants.ADB_PORT: 22222, + constants.DEVICE_SERIAL: "127.0.0.1:22222" + } + fake_report.AddData.assert_called_with(key="devices", value=fake_device_dict) instance_object.display = "999x777 (99)" extra_args_ssh_tunnel = "fake_extra_args_ssh_tunnel" @@ -96,8 +115,22 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): ssh_user=constants.GCE_USER, extra_args_ssh_tunnel=extra_args_ssh_tunnel) utils.LaunchVncClient.assert_called_with(11111, "999", "777") + fake_report.AddData.assert_called_with(key="devices", value=fake_device_dict) + + # test fail reconnect report. + self.Patch(utils, "AutoConnect", + return_value=ForwardedPorts(vnc_port=None, adb_port=None)) + reconnect.ReconnectInstance(ssh_private_key_path, instance_object, fake_report) + fake_device_dict = { + constants.IP: "1.1.1.1", + constants.INSTANCE_NAME: "fake_name", + constants.VNC_PORT: None, + constants.ADB_PORT: None + } + fake_report.AddData.assert_called_with(key="device_failing_reconnect", + value=fake_device_dict) - #test reconnect local instance. + # test reconnect local instance. instance_object.islocal = True instance_object.display = "" instance_object.vnc_port = 5555 @@ -108,10 +141,51 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): fake_report) utils.AutoConnect.assert_not_called() utils.LaunchVncClient.assert_called_with(5555) + fake_device_dict = { + constants.IP: "1.1.1.1", + constants.INSTANCE_NAME: "fake_name", + constants.VNC_PORT: 5555, + constants.ADB_PORT: "8686" + } + fake_report.AddData.assert_called_with(key="devices", value=fake_device_dict) + + # pylint: disable=no-member + def testReconnectInstanceWithWebRTC(self): + """Test reconnect instances with WebRTC.""" + ssh_private_key_path = "/fake/acloud_rsa" + fake_report = mock.MagicMock() + instance_object = mock.MagicMock() + instance_object.ip = "1.1.1.1" + instance_object.islocal = False + instance_object.adb_port = "8686" + instance_object.avd_type = "cuttlefish" + instance_object.webrtc_port = 8443 + self.Patch(subprocess, "check_call", return_value=True) + self.Patch(utils, "LaunchVncClient") + self.Patch(utils, "AutoConnect") + self.Patch(utils, "LaunchBrowser") + self.Patch(utils, "EstablishWebRTCSshTunnel") + self.Patch(AdbTools, "IsAdbConnected", return_value=False) + self.Patch(AdbTools, "IsAdbConnectionAlive", return_value=False) + self.Patch(utils, "IsCommandRunning", return_value=False) + self.Patch(reconnect, "_IsWebrtcEnable", return_value=True) + + # test ssh tunnel not reconnect to the remote instance. + instance_object.vnc_port = 6666 + instance_object.display = "" + utils.AutoConnect.call_count = 0 + reconnect.ReconnectInstance(ssh_private_key_path, instance_object, fake_report) + utils.AutoConnect.assert_not_called() + utils.LaunchVncClient.assert_not_called() + utils.EstablishWebRTCSshTunnel.assert_called_with(extra_args_ssh_tunnel=None, + ip_addr='1.1.1.1', + rsa_key_file='/fake/acloud_rsa', + ssh_user='vsoc-01') + utils.LaunchBrowser.assert_called_with('localhost', 8443) def testReconnectInstanceAvdtype(self): """Test Reconnect Instances of avd_type.""" - ssh_private_key_path = "/fake/acloud_rea" + ssh_private_key_path = "/fake/acloud_rsa" fake_report = mock.MagicMock() instance_object = mock.MagicMock() instance_object.ip = "1.1.1.1" @@ -121,6 +195,7 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): instance_object.ssh_tunnel_is_connected = False self.Patch(utils, "AutoConnect") self.Patch(reconnect, "StartVnc") + self.Patch(reconnect, "_IsWebrtcEnable", return_value=False) #test reconnect remote instance when avd_type as gce. instance_object.avd_type = "gce" reconnect.ReconnectInstance(ssh_private_key_path, instance_object, fake_report) @@ -130,9 +205,11 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): target_adb_port=constants.GCE_ADB_PORT, ssh_user=constants.GCE_USER, extra_args_ssh_tunnel=None) + reconnect.StartVnc.assert_called_once() #test reconnect remote instance when avd_type as cuttlefish. instance_object.avd_type = "cuttlefish" + reconnect.StartVnc.call_count = 0 reconnect.ReconnectInstance(ssh_private_key_path, instance_object, fake_report) utils.AutoConnect.assert_called_with(ip_addr=instance_object.ip, rsa_key_file=ssh_private_key_path, @@ -140,11 +217,11 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): target_adb_port=constants.CF_ADB_PORT, ssh_user=constants.GCE_USER, extra_args_ssh_tunnel=None) - + reconnect.StartVnc.assert_called_once() def testReconnectInstanceUnknownAvdType(self): """Test reconnect instances of unknown avd type.""" - ssh_private_key_path = "/fake/acloud_rea" + ssh_private_key_path = "/fake/acloud_rsa" fake_report = mock.MagicMock() instance_object = mock.MagicMock() instance_object.avd_type = "unknown" @@ -154,10 +231,9 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): instance_object, fake_report) - def testReconnectInstanceNoAvdType(self): """Test reconnect instances with no avd type.""" - ssh_private_key_path = "/fake/acloud_rea" + ssh_private_key_path = "/fake/acloud_rsa" fake_report = mock.MagicMock() instance_object = mock.MagicMock() self.assertRaises(errors.UnknownAvdType, @@ -166,7 +242,6 @@ class ReconnectTest(driver_test_lib.BaseDriverTest): instance_object, fake_report) - def testStartVnc(self): """Test start Vnc.""" self.Patch(subprocess, "check_call", return_value=True) diff --git a/restart/__init__.py b/restart/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/restart/__init__.py diff --git a/restart/restart.py b/restart/restart.py new file mode 100644 index 00000000..5e148941 --- /dev/null +++ b/restart/restart.py @@ -0,0 +1,102 @@ +# Copyright 2021 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r"""Restart entry point. + +This command will restart the CF AVD from a remote instance. +""" + +import logging +import subprocess + +from acloud import errors +from acloud.internal import constants +from acloud.internal.lib import utils +from acloud.internal.lib.ssh import Ssh +from acloud.internal.lib.ssh import IP +from acloud.list import list as list_instances +from acloud.powerwash import powerwash +from acloud.public import config +from acloud.public import report +from acloud.reconnect import reconnect + + +logger = logging.getLogger(__name__) + + +def RestartFromInstance(cfg, instance, instance_id, powerwash_data): + """Restart AVD from remote CF instance. + + Args: + cfg: AcloudConfig object. + instance: list.Instance() object. + instance_id: Integer of the instance id. + powerwash_data: Boolean, True to powerwash AVD data. + + Returns: + A Report instance. + """ + ssh = Ssh(ip=IP(ip=instance.ip), + user=constants.GCE_USER, + ssh_private_key_path=cfg.ssh_private_key_path, + extra_args_ssh_tunnel=cfg.extra_args_ssh_tunnel) + logger.info("Start to restart AVD id (%s) from the instance: %s.", + instance_id, instance.name) + if powerwash_data: + powerwash.PowerwashDevice(ssh, instance_id) + else: + RestartDevice(ssh, instance_id) + reconnect.ReconnectInstance(cfg.ssh_private_key_path, + instance, + report.Report(command="reconnect"), + cfg.extra_args_ssh_tunnel) + return report.Report(command="restart") + + +@utils.TimeExecute(function_description="Waiting for AVD to restart") +def RestartDevice(ssh, instance_id): + """Restart AVD with the instance id. + + Args: + ssh: Ssh object. + instance_id: Integer of the instance id. + """ + ssh_command = "./bin/restart_cvd --instance_num=%d" % (instance_id) + try: + ssh.Run(ssh_command) + except (subprocess.CalledProcessError, errors.DeviceConnectionError) as e: + logger.debug(str(e)) + utils.PrintColorString(str(e), utils.TextColors.FAIL) + + +def Run(args): + """Run restart. + + After restart command executed, tool will return one Report instance. + + Args: + args: Namespace object from argparse.parse_args. + + Returns: + A Report instance. + """ + cfg = config.GetAcloudConfig(args) + if args.instance_name: + instance = list_instances.GetInstancesFromInstanceNames( + cfg, [args.instance_name]) + return RestartFromInstance( + cfg, instance[0], args.instance_id, args.powerwash) + return RestartFromInstance(cfg, + list_instances.ChooseOneRemoteInstance(cfg), + args.instance_id, + args.powerwash) diff --git a/restart/restart_args.py b/restart/restart_args.py new file mode 100644 index 00000000..b63904a3 --- /dev/null +++ b/restart/restart_args.py @@ -0,0 +1,65 @@ +# Copyright 2021 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r"""Restart args. + +Defines the restart arg parser that holds restart specific args. +""" +import argparse + + +CMD_RESTART = "restart" + + +def GetRestartArgParser(subparser): + """Return the restart arg parser. + + Args: + subparser: argparse.ArgumentParser that is attached to main acloud cmd. + + Returns: + argparse.ArgumentParser with restart options defined. + """ + restart_parser = subparser.add_parser(CMD_RESTART) + restart_parser.required = False + restart_parser.set_defaults(which=CMD_RESTART) + restart_group = restart_parser.add_mutually_exclusive_group() + restart_group.add_argument( + "--instance-name", + dest="instance_name", + type=str, + required=False, + help="The name of the remote instance that need to restart the AVDs.") + # TODO(b/118439885): Old arg formats to support transition, delete when + # transistion is done. + restart_group.add_argument( + "--instance_name", + dest="instance_name", + type=str, + required=False, + help=argparse.SUPPRESS) + restart_parser.add_argument( + "--instance-id", + dest="instance_id", + type=int, + required=False, + default=1, + help="The instance id of the remote instance that need to be restart.") + restart_parser.add_argument( + "--powerwash", + dest="powerwash", + action="store_true", + required=False, + help="Erase all userdata in the AVD.") + + return restart_parser diff --git a/restart/restart_test.py b/restart/restart_test.py new file mode 100644 index 00000000..1448a841 --- /dev/null +++ b/restart/restart_test.py @@ -0,0 +1,56 @@ +# Copyright 2021 - The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for restart.""" +import unittest + +from unittest import mock + +from acloud.internal.lib import driver_test_lib +from acloud.list import list as list_instances +from acloud.public import config +from acloud.restart import restart + + +class RestartTest(driver_test_lib.BaseDriverTest): + """Test restart.""" + + @mock.patch.object(restart, "RestartFromInstance") + def testRun(self, mock_restart): + """test Run.""" + cfg = mock.MagicMock() + args = mock.MagicMock() + instance_obj = mock.MagicMock() + # Test case with provided instance name. + args.instance_name = "instance_1" + args.instance_id = 1 + args.powerwash = False + self.Patch(config, "GetAcloudConfig", return_value=cfg) + self.Patch(list_instances, "GetInstancesFromInstanceNames", + return_value=[instance_obj]) + restart.Run(args) + mock_restart.assert_has_calls([ + mock.call(cfg, instance_obj, args.instance_id, args.powerwash)]) + + # Test case for user select one instance to restart AVD. + selected_instance = mock.MagicMock() + self.Patch(list_instances, "ChooseOneRemoteInstance", + return_value=selected_instance) + args.instance_name = None + restart.Run(args) + mock_restart.assert_has_calls([ + mock.call(cfg, selected_instance, args.instance_id, args.powerwash)]) + + +if __name__ == '__main__': + unittest.main() diff --git a/run_tests.sh b/run_tests.sh index 20c430c7..c1ee76f2 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -14,6 +14,8 @@ function get_python_path() { "dateutil" "google-api-python-client" "oauth2client" + "uritemplates" + "rsa" ) for lib in ${third_party_libs[*]}; do @@ -27,8 +29,8 @@ function print_summary() { local test_results=$1 local tmp_dir=$(mktemp -d) local rc_file=${ACLOUD_DIR}/.coveragerc - PYTHONPATH=$(get_python_path) python -m coverage report -m - PYTHONPATH=$(get_python_path) python -m coverage html -d $tmp_dir --rcfile=$rc_file + PYTHONPATH=$(get_python_path) python3 -m coverage report -m + PYTHONPATH=$(get_python_path) python3 -m coverage html -d $tmp_dir --rcfile=$rc_file echo "coverage report available at file://${tmp_dir}/index.html" if [[ $test_results -eq 0 ]]; then @@ -41,16 +43,16 @@ function print_summary() { function run_unittests() { local specified_tests=$@ local rc=0 - local run_cmd="python -m coverage run --append" + local run_cmd="python3 -m coverage run --append" # clear previously collected coverage data. - PYTHONPATH=$(get_python_path) python -m coverage erase + PYTHONPATH=$(get_python_path) python3 -m coverage erase # Get all unit tests under tools/acloud. local all_tests=$(find $ACLOUD_DIR -type f -name "*_test.py" ! -name "acloud_test.py"); local tests_to_run=$all_tests - # Filter out the tests if specifed. + # Filter out the tests if specified. if [[ ! -z $specified_tests ]]; then tests_to_run=() for t in $all_tests; @@ -78,16 +80,20 @@ function run_unittests() { } function check_env() { - if [ -z "$ANDROID_BUILD_TOP" ]; then - echo "Missing ANDROID_BUILD_TOP env variable. Run 'lunch' first." + if [ -z "$ANDROID_HOST_OUT" ]; then + echo "Missing ANDROID_HOST_OUT env variable. Run 'lunch' first." + exit 1 + fi + if [ ! -f "$ANDROID_HOST_OUT/bin/aprotoc" ]; then + echo "Missing aprotoc. Run 'm aprotoc' first." exit 1 fi local missing_py_packages=false for py_lib in {coverage,mock}; do - if ! pip list | grep $py_lib &> /dev/null; then - echo "Missing required python package: $py_lib (pip install $py_lib)" + if ! python3 -m pip list | grep $py_lib &> /dev/null; then + echo "Missing required python package: $py_lib (python3 -m pip install $py_lib)" missing_py_packages=true fi done @@ -98,7 +104,7 @@ function check_env() { function gen_proto_py() { # Use aprotoc to generate python proto files. - local protoc_cmd=$ANDROID_BUILD_TOP/prebuilts/misc/linux-x86/protobuf/aprotoc + local protoc_cmd=$ANDROID_HOST_OUT/bin/aprotoc pushd $ACLOUD_DIR &> /dev/null $protoc_cmd internal/proto/*.proto --python_out=./ touch internal/proto/__init__.py diff --git a/run_tests_py2.sh b/run_tests_py2.sh new file mode 100755 index 00000000..cb524c4f --- /dev/null +++ b/run_tests_py2.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' # No Color +ACLOUD_DIR=$(dirname $(realpath $0)) +TOOLS_DIR=$(dirname $ACLOUD_DIR) +THIRD_PARTY_DIR=$(dirname $TOOLS_DIR)/external/python + +function get_python_path() { + local python_path=$TOOLS_DIR + local third_party_libs=( + "apitools" + "dateutil" + "google-api-python-client" + "oauth2client" + ) + for lib in ${third_party_libs[*]}; + do + python_path=$THIRD_PARTY_DIR/$lib:$python_path + done + python_path=$python_path:$PYTHONPATH + echo $python_path +} + +function print_summary() { + local test_results=$1 + local tmp_dir=$(mktemp -d) + local rc_file=${ACLOUD_DIR}/.coveragerc + PYTHONPATH=$(get_python_path) python -m coverage report -m + PYTHONPATH=$(get_python_path) python -m coverage html -d $tmp_dir --rcfile=$rc_file + echo "coverage report available at file://${tmp_dir}/index.html" + + if [[ $test_results -eq 0 ]]; then + echo -e "${GREEN}All unittests pass${NC}!" + else + echo -e "${RED}There was a unittest failure${NC}" + fi +} + +function run_unittests() { + local specified_tests=$@ + local rc=0 + local run_cmd="python -m coverage run --append" + + # clear previously collected coverage data. + PYTHONPATH=$(get_python_path) python -m coverage erase + + # Get all unit tests under tools/acloud. + local all_tests=$(find $ACLOUD_DIR -type f -name "*_test.py" ! -name "acloud_test.py"); + local tests_to_run=$all_tests + + # Filter out the tests if specified. + if [[ ! -z $specified_tests ]]; then + tests_to_run=() + for t in $all_tests; + do + for t_pattern in $specified_tests; + do + if [[ "$t" =~ "$t_pattern" ]]; then + tests_to_run=("${tests_to_run[@]}" "$t") + fi + done + done + fi + + for t in $tests_to_run; + do + if ! PYTHONPATH=$(get_python_path):$PYTHONPATH $run_cmd $t; then + rc=1 + echo -e "${RED}$t failed${NC}" + fi + done + + print_summary $rc + cleanup + exit $rc +} + +function check_env() { + if [ -z "$ANDROID_HOST_OUT" ]; then + echo "Missing ANDROID_HOST_OUT env variable. Run 'lunch' first." + exit 1 + fi + if [ ! -f "$ANDROID_HOST_OUT/bin/aprotoc" ]; then + echo "Missing aprotoc. Run 'm aprotoc' first." + exit 1 + fi + + local missing_py_packages=false + for py_lib in {coverage,mock}; + do + if ! python -m pip list | grep $py_lib &> /dev/null; then + echo "Missing required python package: $py_lib (python -m pip install $py_lib)" + missing_py_packages=true + fi + done + if $missing_py_packages; then + exit 1 + fi +} + +function gen_proto_py() { + # Use aprotoc to generate python proto files. + local protoc_cmd=$ANDROID_HOST_OUT/bin/aprotoc + pushd $ACLOUD_DIR &> /dev/null + $protoc_cmd internal/proto/*.proto --python_out=./ + touch internal/proto/__init__.py + popd &> /dev/null +} + +function cleanup() { + # Search for *.pyc and delete them. + find $ACLOUD_DIR -name "*.pyc" -exec rm -f {} \; + + # Delete the generated proto files too. + find $ACLOUD_DIR/internal/proto -name "*.py" -exec rm -f {} \; +} + +check_env +cleanup +gen_proto_py +run_unittests $@ diff --git a/setup/gcp_setup_runner.py b/setup/gcp_setup_runner.py index 2d12ccc5..26c1d067 100644 --- a/setup/gcp_setup_runner.py +++ b/setup/gcp_setup_runner.py @@ -166,8 +166,8 @@ class GoogleSDKBins(object): Returns: String, return message after execute gcloud command. """ - return subprocess.check_output([self.gcloud_command_path] + cmd, - env=self._env, **kwargs) + return utils.CheckOutput([self.gcloud_command_path] + cmd, + env=self._env, **kwargs) def RunGsutil(self, cmd, **kwargs): """Run gsutil command. @@ -180,8 +180,8 @@ class GoogleSDKBins(object): Returns: String, return message after execute gsutil command. """ - return subprocess.check_output([self.gsutil_command_path] + cmd, - env=self._env, **kwargs) + return utils.CheckOutput([self.gsutil_command_path] + cmd, + env=self._env, **kwargs) class GoogleAPIService(object): @@ -315,7 +315,7 @@ class GcpTaskRunner(base_task_runner.BaseTaskRunner): def _CreateStableHostImage(self): """Create the stable host image.""" - # Write default stable_host_image_name with dummy value. + # Write default stable_host_image_name with unused value. # TODO(113091773): An additional step to create the host image. if not self.stable_host_image_name: UpdateConfigFile(self.config_path, "stable_host_image_name", "") diff --git a/setup/gcp_setup_runner_test.py b/setup/gcp_setup_runner_test.py index e26ea8cb..e4a0e921 100644 --- a/setup/gcp_setup_runner_test.py +++ b/setup/gcp_setup_runner_test.py @@ -17,7 +17,8 @@ import unittest import os -import mock + +from unittest import mock import six # pylint: disable=no-name-in-module,import-error,no-member @@ -37,7 +38,6 @@ disable_usage_reporting = False project = new_project """ - def _CreateCfgFile(): """A helper method that creates a mock configuration object.""" default_cfg = """ @@ -94,7 +94,7 @@ class AcloudGCPSetupTest(unittest.TestCase): self.assertEqual(cfg.project, "test_project") @mock.patch("os.path.dirname", return_value="") - @mock.patch("subprocess.check_output") + @mock.patch.object(utils, "CheckOutput") def testSeupProjectZone(self, mock_runner, mock_path): """Test setup project and zone.""" gcloud_runner = gcp_setup_runner.GoogleSDKBins(mock_path) @@ -146,7 +146,7 @@ class AcloudGCPSetupTest(unittest.TestCase): @mock.patch.object(gcp_setup_runner, "GoogleSDKBins") def testSetupGcloudInfo(self, mock_sdk, mock_set, mock_run, mock_create): """test setup gcloud info""" - with mock.patch("google_sdk.GoogleSDK"): + with mock.patch("acloud.setup.google_sdk.GoogleSDK"): self.gcp_env_runner._SetupGcloudInfo() mock_sdk.assert_called_once() mock_set.assert_called_once() @@ -226,7 +226,7 @@ class AcloudGCPSetupTest(unittest.TestCase): self.gcp_env_runner.client_secret = "test_client_secret" self.assertFalse(self.gcp_env_runner._NeedClientIDSetup(False)) - @mock.patch("subprocess.check_output") + @mock.patch.object(utils, "CheckOutput") def testEnableGcloudServices(self, mock_run): """test enable Gcloud services.""" mock_run.return_value = "" @@ -239,7 +239,7 @@ class AcloudGCPSetupTest(unittest.TestCase): gcp_setup_runner._COMPUTE_ENGINE_SERVICE], env=self.gcloud_runner._env, stderr=-2)]) - @mock.patch("subprocess.check_output") + @mock.patch.object(utils, "CheckOutput") def testGoogleAPIService(self, mock_run): """Test GoogleAPIService""" api_service = gcp_setup_runner.GoogleAPIService("service_name", @@ -249,7 +249,7 @@ class AcloudGCPSetupTest(unittest.TestCase): mock.call(["gcloud", "services", "enable", "service_name"], env=self.gcloud_runner._env, stderr=-2)]) - @mock.patch("subprocess.check_output") + @mock.patch.object(utils, "CheckOutput") def testCheckBillingEnable(self, mock_run): """Test CheckBillingEnable""" # Test billing account in gcp project already enabled. diff --git a/setup/host_setup_runner.py b/setup/host_setup_runner.py index 9a668e5e..fcc5b744 100644 --- a/setup/host_setup_runner.py +++ b/setup/host_setup_runner.py @@ -39,7 +39,7 @@ logger = logging.getLogger(__name__) # Packages "devscripts" and "equivs" are required for "mk-build-deps". _AVD_REQUIRED_PKGS = [ "devscripts", "equivs", "libvirt-clients", "libvirt-daemon-system"] -_BASE_REQUIRED_PKGS = ["ssvnc", "lzop"] +_BASE_REQUIRED_PKGS = ["ssvnc", "lzop", "python3-tk"] _CUTTLEFISH_COMMOM_PKG = "cuttlefish-common" _CF_COMMOM_FOLDER = "cf-common" _LIST_OF_MODULES = ["kvm_intel", "kvm"] @@ -83,9 +83,8 @@ class BasePkgInstaller(base_task_runner.BaseTaskRunner): if not setup_common.PackageInstalled(pkg)]) if not utils.GetUserAnswerYes("\nStart to install package(s):\n%s" - "\nPress 'y' to continue or anything " - "else to do it myself and run acloud " - "again[y/N]: " % cmd): + "\nEnter 'y' to continue, otherwise N or " + "enter to exit: " % cmd): sys.exit(constants.EXIT_BY_USER) setup_common.CheckCmdOutput(_UPDATE_APT_GET_CMD, shell=True) @@ -145,9 +144,8 @@ class CuttlefishCommonPkgInstaller(base_task_runner.BaseTaskRunner): for sub_cmd in _INSTALL_CUTTLEFISH_COMMOM_CMD) if not utils.GetUserAnswerYes("\nStart to install cuttlefish-common :\n%s" - "\nPress 'y' to continue or anything " - "else to do it myself and run acloud " - "again[y/N]: " % cmd): + "\nEnter 'y' to continue, otherwise N or " + "enter to exit: " % cmd): sys.exit(constants.EXIT_BY_USER) try: setup_common.CheckCmdOutput(cmd, shell=True) @@ -227,6 +225,6 @@ class CuttlefishHostSetup(base_task_runner.BaseTaskRunner): True if user answer yes. """ answer_client = utils.InteractWithQuestion( - "\nPress 'y' to continue or anything else to do it myself[y/N]: ", + "\nEnter 'y' to continue, otherwise N or enter to exit: ", utils.TextColors.WARNING) return answer_client in constants.USER_ANSWER_YES diff --git a/setup/host_setup_runner_test.py b/setup/host_setup_runner_test.py index 111540e2..b98772da 100644 --- a/setup/host_setup_runner_test.py +++ b/setup/host_setup_runner_test.py @@ -16,7 +16,8 @@ import platform import shutil import tempfile import unittest -import mock + +from unittest import mock from acloud.internal.lib import driver_test_lib from acloud.internal.lib import utils diff --git a/setup/setup.py b/setup/setup.py index 39513dc4..c424318b 100644 --- a/setup/setup.py +++ b/setup/setup.py @@ -24,6 +24,7 @@ import sys from acloud.internal import constants from acloud.internal.lib import utils +from acloud.public import config from acloud.setup import host_setup_runner from acloud.setup import gcp_setup_runner @@ -39,6 +40,10 @@ def Run(args): Args: args: Namespace object from argparse.parse_args. """ + if args.update_config: + _UpdateConfig(args.config_file, args.update_config[0], args.update_config[1]) + return + _RunPreSetup() # Setup process will be in the following manner: @@ -113,3 +118,18 @@ def _RunPreSetup(): if os.path.exists(pre_setup_sh): subprocess.call([pre_setup_sh]) + +def _UpdateConfig(config_file, field, value): + """Update the user config. + + Args: + config_file: String of config file path. + field: String, field name in user config. + value: String, the value of field. + """ + config_mgr = config.AcloudConfigManager(config_file) + config_mgr.Load() + user_config = config_mgr.user_config_path + print("Your config (%s) is updated." % user_config) + gcp_setup_runner.UpdateConfigFile(user_config, field, value) + _PrintUsage() diff --git a/setup/setup_args.py b/setup/setup_args.py index e3df4de3..1e3583ce 100644 --- a/setup/setup_args.py +++ b/setup/setup_args.py @@ -18,6 +18,13 @@ r"""Setup args. Defines the setup arg parser that holds setup specific args. """ +from acloud import errors +# pylint: disable=no-name-in-module,import-error +from acloud.internal.proto.user_config_pb2 import UserConfig + + +_FIELD_NAMES = sorted([field.name for field in UserConfig.DESCRIPTOR.fields]) +_CONFIG_FIELD = 0 CMD_SETUP = "setup" @@ -60,5 +67,34 @@ def GetSetupArgParser(subparser): dest="force", required=False, help="Force the setup steps even if it's not required.") + # TODO(157532869): Validate the field name. + setup_parser.add_argument( + "--update-config", + nargs=2, + dest="update_config", + required=False, + help="Update the acloud user config. The first arg is field name in " + "config, and the second arg is the value of the field. Command would " + "like: 'acloud setup --config stable_host_image_family acloud-release'." + " The first arg can be one of following fields:%s" % _FIELD_NAMES) return setup_parser + + +def VerifyArgs(args): + """Verify args. + + One example of command "acloud setup --update-config zone us-central1-c", + then this function would check "zone" is a valid field. + + Args: + args: Namespace object from argparse.parse_args. + + Raises: + errors.NotSupportedFieldName: The field name doesn't support in config. + """ + if args.update_config: + if args.update_config[_CONFIG_FIELD] not in _FIELD_NAMES: + raise errors.NotSupportedFieldName( + "Field[%s] isn't in support list: %s" % (args.update_config[0], + _FIELD_NAMES)) diff --git a/setup/setup_common.py b/setup/setup_common.py index f5ded570..97ea1417 100644 --- a/setup/setup_common.py +++ b/setup/setup_common.py @@ -21,6 +21,7 @@ import re import subprocess from acloud import errors +from acloud.internal.lib import utils logger = logging.getLogger(__name__) @@ -50,7 +51,7 @@ def CheckCmdOutput(cmd, print_cmd=True, **kwargs): print("Run command: %s" % cmd) logger.debug("Run command: %s", cmd) - return subprocess.check_output(cmd, **kwargs) + return utils.CheckOutput(cmd, **kwargs) def InstallPackage(pkg): |