summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTreehugger Robot <android-test-infra-autosubmit@system.gserviceaccount.com>2023-12-07 20:44:12 +0000
committerAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>2023-12-07 20:44:12 +0000
commit9aedeb1d2f028e8f4e2c394761fde24507061069 (patch)
treeed723af0d751a60da4bb668f0d9cc288b9faf8b5
parentf99e28bafe2bbc241d7785d14c39d99610606144 (diff)
parent5f0dd4e2135b82782780dc2c6ed7197904479a27 (diff)
downloadplatform_testing-tmp_amf_315507370.tar.gz
Merge "enable/disable GMS auto updates before/after test." into main am: 5f0dd4e213tmp_amf_315507370
Original change: https://android-review.googlesource.com/c/platform/platform_testing/+/2860405 Change-Id: I83d757d82b2b54ba342d8362b5c7f2f6c7fad078 Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
-rw-r--r--tests/bettertogether/quickstart/performance_test/Android.bp12
-rw-r--r--tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py2
-rw-r--r--tests/bettertogether/quickstart/performance_test/gms_auto_updates_util.py173
-rw-r--r--tests/bettertogether/quickstart/performance_test/nc_base_test.py101
-rw-r--r--tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py2
-rw-r--r--tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py2
-rw-r--r--tests/bettertogether/quickstart/performance_test/setup_utils.py27
7 files changed, 283 insertions, 36 deletions
diff --git a/tests/bettertogether/quickstart/performance_test/Android.bp b/tests/bettertogether/quickstart/performance_test/Android.bp
index cd9fb8c3a..9d4d3d1f4 100644
--- a/tests/bettertogether/quickstart/performance_test/Android.bp
+++ b/tests/bettertogether/quickstart/performance_test/Android.bp
@@ -32,12 +32,24 @@ python_library_host {
}
python_library_host {
+ name: "gms_auto_updates_util",
+ defaults: ["quick_start_perf_test_defaults",],
+ srcs: [
+ "gms_auto_updates_util.py",
+ ],
+ libs: [
+ "mobly",
+ ],
+}
+
+python_library_host {
name: "setup_utils",
defaults: ["quick_start_perf_test_defaults",],
srcs: [
"setup_utils.py",
],
libs: [
+ "gms_auto_updates_util",
"nc_constants",
"mobly",
],
diff --git a/tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py b/tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py
index aab719239..ed3316860 100644
--- a/tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py
+++ b/tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py
@@ -41,7 +41,7 @@ from performance_test import nc_constants
from performance_test import nearby_connection_wrapper
from performance_test import setup_utils
-_TEST_SCRIPT_VERSTION = '1.5'
+_TEST_SCRIPT_VERSTION = '1.6'
_DELAY_BETWEEN_EACH_TEST_CYCLE = datetime.timedelta(seconds=5)
_TRANSFER_FILE_SIZE_1MB = 1024
diff --git a/tests/bettertogether/quickstart/performance_test/gms_auto_updates_util.py b/tests/bettertogether/quickstart/performance_test/gms_auto_updates_util.py
new file mode 100644
index 000000000..79832fd12
--- /dev/null
+++ b/tests/bettertogether/quickstart/performance_test/gms_auto_updates_util.py
@@ -0,0 +1,173 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""class to enable/disable GMS auto update."""
+
+import logging
+import os
+import tempfile
+# from xml import etree has a problem, don't use it.
+from xml.etree import ElementTree
+from mobly.controllers import android_device
+
+
+_FINSKY_CONFIG_FILE = '/data/data/com.android.vending/shared_prefs/finsky.xml'
+_FINSKY_CONFIG_NAME = 'auto_update_enabled'
+_FINSKY_CONFIG_VALUE_DISABLE = 'false'
+_FINSKY_CONFIG_VALUE_ENABLE = 'true'
+_VENDING_CONFIG_FILE = '/data/data/com.android.vending/shared_prefs/com.android.vending_preferences.xml'
+_VENDING_CONFIG_NAME = 'auto-update-mode'
+_VENDING_CONFIG_VALUE_DISABLE = 'AUTO_UPDATE_NEVER'
+_VENDING_CONFIG_VALUE_ENABLE = 'AUTO_UPDATE_WIFI'
+_BLANK_CONFIG = '<?xml version="1.0" encoding="utf-8"?><map></map>'
+_XML_BOOL_TYPE = 'boolean'
+_XML_STRING_TYPE = 'string'
+_ENABLE_GSERVICES_CMD_TEMPLATE = [
+ (
+ 'am broadcast '
+ '-a com.google.gservices.intent.action.GSERVICES_OVERRIDE '
+ '-e finsky.play_services_auto_update_enabled {}'
+ ),
+ (
+ 'am broadcast '
+ '-a com.google.gservices.intent.action.GSERVICES_OVERRIDE '
+ '-e finsky.setup_wizard_additional_account_vpa_enable {}'
+ ),
+]
+
+
+class GmsAutoUpdatesUtil:
+ """class to enable/disable GMS auto updates."""
+
+ def __init__(self, ad: android_device.AndroidDevice):
+ self._device: android_device.AndroidDevice = ad
+
+ def enable_gms_auto_updates(self) -> None:
+ self._config_gms_auto_updates(True)
+
+ def disable_gms_auto_updates(self) -> None:
+ self._config_gms_auto_updates(False)
+
+ def _config_gms_auto_updates(self, enable_updates: bool) -> None:
+ """Configures GMS auto updates."""
+ if not self._device.is_adb_root:
+ self._device.log.info(
+ f'failed to set the play store auto updates as {enable_updates}'
+ 'you should enable/disable it manually on an unrooted device.')
+ else:
+ if enable_updates:
+ self._configure_play_store_updates(
+ _FINSKY_CONFIG_VALUE_ENABLE, _VENDING_CONFIG_VALUE_ENABLE
+ )
+ else:
+ self._configure_play_store_updates(
+ _FINSKY_CONFIG_VALUE_DISABLE, _VENDING_CONFIG_VALUE_DISABLE
+ )
+ self._configure_gservice_updates(enable_updates)
+
+ def _configure_gservice_updates(self, enable_updates: bool) -> None:
+ """Overwites Gservice to enable/disable updates."""
+ for cmd in _ENABLE_GSERVICES_CMD_TEMPLATE:
+ self._device.adb.shell(
+ cmd.format('true' if enable_updates else 'false')
+ )
+
+ def _create_or_update_play_store_config(
+ self,
+ tmp_dir: str,
+ value_type: str,
+ name: str,
+ value: str,
+ device_path: str,
+ ) -> str:
+ """Creates or updates a Play Store configuration file.
+
+ The function retrieves the Play Store configuration file from the device
+ then update it. If the file does not exist, it creates a new one.
+
+ Args:
+ tmp_dir: The temporary directory to store the configuration file.
+ value_type: The type of the configuration field.
+ name: The name of the configuration field.
+ value: The value of the configuration field.
+ device_path: The path to the configuration file on the device.
+
+ Returns:
+ The path to the updated configuration file.
+ """
+ path = os.path.join(tmp_dir, f'play_store_config_{name}.xml')
+ self._device.adb.pull([device_path, path])
+
+ config_doc = ElementTree.parse(path) if os.path.isfile(path) else None
+
+ changing_element = None
+ root = (
+ ElementTree.fromstring(_BLANK_CONFIG.encode())
+ if config_doc is None
+ else config_doc.getroot()
+ )
+
+ # find the element, xPath doesn't work as the name is a reserved word.
+ for child in root:
+ if child.attrib['name'] == name:
+ changing_element = child
+ break
+ if changing_element is None:
+ if value_type == _XML_BOOL_TYPE:
+ changing_element = ElementTree.SubElement(root, 'boolean')
+ else:
+ changing_element = ElementTree.SubElement(root, 'string')
+ logging.info('element for %s is %s, %s', name, changing_element.tag,
+ changing_element.attrib)
+ if value_type == _XML_BOOL_TYPE:
+ changing_element.set('name', name)
+ changing_element.set('value', value)
+ else:
+ changing_element.attrib['name'] = name
+ changing_element.text = value
+
+ tree = ElementTree.ElementTree(root)
+ tree.write(path, xml_declaration=True, encoding='utf-8')
+ return path
+
+ def _configure_play_store_updates(
+ self, finsky_config_value: str, vending_config_value: str
+ ) -> None:
+ """Configures the Play Store update related settings."""
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ finsky_config = self._create_or_update_play_store_config(
+ tmp_dir,
+ _XML_BOOL_TYPE,
+ _FINSKY_CONFIG_NAME,
+ finsky_config_value,
+ _FINSKY_CONFIG_FILE,
+ )
+ self._device.adb.push([finsky_config, _FINSKY_CONFIG_FILE])
+ try:
+ os.remove(finsky_config)
+ except OSError as e:
+ logging.warning('failed to remove %s: %s', finsky_config, e)
+
+ vending_config = self._create_or_update_play_store_config(
+ tmp_dir,
+ _XML_STRING_TYPE,
+ _VENDING_CONFIG_NAME,
+ vending_config_value,
+ _VENDING_CONFIG_FILE,
+ )
+ self._device.adb.push([vending_config, _VENDING_CONFIG_FILE])
+ try:
+ os.remove(vending_config)
+ except OSError as e:
+ logging.warning('failed to remove %s: %s', vending_config, e)
diff --git a/tests/bettertogether/quickstart/performance_test/nc_base_test.py b/tests/bettertogether/quickstart/performance_test/nc_base_test.py
index 81537d108..0c2ec6e28 100644
--- a/tests/bettertogether/quickstart/performance_test/nc_base_test.py
+++ b/tests/bettertogether/quickstart/performance_test/nc_base_test.py
@@ -58,35 +58,43 @@ class NCBaseTestClass(base_test.BaseTestClass):
try:
self.discoverer = android_device.get_device(
- self.ads, role='source_device')
+ self.ads, role='source_device'
+ )
self.advertiser = android_device.get_device(
- self.ads, role='target_device')
+ self.ads, role='target_device'
+ )
except errors.Error:
- logging.warning('The source,target devices are not specified in testbed;'
- 'The result may not be expected.')
+ logging.warning(
+ 'The source,target devices are not specified in testbed;'
+ 'The result may not be expected.'
+ )
self.advertiser, self.discoverer = self.ads
def _disconnect_from_wifi(self, ad: android_device.AndroidDevice) -> None:
- if(not ad.is_adb_root):
+ if not ad.is_adb_root:
ad.log.info("Can't clear wifi network in non-rooted device")
return
ad.nearby.wifiClearConfiguredNetworks()
time.sleep(nc_constants.WIFI_DISCONNECTION_DELAY.total_seconds())
def _setup_android_device(self, ad: android_device.AndroidDevice) -> None:
- if (not ad.is_adb_root):
- if (self.test_parameters.allow_unrooted_device):
+ if not ad.is_adb_root:
+ if self.test_parameters.allow_unrooted_device:
ad.log.info('Unrooted device is detected. Test coverage is limited')
else:
asserts.skip('The test only can run on rooted device.')
+ setup_utils.disable_gms_auto_updates(ad)
+
ad.debug_tag = ad.serial + '(' + ad.adb.getprop('ro.product.model') + ')'
ad.log.info('try to install nearby_snippet_apk')
if self._nearby_snippet_apk_path:
setup_utils.install_apk(ad, self._nearby_snippet_apk_path)
else:
- ad.log.warning('nearby_snippet apk is not specified, '
- 'make sure it is installed in the device')
+ ad.log.warning(
+ 'nearby_snippet apk is not specified, '
+ 'make sure it is installed in the device'
+ )
ad.load_snippet('nearby', NEARBY_SNIPPET_PACKAGE_NAME)
ad.log.info('grant manage external storage permission')
@@ -121,6 +129,7 @@ class NCBaseTestClass(base_test.BaseTestClass):
def _teardown_device(self, ad: android_device.AndroidDevice) -> None:
ad.nearby.transferFilesCleanup()
+ setup_utils.enable_gms_auto_updates(ad)
if self.test_parameters.disconnect_wifi_after_test:
self._disconnect_from_wifi(ad)
ad.unload_snippet('nearby')
@@ -129,7 +138,8 @@ class NCBaseTestClass(base_test.BaseTestClass):
utils.concurrent_exec(
lambda d: d.services.create_output_excerpts_all(self.current_test_info),
param_list=[[ad] for ad in self.ads],
- raise_on_exception=True)
+ raise_on_exception=True,
+ )
def teardown_class(self) -> None:
utils.concurrent_exec(
@@ -148,8 +158,11 @@ class NCBaseTestClass(base_test.BaseTestClass):
field.name for field in dataclasses.fields(nc_constants.TestParameters)
}
test_parameters = nc_constants.TestParameters(
- **{key: val for key, val in self.user_params.items(
- ) if key in test_parameters_names}
+ **{
+ key: val
+ for key, val in self.user_params.items()
+ if key in test_parameters_names
+ }
)
return test_parameters
@@ -170,8 +183,11 @@ class NCBaseTestClass(base_test.BaseTestClass):
) -> nc_constants.ThroughputResultStats:
"""Statistics the throughput test result of all iterations."""
n = self.performance_test_iterations
- filtered = [x for x in throughput_indicators
- if x != nc_constants.UNSET_THROUGHPUT_KBPS]
+ filtered = [
+ x
+ for x in throughput_indicators
+ if x != nc_constants.UNSET_THROUGHPUT_KBPS
+ ]
if not filtered:
# all test cases are failed
return nc_constants.ThroughputResultStats(
@@ -182,18 +198,26 @@ class NCBaseTestClass(base_test.BaseTestClass):
success_count=0,
fail_targets=[
nc_constants.FailTargetSummary(
- f'{medium_name} transfer success rate', 0.0,
- success_rate_target, '%')])
+ f'{medium_name} transfer success rate',
+ 0.0,
+ success_rate_target,
+ '%',
+ )
+ ],
+ )
# use the descenting order of the throughput
filtered.sort(reverse=True)
success_count = len(filtered)
- success_rate = round(success_count * 100.0 / n,
- nc_constants.SUCCESS_RATE_PRECISION_DIGITS)
+ success_rate = round(
+ success_count * 100.0 / n, nc_constants.SUCCESS_RATE_PRECISION_DIGITS
+ )
average_kbps = round(sum(filtered) / len(filtered))
percentile_50_kbps = filtered[
- int(len(filtered) * nc_constants.PERCENTILE_50_FACTOR)]
+ int(len(filtered) * nc_constants.PERCENTILE_50_FACTOR)
+ ]
percentile_95_kbps = filtered[
- int(len(filtered) * nc_constants.PERCENTILE_95_FACTOR)]
+ int(len(filtered) * nc_constants.PERCENTILE_95_FACTOR)
+ ]
fail_targets: list[nc_constants.FailTargetSummary] = []
if success_rate < success_rate_target:
fail_targets.append(
@@ -201,15 +225,16 @@ class NCBaseTestClass(base_test.BaseTestClass):
f'{medium_name} transfer success rate',
success_rate,
success_rate_target,
- '%')
+ '%',
+ )
)
if percentile_50_kbps < median_benchmark_kbps:
fail_targets.append(
nc_constants.FailTargetSummary(
f'{medium_name} median transfer speed (KBps)',
percentile_50_kbps,
- median_benchmark_kbps
- )
+ median_benchmark_kbps,
+ )
)
return nc_constants.ThroughputResultStats(
success_rate,
@@ -217,7 +242,7 @@ class NCBaseTestClass(base_test.BaseTestClass):
percentile_50_kbps,
percentile_95_kbps,
success_count,
- fail_targets
+ fail_targets,
)
def _stats_latency_result(
@@ -232,27 +257,37 @@ class NCBaseTestClass(base_test.BaseTestClass):
if not filtered:
# All test cases are failed.
return nc_constants.LatencyResultStats(
- average_latency=0.0, percentile_50=0.0,
- percentile_95=0.0, failure_count=n)
+ average_latency=0.0,
+ percentile_50=0.0,
+ percentile_95=0.0,
+ failure_count=n,
+ )
filtered.sort()
- average = round(sum(filtered) / len(filtered),
- nc_constants.LATENCY_PRECISION_DIGITS) / n
+ average = (
+ round(
+ sum(filtered) / len(filtered), nc_constants.LATENCY_PRECISION_DIGITS
+ )
+ / n
+ )
percentile_50 = round(
filtered[int(len(filtered) * nc_constants.PERCENTILE_50_FACTOR)],
- nc_constants.LATENCY_PRECISION_DIGITS)
+ nc_constants.LATENCY_PRECISION_DIGITS,
+ )
percentile_95 = round(
filtered[int(len(filtered) * nc_constants.PERCENTILE_95_FACTOR)],
- nc_constants.LATENCY_PRECISION_DIGITS)
+ nc_constants.LATENCY_PRECISION_DIGITS,
+ )
return nc_constants.LatencyResultStats(
average, percentile_50, percentile_95, n - len(filtered)
)
def _generate_target_fail_message(
- self,
- fail_targets: list[nc_constants.FailTargetSummary]) -> str:
+ self, fail_targets: list[nc_constants.FailTargetSummary]
+ ) -> str:
return ''.join(
f'{fail_target.title}: {fail_target.actual}{fail_target.unit}'
f' < {fail_target.goal}{fail_target.unit}\n'
- for fail_target in fail_targets)
+ for fail_target in fail_targets
+ )
diff --git a/tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py b/tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py
index 20e0819db..a7a908a50 100644
--- a/tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py
+++ b/tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py
@@ -41,7 +41,7 @@ from performance_test import nc_constants
from performance_test import nearby_connection_wrapper
from performance_test import setup_utils
-_TEST_SCRIPT_VERSTION = '1.5'
+_TEST_SCRIPT_VERSTION = '1.6'
_DELAY_BETWEEN_EACH_TEST_CYCLE = datetime.timedelta(seconds=5)
_TRANSFER_FILE_SIZE_1GB = 1024 * 1024
diff --git a/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py b/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py
index 7b81d6ae0..cbd521521 100644
--- a/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py
+++ b/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py
@@ -41,7 +41,7 @@ from performance_test import nc_constants
from performance_test import nearby_connection_wrapper
from performance_test import setup_utils
-_TEST_SCRIPT_VERSION = '1.5'
+_TEST_SCRIPT_VERSION = '1.6'
_NEARBY_SNIPPET_2_PACKAGE_NAME = 'com.google.android.nearby.mobly.snippet.second'
diff --git a/tests/bettertogether/quickstart/performance_test/setup_utils.py b/tests/bettertogether/quickstart/performance_test/setup_utils.py
index 2f7d7fbfc..9b8ac4b52 100644
--- a/tests/bettertogether/quickstart/performance_test/setup_utils.py
+++ b/tests/bettertogether/quickstart/performance_test/setup_utils.py
@@ -20,10 +20,14 @@ from typing import Mapping
from mobly.controllers import android_device
+from performance_test import gms_auto_updates_util
+
WIFI_COUNTRYCODE_CONFIG_TIME_SEC = 3
TOGGLE_AIRPLANE_MODE_WAIT_TIME_SEC = 2
PH_FLAG_WRITE_WAIT_TIME_SEC = 3
+_DISABLE_ENABLE_GMS_UPDATE_WAIT_TIME_SEC = 2
+
LOG_TAGS = [
'Nearby',
'NearbyMessages',
@@ -53,6 +57,7 @@ def set_wifi_country_code(
ad.log.info(f'Skipped setting wifi country code on device "{ad.serial}" '
'because we do not set wifi country code on unrooted phone.')
return
+
ad.log.info(f'Set Wi-Fi country code to {country_code}.')
ad.adb.shell('cmd wifi set-wifi-enabled disabled')
time.sleep(WIFI_COUNTRYCODE_CONFIG_TIME_SEC)
@@ -293,3 +298,25 @@ def disable_redaction(ad: android_device.AndroidDevice) -> None:
def install_apk(ad: android_device.AndroidDevice, apk_path: str) -> None:
"""Installs the apk on the given device."""
ad.adb.install(['-r', '-g', '-t', apk_path])
+
+
+def disable_gms_auto_updates(ad: android_device.AndroidDevice) -> None:
+ """Disable GMS auto updates on the given device."""
+ if not ad.is_adb_root:
+ ad.log.warning(
+ 'You should disable the play store auto updates manually on a'
+ 'unrooted device, otherwise the test may be broken unexpected')
+ ad.log.info('try to disable GMS Auto Updates.')
+ gms_auto_updates_util.GmsAutoUpdatesUtil(ad).disable_gms_auto_updates()
+ time.sleep(_DISABLE_ENABLE_GMS_UPDATE_WAIT_TIME_SEC)
+
+
+def enable_gms_auto_updates(ad: android_device.AndroidDevice) -> None:
+ """Enable GMS auto updates on the given device."""
+ if not ad.is_adb_root:
+ ad.log.warning(
+ 'You may enable the play store auto updates manually on a'
+ 'unrooted device after test.')
+ ad.log.info('try to enable GMS Auto Updates.')
+ gms_auto_updates_util.GmsAutoUpdatesUtil(ad).enable_gms_auto_updates()
+ time.sleep(_DISABLE_ENABLE_GMS_UPDATE_WAIT_TIME_SEC)