diff options
author | Treehugger Robot <android-test-infra-autosubmit@system.gserviceaccount.com> | 2024-02-09 00:53:03 +0000 |
---|---|---|
committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2024-02-09 00:53:03 +0000 |
commit | 3b41d352624f42cf9dd6c9f431f31165731301b0 (patch) | |
tree | 2e26352607d0f33e804b7a6f2b018a28a5cf0ade | |
parent | 4ed5455a2ac5574ccbdaaddffa8732e85cf3c139 (diff) | |
parent | c43b382148b29fef604c86fbc76a27d5c9f4f8f6 (diff) | |
download | platform_testing-3b41d352624f42cf9dd6c9f431f31165731301b0.tar.gz |
Merge "Set a 'run_identifier' property for QS perf tests" into main
5 files changed, 61 insertions, 19 deletions
diff --git a/tests/bettertogether/quickstart/performance_test/CHANGELOG.md b/tests/bettertogether/quickstart/performance_test/CHANGELOG.md new file mode 100644 index 000000000..1e51c2886 --- /dev/null +++ b/tests/bettertogether/quickstart/performance_test/CHANGELOG.md @@ -0,0 +1,18 @@ +# Quickstart performance test suite release history + +## 1.6 + +### New +* `nearby_share_stress_test.py` for testing Nearby Share using Wifi only. + +### Fixes +* Change discovery medium to BLE only. +* Increase 1G file transfer timeout to 400s. +* Disable GMS auto-updates for the duration of the test. + +## 1.5 + +### New +* `esim_transfer_stress_test.py` for testing eSIM transfer using Bluetooth only. +* `quick_start_stress_test.py` for testing the Quickstart flow using both + Bluetooth and Wifi.
\ No newline at end of file diff --git a/tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py b/tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py index 2125b45b0..d14954b82 100644 --- a/tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py +++ b/tests/bettertogether/quickstart/performance_test/esim_transfer_stress_test.py @@ -41,7 +41,7 @@ from performance_test import nc_constants from performance_test import nearby_connection_wrapper from performance_test import setup_utils -_TEST_SCRIPT_VERSTION = '1.6' +_TEST_SCRIPT_VERSION = '1.6' _DELAY_BETWEEN_EACH_TEST_CYCLE = datetime.timedelta(seconds=5) _TRANSFER_FILE_SIZE_1MB = 1024 @@ -68,6 +68,7 @@ class EsimTransferStressTest(nc_base_test.NCBaseTestClass): super().__init__(configs) self._test_result = nc_constants.SingleTestResult() self._esim_transfer_test_metrics = self.EsimTransferTestMetrics() + self._test_script_version = _TEST_SCRIPT_VERSION # @typing.override def setup_class(self): @@ -228,7 +229,7 @@ class EsimTransferStressTest(nc_base_test.NCBaseTestClass): self.record_data({ 'Test Class': self.TAG, 'sponge_properties': { - 'test_script_verion': _TEST_SCRIPT_VERSTION, + 'test_script_version': self._test_script_version, '00_test_report_alias_name': ( self.test_parameters.test_report_alias_name), '01_test_result': result_message, diff --git a/tests/bettertogether/quickstart/performance_test/nc_base_test.py b/tests/bettertogether/quickstart/performance_test/nc_base_test.py index 8467cfcf7..96c8ed73f 100644 --- a/tests/bettertogether/quickstart/performance_test/nc_base_test.py +++ b/tests/bettertogether/quickstart/performance_test/nc_base_test.py @@ -41,6 +41,7 @@ class NCBaseTestClass(base_test.BaseTestClass): self.advertiser: android_device.AndroidDevice = None self.discoverer: android_device.AndroidDevice = None self.test_parameters: nc_constants.TestParameters = None + self._test_script_version = None self._nearby_snippet_apk_path: str = None self.performance_test_iterations: int = 1 self.num_bug_reports: int = 0 @@ -51,6 +52,9 @@ class NCBaseTestClass(base_test.BaseTestClass): self._nearby_snippet_apk_path = self.user_params.get('files', {}).get( 'nearby_snippet', [''])[0] + # set run identifier property + self._set_run_identifier() + utils.concurrent_exec( self._setup_android_device, param_list=[[ad] for ad in self.ads], @@ -71,6 +75,22 @@ class NCBaseTestClass(base_test.BaseTestClass): ) self.advertiser, self.discoverer = self.ads + def _set_run_identifier(self) -> None: + """Set a run_identifier property describing the test run context.""" + run_identifier = {} + run_identifier['test_version'] = self._test_script_version + run_identifier['alias'] = self.test_parameters.test_report_alias_name + run_identifier['devices'] = [ + f'{ad.model}({ad.build_info["build_id"]})' for ad in self.ads + ] + run_identifier_str = ', '.join( + [f'{key}:{value}' for key, value in run_identifier.items()] + ) + run_identifier_str = f'{{{run_identifier_str}}}' + self.record_data( + {'resultstore_properties': {'run_identifier': run_identifier_str}} + ) + def _disconnect_from_wifi(self, ad: android_device.AndroidDevice) -> None: if not ad.is_adb_root: ad.log.info("Can't clear wifi network in non-rooted device") diff --git a/tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py b/tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py index f32d989e3..a96750d17 100644 --- a/tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py +++ b/tests/bettertogether/quickstart/performance_test/nearby_share_stress_test.py @@ -41,7 +41,7 @@ from performance_test import nc_constants from performance_test import nearby_connection_wrapper from performance_test import setup_utils -_TEST_SCRIPT_VERSTION = '1.6' +_TEST_SCRIPT_VERSION = '1.6' _DELAY_BETWEEN_EACH_TEST_CYCLE = datetime.timedelta(seconds=5) _TRANSFER_FILE_SIZE_1GB = 1024 * 1024 @@ -75,6 +75,7 @@ class NearbyShareStressTest(nc_base_test.NCBaseTestClass): super().__init__(configs) self._test_result = nc_constants.SingleTestResult() self._nearby_share_test_metrics = self.NearbyShareTestMetrics() + self._test_script_version = _TEST_SCRIPT_VERSION # @typing.override def setup_class(self): @@ -198,7 +199,7 @@ class NearbyShareStressTest(nc_base_test.NCBaseTestClass): wifi_transfer_stats = self._stats_throughput_result( 'WiFi', self._nearby_share_test_metrics.wifi_transfer_throughputs_kbps, - nc_constants.BT_TRANSFER_SUCCESS_RATE_TARGET_PERCENTAGE, + nc_constants.WIFI_TRANSFER_SUCCESS_RATE_TARGET_PERCENTAGE, self.test_parameters.wifi_transfer_throughput_median_benchmark_kbps) discovery_stats = self._stats_latency_result( @@ -246,7 +247,7 @@ class NearbyShareStressTest(nc_base_test.NCBaseTestClass): self.record_data({ 'Test Class': self.TAG, 'sponge_properties': { - 'test_script_verion': _TEST_SCRIPT_VERSTION, + 'test_script_verion': self._test_script_version, '00_test_report_alias_name': ( self.test_parameters.test_report_alias_name), '01_test_result': result_message, diff --git a/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py b/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py index 5cff00e3a..8799c612b 100644 --- a/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py +++ b/tests/bettertogether/quickstart/performance_test/quick_start_stress_test.py @@ -66,6 +66,7 @@ class QuickStartStressTest(nc_base_test.NCBaseTestClass): nc_constants.SingleTestResult()) self._quick_start_test_metrics: nc_constants.QuickStartTestMetrics = ( nc_constants.QuickStartTestMetrics()) + self._test_script_version = _TEST_SCRIPT_VERSION # @typing.override def setup_test(self): @@ -317,13 +318,13 @@ class QuickStartStressTest(nc_base_test.NCBaseTestClass): # @typing.override def _summary_test_results(self) -> None: """Summarizes test results of all iterations.""" - bt_transfer_stats = self._stats_throughput_result( + first_bt_transfer_stats = self._stats_throughput_result( 'BT', self._quick_start_test_metrics.bt_transfer_throughputs_kbps, nc_constants.BT_TRANSFER_SUCCESS_RATE_TARGET_PERCENTAGE, self.test_parameters.bt_transfer_throughput_median_benchmark_kbps) - wifi_transfer_stats = self._stats_throughput_result( + second_wifi_transfer_stats = self._stats_throughput_result( 'Wi-Fi', self._quick_start_test_metrics.wifi_transfer_throughputs_kbps, nc_constants.WIFI_TRANSFER_SUCCESS_RATE_TARGET_PERCENTAGE, @@ -343,12 +344,12 @@ class QuickStartStressTest(nc_base_test.NCBaseTestClass): passed = True result_message = 'Passed' fail_message = '' - if bt_transfer_stats.fail_targets: + if first_bt_transfer_stats.fail_targets: fail_message += self._generate_target_fail_message( - bt_transfer_stats.fail_targets) - if wifi_transfer_stats.fail_targets: + first_bt_transfer_stats.fail_targets) + if second_wifi_transfer_stats.fail_targets: fail_message += self._generate_target_fail_message( - wifi_transfer_stats.fail_targets) + second_wifi_transfer_stats.fail_targets) if fail_message: passed = False result_message = 'Test Failed due to:\n' + fail_message @@ -356,29 +357,29 @@ class QuickStartStressTest(nc_base_test.NCBaseTestClass): detailed_stats = { '0 test iterations': self.performance_test_iterations, '1 Completed BT/Wi-Fi transfer': ( - f'{bt_transfer_stats.success_count}' - f' / {wifi_transfer_stats.success_count}'), + f'{first_bt_transfer_stats.success_count}' + f' / {second_wifi_transfer_stats.success_count}'), '2 BT transfer failures': { '1 discovery': first_discovery_stats.failure_count, '2 connection': first_connection_stats.failure_count, '3 transfer': self.performance_test_iterations - ( - bt_transfer_stats.success_count), + first_bt_transfer_stats.success_count), }, '3 Wi-Fi transfer failures': { '1 discovery': second_discovery_stats.failure_count, '2 connection': second_connection_stats.failure_count, '3 upgrade': medium_upgrade_stats.failure_count, '4 transfer': self.performance_test_iterations - ( - wifi_transfer_stats.success_count), + second_wifi_transfer_stats.success_count), }, '4 Medium upgrade count': ( self._summary_upgraded_wifi_transfer_mediums()), '5 50% and 95% of BT transfer speed (KBps)': ( - f'{bt_transfer_stats.percentile_50_kbps}' - f' / {bt_transfer_stats.percentile_95_kbps}'), + f'{first_bt_transfer_stats.percentile_50_kbps}' + f' / {first_bt_transfer_stats.percentile_95_kbps}'), '6 50% and 95% of Wi-Fi transfer speed(KBps)': ( - f'{wifi_transfer_stats.percentile_50_kbps}' - f' / {wifi_transfer_stats.percentile_95_kbps}'), + f'{second_wifi_transfer_stats.percentile_50_kbps}' + f' / {second_wifi_transfer_stats.percentile_95_kbps}'), '7 50% and 95% of discovery latency(sec)': ( f'{first_discovery_stats.percentile_50}' f' / {first_discovery_stats.percentile_95} (1st), ' @@ -413,5 +414,6 @@ class QuickStartStressTest(nc_base_test.NCBaseTestClass): asserts.assert_true(passed, result_message) + if __name__ == '__main__': test_runner.main() |