diff options
author | Xin Li <delphij@google.com> | 2023-08-14 15:45:04 -0700 |
---|---|---|
committer | Xin Li <delphij@google.com> | 2023-08-14 15:45:04 -0700 |
commit | 59ca87a5634f74c2bdaccc45824c7308063c51e8 (patch) | |
tree | 97ec3df55a6e23969024f73faeed7fc5cc5f1de8 | |
parent | 0e49165c16dcb54468c0ffc47eed945dcca055f9 (diff) | |
parent | 195d6530c95de9d8b12c8354316cf865471355bf (diff) | |
download | catbox-tmp_amf_298295554.tar.gz |
Merge Android U (ab/10368041)tmp_amf_298295554
Bug: 291102124
Merged-In: I7d1845e7623f3acd3516be511927a2950d235b6a
Change-Id: Iefebd0c3b952a04dc359fa75faa39d424400d4c6
23 files changed, 623 insertions, 83 deletions
diff --git a/prebuilts/Android.mk b/prebuilts/Android.mk new file mode 100644 index 0000000..3b01d73 --- /dev/null +++ b/prebuilts/Android.mk @@ -0,0 +1,23 @@ +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LOCAL_PATH := $(call my-dir) + +include $(CLEAR_VARS) + +# tag this module as a catbox test artifact +LOCAL_COMPATIBILITY_SUITE := CATBOX + +# Build the test APKs using their own makefiles +include $(call all-makefiles-under,$(LOCAL_PATH)) diff --git a/prebuilts/moped_bluev/Android.mk b/prebuilts/moped_bluev/Android.mk new file mode 100644 index 0000000..e26f013 --- /dev/null +++ b/prebuilts/moped_bluev/Android.mk @@ -0,0 +1,26 @@ +# +# Copyright (C) 2023 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### +LOCAL_PATH := $(call my-dir) +MOPED_NAME := moped.tar.gz +MOPED_TAR_PATH := $(LOCAL_PATH)/$(MOPED_NAME) + +TESTCASES_DIR := $(HOST_OUT)/catbox/android-catbox/testcases + +# Moped Test Binary +COMPATIBILITY.catbox.FILES += $(call copy-many-files,\ + $(MOPED_TAR_PATH):$(TESTCASES_DIR)/$(MOPED_NAME)) + +############################################################################### diff --git a/prebuilts/moped_bluev/moped.tar.gz b/prebuilts/moped_bluev/moped.tar.gz Binary files differnew file mode 100644 index 0000000..38a9bbb --- /dev/null +++ b/prebuilts/moped_bluev/moped.tar.gz diff --git a/report/src/com/android/catbox/result/JsonResultReporter.java b/report/src/com/android/catbox/result/JsonResultReporter.java index 90a71ec..9363f9d 100644 --- a/report/src/com/android/catbox/result/JsonResultReporter.java +++ b/report/src/com/android/catbox/result/JsonResultReporter.java @@ -178,6 +178,12 @@ public class JsonResultReporter implements ITestInvocationListener { mTestMetricsUtil.setIterationSeparator(mTestIterationSeparator); } + /** Re-initialize object to erase all existing test metrics */ + private void reInitializeTestMetricsUtil() { + mTestMetricsUtil = initializeTestMetricsUtil(); + mTestMetricsUtil.setIterationSeparator(mTestIterationSeparator); + } + /** Write Test Metrics to JSON */ private void writeTestMetrics( String classMethodName, Map<String, String> metrics) { @@ -332,6 +338,9 @@ public class JsonResultReporter implements ITestInvocationListener { writeTestMetrics(testName, aggregatedMetrics.get(testName)); } } + + // Avoid reporting duplicate metrics by erasing metrics from previous runs + reInitializeTestMetricsUtil(); } /** {@inheritDoc} */ diff --git a/runner/Android.bp b/runner/Android.bp new file mode 100644 index 0000000..026d429 --- /dev/null +++ b/runner/Android.bp @@ -0,0 +1,27 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +java_library_host { + name: "catbox-test-runner-lib", + + srcs: ["src/**/*.java"], + + libs: [ + "tradefed", + ], +} diff --git a/runner/src/com/android/catbox/runner/MopedRunner.java b/runner/src/com/android/catbox/runner/MopedRunner.java new file mode 100644 index 0000000..c1148b6 --- /dev/null +++ b/runner/src/com/android/catbox/runner/MopedRunner.java @@ -0,0 +1,147 @@ + +// Copyright 2023 Google Inc. All Rights Reserved. +package com.google.android.car.aaosbt; +import com.android.ddmlib.Log.LogLevel; +import com.android.tradefed.config.Option; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.invoker.TestInformation; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.targetprep.TargetSetupError; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.io.IOException; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.TimeUnit; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.testtype.IRemoteTest; +import java.net.URISyntaxException; +import com.android.tradefed.device.ITestDevice; + +@OptionClass(alias = "aaos-moped-test") +public class MopedRunner implements IRemoteTest { + @Option(name = "test-artifact", description = "test artifact") + private File test_artifact = null; + + @Option(name = "artifact", description = "test artifact") + private String artifact_str = null; + + @Option(name = "unzip-build-timeout-min", description = "unzip build timeout in minutes") + private int unzip_build_timeout_min = 10; + + @Option(name = "test-timeout-min", description = "test timeout in minutes") + private int test_timeout_min = 60; + + @Option(name = "testcase", description = "which moped test binary to run") + private String test_case = null; + + private File mLocalDestFile; + private File mLocalSrcFile; + private String mArtifactLocation; + + private void unTarTestArtifact(TestInformation testInfo) throws TargetSetupError, TimeoutException, URISyntaxException { + if (test_artifact != null && artifact_str == null) { + artifact_str = test_artifact.toPath().toString(); + } + String jarPath = getClass() + .getProtectionDomain() + .getCodeSource() + .getLocation() + .toURI() + .getPath(); + File jarFile = new File(jarPath); + mLocalSrcFile = new File(jarFile.getParent() + "/../testcases/" + artifact_str); + mLocalDestFile = new File(testInfo.dependenciesFolder().toString()); + mArtifactLocation = + mLocalDestFile.getPath() + + "/" + + mLocalSrcFile.getName().replaceAll(".tar.*gz", "/"); + Path localArtifactPath = Paths.get(mArtifactLocation); + if (!Files.exists(localArtifactPath)) { + // untar file + executeHostCommand( + new String[] { + "bash", + "-c", + "tar xf " + mLocalSrcFile.getPath() + " -C " + mLocalDestFile.getPath() + }, + unzip_build_timeout_min); + } + } + + private String getDevicesString(TestInformation testInfo) throws DeviceNotAvailableException { + StringBuilder deviceString = new StringBuilder(); + int deviceNum = 0; + for(ITestDevice device : testInfo.getDevices()) { + if (deviceNum==0) { // by default the first device is head unit. + deviceString.append(String.format(" --hu %s", device.getSerialNumber())); + } else { + deviceString.append(String.format(" --phone%s %s", String.valueOf(deviceNum), device.getSerialNumber())); + } + deviceNum++; + } + deviceString.append(String.format(" --devicenum %s", String.valueOf(deviceNum))); + return deviceString.toString(); + } + + @Override + public void run(TestInformation testInfo, ITestInvocationListener listener) + throws DeviceNotAvailableException { + // Download Moped binanry / config and run test + try { + unTarTestArtifact(testInfo); + executeHostCommand( + new String[] {"bash", "-c", "bash " + + String.format("%s/run.sh %s --testcase %s", mArtifactLocation, + getDevicesString(testInfo), + test_case)}, + test_timeout_min); + } catch (TargetSetupError e) { + CLog.logAndDisplay(LogLevel.VERBOSE, "There are problems running tests! %s", e); + } catch (TimeoutException e) { + CLog.logAndDisplay(LogLevel.VERBOSE, "Test execution timeout! %s", e); + } catch (URISyntaxException e) { + CLog.logAndDisplay(LogLevel.VERBOSE, "Test artifact not found! %s", e); + } + } + + private ArrayList<String> executeHostCommand(String[] command, int timeout) + throws TargetSetupError, TimeoutException { + ArrayList<String> ret = new ArrayList<String>(); + try { + CLog.logAndDisplay( + LogLevel.VERBOSE, "Output of running %s is:", Arrays.toString(command)); + Process p = Runtime.getRuntime().exec(command); + if (!p.waitFor(timeout, TimeUnit.MINUTES)) { + p.destroy(); + throw new TimeoutException(); + } + InputStream is = p.getInputStream(); + InputStreamReader isr = new InputStreamReader(is); + BufferedReader br = new BufferedReader(isr); + String line; + while ((line = br.readLine()) != null) { + CLog.logAndDisplay(LogLevel.VERBOSE, line); + ret.add(line); + } + int exitCode = p.waitFor(); + if (exitCode != 0) { + throw new TargetSetupError( + "Execution of command " + Arrays.toString(command) + " failed!"); + } + } catch (IOException e) { + CLog.logAndDisplay(LogLevel.VERBOSE, "There are problems with IO! %s", e); + } catch (InterruptedException e) { + CLog.logAndDisplay(LogLevel.VERBOSE, "User interruption!"); + } + return ret; + } +} + diff --git a/tools/catbox-tradefed/Android.bp b/tools/catbox-tradefed/Android.bp index 493e19c..c94cc9f 100644 --- a/tools/catbox-tradefed/Android.bp +++ b/tools/catbox-tradefed/Android.bp @@ -39,7 +39,8 @@ tradefed_binary_host { static_libs: [ "cts-tradefed-harness", "catbox-common", - "catbox-preparer-lib" + "catbox-preparer-lib", + "catbox-test-runner-lib" ], required: [ "compatibility-host-util", diff --git a/tools/catbox-tradefed/res/config/catbox-functional-app-info-setting-ui-elements.xml b/tools/catbox-tradefed/res/config/catbox-functional-app-info-setting-ui-elements.xml new file mode 100644 index 0000000..17ce11d --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-app-info-setting-ui-elements.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - App Info Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-app-info-setting"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.AppInfoVerifyUIElementsTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-btmoped.xml b/tools/catbox-tradefed/res/config/catbox-functional-btmoped.xml new file mode 100644 index 0000000..f5113b3 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-btmoped.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Bluetooth Moped based Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-btmoped"/> + + <!-- Artificially disabling tests in CompatibilityTestSuite --> + <!-- See com.android.tradefed.testtype.suite.CompatibilityTestSuite:loadTests() --> + <option name="compatibility:reverse-exclude-filters" value="true" /> + + <test class="com.google.android.car.aaosbt.MopedRunner"> + <option name="test-artifact" value="moped.tar.gz" /> + </test> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-functional-display-setting.xml b/tools/catbox-tradefed/res/config/catbox-functional-display-setting.xml new file mode 100644 index 0000000..aa2bf20 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-display-setting.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2021 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Display Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-display-setting"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.DisplaySettingTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-driving-optimized-apps.xml b/tools/catbox-tradefed/res/config/catbox-functional-driving-optimized-apps.xml new file mode 100644 index 0000000..a21e086 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-driving-optimized-apps.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Ux Restriction Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-ux-restriction"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests android.platform.tests.UxRestrictionDOAppsTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-microphone-recent-apps.xml b/tools/catbox-tradefed/res/config/catbox-functional-microphone-recent-apps.xml new file mode 100644 index 0000000..188ebed --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-microphone-recent-apps.xml @@ -0,0 +1,34 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Microphone Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-microphone-recent-apps"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.MicroPhoneRecentAppsTest" /> +</configuration> + diff --git a/tools/catbox-tradefed/res/config/catbox-functional-microphone-setting.xml b/tools/catbox-tradefed/res/config/catbox-functional-microphone-setting.xml new file mode 100644 index 0000000..d837487 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-microphone-setting.xml @@ -0,0 +1,34 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Microphone Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-microphone-setting"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.MicroPhoneSettingTest" /> +</configuration> + diff --git a/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting.xml b/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting.xml new file mode 100644 index 0000000..3f2bde1 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting.xml @@ -0,0 +1,34 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Microphone Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-privacy-setting"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.PrivacySettingTest" /> +</configuration> + diff --git a/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction-facet-bar.xml b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction-facet-bar.xml new file mode 100644 index 0000000..06ab2c7 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction-facet-bar.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2021 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Ux Restriction Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-ux-restriction-facet-bar"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests android.platform.tests.UxRestrictionFacetBarTest" /> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml index 5bc518b..0141ecb 100644 --- a/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml +++ b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml @@ -29,5 +29,5 @@ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> <!-- Tests --> - <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests" /> -</configuration>
\ No newline at end of file + <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests android.platform.tests.UxRestrictionTest" /> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml index 9dd7470..e4deff5 100644 --- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml +++ b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml @@ -19,7 +19,6 @@ <!-- Clear pre-created guests --> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_guests 0" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" /> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml index 97454a0..c066a6c 100644 --- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml +++ b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml @@ -19,7 +19,6 @@ <!-- Clear pre-created users --> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_users 0" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" /> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-guest.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-guest.xml deleted file mode 100644 index bfc140a..0000000 --- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-guest.xml +++ /dev/null @@ -1,38 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright (C) 2021 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<configuration description="CATBox Performance Test to measure latency to switch to a Pre-created Guest"> - <include name="catbox-performance-multiuser-base" /> - - <!-- Set pre-created guests --> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_guests 5" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" /> - - <!--Test --> - <option name="compatibility:include-filter" value="AndroidAutomotiveMultiuserScenarioTests android.platform.scenario.multiuser.SwitchToNewGuestBenchmark" /> - <option name="compatibility:module-arg" value="AndroidAutomotiveMultiuserScenarioTests:instrumentation-arg:include_creation_time:=true" /> - - <!-- Test Metrics Report --> - <option name="report-log-name" value="CatboxPerformanceTests" /> - <option name="report-test-name-mapping" key="android.platform.scenario.multiuser.SwitchToNewGuestBenchmark#testSwitch" value="switch_to_pre_created_guest" /> - <option name="report-all-metrics" value="false" /> - <option name="report-metric-key-mapping" key="duration_ms_android.platform.scenario.multiuser.SwitchToNewGuestBenchmark#testSwitch-median" value="duration_ms" /> - - <option name="plan" value="catbox-performance-create-and-switch-to-precreated-guest" /> -</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-user.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-user.xml deleted file mode 100644 index c0acabb..0000000 --- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-user.xml +++ /dev/null @@ -1,38 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright (C) 2018 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<configuration description="CATBox Performance Test to measure latency to switch to a Pre-created User"> - <include name="catbox-performance-multiuser-base" /> - - <!-- Clear pre-created guests --> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_users 5" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" /> - - <!--Test --> - <option name="compatibility:include-filter" value="AndroidAutomotiveMultiuserScenarioTests android.platform.scenario.multiuser.SwitchToNewSecondaryUserBenchmark" /> - <option name="compatibility:module-arg" value="AndroidAutomotiveMultiuserScenarioTests:instrumentation-arg:include_creation_time:=true" /> - - <!-- Test Metrics Report --> - <option name="report-log-name" value="CatboxPerformanceTests" /> - <option name="report-test-name-mapping" key="android.platform.scenario.multiuser.SwitchToNewSecondaryUserBenchmark#testSwitch" value="switch_to_pre_created_user" /> - <option name="report-all-metrics" value="false" /> - <option name="report-metric-key-mapping" key="duration_ms_android.platform.scenario.multiuser.SwitchToNewSecondaryUserBenchmark#testSwitch-median" value="duration_ms" /> - - <option name="plan" value="catbox-performance-create-and-switch-to-precreated-user" /> -</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml b/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml index 0c14d83..be24379 100644 --- a/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml +++ b/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml @@ -37,14 +37,14 @@ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.test.scenario.appgrid" /> <!-- Test --> - <option name="compatibility:include-filter" value="AndroidAutomotiveAppGridScenarioTests android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollDownAndUp" /> + <option name="compatibility:include-filter" value="AndroidAutomotiveAppGridScenarioTests android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollForwardAndBackward" /> <!-- Jank Package --> <option name="compatibility:module-arg" value="AndroidAutomotiveAppGridScenarioTests:instrumentation-arg:jank-package-names:=com.android.car.carlauncher" /> <!-- Test Metrics Report Options --> <option name="report-log-name" value="CatboxPerformanceTests" /> - <option name="report-test-name-mapping" key="android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollDownAndUp" value="appgrid_scroll_jank" /> + <option name="report-test-name-mapping" key="android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollForwardAndBackward" value="appgrid_scroll_jank" /> <option name="report-all-metrics" value="false" /> <option name="report-metric-key-mapping" key="gfxinfo_com.android.car.carlauncher_janky_frames_percent-mean" value="jank_frames_percent" /> </configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-jank-media-switch-playback.xml b/tools/catbox-tradefed/res/config/catbox-performance-jank-media-switch-playback.xml new file mode 100644 index 0000000..f0fea15 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-performance-jank-media-switch-playback.xml @@ -0,0 +1,52 @@ +<!-- + ~ Copyright (C) 2023 The Android Open Source Project + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration description="CATBox Jank Performance Test For Media Switch Playback."> + <!-- Plan --> + <option name="plan" value="catbox-performance-jank-media-switch-playback" /> + + <!-- Base File --> + <include name="catbox-performance-jank-base" /> + + <!-- Proc Load Options --> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:per_run:=true" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-interval:=20000" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-threshold:=2.0" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-timeout:=900000" /> + + <!-- Jank Options --> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:listener:=android.device.collectors.ProcLoadListener,android.device.collectors.JankListener" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:iterations:=20" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:favor-shell-commands:=true" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:rename-iterations:=true" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:iteration-separator:=$" /> + + <!-- Test Package --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.test.scenario.mediacenter" /> + + <!-- Test --> + <option name="compatibility:include-filter" value="AndroidAutomotiveMediaCenterScenarioTests android.platform.test.scenario.mediacenter.SwitchPlaybackMicrobenchmark#testSwitchPlayback" /> + + <!-- Jank Package --> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:jank-package-names:=com.android.car.media" /> + + <!-- Test Metrics Report Options --> + <option name="report-log-name" value="CatboxPerformanceTests" /> + <option name="report-test-name-mapping" key="android.platform.test.scenario.mediacenter.SwitchPlaybackMicrobenchmark#testSwitchPlayback" value="media_switchplayback_jank" /> + <option name="report-all-metrics" value="false" /> + <!-- TODO: b/247588415 Add metrics to APFE --> + <option name="report-metric-key-mapping" key="gfxinfo_com.android.car.media_janky_frames_percent-mean" value="jank_frames_percent" /> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml b/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml new file mode 100644 index 0000000..863b7e1 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml @@ -0,0 +1,66 @@ +<!-- + ~ Copyright (C) 2022 The Android Open Source Project + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration> + <!-- Extend existing boottime config--> + <include name="google/test/boottime-successive"/> + + <include name="catbox-performance-base" /> + + <!-- Artificially disabling tests in CompatibilityTestSuite --> + <!-- See com.android.tradefed.testtype.suite.CompatibilityTestSuite:loadTests() --> + <option name="compatibility:reverse-exclude-filters" value="true" /> + + <option name="boot-delay" value="20000"/> + <option name="boot-count" value="20"/> + <option name="dmesg-info" value="true"/> + <option name="granular-boot-info" value="true"/> + <option name="online-timeout" value="120000"/> + <option name="successive-boot" value="true"/> + + <!-- Prepare commands --> + <option name="successive-boot-prepare-cmd" + value="am start -n com.google.android.car.setupwizard/.ExitActivity"/> + <option name="successive-boot-prepare-cmd" + value="setprop persist.aae.suw.disable_short_notice true"/> + <option name="successive-boot-prepare-cmd" + value="cmd car_service enable-feature car_evs_service"/> + + <!-- For custom boot time metrics --> + <option name="boot-time-pattern" key="boot_to_ble_on" + value="BluetoothAdapterService: updateAdapterState\(\) \- Broadcasting state BLE_ON"/> + <option name="boot-time-pattern" key="boot_to_carlauncher_fully_drawn" + value="Fully drawn com.android.car.carlauncher/.CarLauncher"/> + <option name="boot-time-pattern" key="boot_to_first_user_unlocked" + value="CarServiceHelper: Time to unlock 1st user"/> + <option name="boot-time-pattern" key="boot_to_maps_displayed" + value="Displayed com.google.android.apps.maps"/> + <option name="boot-time-pattern" key="boot_to_settings_fallbackhome_displayed" + value="Displayed com.android.car.settings/.FallbackHome"/> + <option name="boot-time-pattern" key="boot_to_user0_unlocked" + value="SystemServerTiming: SystemUserUnlock took to complete"/> + <option name="boot-time-pattern" key="boot_to_user10_unlocked" + value="SystemServerTiming: User10Unlock took to complete"/> + <option name="boot-time-pattern" key="boot_to_gmscore_started_experimental" + value="ActivityManager: Start proc \d+:com.google.android.gms/u0"/> + <option name="boot-time-pattern" key="boot_to_car_evs_service_started" + value="CAR.InitTiming: CarEvsService took to complete"/> + + <option name="report-log-name" value="CatboxPerformanceTests" /> + <option name="report-test-name-mapping" key="BootTimeTest.BootTimeTest#SuccessiveBootTest" value="boot_time_test" /> + <option name="report-all-metrics" value="false" /> + <option name="report-metric-key-mapping" key="dmesg_action_sys.boot_completed_first_timestamp-median" value="successive_boot_time" /> +</configuration> |