summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorXin Li <delphij@google.com>2023-08-14 15:45:04 -0700
committerXin Li <delphij@google.com>2023-08-14 15:45:04 -0700
commit59ca87a5634f74c2bdaccc45824c7308063c51e8 (patch)
tree97ec3df55a6e23969024f73faeed7fc5cc5f1de8
parent0e49165c16dcb54468c0ffc47eed945dcca055f9 (diff)
parent195d6530c95de9d8b12c8354316cf865471355bf (diff)
downloadcatbox-tmp_amf_298295554.tar.gz
Merge Android U (ab/10368041)tmp_amf_298295554
Bug: 291102124 Merged-In: I7d1845e7623f3acd3516be511927a2950d235b6a Change-Id: Iefebd0c3b952a04dc359fa75faa39d424400d4c6
-rw-r--r--prebuilts/Android.mk23
-rw-r--r--prebuilts/moped_bluev/Android.mk26
-rw-r--r--prebuilts/moped_bluev/moped.tar.gzbin0 -> 24727829 bytes
-rw-r--r--report/src/com/android/catbox/result/JsonResultReporter.java9
-rw-r--r--runner/Android.bp27
-rw-r--r--runner/src/com/android/catbox/runner/MopedRunner.java147
-rw-r--r--tools/catbox-tradefed/Android.bp3
-rw-r--r--tools/catbox-tradefed/res/config/catbox-functional-app-info-setting-ui-elements.xml33
-rw-r--r--tools/catbox-tradefed/res/config/catbox-functional-btmoped.xml33
-rw-r--r--tools/catbox-tradefed/res/config/catbox-functional-display-setting.xml33
-rw-r--r--tools/catbox-tradefed/res/config/catbox-functional-driving-optimized-apps.xml33
-rw-r--r--tools/catbox-tradefed/res/config/catbox-functional-microphone-recent-apps.xml34
-rw-r--r--tools/catbox-tradefed/res/config/catbox-functional-microphone-setting.xml34
-rw-r--r--tools/catbox-tradefed/res/config/catbox-functional-privacy-setting.xml34
-rw-r--r--tools/catbox-tradefed/res/config/catbox-functional-ux-restriction-facet-bar.xml33
-rw-r--r--tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml4
-rw-r--r--tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml1
-rw-r--r--tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml1
-rw-r--r--tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-guest.xml38
-rw-r--r--tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-user.xml38
-rw-r--r--tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml4
-rw-r--r--tools/catbox-tradefed/res/config/catbox-performance-jank-media-switch-playback.xml52
-rw-r--r--tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml66
23 files changed, 623 insertions, 83 deletions
diff --git a/prebuilts/Android.mk b/prebuilts/Android.mk
new file mode 100644
index 0000000..3b01d73
--- /dev/null
+++ b/prebuilts/Android.mk
@@ -0,0 +1,23 @@
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LOCAL_PATH := $(call my-dir)
+
+include $(CLEAR_VARS)
+
+# tag this module as a catbox test artifact
+LOCAL_COMPATIBILITY_SUITE := CATBOX
+
+# Build the test APKs using their own makefiles
+include $(call all-makefiles-under,$(LOCAL_PATH))
diff --git a/prebuilts/moped_bluev/Android.mk b/prebuilts/moped_bluev/Android.mk
new file mode 100644
index 0000000..e26f013
--- /dev/null
+++ b/prebuilts/moped_bluev/Android.mk
@@ -0,0 +1,26 @@
+#
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+LOCAL_PATH := $(call my-dir)
+MOPED_NAME := moped.tar.gz
+MOPED_TAR_PATH := $(LOCAL_PATH)/$(MOPED_NAME)
+
+TESTCASES_DIR := $(HOST_OUT)/catbox/android-catbox/testcases
+
+# Moped Test Binary
+COMPATIBILITY.catbox.FILES += $(call copy-many-files,\
+ $(MOPED_TAR_PATH):$(TESTCASES_DIR)/$(MOPED_NAME))
+
+###############################################################################
diff --git a/prebuilts/moped_bluev/moped.tar.gz b/prebuilts/moped_bluev/moped.tar.gz
new file mode 100644
index 0000000..38a9bbb
--- /dev/null
+++ b/prebuilts/moped_bluev/moped.tar.gz
Binary files differ
diff --git a/report/src/com/android/catbox/result/JsonResultReporter.java b/report/src/com/android/catbox/result/JsonResultReporter.java
index 90a71ec..9363f9d 100644
--- a/report/src/com/android/catbox/result/JsonResultReporter.java
+++ b/report/src/com/android/catbox/result/JsonResultReporter.java
@@ -178,6 +178,12 @@ public class JsonResultReporter implements ITestInvocationListener {
mTestMetricsUtil.setIterationSeparator(mTestIterationSeparator);
}
+ /** Re-initialize object to erase all existing test metrics */
+ private void reInitializeTestMetricsUtil() {
+ mTestMetricsUtil = initializeTestMetricsUtil();
+ mTestMetricsUtil.setIterationSeparator(mTestIterationSeparator);
+ }
+
/** Write Test Metrics to JSON */
private void writeTestMetrics(
String classMethodName, Map<String, String> metrics) {
@@ -332,6 +338,9 @@ public class JsonResultReporter implements ITestInvocationListener {
writeTestMetrics(testName, aggregatedMetrics.get(testName));
}
}
+
+ // Avoid reporting duplicate metrics by erasing metrics from previous runs
+ reInitializeTestMetricsUtil();
}
/** {@inheritDoc} */
diff --git a/runner/Android.bp b/runner/Android.bp
new file mode 100644
index 0000000..026d429
--- /dev/null
+++ b/runner/Android.bp
@@ -0,0 +1,27 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+ default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+java_library_host {
+ name: "catbox-test-runner-lib",
+
+ srcs: ["src/**/*.java"],
+
+ libs: [
+ "tradefed",
+ ],
+}
diff --git a/runner/src/com/android/catbox/runner/MopedRunner.java b/runner/src/com/android/catbox/runner/MopedRunner.java
new file mode 100644
index 0000000..c1148b6
--- /dev/null
+++ b/runner/src/com/android/catbox/runner/MopedRunner.java
@@ -0,0 +1,147 @@
+
+// Copyright 2023 Google Inc. All Rights Reserved.
+package com.google.android.car.aaosbt;
+import com.android.ddmlib.Log.LogLevel;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.invoker.TestInformation;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.targetprep.TargetSetupError;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.TimeUnit;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.testtype.IRemoteTest;
+import java.net.URISyntaxException;
+import com.android.tradefed.device.ITestDevice;
+
+@OptionClass(alias = "aaos-moped-test")
+public class MopedRunner implements IRemoteTest {
+ @Option(name = "test-artifact", description = "test artifact")
+ private File test_artifact = null;
+
+ @Option(name = "artifact", description = "test artifact")
+ private String artifact_str = null;
+
+ @Option(name = "unzip-build-timeout-min", description = "unzip build timeout in minutes")
+ private int unzip_build_timeout_min = 10;
+
+ @Option(name = "test-timeout-min", description = "test timeout in minutes")
+ private int test_timeout_min = 60;
+
+ @Option(name = "testcase", description = "which moped test binary to run")
+ private String test_case = null;
+
+ private File mLocalDestFile;
+ private File mLocalSrcFile;
+ private String mArtifactLocation;
+
+ private void unTarTestArtifact(TestInformation testInfo) throws TargetSetupError, TimeoutException, URISyntaxException {
+ if (test_artifact != null && artifact_str == null) {
+ artifact_str = test_artifact.toPath().toString();
+ }
+ String jarPath = getClass()
+ .getProtectionDomain()
+ .getCodeSource()
+ .getLocation()
+ .toURI()
+ .getPath();
+ File jarFile = new File(jarPath);
+ mLocalSrcFile = new File(jarFile.getParent() + "/../testcases/" + artifact_str);
+ mLocalDestFile = new File(testInfo.dependenciesFolder().toString());
+ mArtifactLocation =
+ mLocalDestFile.getPath()
+ + "/"
+ + mLocalSrcFile.getName().replaceAll(".tar.*gz", "/");
+ Path localArtifactPath = Paths.get(mArtifactLocation);
+ if (!Files.exists(localArtifactPath)) {
+ // untar file
+ executeHostCommand(
+ new String[] {
+ "bash",
+ "-c",
+ "tar xf " + mLocalSrcFile.getPath() + " -C " + mLocalDestFile.getPath()
+ },
+ unzip_build_timeout_min);
+ }
+ }
+
+ private String getDevicesString(TestInformation testInfo) throws DeviceNotAvailableException {
+ StringBuilder deviceString = new StringBuilder();
+ int deviceNum = 0;
+ for(ITestDevice device : testInfo.getDevices()) {
+ if (deviceNum==0) { // by default the first device is head unit.
+ deviceString.append(String.format(" --hu %s", device.getSerialNumber()));
+ } else {
+ deviceString.append(String.format(" --phone%s %s", String.valueOf(deviceNum), device.getSerialNumber()));
+ }
+ deviceNum++;
+ }
+ deviceString.append(String.format(" --devicenum %s", String.valueOf(deviceNum)));
+ return deviceString.toString();
+ }
+
+ @Override
+ public void run(TestInformation testInfo, ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ // Download Moped binanry / config and run test
+ try {
+ unTarTestArtifact(testInfo);
+ executeHostCommand(
+ new String[] {"bash", "-c", "bash " +
+ String.format("%s/run.sh %s --testcase %s", mArtifactLocation,
+ getDevicesString(testInfo),
+ test_case)},
+ test_timeout_min);
+ } catch (TargetSetupError e) {
+ CLog.logAndDisplay(LogLevel.VERBOSE, "There are problems running tests! %s", e);
+ } catch (TimeoutException e) {
+ CLog.logAndDisplay(LogLevel.VERBOSE, "Test execution timeout! %s", e);
+ } catch (URISyntaxException e) {
+ CLog.logAndDisplay(LogLevel.VERBOSE, "Test artifact not found! %s", e);
+ }
+ }
+
+ private ArrayList<String> executeHostCommand(String[] command, int timeout)
+ throws TargetSetupError, TimeoutException {
+ ArrayList<String> ret = new ArrayList<String>();
+ try {
+ CLog.logAndDisplay(
+ LogLevel.VERBOSE, "Output of running %s is:", Arrays.toString(command));
+ Process p = Runtime.getRuntime().exec(command);
+ if (!p.waitFor(timeout, TimeUnit.MINUTES)) {
+ p.destroy();
+ throw new TimeoutException();
+ }
+ InputStream is = p.getInputStream();
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+ String line;
+ while ((line = br.readLine()) != null) {
+ CLog.logAndDisplay(LogLevel.VERBOSE, line);
+ ret.add(line);
+ }
+ int exitCode = p.waitFor();
+ if (exitCode != 0) {
+ throw new TargetSetupError(
+ "Execution of command " + Arrays.toString(command) + " failed!");
+ }
+ } catch (IOException e) {
+ CLog.logAndDisplay(LogLevel.VERBOSE, "There are problems with IO! %s", e);
+ } catch (InterruptedException e) {
+ CLog.logAndDisplay(LogLevel.VERBOSE, "User interruption!");
+ }
+ return ret;
+ }
+}
+
diff --git a/tools/catbox-tradefed/Android.bp b/tools/catbox-tradefed/Android.bp
index 493e19c..c94cc9f 100644
--- a/tools/catbox-tradefed/Android.bp
+++ b/tools/catbox-tradefed/Android.bp
@@ -39,7 +39,8 @@ tradefed_binary_host {
static_libs: [
"cts-tradefed-harness",
"catbox-common",
- "catbox-preparer-lib"
+ "catbox-preparer-lib",
+ "catbox-test-runner-lib"
],
required: [
"compatibility-host-util",
diff --git a/tools/catbox-tradefed/res/config/catbox-functional-app-info-setting-ui-elements.xml b/tools/catbox-tradefed/res/config/catbox-functional-app-info-setting-ui-elements.xml
new file mode 100644
index 0000000..17ce11d
--- /dev/null
+++ b/tools/catbox-tradefed/res/config/catbox-functional-app-info-setting-ui-elements.xml
@@ -0,0 +1,33 @@
+<!--
+ Copyright (C) 2023 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Complete Automotive Tests - App Info Setting Functional Tests.">
+ <!-- Common Base -->
+ <include name="catbox-common"/>
+
+ <!-- Device Preparers -->
+ <include name="catbox-preparer"/>
+
+ <!-- Plan -->
+ <option name="plan" value="catbox-functional-app-info-setting"/>
+
+ <!-- Test Args -->
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" />
+
+ <!-- Tests -->
+ <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.AppInfoVerifyUIElementsTest" />
+</configuration> \ No newline at end of file
diff --git a/tools/catbox-tradefed/res/config/catbox-functional-btmoped.xml b/tools/catbox-tradefed/res/config/catbox-functional-btmoped.xml
new file mode 100644
index 0000000..f5113b3
--- /dev/null
+++ b/tools/catbox-tradefed/res/config/catbox-functional-btmoped.xml
@@ -0,0 +1,33 @@
+<!--
+ Copyright (C) 2023 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Complete Automotive Tests - Bluetooth Moped based Functional Tests.">
+ <!-- Common Base -->
+ <include name="catbox-common"/>
+
+ <!-- Device Preparers -->
+ <include name="catbox-preparer"/>
+
+ <!-- Plan -->
+ <option name="plan" value="catbox-functional-btmoped"/>
+
+ <!-- Artificially disabling tests in CompatibilityTestSuite -->
+ <!-- See com.android.tradefed.testtype.suite.CompatibilityTestSuite:loadTests() -->
+ <option name="compatibility:reverse-exclude-filters" value="true" />
+
+ <test class="com.google.android.car.aaosbt.MopedRunner">
+ <option name="test-artifact" value="moped.tar.gz" />
+ </test>
+</configuration>
diff --git a/tools/catbox-tradefed/res/config/catbox-functional-display-setting.xml b/tools/catbox-tradefed/res/config/catbox-functional-display-setting.xml
new file mode 100644
index 0000000..aa2bf20
--- /dev/null
+++ b/tools/catbox-tradefed/res/config/catbox-functional-display-setting.xml
@@ -0,0 +1,33 @@
+<!--
+ Copyright (C) 2021 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Complete Automotive Tests - Display Setting Functional Tests.">
+ <!-- Common Base -->
+ <include name="catbox-common"/>
+
+ <!-- Device Preparers -->
+ <include name="catbox-preparer"/>
+
+ <!-- Plan -->
+ <option name="plan" value="catbox-functional-display-setting"/>
+
+ <!-- Test Args -->
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" />
+
+ <!-- Tests -->
+ <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.DisplaySettingTest" />
+</configuration> \ No newline at end of file
diff --git a/tools/catbox-tradefed/res/config/catbox-functional-driving-optimized-apps.xml b/tools/catbox-tradefed/res/config/catbox-functional-driving-optimized-apps.xml
new file mode 100644
index 0000000..a21e086
--- /dev/null
+++ b/tools/catbox-tradefed/res/config/catbox-functional-driving-optimized-apps.xml
@@ -0,0 +1,33 @@
+<!--
+ Copyright (C) 2023 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Complete Automotive Tests - Ux Restriction Functional Tests.">
+ <!-- Common Base -->
+ <include name="catbox-common"/>
+
+ <!-- Device Preparers -->
+ <include name="catbox-preparer"/>
+
+ <!-- Plan -->
+ <option name="plan" value="catbox-functional-ux-restriction"/>
+
+ <!-- Test Args -->
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" />
+
+ <!-- Tests -->
+ <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests android.platform.tests.UxRestrictionDOAppsTest" />
+</configuration> \ No newline at end of file
diff --git a/tools/catbox-tradefed/res/config/catbox-functional-microphone-recent-apps.xml b/tools/catbox-tradefed/res/config/catbox-functional-microphone-recent-apps.xml
new file mode 100644
index 0000000..188ebed
--- /dev/null
+++ b/tools/catbox-tradefed/res/config/catbox-functional-microphone-recent-apps.xml
@@ -0,0 +1,34 @@
+<!--
+ Copyright (C) 2023 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Complete Automotive Tests - Microphone Setting Functional Tests.">
+ <!-- Common Base -->
+ <include name="catbox-common"/>
+
+ <!-- Device Preparers -->
+ <include name="catbox-preparer"/>
+
+ <!-- Plan -->
+ <option name="plan" value="catbox-functional-microphone-recent-apps"/>
+
+ <!-- Test Args -->
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" />
+
+ <!-- Tests -->
+ <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.MicroPhoneRecentAppsTest" />
+</configuration>
+
diff --git a/tools/catbox-tradefed/res/config/catbox-functional-microphone-setting.xml b/tools/catbox-tradefed/res/config/catbox-functional-microphone-setting.xml
new file mode 100644
index 0000000..d837487
--- /dev/null
+++ b/tools/catbox-tradefed/res/config/catbox-functional-microphone-setting.xml
@@ -0,0 +1,34 @@
+<!--
+ Copyright (C) 2023 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Complete Automotive Tests - Microphone Setting Functional Tests.">
+ <!-- Common Base -->
+ <include name="catbox-common"/>
+
+ <!-- Device Preparers -->
+ <include name="catbox-preparer"/>
+
+ <!-- Plan -->
+ <option name="plan" value="catbox-functional-microphone-setting"/>
+
+ <!-- Test Args -->
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" />
+
+ <!-- Tests -->
+ <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.MicroPhoneSettingTest" />
+</configuration>
+
diff --git a/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting.xml b/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting.xml
new file mode 100644
index 0000000..3f2bde1
--- /dev/null
+++ b/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting.xml
@@ -0,0 +1,34 @@
+<!--
+ Copyright (C) 2023 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Complete Automotive Tests - Microphone Setting Functional Tests.">
+ <!-- Common Base -->
+ <include name="catbox-common"/>
+
+ <!-- Device Preparers -->
+ <include name="catbox-preparer"/>
+
+ <!-- Plan -->
+ <option name="plan" value="catbox-functional-privacy-setting"/>
+
+ <!-- Test Args -->
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" />
+
+ <!-- Tests -->
+ <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.PrivacySettingTest" />
+</configuration>
+
diff --git a/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction-facet-bar.xml b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction-facet-bar.xml
new file mode 100644
index 0000000..06ab2c7
--- /dev/null
+++ b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction-facet-bar.xml
@@ -0,0 +1,33 @@
+<!--
+ Copyright (C) 2021 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration description="Complete Automotive Tests - Ux Restriction Functional Tests.">
+ <!-- Common Base -->
+ <include name="catbox-common"/>
+
+ <!-- Device Preparers -->
+ <include name="catbox-preparer"/>
+
+ <!-- Plan -->
+ <option name="plan" value="catbox-functional-ux-restriction-facet-bar"/>
+
+ <!-- Test Args -->
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" />
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" />
+
+ <!-- Tests -->
+ <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests android.platform.tests.UxRestrictionFacetBarTest" />
+</configuration>
diff --git a/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml
index 5bc518b..0141ecb 100644
--- a/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml
+++ b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml
@@ -29,5 +29,5 @@
<option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" />
<!-- Tests -->
- <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests" />
-</configuration> \ No newline at end of file
+ <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests android.platform.tests.UxRestrictionTest" />
+</configuration>
diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml
index 9dd7470..e4deff5 100644
--- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml
+++ b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml
@@ -19,7 +19,6 @@
<!-- Clear pre-created guests -->
<option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" />
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_guests 0" />
<option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" />
<option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" />
<option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" />
diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml
index 97454a0..c066a6c 100644
--- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml
+++ b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml
@@ -19,7 +19,6 @@
<!-- Clear pre-created users -->
<option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" />
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_users 0" />
<option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" />
<option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" />
<option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" />
diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-guest.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-guest.xml
deleted file mode 100644
index bfc140a..0000000
--- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-guest.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Copyright (C) 2021 Google Inc.
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration description="CATBox Performance Test to measure latency to switch to a Pre-created Guest">
- <include name="catbox-performance-multiuser-base" />
-
- <!-- Set pre-created guests -->
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" />
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_guests 5" />
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" />
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" />
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" />
-
- <!--Test -->
- <option name="compatibility:include-filter" value="AndroidAutomotiveMultiuserScenarioTests android.platform.scenario.multiuser.SwitchToNewGuestBenchmark" />
- <option name="compatibility:module-arg" value="AndroidAutomotiveMultiuserScenarioTests:instrumentation-arg:include_creation_time:=true" />
-
- <!-- Test Metrics Report -->
- <option name="report-log-name" value="CatboxPerformanceTests" />
- <option name="report-test-name-mapping" key="android.platform.scenario.multiuser.SwitchToNewGuestBenchmark#testSwitch" value="switch_to_pre_created_guest" />
- <option name="report-all-metrics" value="false" />
- <option name="report-metric-key-mapping" key="duration_ms_android.platform.scenario.multiuser.SwitchToNewGuestBenchmark#testSwitch-median" value="duration_ms" />
-
- <option name="plan" value="catbox-performance-create-and-switch-to-precreated-guest" />
-</configuration> \ No newline at end of file
diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-user.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-user.xml
deleted file mode 100644
index c0acabb..0000000
--- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-user.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Copyright (C) 2018 Google Inc.
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration description="CATBox Performance Test to measure latency to switch to a Pre-created User">
- <include name="catbox-performance-multiuser-base" />
-
- <!-- Clear pre-created guests -->
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" />
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_users 5" />
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" />
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" />
- <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" />
-
- <!--Test -->
- <option name="compatibility:include-filter" value="AndroidAutomotiveMultiuserScenarioTests android.platform.scenario.multiuser.SwitchToNewSecondaryUserBenchmark" />
- <option name="compatibility:module-arg" value="AndroidAutomotiveMultiuserScenarioTests:instrumentation-arg:include_creation_time:=true" />
-
- <!-- Test Metrics Report -->
- <option name="report-log-name" value="CatboxPerformanceTests" />
- <option name="report-test-name-mapping" key="android.platform.scenario.multiuser.SwitchToNewSecondaryUserBenchmark#testSwitch" value="switch_to_pre_created_user" />
- <option name="report-all-metrics" value="false" />
- <option name="report-metric-key-mapping" key="duration_ms_android.platform.scenario.multiuser.SwitchToNewSecondaryUserBenchmark#testSwitch-median" value="duration_ms" />
-
- <option name="plan" value="catbox-performance-create-and-switch-to-precreated-user" />
-</configuration>
diff --git a/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml b/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml
index 0c14d83..be24379 100644
--- a/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml
+++ b/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml
@@ -37,14 +37,14 @@
<option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.test.scenario.appgrid" />
<!-- Test -->
- <option name="compatibility:include-filter" value="AndroidAutomotiveAppGridScenarioTests android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollDownAndUp" />
+ <option name="compatibility:include-filter" value="AndroidAutomotiveAppGridScenarioTests android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollForwardAndBackward" />
<!-- Jank Package -->
<option name="compatibility:module-arg" value="AndroidAutomotiveAppGridScenarioTests:instrumentation-arg:jank-package-names:=com.android.car.carlauncher" />
<!-- Test Metrics Report Options -->
<option name="report-log-name" value="CatboxPerformanceTests" />
- <option name="report-test-name-mapping" key="android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollDownAndUp" value="appgrid_scroll_jank" />
+ <option name="report-test-name-mapping" key="android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollForwardAndBackward" value="appgrid_scroll_jank" />
<option name="report-all-metrics" value="false" />
<option name="report-metric-key-mapping" key="gfxinfo_com.android.car.carlauncher_janky_frames_percent-mean" value="jank_frames_percent" />
</configuration>
diff --git a/tools/catbox-tradefed/res/config/catbox-performance-jank-media-switch-playback.xml b/tools/catbox-tradefed/res/config/catbox-performance-jank-media-switch-playback.xml
new file mode 100644
index 0000000..f0fea15
--- /dev/null
+++ b/tools/catbox-tradefed/res/config/catbox-performance-jank-media-switch-playback.xml
@@ -0,0 +1,52 @@
+<!--
+ ~ Copyright (C) 2023 The Android Open Source Project
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration description="CATBox Jank Performance Test For Media Switch Playback.">
+ <!-- Plan -->
+ <option name="plan" value="catbox-performance-jank-media-switch-playback" />
+
+ <!-- Base File -->
+ <include name="catbox-performance-jank-base" />
+
+ <!-- Proc Load Options -->
+ <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:per_run:=true" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-interval:=20000" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-threshold:=2.0" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-timeout:=900000" />
+
+ <!-- Jank Options -->
+ <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:listener:=android.device.collectors.ProcLoadListener,android.device.collectors.JankListener" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:iterations:=20" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:favor-shell-commands:=true" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:rename-iterations:=true" />
+ <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:iteration-separator:=$" />
+
+ <!-- Test Package -->
+ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.test.scenario.mediacenter" />
+
+ <!-- Test -->
+ <option name="compatibility:include-filter" value="AndroidAutomotiveMediaCenterScenarioTests android.platform.test.scenario.mediacenter.SwitchPlaybackMicrobenchmark#testSwitchPlayback" />
+
+ <!-- Jank Package -->
+ <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:jank-package-names:=com.android.car.media" />
+
+ <!-- Test Metrics Report Options -->
+ <option name="report-log-name" value="CatboxPerformanceTests" />
+ <option name="report-test-name-mapping" key="android.platform.test.scenario.mediacenter.SwitchPlaybackMicrobenchmark#testSwitchPlayback" value="media_switchplayback_jank" />
+ <option name="report-all-metrics" value="false" />
+ <!-- TODO: b/247588415 Add metrics to APFE -->
+ <option name="report-metric-key-mapping" key="gfxinfo_com.android.car.media_janky_frames_percent-mean" value="jank_frames_percent" />
+</configuration>
diff --git a/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml b/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml
new file mode 100644
index 0000000..863b7e1
--- /dev/null
+++ b/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml
@@ -0,0 +1,66 @@
+<!--
+ ~ Copyright (C) 2022 The Android Open Source Project
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <!-- Extend existing boottime config-->
+ <include name="google/test/boottime-successive"/>
+
+ <include name="catbox-performance-base" />
+
+ <!-- Artificially disabling tests in CompatibilityTestSuite -->
+ <!-- See com.android.tradefed.testtype.suite.CompatibilityTestSuite:loadTests() -->
+ <option name="compatibility:reverse-exclude-filters" value="true" />
+
+ <option name="boot-delay" value="20000"/>
+ <option name="boot-count" value="20"/>
+ <option name="dmesg-info" value="true"/>
+ <option name="granular-boot-info" value="true"/>
+ <option name="online-timeout" value="120000"/>
+ <option name="successive-boot" value="true"/>
+
+ <!-- Prepare commands -->
+ <option name="successive-boot-prepare-cmd"
+ value="am start -n com.google.android.car.setupwizard/.ExitActivity"/>
+ <option name="successive-boot-prepare-cmd"
+ value="setprop persist.aae.suw.disable_short_notice true"/>
+ <option name="successive-boot-prepare-cmd"
+ value="cmd car_service enable-feature car_evs_service"/>
+
+ <!-- For custom boot time metrics -->
+ <option name="boot-time-pattern" key="boot_to_ble_on"
+ value="BluetoothAdapterService: updateAdapterState\(\) \- Broadcasting state BLE_ON"/>
+ <option name="boot-time-pattern" key="boot_to_carlauncher_fully_drawn"
+ value="Fully drawn com.android.car.carlauncher/.CarLauncher"/>
+ <option name="boot-time-pattern" key="boot_to_first_user_unlocked"
+ value="CarServiceHelper: Time to unlock 1st user"/>
+ <option name="boot-time-pattern" key="boot_to_maps_displayed"
+ value="Displayed com.google.android.apps.maps"/>
+ <option name="boot-time-pattern" key="boot_to_settings_fallbackhome_displayed"
+ value="Displayed com.android.car.settings/.FallbackHome"/>
+ <option name="boot-time-pattern" key="boot_to_user0_unlocked"
+ value="SystemServerTiming: SystemUserUnlock took to complete"/>
+ <option name="boot-time-pattern" key="boot_to_user10_unlocked"
+ value="SystemServerTiming: User10Unlock took to complete"/>
+ <option name="boot-time-pattern" key="boot_to_gmscore_started_experimental"
+ value="ActivityManager: Start proc \d+:com.google.android.gms/u0"/>
+ <option name="boot-time-pattern" key="boot_to_car_evs_service_started"
+ value="CAR.InitTiming: CarEvsService took to complete"/>
+
+ <option name="report-log-name" value="CatboxPerformanceTests" />
+ <option name="report-test-name-mapping" key="BootTimeTest.BootTimeTest#SuccessiveBootTest" value="boot_time_test" />
+ <option name="report-all-metrics" value="false" />
+ <option name="report-metric-key-mapping" key="dmesg_action_sys.boot_completed_first_timestamp-median" value="successive_boot_time" />
+</configuration>