diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2024-05-10 15:49:51 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2024-05-10 15:49:51 +0000 |
commit | 349ca5b665d0d61df459e31f7ef4b45ba33cd5ba (patch) | |
tree | a9cca53b197b68421e0ff9ee9b10faa3a2453233 | |
parent | c595456b40b3176de8aadde8659a3a4713fea0ac (diff) | |
parent | 3b583f239744e09af52f6f294e5643d48d4b0710 (diff) | |
download | catbox-busytown-mac-infra-release.tar.gz |
Snap for 11819167 from 3b583f239744e09af52f6f294e5643d48d4b0710 to busytown-mac-infra-releasebusytown-mac-infra-release
Change-Id: Iaa04d10250640b43e8c62e51010859c27b24ed7d
43 files changed, 1567 insertions, 116 deletions
diff --git a/report/src/com/android/catbox/result/JsonResultReporter.java b/report/src/com/android/catbox/result/JsonResultReporter.java index 6e9b9c3..9363f9d 100644 --- a/report/src/com/android/catbox/result/JsonResultReporter.java +++ b/report/src/com/android/catbox/result/JsonResultReporter.java @@ -178,6 +178,12 @@ public class JsonResultReporter implements ITestInvocationListener { mTestMetricsUtil.setIterationSeparator(mTestIterationSeparator); } + /** Re-initialize object to erase all existing test metrics */ + private void reInitializeTestMetricsUtil() { + mTestMetricsUtil = initializeTestMetricsUtil(); + mTestMetricsUtil.setIterationSeparator(mTestIterationSeparator); + } + /** Write Test Metrics to JSON */ private void writeTestMetrics( String classMethodName, Map<String, String> metrics) { @@ -287,9 +293,9 @@ public class JsonResultReporter implements ITestInvocationListener { String.format("%s is not a directory", hostReportDir.getAbsolutePath())); return; } - // Copy the report logs from temp directory and to the results directory + // Copy/Merge the report logs from temp directory and to the results directory + CollectorUtil.reformatRepeatedStreams(hostReportDir); CollectorUtil.pullFromHost(hostReportDir, resultDir); - CollectorUtil.reformatRepeatedStreams(resultDir); CLog.logAndDisplay(LogLevel.INFO, "Copying the report log completed successfully."); } catch (IOException exception) { CLog.logAndDisplay(LogLevel.ERROR, exception.getMessage()); @@ -332,6 +338,9 @@ public class JsonResultReporter implements ITestInvocationListener { writeTestMetrics(testName, aggregatedMetrics.get(testName)); } } + + // Avoid reporting duplicate metrics by erasing metrics from previous runs + reInitializeTestMetricsUtil(); } /** {@inheritDoc} */ diff --git a/runner/Android.bp b/runner/Android.bp new file mode 100644 index 0000000..026d429 --- /dev/null +++ b/runner/Android.bp @@ -0,0 +1,27 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +java_library_host { + name: "catbox-test-runner-lib", + + srcs: ["src/**/*.java"], + + libs: [ + "tradefed", + ], +} diff --git a/runner/src/com/android/catbox/runner/MopedRunner.java b/runner/src/com/android/catbox/runner/MopedRunner.java new file mode 100644 index 0000000..90d84bd --- /dev/null +++ b/runner/src/com/android/catbox/runner/MopedRunner.java @@ -0,0 +1,168 @@ + +// Copyright 2023 Google Inc. All Rights Reserved. +package com.google.android.car.aaosbt; +import com.android.ddmlib.Log.LogLevel; +import com.android.tradefed.config.Option; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.invoker.TestInformation; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.targetprep.TargetSetupError; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.io.IOException; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.TimeUnit; +import com.android.tradefed.result.ITestInvocationListener; +import com.android.tradefed.testtype.IRemoteTest; +import java.net.URISyntaxException; +import com.android.tradefed.device.ITestDevice; +import java.util.Map; +import java.util.HashMap; +import java.util.Collections; + +@OptionClass(alias = "aaos-moped-test") +public class MopedRunner implements IRemoteTest { + @Option(name = "test-artifact", description = "test artifact") + private File test_artifact = null; + + @Option(name = "artifact", description = "test artifact") + private String artifact_str = null; + + @Option(name = "unzip-build-timeout-min", description = "unzip build timeout in minutes") + private int unzip_build_timeout_min = 10; + + @Option(name = "test-timeout-min", description = "test timeout in minutes") + private int test_timeout_min = 60; + + @Option(name = "testcase", description = "which moped test binary to run") + private String test_case = null; + + private File mLocalDestFile; + private File mLocalSrcFile; + private String mArtifactLocation; + + private static final Map<String, String> autoDic = initDeviceMap(); + + private void unTarTestArtifact(TestInformation testInfo) throws TargetSetupError, TimeoutException, URISyntaxException { + if (test_artifact != null && artifact_str == null) { + artifact_str = test_artifact.toPath().toString(); + } + String jarPath = getClass() + .getProtectionDomain() + .getCodeSource() + .getLocation() + .toURI() + .getPath(); + File jarFile = new File(jarPath); + mLocalSrcFile = new File(jarFile.getParent() + "/../testcases/" + artifact_str); + mLocalDestFile = new File(testInfo.dependenciesFolder().toString()); + mArtifactLocation = + mLocalDestFile.getPath() + + "/" + + mLocalSrcFile.getName().replaceAll(".tar.*gz", "/"); + Path localArtifactPath = Paths.get(mArtifactLocation); + if (!Files.exists(localArtifactPath)) { + // untar file + executeHostCommand( + new String[] { + "bash", + "-c", + "tar xf " + mLocalSrcFile.getPath() + " -C " + mLocalDestFile.getPath() + }, + unzip_build_timeout_min); + } + } + + private static Map<String, String> initDeviceMap() { + Map<String, String> map = new HashMap<>(); + map.put("seahawk", "AUTO"); + map.put("seahawk_hwasan", "AUTO"); + map.put("cf_x86_auto", "AUTO"); + return Collections.unmodifiableMap(map); + } + + private String checkDevice(ITestDevice device) throws DeviceNotAvailableException { + String buildFlavor = device.getBuildFlavor().split("-")[0]; + return autoDic.get(buildFlavor); + } + + private String getDevicesString(TestInformation testInfo) throws DeviceNotAvailableException { + StringBuilder deviceString = new StringBuilder(); + int deviceNum = 0; + int companionDeviceNum = 0; + for(ITestDevice device : testInfo.getDevices()) { + String deviceType = checkDevice(device); + if (deviceType.equals("AUTO")) { + deviceString.append(String.format(" --hu %s", device.getSerialNumber())); + } else { + companionDeviceNum++; + deviceString.append(String.format(" --phone%s %s", String.valueOf(companionDeviceNum), device.getSerialNumber())); + } + deviceNum++; + } + deviceString.append(String.format(" --devicenum %s", String.valueOf(deviceNum))); + return deviceString.toString(); + } + + @Override + public void run(TestInformation testInfo, ITestInvocationListener listener) + throws DeviceNotAvailableException { + // Download Moped binanry / config and run test + try { + unTarTestArtifact(testInfo); + executeHostCommand( + new String[] {"bash", "-c", "bash " + + String.format("%s/run.sh %s --testcase %s", mArtifactLocation, + getDevicesString(testInfo), + test_case)}, + test_timeout_min); + } catch (TargetSetupError e) { + CLog.logAndDisplay(LogLevel.VERBOSE, "There are problems running tests! %s", e); + } catch (TimeoutException e) { + CLog.logAndDisplay(LogLevel.VERBOSE, "Test execution timeout! %s", e); + } catch (URISyntaxException e) { + CLog.logAndDisplay(LogLevel.VERBOSE, "Test artifact not found! %s", e); + } + } + + private ArrayList<String> executeHostCommand(String[] command, int timeout) + throws TargetSetupError, TimeoutException { + ArrayList<String> ret = new ArrayList<String>(); + try { + CLog.logAndDisplay( + LogLevel.VERBOSE, "Output of running %s is:", Arrays.toString(command)); + Process p = Runtime.getRuntime().exec(command); + if (!p.waitFor(timeout, TimeUnit.MINUTES)) { + p.destroy(); + throw new TimeoutException(); + } + InputStream is = p.getInputStream(); + InputStreamReader isr = new InputStreamReader(is); + BufferedReader br = new BufferedReader(isr); + String line; + while ((line = br.readLine()) != null) { + CLog.logAndDisplay(LogLevel.VERBOSE, line); + ret.add(line); + } + int exitCode = p.waitFor(); + if (exitCode != 0) { + throw new TargetSetupError( + "Execution of command " + Arrays.toString(command) + " failed!"); + } + } catch (IOException e) { + CLog.logAndDisplay(LogLevel.VERBOSE, "There are problems with IO! %s", e); + } catch (InterruptedException e) { + CLog.logAndDisplay(LogLevel.VERBOSE, "User interruption!"); + } + return ret; + } +} + diff --git a/target_preparers/src/com/android/catbox/targetpreparer/ChromeMdPassengerLoadPreparer.java b/target_preparers/src/com/android/catbox/targetpreparer/ChromeMdPassengerLoadPreparer.java new file mode 100644 index 0000000..9fc4e88 --- /dev/null +++ b/target_preparers/src/com/android/catbox/targetpreparer/ChromeMdPassengerLoadPreparer.java @@ -0,0 +1,177 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.catbox.targetpreparer; + +import com.android.tradefed.build.IBuildInfo; +import com.android.tradefed.config.Option; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.device.CollectingOutputReceiver; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; +import com.android.tradefed.invoker.TestInformation; +import com.android.tradefed.log.LogUtil; +import com.android.tradefed.result.error.DeviceErrorIdentifier; +import com.android.tradefed.targetprep.BaseTargetPreparer; +import com.android.tradefed.targetprep.BuildError; +import com.android.tradefed.targetprep.TargetSetupError; +import com.android.tradefed.targetprep.TestAppInstallSetup; +import com.android.tradefed.util.CommandResult; +import com.android.tradefed.util.CommandStatus; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@OptionClass(alias = "chrome-md-passenger-load") +public class ChromeMdPassengerLoadPreparer extends BaseTargetPreparer { + private static final String INSTR_SUCCESS = "OK (1 test)"; + + @Option(name = "skip-display-id", description = "Display id to skip passenger load for") + private List<Integer> mSkipDisplayIds = new ArrayList<>(); + + @Option(name = "url", description = "Url to open in Chrome browser", mandatory = true) + private String mUrl; + + @Option(name = "package", description = "Chrome package") + private String mPackage = "com.android.chrome"; + + @Option(name = "activity", description = "Chrome activity") + private String mActivity = "com.google.android.apps.chrome.Main"; + + @Option( + name = "test-app-file-name", + description = + "the name of an apk file to be installed in the user profiles.") + private List<String> mTestFiles = new ArrayList<>(); + + Map<Integer, Integer> mDisplayToCreatedUsers = new HashMap<>(); + private final ArrayList<TestAppInstallSetup> mInstallPreparers = + new ArrayList<TestAppInstallSetup>(); + + @Override + public void setUp(TestInformation testInfo) throws TargetSetupError, BuildError, + DeviceNotAvailableException { + ITestDevice device = testInfo.getDevice(); + Set<Integer> displayIds = device.listDisplayIdsForStartingVisibleBackgroundUsers(); + for (Integer displayId : displayIds) { + if (mSkipDisplayIds.contains(displayId)) { + LogUtil.CLog.d("Skipping load on display %d", displayId); + continue; + } + int userId = createUser(device, displayId); + mDisplayToCreatedUsers.put(displayId, userId); + } + + if (mDisplayToCreatedUsers.size() == 0) { + LogUtil.CLog.w("Won't create any passenger load. No display ids matched."); + throw new TargetSetupError( + String.format("Available displays on the device %s. Skipped displays %s", + displayIds, mSkipDisplayIds), + device.getDeviceDescriptor()); + } + + installApks(testInfo); + + for (Integer displayId : mDisplayToCreatedUsers.keySet()) { + int userId = mDisplayToCreatedUsers.get(displayId); + dismissInitialDialog(device, userId); + simulatePassengerLoad(device, userId); + } + } + + @Override + public void tearDown(TestInformation testInfo, Throwable e) + throws DeviceNotAvailableException { + ITestDevice device = testInfo.getDevice(); + for (TestAppInstallSetup installPreparer : mInstallPreparers) { + installPreparer.tearDown(testInfo, e); + } + for (int userId : mDisplayToCreatedUsers.values()) { + device.removeUser(userId); + } + } + + private int createUser(ITestDevice device, int displayId) + throws TargetSetupError, + DeviceNotAvailableException { + int userId = device.createUser(String.format("user-display-%d", displayId)); + LogUtil.CLog.d( + String.format("Created user with id %d for display %d", userId, displayId)); + if (!device.startVisibleBackgroundUser(userId, displayId, true)) { + throw new TargetSetupError( + String.format("Device failed to switch to user %d", userId), + device.getDeviceDescriptor()); + } + LogUtil.CLog.d( + String.format("Started background user %d for display %d", userId, displayId)); + return userId; + } + + private void installApks(TestInformation testInfo) + throws TargetSetupError, BuildError, DeviceNotAvailableException { + for (int userId : mDisplayToCreatedUsers.values()) { + TestAppInstallSetup installPreparer = new TestAppInstallSetup(); + LogUtil.CLog.d( + String.format("Installing the following test APKs in user %d: \n%s", userId, + mTestFiles)); + installPreparer.setUserId(userId); + installPreparer.setShouldGrantPermission(true); + for (String file : mTestFiles) { + installPreparer.addTestFileName(file); + } + installPreparer.addInstallArg("-r"); + installPreparer.addInstallArg("-d"); + installPreparer.setUp(testInfo); + mInstallPreparers.add(installPreparer); + } + } + + private void simulatePassengerLoad(ITestDevice device, int userId) + throws TargetSetupError, DeviceNotAvailableException { + LogUtil.CLog.d(String.format("Launching Chrome for User %d with url %s", userId, mUrl)); + String launchChromeActivityWithUrlCommand = String.format( + "am start -n %s/%s --user %d -a android.intent.action.VIEW -d %s", mPackage, + mActivity, userId, mUrl); + CommandResult result = device.executeShellV2Command(launchChromeActivityWithUrlCommand); + if (!CommandStatus.SUCCESS.equals(result.getStatus())) { + throw new TargetSetupError( + String.format("Chrome activity failed to launch for user %d", userId), + device.getDeviceDescriptor()); + } + } + + private void dismissInitialDialog(ITestDevice device, int userId) + throws DeviceNotAvailableException, TargetSetupError { + OutputStream output = new ByteArrayOutputStream(); + String dismissCommand = String.format( + "am instrument -w --user %d -e class android.platform.tests" + + ".ChromeDismissDialogsTest android.platform.tests/androidx.test.runner" + + ".AndroidJUnitRunner", + userId); + device.executeShellV2Command(dismissCommand, output); + if (!output.toString().contains(INSTR_SUCCESS)) { + throw new TargetSetupError( + String.format("Failed dismissal.\nCommand output: %s", output), + device.getDeviceDescriptor(), + DeviceErrorIdentifier.DEVICE_UNEXPECTED_RESPONSE); + } + } +} diff --git a/target_preparers/src/com/android/catbox/targetpreparer/SkipTestPreparer.java b/target_preparers/src/com/android/catbox/targetpreparer/SkipTestPreparer.java new file mode 100644 index 0000000..efe97e0 --- /dev/null +++ b/target_preparers/src/com/android/catbox/targetpreparer/SkipTestPreparer.java @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.catbox.targetpreparer; + +import com.android.ddmlib.Log.LogLevel; + +import com.android.tradefed.build.IBuildInfo; +import com.android.tradefed.config.Option; +import com.android.tradefed.config.OptionClass; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.device.ITestDevice; + +import com.android.tradefed.log.LogUtil.CLog; + +import com.android.tradefed.targetprep.BaseTargetPreparer; +import com.android.tradefed.targetprep.BuildError; +import com.android.tradefed.targetprep.TargetSetupError; + +import com.google.common.collect.ImmutableSet; + +import java.util.Set; + +/** + * SkipTestPreparer is an {@link ITargetPreparer} that skips tests based on values of + * certain ADB properties on the device + */ +@OptionClass(alias = "skip-test-preparer") +public class SkipTestPreparer extends BaseTargetPreparer { + + @Option(name = "comp-property", description = "ADB property of device to check against") + private String mCompProp; + + @Option( + name = "comp-property-int-value", + description = "Integer value of ADB property to check against") + private int mPropVal; + + @Option( + name = "int-comparison-operator", + description = "Operator to compare expected and actual int values") + private String mCompOperator; + + private static final Set<String> supportedOperators = ImmutableSet.of("lt", "gt", "eq", "neq"); + + @Override + public void setUp(ITestDevice device, IBuildInfo buildInfo) throws TargetSetupError, + BuildError, DeviceNotAvailableException { + + // Skip this preparer if @Option values are not provided + if (mCompProp == null || mCompOperator == null) { + CLog.logAndDisplay(LogLevel.INFO, + "Missing value for comp-property or comp-property-int-value. Skipping preparer."); + return; + } + + boolean skipTestFlag = false; + + if (!supportedOperators.contains(mCompOperator)) { + CLog.logAndDisplay(LogLevel.WARN, + String.format("Incompatible operator %s. Skipping preparer", mCompOperator)); + CLog.logAndDisplay(LogLevel.INFO, + String.format("Supported operators are %s", String.join(",", supportedOperators))); + return; + } + + int devicePropertyValue = + Integer.parseInt( + device.executeShellCommand(String.format("getprop %s", mCompProp)).trim()); + CLog.logAndDisplay(LogLevel.INFO, + String.format("%s returned %d", mCompProp, devicePropertyValue) + ); + + skipTestFlag = getSkipTestFlag(devicePropertyValue); + + if (skipTestFlag) { + CLog.logAndDisplay(LogLevel.INFO, "Skip condition satisfied. Skipping test module."); + throw new TargetSetupError( + String.format("Test incompatible with %s = %d", mCompProp, devicePropertyValue), + device.getDeviceDescriptor()); + } else { + CLog.logAndDisplay(LogLevel.INFO, + "Skip condition not satisfied. Proceeding with test module."); + } + } + + private boolean getSkipTestFlag(int devicePropertyValue) { + switch (mCompOperator) { + case "lt": + CLog.logAndDisplay(LogLevel.INFO, + "Checking skip condition %d < %d", devicePropertyValue, mPropVal); + return devicePropertyValue < mPropVal; + + case "gt": + CLog.logAndDisplay(LogLevel.INFO, + "Checking skip condition %d > %d", devicePropertyValue, mPropVal); + return devicePropertyValue > mPropVal; + + case "eq": + CLog.logAndDisplay(LogLevel.INFO, + "Checking skip condition %d == %d", devicePropertyValue, mPropVal); + return devicePropertyValue == mPropVal; + + case "neq": + CLog.logAndDisplay(LogLevel.INFO, + "Checking condition %d != %d", devicePropertyValue, mPropVal); + return devicePropertyValue != mPropVal; + + default: + return false; + } + } +} diff --git a/tools/catbox-common/res/config/catbox-performance-base.xml b/tools/catbox-common/res/config/catbox-performance-base.xml index 6109552..1b1bec5 100644 --- a/tools/catbox-common/res/config/catbox-performance-base.xml +++ b/tools/catbox-common/res/config/catbox-performance-base.xml @@ -14,29 +14,37 @@ limitations under the License. --> <configuration description="Automotive Performance Test Base."> - <!-- Common Target Preparers --> - <include name="catbox-preparer" /> - - <!-- Result Reporter --> - <result_reporter class="com.android.catbox.result.JsonResultReporter"> - <option name="dest-dir" value="report-log-files/"/> - <option name="test-iteration-separator" value="$"/> - <option name="aggregate-similar-tests" value="true"/> - </result_reporter> - - <!-- Common Config --> - <include name="catbox-common" /> - - <!-- Test Tag Suffix --> - <option name="test-tag-suffix" value="performance" /> - - <!-- Performance Specific Target Preparers --> - <target_preparer class="com.android.tradefed.targetprep.PerfettoPreparer" /> - <target_preparer class="com.android.tradefed.targetprep.TimeWaster" /> - <target_preparer class="com.android.compatibility.common.tradefed.targetprep.ReportLogCollector"> - <option name="src-dir" value="/sdcard/report-log-files/"/> - <option name="dest-dir" value="report-log-files/"/> - <option name="temp-dir" value="temp-report-logs/"/> - </target_preparer> + <!-- Common Target Preparers --> + <include name="catbox-preparer"/> + + <!-- Result Reporter --> + <result_reporter class="com.android.catbox.result.JsonResultReporter"> + <option name="dest-dir" value="report-log-files/"/> + <option name="test-iteration-separator" value="$"/> + <option name="aggregate-similar-tests" value="true"/> + </result_reporter> + + <!-- Common Config --> + <include name="catbox-common"/> + + <!-- Test Tag Suffix --> + <option name="test-tag-suffix" value="performance"/> + + <!-- Performance Specific Target Preparers --> + <target_preparer class="com.android.tradefed.targetprep.PerfettoPreparer"/> + <target_preparer class="com.android.tradefed.targetprep.TimeWaster"/> + <target_preparer + class="com.android.compatibility.common.tradefed.targetprep.ReportLogCollector"> + <option name="src-dir" value="/sdcard/report-log-files/"/> + <option name="dest-dir" value="report-log-files/"/> + <option name="temp-dir" value="temp-report-logs/"/> + </target_preparer> + <target_preparer class="com.android.catbox.targetpreparer.ChromeMdPassengerLoadPreparer"> + <option name="disable" value="true"/> + <option name="url" value="https://youtu.be/dQw4w9WgXcQ"/> + </target_preparer> + + <!-- Enable perfetto host side metric collector --> + <include name="perfetto-puller"/> </configuration> diff --git a/tools/catbox-common/res/config/catbox-performance-multiuser-base.xml b/tools/catbox-common/res/config/catbox-performance-multiuser-base.xml index e9fea7b..2929ac9 100644 --- a/tools/catbox-common/res/config/catbox-performance-multiuser-base.xml +++ b/tools/catbox-common/res/config/catbox-performance-multiuser-base.xml @@ -23,11 +23,6 @@ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:shell-timeout:1000000" /> - <!-- Switch to User 0 and wait for a some time (milliseconds) until system idle --> - <target_preparer class="com.android.tradefed.targetprep.SwitchUserTargetPreparer" > - <option name="user-type" value="SYSTEM" /> - </target_preparer> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.TimeWaster:delay:20000" /> <option name="compatibility:module-arg" value="AndroidAutomotiveMultiuserScenarioTests:instrumentation-arg:iterations:=5" /> @@ -36,4 +31,4 @@ <!-- Default metrics post processor --> <include name="catbox-performance-postprocessors" /> -</configuration>
\ No newline at end of file +</configuration> diff --git a/tools/catbox-common/res/config/catbox-preparer.xml b/tools/catbox-common/res/config/catbox-preparer.xml index db9aaf7..21109c5 100644 --- a/tools/catbox-common/res/config/catbox-preparer.xml +++ b/tools/catbox-common/res/config/catbox-preparer.xml @@ -68,7 +68,9 @@ <target_preparer class="com.android.compatibility.common.tradefed.targetprep.FilePusher" /> <!-- Target Preparers - Run Shell Commands --> - <target_preparer class="com.android.tradefed.targetprep.RunCommandTargetPreparer" /> + <target_preparer class="com.android.tradefed.targetprep.RunCommandTargetPreparer"> + <option name="run-command" value="am start -n com.google.android.car.setupwizard/.ExitActivity" /> + </target_preparer> <!-- Target Preparers - Set Properties --> <target_preparer class="com.android.catbox.targetpreparer.SetDevicePropertyPreparer" /> diff --git a/tools/catbox-common/res/config/perfetto-puller.xml b/tools/catbox-common/res/config/perfetto-puller.xml new file mode 100644 index 0000000..a29cf72 --- /dev/null +++ b/tools/catbox-common/res/config/perfetto-puller.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + ~ Copyright (C) 2023 The Android Open Source Project + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<configuration description="Pull the files from the device and add it to the test logs."> + <!-- Pulls the perfetto files from the device and collects the metrics from the trace files --> + <metrics_collector class="com.android.tradefed.device.metric.PerfettoPullerMetricCollector" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/Android.bp b/tools/catbox-tradefed/Android.bp index 493e19c..c94cc9f 100644 --- a/tools/catbox-tradefed/Android.bp +++ b/tools/catbox-tradefed/Android.bp @@ -39,7 +39,8 @@ tradefed_binary_host { static_libs: [ "cts-tradefed-harness", "catbox-common", - "catbox-preparer-lib" + "catbox-preparer-lib", + "catbox-test-runner-lib" ], required: [ "compatibility-host-util", diff --git a/tools/catbox-tradefed/res/config/catbox-functional-admin-user-grant-permissions.xml b/tools/catbox-tradefed/res/config/catbox-functional-admin-user-grant-permissions.xml new file mode 100644 index 0000000..ea1afcd --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-admin-user-grant-permissions.xml @@ -0,0 +1,34 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - AdminUserGrantPermissions Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value=" functional_tests.catbox_functional_admin_user_grant_permissions +"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveAdminUserGrantPermissionsTests android.platform.tests.GrantPermissionsToNonAdminUserTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-app-info-setting-ui-elements.xml b/tools/catbox-tradefed/res/config/catbox-functional-app-info-setting-ui-elements.xml new file mode 100644 index 0000000..17ce11d --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-app-info-setting-ui-elements.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - App Info Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-app-info-setting"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.AppInfoVerifyUIElementsTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-bluetooth-palette.xml b/tools/catbox-tradefed/res/config/catbox-functional-bluetooth-palette.xml new file mode 100644 index 0000000..86413bd --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-bluetooth-palette.xml @@ -0,0 +1,31 @@ +<!-- + Copyright (C) 2023 Google Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration + description="Complete Automotive Tests - Status Bar Functional Tests."> + <!-- Common Base --> + <include name="catbox-common" /> + <!-- Device Preparers --> + <include name="catbox-preparer" /> + <!-- Plan --> + <option name="plan" value="catbox-functional-bluetooth-palette" /> + <!-- Test Args --> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + <!-- Tests --> + <option name="compatibility:include-filter" + value="AndroidAutomotiveStatusBarTests android.platform.tests.BluetoothPaletteTest" /> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-functional-bluetooth-tests.xml b/tools/catbox-tradefed/res/config/catbox-functional-bluetooth-tests.xml new file mode 100644 index 0000000..b1dcd97 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-bluetooth-tests.xml @@ -0,0 +1,55 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Bluetooth Tests Using Mobly"> + <!-- Plan --> + <!-- TODO(@vitalidim) rename test plan since it's not for bt discovery only --> + <option name="plan" value="catbox-functional-bluetooth-tests"/> + + <include name="everything" /> + + <!-- Template for Result Reporters --> + <template-include name="reporters" default="empty" /> + + <!-- Test Tag --> + <option name="test-tag" value="catbox" /> + + <!-- Basic Reporters --> + <include name="basic-reporters" /> + + <!-- Template for Metadata Reporters --> + <template-include name="metadata-reporters" default="empty" /> + + <!-- Default ABI --> + <option name="compatibility:primary-abi-only" value="true" /> + + <!-- Template for Device Configuration --> + <template-include name="device-config" default="device-config/two-devices" /> + + <!-- + CATBox Runs all modules if it is not specified. + So, we need to skip these modules in case of Mobly tests + which can be done by adding following option. + --> + <option name="compatibility:reverse-exclude-filters" value="true" /> + + <test class="com.android.tradefed.testtype.mobly.MoblyBinaryHostTest"> + <!-- The mobly-par-file-name should match the module name , it is passed on runtime--> + <!-- Timeout limit in milliseconds for all test cases of the python binary --> + <option name="mobly-test-timeout" value="300000" /> + <!-- Testbed config file --> + <option name="mobly-config-file-name" value="bt_discovery_config.yaml" /> + </test> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-functional-brightness-palette.xml b/tools/catbox-tradefed/res/config/catbox-functional-brightness-palette.xml new file mode 100644 index 0000000..633828a --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-brightness-palette.xml @@ -0,0 +1,31 @@ +<!-- + Copyright (C) 2023 Google Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration + description="Complete Automotive Tests - Status Bar Functional Tests."> + <!-- Common Base --> + <include name="catbox-common" /> + <!-- Device Preparers --> + <include name="catbox-preparer" /> + <!-- Plan --> + <option name="plan" value="catbox-functional-brightness-palette" /> + <!-- Test Args --> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + <!-- Tests --> + <option name="compatibility:include-filter" + value="AndroidAutomotiveNavigationBarTests android.platform.tests.BrightnessPaletteTest" /> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-functional-btmoped.xml b/tools/catbox-tradefed/res/config/catbox-functional-btmoped.xml new file mode 100644 index 0000000..f5113b3 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-btmoped.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Bluetooth Moped based Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-btmoped"/> + + <!-- Artificially disabling tests in CompatibilityTestSuite --> + <!-- See com.android.tradefed.testtype.suite.CompatibilityTestSuite:loadTests() --> + <option name="compatibility:reverse-exclude-filters" value="true" /> + + <test class="com.google.android.car.aaosbt.MopedRunner"> + <option name="test-artifact" value="moped.tar.gz" /> + </test> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-functional-display-setting.xml b/tools/catbox-tradefed/res/config/catbox-functional-display-setting.xml new file mode 100644 index 0000000..aa2bf20 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-display-setting.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2021 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Display Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-display-setting"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.DisplaySettingTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-driving-optimized-apps.xml b/tools/catbox-tradefed/res/config/catbox-functional-driving-optimized-apps.xml new file mode 100644 index 0000000..01b61d7 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-driving-optimized-apps.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Ux Restriction Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-ux-restriction"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests android.platform.tests.DrivingOptimizedAppsTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-enable-developers-option.xml b/tools/catbox-tradefed/res/config/catbox-functional-enable-developers-option.xml new file mode 100644 index 0000000..abd22d7 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-enable-developers-option.xml @@ -0,0 +1,34 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Enable Developers Option Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-enable-developers-option"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.EnableDevelopersOption" /> +</configuration> + diff --git a/tools/catbox-tradefed/res/config/catbox-functional-microphone-recent-apps.xml b/tools/catbox-tradefed/res/config/catbox-functional-microphone-recent-apps.xml new file mode 100644 index 0000000..188ebed --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-microphone-recent-apps.xml @@ -0,0 +1,34 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Microphone Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-microphone-recent-apps"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.MicroPhoneRecentAppsTest" /> +</configuration> + diff --git a/tools/catbox-tradefed/res/config/catbox-functional-microphone-setting.xml b/tools/catbox-tradefed/res/config/catbox-functional-microphone-setting.xml new file mode 100644 index 0000000..d837487 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-microphone-setting.xml @@ -0,0 +1,34 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Microphone Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-microphone-setting"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.MicroPhoneSettingTest" /> +</configuration> + diff --git a/tools/catbox-tradefed/res/config/catbox-functional-network-palette.xml b/tools/catbox-tradefed/res/config/catbox-functional-network-palette.xml new file mode 100644 index 0000000..dd661a8 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-network-palette.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Status Bar Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-network-palette"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveStatusBarTests android.platform.tests.NetworkPaletteTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-privacy-permission-manager.xml b/tools/catbox-tradefed/res/config/catbox-functional-privacy-permission-manager.xml new file mode 100644 index 0000000..1406305 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-privacy-permission-manager.xml @@ -0,0 +1,34 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Privacy App Permission Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-privacy-permission-manager"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.PrivacyPermissionManagerTest" /> +</configuration> + diff --git a/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting-ui-elements.xml b/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting-ui-elements.xml new file mode 100644 index 0000000..644aca5 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting-ui-elements.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Privacy Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-privacy-setting-ui-elements"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.PrivacySettingVerifyUIElementsTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting.xml b/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting.xml new file mode 100644 index 0000000..3f2bde1 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-privacy-setting.xml @@ -0,0 +1,34 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Microphone Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-privacy-setting"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.PrivacySettingTest" /> +</configuration> + diff --git a/tools/catbox-tradefed/res/config/catbox-functional-profile-icon.xml b/tools/catbox-tradefed/res/config/catbox-functional-profile-icon.xml new file mode 100644 index 0000000..b7e0c47 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-profile-icon.xml @@ -0,0 +1,40 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration + description="Complete Automotive Tests - profile icon Bar Functional Tests."> + <!-- Common Base --> + <include name="catbox-common" /> + + <!-- Device Preparers --> + <include name="catbox-preparer" /> + + <!-- Enable system user target preparer to run as user 0 --> + <option name="run-on-system-user:disable" value="false" /> + + <!-- Plan --> + <option name="plan" value="catbox-functional-profile-icon" /> + + <!-- Test Args --> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveProfileIconTests android.platform.tests.ProfileIconTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-profiles-icon-list.xml b/tools/catbox-tradefed/res/config/catbox-functional-profiles-icon-list.xml new file mode 100644 index 0000000..a62ead1 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-profiles-icon-list.xml @@ -0,0 +1,38 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Profiles Icon List Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Enable system user target preparer to run as user 0 --> + <option name="run-on-system-user:disable" value="false" /> + + <!-- Plan --> + <option name="plan" value="catbox-functional-profiles-icon-list"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveProfileIconListTests android.platform.tests.ProfileIconsListTest" /> +</configuration> + + diff --git a/tools/catbox-tradefed/res/config/catbox-functional-settings-location.xml b/tools/catbox-tradefed/res/config/catbox-functional-settings-location.xml new file mode 100644 index 0000000..8b5ab15 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-settings-location.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Setting Location Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-settings-location"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsLocationTests android.platform.tests.SettingsLocationTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-status-bar.xml b/tools/catbox-tradefed/res/config/catbox-functional-status-bar.xml new file mode 100644 index 0000000..2ab8336 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-status-bar.xml @@ -0,0 +1,38 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration + description="Complete Automotive Tests - status Bar Functional Tests."> + <!-- Common Base --> + <include name="catbox-common" /> + + <!-- Device Preparers --> + <include name="catbox-preparer" /> + + <!-- Plan --> + <option name="plan" value="catbox-functional-status-bar" /> + + <!-- Test Args --> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" + value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" + value="AndroidAutomotiveStatusBarTests android.platform.tests.StatusBarTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-storage-setting.xml b/tools/catbox-tradefed/res/config/catbox-functional-storage-setting.xml new file mode 100644 index 0000000..c21fb4a --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-storage-setting.xml @@ -0,0 +1,27 @@ + +<!-- + Copyright (C) 2023 Google Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Storage Setting Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + <!-- Device Preparers --> + <include name="catbox-preparer"/> + <!-- Plan --> + <option name="plan" value="catbox-functional-storage-setting"/> + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveSettingsTests android.platform.tests.StorageSettingTest" /> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-functional-systemui.xml b/tools/catbox-tradefed/res/config/catbox-functional-systemui.xml new file mode 100644 index 0000000..fcaae21 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-systemui.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2023 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - System UI Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-systemui"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveHomeTests android.platform.tests.SystemUiTest" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction-facet-bar.xml b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction-facet-bar.xml new file mode 100644 index 0000000..06ab2c7 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction-facet-bar.xml @@ -0,0 +1,33 @@ +<!-- + Copyright (C) 2021 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Complete Automotive Tests - Ux Restriction Functional Tests."> + <!-- Common Base --> + <include name="catbox-common"/> + + <!-- Device Preparers --> + <include name="catbox-preparer"/> + + <!-- Plan --> + <option name="plan" value="catbox-functional-ux-restriction-facet-bar"/> + + <!-- Test Args --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:runner:androidx.test.runner.AndroidJUnitRunner" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.tests" /> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> + + <!-- Tests --> + <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests android.platform.tests.UxRestrictionFacetBarTest" /> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml index 5bc518b..0141ecb 100644 --- a/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml +++ b/tools/catbox-tradefed/res/config/catbox-functional-ux-restriction.xml @@ -29,5 +29,5 @@ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:no-rerun:true" /> <!-- Tests --> - <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests" /> -</configuration>
\ No newline at end of file + <option name="compatibility:include-filter" value="AndroidAutomotiveUxRestrictionTests android.platform.tests.UxRestrictionTest" /> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml index 9dd7470..e4deff5 100644 --- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml +++ b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-guest.xml @@ -19,7 +19,6 @@ <!-- Clear pre-created guests --> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_guests 0" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" /> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml index 97454a0..c066a6c 100644 --- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml +++ b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-new-user.xml @@ -19,7 +19,6 @@ <!-- Clear pre-created users --> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_users 0" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" /> <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" /> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-guest.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-guest.xml deleted file mode 100644 index bfc140a..0000000 --- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-guest.xml +++ /dev/null @@ -1,38 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright (C) 2021 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<configuration description="CATBox Performance Test to measure latency to switch to a Pre-created Guest"> - <include name="catbox-performance-multiuser-base" /> - - <!-- Set pre-created guests --> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_guests 5" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" /> - - <!--Test --> - <option name="compatibility:include-filter" value="AndroidAutomotiveMultiuserScenarioTests android.platform.scenario.multiuser.SwitchToNewGuestBenchmark" /> - <option name="compatibility:module-arg" value="AndroidAutomotiveMultiuserScenarioTests:instrumentation-arg:include_creation_time:=true" /> - - <!-- Test Metrics Report --> - <option name="report-log-name" value="CatboxPerformanceTests" /> - <option name="report-test-name-mapping" key="android.platform.scenario.multiuser.SwitchToNewGuestBenchmark#testSwitch" value="switch_to_pre_created_guest" /> - <option name="report-all-metrics" value="false" /> - <option name="report-metric-key-mapping" key="duration_ms_android.platform.scenario.multiuser.SwitchToNewGuestBenchmark#testSwitch-median" value="duration_ms" /> - - <option name="plan" value="catbox-performance-create-and-switch-to-precreated-guest" /> -</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-user.xml b/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-user.xml deleted file mode 100644 index c0acabb..0000000 --- a/tools/catbox-tradefed/res/config/catbox-performance-create-and-switch-to-precreated-user.xml +++ /dev/null @@ -1,38 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright (C) 2018 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> - -<configuration description="CATBox Performance Test to measure latency to switch to a Pre-created User"> - <include name="catbox-performance-multiuser-base" /> - - <!-- Clear pre-created guests --> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop fw.max_users 15" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:setprop android.car.number_pre_created_users 5" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:stop" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:run-command:start" /> - <option name="compatibility:test-arg" value="com.android.tradefed.targetprep.RunCommandTargetPreparer:delay-after-commands:120000" /> - - <!--Test --> - <option name="compatibility:include-filter" value="AndroidAutomotiveMultiuserScenarioTests android.platform.scenario.multiuser.SwitchToNewSecondaryUserBenchmark" /> - <option name="compatibility:module-arg" value="AndroidAutomotiveMultiuserScenarioTests:instrumentation-arg:include_creation_time:=true" /> - - <!-- Test Metrics Report --> - <option name="report-log-name" value="CatboxPerformanceTests" /> - <option name="report-test-name-mapping" key="android.platform.scenario.multiuser.SwitchToNewSecondaryUserBenchmark#testSwitch" value="switch_to_pre_created_user" /> - <option name="report-all-metrics" value="false" /> - <option name="report-metric-key-mapping" key="duration_ms_android.platform.scenario.multiuser.SwitchToNewSecondaryUserBenchmark#testSwitch-median" value="duration_ms" /> - - <option name="plan" value="catbox-performance-create-and-switch-to-precreated-user" /> -</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml b/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml index 0c14d83..be24379 100644 --- a/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml +++ b/tools/catbox-tradefed/res/config/catbox-performance-jank-appgrid.xml @@ -37,14 +37,14 @@ <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.test.scenario.appgrid" /> <!-- Test --> - <option name="compatibility:include-filter" value="AndroidAutomotiveAppGridScenarioTests android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollDownAndUp" /> + <option name="compatibility:include-filter" value="AndroidAutomotiveAppGridScenarioTests android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollForwardAndBackward" /> <!-- Jank Package --> <option name="compatibility:module-arg" value="AndroidAutomotiveAppGridScenarioTests:instrumentation-arg:jank-package-names:=com.android.car.carlauncher" /> <!-- Test Metrics Report Options --> <option name="report-log-name" value="CatboxPerformanceTests" /> - <option name="report-test-name-mapping" key="android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollDownAndUp" value="appgrid_scroll_jank" /> + <option name="report-test-name-mapping" key="android.platform.test.scenario.appgrid.ScrollMicrobenchmark#testScrollForwardAndBackward" value="appgrid_scroll_jank" /> <option name="report-all-metrics" value="false" /> <option name="report-metric-key-mapping" key="gfxinfo_com.android.car.carlauncher_janky_frames_percent-mean" value="jank_frames_percent" /> </configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-jank-media-switch-playback.xml b/tools/catbox-tradefed/res/config/catbox-performance-jank-media-switch-playback.xml new file mode 100644 index 0000000..f0fea15 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-performance-jank-media-switch-playback.xml @@ -0,0 +1,52 @@ +<!-- + ~ Copyright (C) 2023 The Android Open Source Project + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration description="CATBox Jank Performance Test For Media Switch Playback."> + <!-- Plan --> + <option name="plan" value="catbox-performance-jank-media-switch-playback" /> + + <!-- Base File --> + <include name="catbox-performance-jank-base" /> + + <!-- Proc Load Options --> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:per_run:=true" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-interval:=20000" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-threshold:=2.0" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-timeout:=900000" /> + + <!-- Jank Options --> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:listener:=android.device.collectors.ProcLoadListener,android.device.collectors.JankListener" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:iterations:=20" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:favor-shell-commands:=true" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:rename-iterations:=true" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:iteration-separator:=$" /> + + <!-- Test Package --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.test.scenario.mediacenter" /> + + <!-- Test --> + <option name="compatibility:include-filter" value="AndroidAutomotiveMediaCenterScenarioTests android.platform.test.scenario.mediacenter.SwitchPlaybackMicrobenchmark#testSwitchPlayback" /> + + <!-- Jank Package --> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:jank-package-names:=com.android.car.media" /> + + <!-- Test Metrics Report Options --> + <option name="report-log-name" value="CatboxPerformanceTests" /> + <option name="report-test-name-mapping" key="android.platform.test.scenario.mediacenter.SwitchPlaybackMicrobenchmark#testSwitchPlayback" value="media_switchplayback_jank" /> + <option name="report-all-metrics" value="false" /> + <!-- TODO: b/247588415 Add metrics to APFE --> + <option name="report-metric-key-mapping" key="gfxinfo_com.android.car.media_janky_frames_percent-mean" value="jank_frames_percent" /> +</configuration> diff --git a/tools/catbox-tradefed/res/config/catbox-performance-jank-media.xml b/tools/catbox-tradefed/res/config/catbox-performance-jank-media.xml new file mode 100644 index 0000000..4212fa3 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-performance-jank-media.xml @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2018 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="CATBox Jank Performance Test For Media."> + <!-- Plan --> + <option name="plan" value="catbox-performance-jank-media" /> + + <!-- Base File --> + <include name="catbox-performance-jank-base" /> + + <!-- Proc Load Options --> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:per_run:=true" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-interval:=20000" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-threshold:=2.0" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:procload-collector:proc-loadavg-timeout:=900000" /> + + <!-- Jank Options --> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:listener:=android.device.collectors.ProcLoadListener,android.device.collectors.JankListener" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:iterations:=20" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:favor-shell-commands:=true" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:rename-iterations:=true" /> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:iteration-separator:=$" /> + + <!-- Test Package --> + <option name="compatibility:test-arg" value="com.android.tradefed.testtype.AndroidJUnitTest:package:android.platform.test.scenario.mediacenter" /> + + <!-- Test --> + <option name="compatibility:include-filter" value="AndroidAutomotiveMediaCenterScenarioTests android.platform.test.scenario.mediacenter.ScrollMicrobenchmark#testScrollDownAndUp" /> + + <!-- Jank Package --> + <option name="compatibility:module-arg" value="AndroidAutomotiveMediaCenterScenarioTests:instrumentation-arg:jank-package-names:=com.android.car.media" /> + + <!-- Test Metrics Report Options --> + <option name="report-log-name" value="CatboxPerformanceTests" /> + <option name="report-test-name-mapping" key="android.platform.test.scenario.mediacenter.ScrollMicrobenchmark#testScrollDownAndUp" value="media_scroll_jank" /> + <option name="report-all-metrics" value="false" /> + <!-- TODO: b/247588415 Add metrics to APFE --> + <option name="report-metric-key-mapping" key="gfxinfo_com.android.car.media_janky_frames_percent-mean" value="jank_frames_percent" /> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml b/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml new file mode 100644 index 0000000..863b7e1 --- /dev/null +++ b/tools/catbox-tradefed/res/config/catbox-performance-successive-boottime.xml @@ -0,0 +1,66 @@ +<!-- + ~ Copyright (C) 2022 The Android Open Source Project + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration> + <!-- Extend existing boottime config--> + <include name="google/test/boottime-successive"/> + + <include name="catbox-performance-base" /> + + <!-- Artificially disabling tests in CompatibilityTestSuite --> + <!-- See com.android.tradefed.testtype.suite.CompatibilityTestSuite:loadTests() --> + <option name="compatibility:reverse-exclude-filters" value="true" /> + + <option name="boot-delay" value="20000"/> + <option name="boot-count" value="20"/> + <option name="dmesg-info" value="true"/> + <option name="granular-boot-info" value="true"/> + <option name="online-timeout" value="120000"/> + <option name="successive-boot" value="true"/> + + <!-- Prepare commands --> + <option name="successive-boot-prepare-cmd" + value="am start -n com.google.android.car.setupwizard/.ExitActivity"/> + <option name="successive-boot-prepare-cmd" + value="setprop persist.aae.suw.disable_short_notice true"/> + <option name="successive-boot-prepare-cmd" + value="cmd car_service enable-feature car_evs_service"/> + + <!-- For custom boot time metrics --> + <option name="boot-time-pattern" key="boot_to_ble_on" + value="BluetoothAdapterService: updateAdapterState\(\) \- Broadcasting state BLE_ON"/> + <option name="boot-time-pattern" key="boot_to_carlauncher_fully_drawn" + value="Fully drawn com.android.car.carlauncher/.CarLauncher"/> + <option name="boot-time-pattern" key="boot_to_first_user_unlocked" + value="CarServiceHelper: Time to unlock 1st user"/> + <option name="boot-time-pattern" key="boot_to_maps_displayed" + value="Displayed com.google.android.apps.maps"/> + <option name="boot-time-pattern" key="boot_to_settings_fallbackhome_displayed" + value="Displayed com.android.car.settings/.FallbackHome"/> + <option name="boot-time-pattern" key="boot_to_user0_unlocked" + value="SystemServerTiming: SystemUserUnlock took to complete"/> + <option name="boot-time-pattern" key="boot_to_user10_unlocked" + value="SystemServerTiming: User10Unlock took to complete"/> + <option name="boot-time-pattern" key="boot_to_gmscore_started_experimental" + value="ActivityManager: Start proc \d+:com.google.android.gms/u0"/> + <option name="boot-time-pattern" key="boot_to_car_evs_service_started" + value="CAR.InitTiming: CarEvsService took to complete"/> + + <option name="report-log-name" value="CatboxPerformanceTests" /> + <option name="report-test-name-mapping" key="BootTimeTest.BootTimeTest#SuccessiveBootTest" value="boot_time_test" /> + <option name="report-all-metrics" value="false" /> + <option name="report-metric-key-mapping" key="dmesg_action_sys.boot_completed_first_timestamp-median" value="successive_boot_time" /> +</configuration> diff --git a/tools/catbox-tradefed/res/config/device-config/three-devices.xml b/tools/catbox-tradefed/res/config/device-config/three-devices.xml new file mode 100644 index 0000000..ebf0151 --- /dev/null +++ b/tools/catbox-tradefed/res/config/device-config/three-devices.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Bluetooth Device Configuration For Threee Devices"> + <device name="auto" > + <!-- Build Provider --> + <build_provider class="com.android.compatibility.common.tradefed.build.CompatibilityBuildProvider" /> + <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller" > + <option name="test-file-name" value="AutomotiveSnippet.apk" /> + </target_preparer> + </device> + + <device name="phone"> + <!-- Build Provider --> + <build_provider class="com.android.compatibility.common.tradefed.build.CompatibilityBuildProvider" /> + <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller" > + <option name="test-file-name" value="PhoneSnippet.apk" /> + </target_preparer> + </device> + + <device name="phone_notpaired"> + <!-- Build Provider --> + <build_provider class="com.android.compatibility.common.tradefed.build.CompatibilityBuildProvider" /> + <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller" > + <option name="test-file-name" value="PhoneSnippet.apk" /> + </target_preparer> + </device> +</configuration>
\ No newline at end of file diff --git a/tools/catbox-tradefed/res/config/device-config/two-devices.xml b/tools/catbox-tradefed/res/config/device-config/two-devices.xml new file mode 100644 index 0000000..9c2ab57 --- /dev/null +++ b/tools/catbox-tradefed/res/config/device-config/two-devices.xml @@ -0,0 +1,29 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Bluetooth Device Configuration For Two Devices"> + <device name="auto" > + <!-- Build Provider --> + <build_provider class="com.android.compatibility.common.tradefed.build.CompatibilityBuildProvider" /> + <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller" > + <option name="test-file-name" value="AutomotiveSnippet.apk" /> + </target_preparer> + </device> + + <device name="phone"> + <!-- Build Provider --> + <build_provider class="com.android.compatibility.common.tradefed.build.CompatibilityBuildProvider" /> + <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller" > + <option name="test-file-name" value="PhoneSnippet.apk" /> + </target_preparer> + </device> +</configuration>
\ No newline at end of file |