diff options
8 files changed, 370 insertions, 6 deletions
diff --git a/test_scripts/src/main/java/com/android/pixel/OWNERS b/test_scripts/src/main/java/com/android/pixel/OWNERS index 05ffe9a..aa4bbf9 100644 --- a/test_scripts/src/main/java/com/android/pixel/OWNERS +++ b/test_scripts/src/main/java/com/android/pixel/OWNERS @@ -1,2 +1,2 @@ murphykuo@google.com -huilingchi@google.com +elisahsu@google.com diff --git a/test_scripts/src/main/java/com/android/pixel/tests/PixelAppCompatTestBase.java b/test_scripts/src/main/java/com/android/pixel/tests/PixelAppCompatTestBase.java index 6c44a88..aa79bb5 100644 --- a/test_scripts/src/main/java/com/android/pixel/tests/PixelAppCompatTestBase.java +++ b/test_scripts/src/main/java/com/android/pixel/tests/PixelAppCompatTestBase.java @@ -33,7 +33,6 @@ import org.junit.Before; /** Base class for Pixel app compatibility tests. */ public abstract class PixelAppCompatTestBase { private static final String KEY_PACKAGE_NAME = "package"; - private DeviceUtils mDeviceUtils; private UiDevice mDevice; private KeyguardManager mKeyguardManager; @@ -41,6 +40,7 @@ public abstract class PixelAppCompatTestBase { @Before public void setUp() throws Exception { + getDeviceUtils().setTestName(this.getClass().getSimpleName()); getDeviceUtils().createLogDataDir(); getDeviceUtils().wakeAndUnlockScreen(); // Start from the home screen diff --git a/test_scripts/src/main/java/com/android/pixel/utils/DeviceUtils.java b/test_scripts/src/main/java/com/android/pixel/utils/DeviceUtils.java index 9d29dbc..1b665e3 100644 --- a/test_scripts/src/main/java/com/android/pixel/utils/DeviceUtils.java +++ b/test_scripts/src/main/java/com/android/pixel/utils/DeviceUtils.java @@ -25,11 +25,14 @@ import android.support.test.uiautomator.By; import android.support.test.uiautomator.UiDevice; import android.util.Log; +import com.google.common.base.Preconditions; + import org.junit.Assert; import java.io.File; import java.io.IOException; import java.nio.file.Paths; +import java.util.Optional; public class DeviceUtils { private static final String TAG = DeviceUtils.class.getSimpleName(); @@ -39,6 +42,8 @@ public class DeviceUtils { private static final long VIDEO_TAIL_BUFFER = 500; private static final String DISMISS_KEYGUARD = "wm dismiss-keyguard"; + private String mFolderDir = LOG_DATA_DIR; + private String mTestName = TAG; private RecordingThread mCurrentThread; private File mLogDataDir; private UiDevice mDevice; @@ -47,9 +52,24 @@ public class DeviceUtils { mDevice = device; } + /** + * Sets the test name and the folder path for the current test. + * + * @param testName The test name. + */ + public void setTestName(String testName) { + Optional<String> optionalTestName = Optional.ofNullable(testName); + if (optionalTestName.isPresent()) { + mTestName = optionalTestName.get(); + mFolderDir = String.join("/", LOG_DATA_DIR, optionalTestName.get()); + } else { + Preconditions.checkNotNull(testName, "testName cannot be null"); + } + } + /** Create a directory to save test screenshots, screenrecord and text files. */ public void createLogDataDir() { - mLogDataDir = new File(LOG_DATA_DIR); + mLogDataDir = new File(mFolderDir); if (mLogDataDir.exists()) { String[] children = mLogDataDir.list(); for (String file : children) { @@ -104,8 +124,9 @@ public class DeviceUtils { public void takeScreenshot(String packageName, String description) { File screenshot = new File( - LOG_DATA_DIR, - String.format("%s_screenshot_%s.png", packageName, description)); + mFolderDir, + String.format( + "%s_%s_screenshot_%s.png", mTestName, packageName, description)); mDevice.takeScreenshot(screenshot); } @@ -118,7 +139,8 @@ public class DeviceUtils { Log.v(TAG, "Started Recording"); mCurrentThread = new RecordingThread( - "test-screen-record", String.format("%s_screenrecord", packageName)); + "test-screen-record", + String.format("%s_%s_screenrecord", mTestName, packageName)); mCurrentThread.start(); } diff --git a/test_scripts/src/main/java/com/android/webview/tests/WebviewAppCrawlTest.java b/test_scripts/src/main/java/com/android/webview/tests/WebviewAppCrawlTest.java new file mode 100644 index 0000000..6ea0e3b --- /dev/null +++ b/test_scripts/src/main/java/com/android/webview/tests/WebviewAppCrawlTest.java @@ -0,0 +1,267 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.webview.tests; + +import com.android.csuite.core.ApkInstaller; +import com.android.csuite.core.ApkInstaller.ApkInstallerException; +import com.android.csuite.core.AppCrawlTester; +import com.android.csuite.core.DeviceUtils; +import com.android.csuite.core.TestUtils; +import com.android.tradefed.config.Option; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.testtype.DeviceJUnit4ClassRunner; +import com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestLogData; +import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test; + +import com.google.common.base.Preconditions; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import javax.annotation.Nullable; + +/** A test that verifies that a single app can be successfully launched. */ +@RunWith(DeviceJUnit4ClassRunner.class) +public class WebviewAppCrawlTest extends BaseHostJUnit4Test { + @Rule public TestLogData mLogData = new TestLogData(); + + private static final String COLLECT_APP_VERSION = "collect-app-version"; + private static final String COLLECT_GMS_VERSION = "collect-gms-version"; + private static final long COMMAND_TIMEOUT_MILLIS = 5 * 60 * 1000; + + private WebviewUtils mWebviewUtils; + private WebviewPackage mPreInstalledWebview; + private ApkInstaller mApkInstaller; + private AppCrawlTester mCrawler; + + @Option(name = "record-screen", description = "Whether to record screen during test.") + private boolean mRecordScreen; + + @Option(name = "webview-version-to-test", description = "Version of Webview to test.") + private String mWebviewVersionToTest; + + @Option( + name = "release-channel", + description = "Release channel to fetch Webview from, i.e. stable.") + private String mReleaseChannel; + + @Option(name = "package-name", description = "Package name of testing app.") + private String mPackageName; + + @Option( + name = "install-apk", + description = + "The path to an apk file or a directory of apk files of a singe package to be" + + " installed on device. Can be repeated.") + private List<File> mApkPaths = new ArrayList<>(); + + @Option( + name = "install-arg", + description = "Arguments for the 'adb install-multiple' package installation command.") + private final List<String> mInstallArgs = new ArrayList<>(); + + @Option( + name = "app-launch-timeout-ms", + description = "Time to wait for an app to launch in msecs.") + private int mAppLaunchTimeoutMs = 20000; + + @Option( + name = COLLECT_APP_VERSION, + description = + "Whether to collect package version information and store the information in" + + " test log files.") + private boolean mCollectAppVersion; + + @Option( + name = COLLECT_GMS_VERSION, + description = + "Whether to collect GMS core version information and store the information in" + + " test log files.") + private boolean mCollectGmsVersion; + + @Option( + name = "repack-apk", + mandatory = false, + description = + "Path to an apk file or a directory containing apk files of a single package " + + "to repack and install in Espresso mode") + private File mRepackApk; + + @Option( + name = "crawl-controller-endpoint", + mandatory = false, + description = "The crawl controller endpoint to target.") + private String mCrawlControllerEndpoint; + + @Option( + name = "ui-automator-mode", + mandatory = false, + description = + "Run the crawler with UIAutomator mode. Apk option is not required in this" + + " mode.") + private boolean mUiAutomatorMode = false; + + @Option( + name = "robo-script-file", + description = "A Roboscript file to be executed by the crawler.") + private File mRoboscriptFile; + + // TODO(b/234512223): add support for contextual roboscript files + + @Option( + name = "crawl-guidance-proto-file", + description = "A CrawlGuidance file to be executed by the crawler.") + private File mCrawlGuidanceProtoFile; + + @Option( + name = "timeout-sec", + mandatory = false, + description = "The timeout for the crawl test.") + private int mTimeoutSec = 60; + + @Option( + name = "save-apk-when", + description = "When to save apk files to the test result artifacts.") + private TestUtils.TakeEffectWhen mSaveApkWhen = TestUtils.TakeEffectWhen.NEVER; + + @Option( + name = "login-config-dir", + description = + "A directory containing Roboscript and CrawlGuidance files with login" + + " credentials that are passed to the crawler. There should be one config" + + " file per package name. If both Roboscript and CrawlGuidance files are" + + " present, only the Roboscript file will be used.") + private File mLoginConfigDir; + + @Before + public void setUp() throws DeviceNotAvailableException, ApkInstallerException, IOException { + Assert.assertNotNull("Package name cannot be null", mPackageName); + Assert.assertTrue( + "Either the --release-channel or --webview-version-to-test arguments " + + "must be used", + mWebviewVersionToTest != null || mReleaseChannel != null); + + mCrawler = AppCrawlTester.newInstance(mPackageName, getTestInformation(), mLogData); + if (!mUiAutomatorMode) { + setApkForEspressoMode(); + } + mCrawler.setCrawlControllerEndpoint(mCrawlControllerEndpoint); + mCrawler.setRecordScreen(mRecordScreen); + mCrawler.setCollectGmsVersion(mCollectGmsVersion); + mCrawler.setCollectAppVersion(mCollectAppVersion); + mCrawler.setUiAutomatorMode(mUiAutomatorMode); + mCrawler.setRoboscriptFile(toPathOrNull(mRoboscriptFile)); + mCrawler.setCrawlGuidanceProtoFile(toPathOrNull(mCrawlGuidanceProtoFile)); + mCrawler.setLoginConfigDir(toPathOrNull(mLoginConfigDir)); + mCrawler.setTimeoutSec(mTimeoutSec); + + mApkInstaller = ApkInstaller.getInstance(getDevice()); + mWebviewUtils = new WebviewUtils(getTestInformation()); + mPreInstalledWebview = mWebviewUtils.getCurrentWebviewPackage(); + + for (File apkPath : mApkPaths) { + CLog.d("Installing " + apkPath); + mApkInstaller.install(apkPath.toPath(), mInstallArgs); + } + + DeviceUtils.getInstance(getDevice()).freezeRotation(); + mWebviewUtils.printWebviewVersion(); + } + + /** + * For Espresso mode, checks that a path with the location of the apk to repackage was provided + */ + private void setApkForEspressoMode() { + Preconditions.checkNotNull( + mRepackApk, "Apk file path is required when not running in UIAutomator mode"); + // set the root path of the target apk for Espresso mode + mCrawler.setApkPath(mRepackApk.toPath()); + } + + private static Path toPathOrNull(@Nullable File f) { + return f == null ? null : f.toPath(); + } + + @Test + public void testAppCrawl() + throws DeviceNotAvailableException, InterruptedException, ApkInstallerException, + IOException { + AssertionError lastError = null; + WebviewPackage lastWebviewInstalled = + mWebviewUtils.installWebview(mWebviewVersionToTest, mReleaseChannel); + + try { + mCrawler.startAndAssertAppNoCrash(); + } catch (AssertionError e) { + lastError = e; + } finally { + mWebviewUtils.uninstallWebview(lastWebviewInstalled, mPreInstalledWebview); + } + + // If the app doesn't crash, complete the test. + if (lastError == null) { + return; + } + + // If the app crashes, try the app with the original webview version that comes with the + // device. + try { + mCrawler.startAndAssertAppNoCrash(); + } catch (AssertionError newError) { + CLog.w( + "The app %s crashed both with and without the webview installation," + + " ignoring the failure...", + mPackageName); + return; + } + throw new AssertionError( + String.format( + "Package %s crashed since webview version %s", + mPackageName, lastWebviewInstalled.getVersion()), + lastError); + } + + @After + public void tearDown() throws DeviceNotAvailableException, ApkInstallerException { + TestUtils testUtils = TestUtils.getInstance(getTestInformation(), mLogData); + testUtils.collectScreenshot(mPackageName); + + DeviceUtils deviceUtils = DeviceUtils.getInstance(getDevice()); + deviceUtils.stopPackage(mPackageName); + deviceUtils.unfreezeRotation(); + + mApkInstaller.uninstallAllInstalledPackages(); + mWebviewUtils.printWebviewVersion(); + + if (!mUiAutomatorMode) { + getDevice().uninstallPackage(mPackageName); + } + + mCrawler.cleanUp(); + } +} diff --git a/test_targets/webview-app-crawl/Android.bp b/test_targets/webview-app-crawl/Android.bp new file mode 100644 index 0000000..57d4192 --- /dev/null +++ b/test_targets/webview-app-crawl/Android.bp @@ -0,0 +1,23 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +csuite_test { + name: "webview-app-crawl", + test_plan_include: "plan.xml", + test_config_template: "ui-automator-mode.xml", +} diff --git a/test_targets/webview-app-crawl/OWNERS b/test_targets/webview-app-crawl/OWNERS new file mode 100644 index 0000000..af3a7c8 --- /dev/null +++ b/test_targets/webview-app-crawl/OWNERS @@ -0,0 +1,2 @@ +amitku@google.com +rmhasan@google.com
\ No newline at end of file diff --git a/test_targets/webview-app-crawl/plan.xml b/test_targets/webview-app-crawl/plan.xml new file mode 100644 index 0000000..83c3e05 --- /dev/null +++ b/test_targets/webview-app-crawl/plan.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="WebView C-Suite Crawler Test Plan"> + <target_preparer class="com.android.webview.tests.WebviewInstallerToolPreparer"/> + <target_preparer class="com.android.csuite.core.AppCrawlTesterHostPreparer"/> +</configuration> diff --git a/test_targets/webview-app-crawl/ui-automator-mode.xml b/test_targets/webview-app-crawl/ui-automator-mode.xml new file mode 100644 index 0000000..ec44190 --- /dev/null +++ b/test_targets/webview-app-crawl/ui-automator-mode.xml @@ -0,0 +1,31 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2023 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration description="Crawl's an app after installing WebView"> + <target_preparer class="com.android.compatibility.targetprep.CheckGmsPreparer"/> + <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller" /> + <target_preparer class="com.android.tradefed.targetprep.RunCommandTargetPreparer"> + <option name="run-command" value="input keyevent KEYCODE_WAKEUP"/> + <option name="run-command" value="input keyevent KEYCODE_MENU"/> + <option name="run-command" value="input keyevent KEYCODE_HOME"/> + </target_preparer> + <test class="com.android.tradefed.testtype.HostTest" > + <option name="set-option" value="package-name:{package}"/> + <option name="set-option" value="install-apk:app\://{package}"/> + <option name="set-option" value="ui-automator-mode:true"/> + <option name="set-option" value="install-arg:-g"/> + <option name="class" value="com.android.webview.tests.WebviewAppCrawlTest" /> + </test> +</configuration> |