From 391e3c98c74fd0c82b83930ad2212650d40582c0 Mon Sep 17 00:00:00 2001 From: "Rakib M. Hasan" Date: Fri, 24 Mar 2023 18:26:22 +0000 Subject: WebView: Add an Robo app crawler c-suite test suite This test suite will install WebView and the third party app and then crawl it with the Robo app crawler. Test: csuite-tradefed run webview-app-launch Bug: 206871132 Change-Id: I56fa75683a51c03908956b821312469be5c47975 --- .../android/webview/tests/WebviewAppCrawlTest.java | 267 +++++++++++++++++++++ test_targets/webview-app-crawl/Android.bp | 23 ++ test_targets/webview-app-crawl/OWNERS | 2 + test_targets/webview-app-crawl/plan.xml | 19 ++ .../webview-app-crawl/ui-automator-mode.xml | 31 +++ 5 files changed, 342 insertions(+) create mode 100644 test_scripts/src/main/java/com/android/webview/tests/WebviewAppCrawlTest.java create mode 100644 test_targets/webview-app-crawl/Android.bp create mode 100644 test_targets/webview-app-crawl/OWNERS create mode 100644 test_targets/webview-app-crawl/plan.xml create mode 100644 test_targets/webview-app-crawl/ui-automator-mode.xml diff --git a/test_scripts/src/main/java/com/android/webview/tests/WebviewAppCrawlTest.java b/test_scripts/src/main/java/com/android/webview/tests/WebviewAppCrawlTest.java new file mode 100644 index 0000000..6ea0e3b --- /dev/null +++ b/test_scripts/src/main/java/com/android/webview/tests/WebviewAppCrawlTest.java @@ -0,0 +1,267 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.webview.tests; + +import com.android.csuite.core.ApkInstaller; +import com.android.csuite.core.ApkInstaller.ApkInstallerException; +import com.android.csuite.core.AppCrawlTester; +import com.android.csuite.core.DeviceUtils; +import com.android.csuite.core.TestUtils; +import com.android.tradefed.config.Option; +import com.android.tradefed.device.DeviceNotAvailableException; +import com.android.tradefed.log.LogUtil.CLog; +import com.android.tradefed.testtype.DeviceJUnit4ClassRunner; +import com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestLogData; +import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test; + +import com.google.common.base.Preconditions; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import javax.annotation.Nullable; + +/** A test that verifies that a single app can be successfully launched. */ +@RunWith(DeviceJUnit4ClassRunner.class) +public class WebviewAppCrawlTest extends BaseHostJUnit4Test { + @Rule public TestLogData mLogData = new TestLogData(); + + private static final String COLLECT_APP_VERSION = "collect-app-version"; + private static final String COLLECT_GMS_VERSION = "collect-gms-version"; + private static final long COMMAND_TIMEOUT_MILLIS = 5 * 60 * 1000; + + private WebviewUtils mWebviewUtils; + private WebviewPackage mPreInstalledWebview; + private ApkInstaller mApkInstaller; + private AppCrawlTester mCrawler; + + @Option(name = "record-screen", description = "Whether to record screen during test.") + private boolean mRecordScreen; + + @Option(name = "webview-version-to-test", description = "Version of Webview to test.") + private String mWebviewVersionToTest; + + @Option( + name = "release-channel", + description = "Release channel to fetch Webview from, i.e. stable.") + private String mReleaseChannel; + + @Option(name = "package-name", description = "Package name of testing app.") + private String mPackageName; + + @Option( + name = "install-apk", + description = + "The path to an apk file or a directory of apk files of a singe package to be" + + " installed on device. Can be repeated.") + private List mApkPaths = new ArrayList<>(); + + @Option( + name = "install-arg", + description = "Arguments for the 'adb install-multiple' package installation command.") + private final List mInstallArgs = new ArrayList<>(); + + @Option( + name = "app-launch-timeout-ms", + description = "Time to wait for an app to launch in msecs.") + private int mAppLaunchTimeoutMs = 20000; + + @Option( + name = COLLECT_APP_VERSION, + description = + "Whether to collect package version information and store the information in" + + " test log files.") + private boolean mCollectAppVersion; + + @Option( + name = COLLECT_GMS_VERSION, + description = + "Whether to collect GMS core version information and store the information in" + + " test log files.") + private boolean mCollectGmsVersion; + + @Option( + name = "repack-apk", + mandatory = false, + description = + "Path to an apk file or a directory containing apk files of a single package " + + "to repack and install in Espresso mode") + private File mRepackApk; + + @Option( + name = "crawl-controller-endpoint", + mandatory = false, + description = "The crawl controller endpoint to target.") + private String mCrawlControllerEndpoint; + + @Option( + name = "ui-automator-mode", + mandatory = false, + description = + "Run the crawler with UIAutomator mode. Apk option is not required in this" + + " mode.") + private boolean mUiAutomatorMode = false; + + @Option( + name = "robo-script-file", + description = "A Roboscript file to be executed by the crawler.") + private File mRoboscriptFile; + + // TODO(b/234512223): add support for contextual roboscript files + + @Option( + name = "crawl-guidance-proto-file", + description = "A CrawlGuidance file to be executed by the crawler.") + private File mCrawlGuidanceProtoFile; + + @Option( + name = "timeout-sec", + mandatory = false, + description = "The timeout for the crawl test.") + private int mTimeoutSec = 60; + + @Option( + name = "save-apk-when", + description = "When to save apk files to the test result artifacts.") + private TestUtils.TakeEffectWhen mSaveApkWhen = TestUtils.TakeEffectWhen.NEVER; + + @Option( + name = "login-config-dir", + description = + "A directory containing Roboscript and CrawlGuidance files with login" + + " credentials that are passed to the crawler. There should be one config" + + " file per package name. If both Roboscript and CrawlGuidance files are" + + " present, only the Roboscript file will be used.") + private File mLoginConfigDir; + + @Before + public void setUp() throws DeviceNotAvailableException, ApkInstallerException, IOException { + Assert.assertNotNull("Package name cannot be null", mPackageName); + Assert.assertTrue( + "Either the --release-channel or --webview-version-to-test arguments " + + "must be used", + mWebviewVersionToTest != null || mReleaseChannel != null); + + mCrawler = AppCrawlTester.newInstance(mPackageName, getTestInformation(), mLogData); + if (!mUiAutomatorMode) { + setApkForEspressoMode(); + } + mCrawler.setCrawlControllerEndpoint(mCrawlControllerEndpoint); + mCrawler.setRecordScreen(mRecordScreen); + mCrawler.setCollectGmsVersion(mCollectGmsVersion); + mCrawler.setCollectAppVersion(mCollectAppVersion); + mCrawler.setUiAutomatorMode(mUiAutomatorMode); + mCrawler.setRoboscriptFile(toPathOrNull(mRoboscriptFile)); + mCrawler.setCrawlGuidanceProtoFile(toPathOrNull(mCrawlGuidanceProtoFile)); + mCrawler.setLoginConfigDir(toPathOrNull(mLoginConfigDir)); + mCrawler.setTimeoutSec(mTimeoutSec); + + mApkInstaller = ApkInstaller.getInstance(getDevice()); + mWebviewUtils = new WebviewUtils(getTestInformation()); + mPreInstalledWebview = mWebviewUtils.getCurrentWebviewPackage(); + + for (File apkPath : mApkPaths) { + CLog.d("Installing " + apkPath); + mApkInstaller.install(apkPath.toPath(), mInstallArgs); + } + + DeviceUtils.getInstance(getDevice()).freezeRotation(); + mWebviewUtils.printWebviewVersion(); + } + + /** + * For Espresso mode, checks that a path with the location of the apk to repackage was provided + */ + private void setApkForEspressoMode() { + Preconditions.checkNotNull( + mRepackApk, "Apk file path is required when not running in UIAutomator mode"); + // set the root path of the target apk for Espresso mode + mCrawler.setApkPath(mRepackApk.toPath()); + } + + private static Path toPathOrNull(@Nullable File f) { + return f == null ? null : f.toPath(); + } + + @Test + public void testAppCrawl() + throws DeviceNotAvailableException, InterruptedException, ApkInstallerException, + IOException { + AssertionError lastError = null; + WebviewPackage lastWebviewInstalled = + mWebviewUtils.installWebview(mWebviewVersionToTest, mReleaseChannel); + + try { + mCrawler.startAndAssertAppNoCrash(); + } catch (AssertionError e) { + lastError = e; + } finally { + mWebviewUtils.uninstallWebview(lastWebviewInstalled, mPreInstalledWebview); + } + + // If the app doesn't crash, complete the test. + if (lastError == null) { + return; + } + + // If the app crashes, try the app with the original webview version that comes with the + // device. + try { + mCrawler.startAndAssertAppNoCrash(); + } catch (AssertionError newError) { + CLog.w( + "The app %s crashed both with and without the webview installation," + + " ignoring the failure...", + mPackageName); + return; + } + throw new AssertionError( + String.format( + "Package %s crashed since webview version %s", + mPackageName, lastWebviewInstalled.getVersion()), + lastError); + } + + @After + public void tearDown() throws DeviceNotAvailableException, ApkInstallerException { + TestUtils testUtils = TestUtils.getInstance(getTestInformation(), mLogData); + testUtils.collectScreenshot(mPackageName); + + DeviceUtils deviceUtils = DeviceUtils.getInstance(getDevice()); + deviceUtils.stopPackage(mPackageName); + deviceUtils.unfreezeRotation(); + + mApkInstaller.uninstallAllInstalledPackages(); + mWebviewUtils.printWebviewVersion(); + + if (!mUiAutomatorMode) { + getDevice().uninstallPackage(mPackageName); + } + + mCrawler.cleanUp(); + } +} diff --git a/test_targets/webview-app-crawl/Android.bp b/test_targets/webview-app-crawl/Android.bp new file mode 100644 index 0000000..57d4192 --- /dev/null +++ b/test_targets/webview-app-crawl/Android.bp @@ -0,0 +1,23 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +csuite_test { + name: "webview-app-crawl", + test_plan_include: "plan.xml", + test_config_template: "ui-automator-mode.xml", +} diff --git a/test_targets/webview-app-crawl/OWNERS b/test_targets/webview-app-crawl/OWNERS new file mode 100644 index 0000000..af3a7c8 --- /dev/null +++ b/test_targets/webview-app-crawl/OWNERS @@ -0,0 +1,2 @@ +amitku@google.com +rmhasan@google.com \ No newline at end of file diff --git a/test_targets/webview-app-crawl/plan.xml b/test_targets/webview-app-crawl/plan.xml new file mode 100644 index 0000000..83c3e05 --- /dev/null +++ b/test_targets/webview-app-crawl/plan.xml @@ -0,0 +1,19 @@ + + + + + + diff --git a/test_targets/webview-app-crawl/ui-automator-mode.xml b/test_targets/webview-app-crawl/ui-automator-mode.xml new file mode 100644 index 0000000..ec44190 --- /dev/null +++ b/test_targets/webview-app-crawl/ui-automator-mode.xml @@ -0,0 +1,31 @@ + + + + + + + + + + -- cgit v1.2.3