Merge "WebView: Add an Robo app crawler c-suite test suite" am: 479dd74796 am: 104168fae8 am: 82bbd3e125android14-dev

Original change: https://android-review.googlesource.com/c/platform/test/app_compat/csuite/+/2507915

Change-Id: I75c4ecb700de9bc7d7912e7e7a393775b257f894
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

5 files changed, 342 insertions, 0 deletions

diff --git a/test_scripts/src/main/java/com/android/webview/tests/WebviewAppCrawlTest.java b/test_scripts/src/main/java/com/android/webview/tests/WebviewAppCrawlTest.java
new file mode 100644
index 0000000..6ea0e3b
--- /dev/null
+++ b/test_scripts/src/main/java/com/android/webview/tests/WebviewAppCrawlTest.java
@@ -0,0 +1,267 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.webview.tests;
+
+import com.android.csuite.core.ApkInstaller;
+import com.android.csuite.core.ApkInstaller.ApkInstallerException;
+import com.android.csuite.core.AppCrawlTester;
+import com.android.csuite.core.DeviceUtils;
+import com.android.csuite.core.TestUtils;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.testtype.DeviceJUnit4ClassRunner;
+import com.android.tradefed.testtype.DeviceJUnit4ClassRunner.TestLogData;
+import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test;
+
+import com.google.common.base.Preconditions;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+/** A test that verifies that a single app can be successfully launched. */
+@RunWith(DeviceJUnit4ClassRunner.class)
+public class WebviewAppCrawlTest extends BaseHostJUnit4Test {
+    @Rule public TestLogData mLogData = new TestLogData();
+
+    private static final String COLLECT_APP_VERSION = "collect-app-version";
+    private static final String COLLECT_GMS_VERSION = "collect-gms-version";
+    private static final long COMMAND_TIMEOUT_MILLIS = 5 * 60 * 1000;
+
+    private WebviewUtils mWebviewUtils;
+    private WebviewPackage mPreInstalledWebview;
+    private ApkInstaller mApkInstaller;
+    private AppCrawlTester mCrawler;
+
+    @Option(name = "record-screen", description = "Whether to record screen during test.")
+    private boolean mRecordScreen;
+
+    @Option(name = "webview-version-to-test", description = "Version of Webview to test.")
+    private String mWebviewVersionToTest;
+
+    @Option(
+            name = "release-channel",
+            description = "Release channel to fetch Webview from, i.e. stable.")
+    private String mReleaseChannel;
+
+    @Option(name = "package-name", description = "Package name of testing app.")
+    private String mPackageName;
+
+    @Option(
+            name = "install-apk",
+            description =
+                    "The path to an apk file or a directory of apk files of a singe package to be"
+                            + " installed on device. Can be repeated.")
+    private List<File> mApkPaths = new ArrayList<>();
+
+    @Option(
+            name = "install-arg",
+            description = "Arguments for the 'adb install-multiple' package installation command.")
+    private final List<String> mInstallArgs = new ArrayList<>();
+
+    @Option(
+            name = "app-launch-timeout-ms",
+            description = "Time to wait for an app to launch in msecs.")
+    private int mAppLaunchTimeoutMs = 20000;
+
+    @Option(
+            name = COLLECT_APP_VERSION,
+            description =
+                    "Whether to collect package version information and store the information in"
+                            + " test log files.")
+    private boolean mCollectAppVersion;
+
+    @Option(
+            name = COLLECT_GMS_VERSION,
+            description =
+                    "Whether to collect GMS core version information and store the information in"
+                            + " test log files.")
+    private boolean mCollectGmsVersion;
+
+    @Option(
+            name = "repack-apk",
+            mandatory = false,
+            description =
+                    "Path to an apk file or a directory containing apk files of a single package "
+                            + "to repack and install in Espresso mode")
+    private File mRepackApk;
+
+    @Option(
+            name = "crawl-controller-endpoint",
+            mandatory = false,
+            description = "The crawl controller endpoint to target.")
+    private String mCrawlControllerEndpoint;
+
+    @Option(
+            name = "ui-automator-mode",
+            mandatory = false,
+            description =
+                    "Run the crawler with UIAutomator mode. Apk option is not required in this"
+                            + " mode.")
+    private boolean mUiAutomatorMode = false;
+
+    @Option(
+            name = "robo-script-file",
+            description = "A Roboscript file to be executed by the crawler.")
+    private File mRoboscriptFile;
+
+    // TODO(b/234512223): add support for contextual roboscript files
+
+    @Option(
+            name = "crawl-guidance-proto-file",
+            description = "A CrawlGuidance file to be executed by the crawler.")
+    private File mCrawlGuidanceProtoFile;
+
+    @Option(
+            name = "timeout-sec",
+            mandatory = false,
+            description = "The timeout for the crawl test.")
+    private int mTimeoutSec = 60;
+
+    @Option(
+            name = "save-apk-when",
+            description = "When to save apk files to the test result artifacts.")
+    private TestUtils.TakeEffectWhen mSaveApkWhen = TestUtils.TakeEffectWhen.NEVER;
+
+    @Option(
+            name = "login-config-dir",
+            description =
+                    "A directory containing Roboscript and CrawlGuidance files with login"
+                        + " credentials that are passed to the crawler. There should be one config"
+                        + " file per package name. If both Roboscript and CrawlGuidance files are"
+                        + " present, only the Roboscript file will be used.")
+    private File mLoginConfigDir;
+
+    @Before
+    public void setUp() throws DeviceNotAvailableException, ApkInstallerException, IOException {
+        Assert.assertNotNull("Package name cannot be null", mPackageName);
+        Assert.assertTrue(
+                "Either the --release-channel or --webview-version-to-test arguments "
+                        + "must be used",
+                mWebviewVersionToTest != null || mReleaseChannel != null);
+
+        mCrawler = AppCrawlTester.newInstance(mPackageName, getTestInformation(), mLogData);
+        if (!mUiAutomatorMode) {
+            setApkForEspressoMode();
+        }
+        mCrawler.setCrawlControllerEndpoint(mCrawlControllerEndpoint);
+        mCrawler.setRecordScreen(mRecordScreen);
+        mCrawler.setCollectGmsVersion(mCollectGmsVersion);
+        mCrawler.setCollectAppVersion(mCollectAppVersion);
+        mCrawler.setUiAutomatorMode(mUiAutomatorMode);
+        mCrawler.setRoboscriptFile(toPathOrNull(mRoboscriptFile));
+        mCrawler.setCrawlGuidanceProtoFile(toPathOrNull(mCrawlGuidanceProtoFile));
+        mCrawler.setLoginConfigDir(toPathOrNull(mLoginConfigDir));
+        mCrawler.setTimeoutSec(mTimeoutSec);
+
+        mApkInstaller = ApkInstaller.getInstance(getDevice());
+        mWebviewUtils = new WebviewUtils(getTestInformation());
+        mPreInstalledWebview = mWebviewUtils.getCurrentWebviewPackage();
+
+        for (File apkPath : mApkPaths) {
+            CLog.d("Installing " + apkPath);
+            mApkInstaller.install(apkPath.toPath(), mInstallArgs);
+        }
+
+        DeviceUtils.getInstance(getDevice()).freezeRotation();
+        mWebviewUtils.printWebviewVersion();
+    }
+
+    /**
+     * For Espresso mode, checks that a path with the location of the apk to repackage was provided
+     */
+    private void setApkForEspressoMode() {
+        Preconditions.checkNotNull(
+                mRepackApk, "Apk file path is required when not running in UIAutomator mode");
+        // set the root path of the target apk for Espresso mode
+        mCrawler.setApkPath(mRepackApk.toPath());
+    }
+
+    private static Path toPathOrNull(@Nullable File f) {
+        return f == null ? null : f.toPath();
+    }
+
+    @Test
+    public void testAppCrawl()
+            throws DeviceNotAvailableException, InterruptedException, ApkInstallerException,
+                    IOException {
+        AssertionError lastError = null;
+        WebviewPackage lastWebviewInstalled =
+                mWebviewUtils.installWebview(mWebviewVersionToTest, mReleaseChannel);
+
+        try {
+            mCrawler.startAndAssertAppNoCrash();
+        } catch (AssertionError e) {
+            lastError = e;
+        } finally {
+            mWebviewUtils.uninstallWebview(lastWebviewInstalled, mPreInstalledWebview);
+        }
+
+        // If the app doesn't crash, complete the test.
+        if (lastError == null) {
+            return;
+        }
+
+        // If the app crashes, try the app with the original webview version that comes with the
+        // device.
+        try {
+            mCrawler.startAndAssertAppNoCrash();
+        } catch (AssertionError newError) {
+            CLog.w(
+                    "The app %s crashed both with and without the webview installation,"
+                            + " ignoring the failure...",
+                    mPackageName);
+            return;
+        }
+        throw new AssertionError(
+                String.format(
+                        "Package %s crashed since webview version %s",
+                        mPackageName, lastWebviewInstalled.getVersion()),
+                lastError);
+    }
+
+    @After
+    public void tearDown() throws DeviceNotAvailableException, ApkInstallerException {
+        TestUtils testUtils = TestUtils.getInstance(getTestInformation(), mLogData);
+        testUtils.collectScreenshot(mPackageName);
+
+        DeviceUtils deviceUtils = DeviceUtils.getInstance(getDevice());
+        deviceUtils.stopPackage(mPackageName);
+        deviceUtils.unfreezeRotation();
+
+        mApkInstaller.uninstallAllInstalledPackages();
+        mWebviewUtils.printWebviewVersion();
+
+        if (!mUiAutomatorMode) {
+            getDevice().uninstallPackage(mPackageName);
+        }
+
+        mCrawler.cleanUp();
+    }
+}
diff --git a/test_targets/webview-app-crawl/Android.bp b/test_targets/webview-app-crawl/Android.bp
new file mode 100644
index 0000000..57d4192
--- /dev/null
+++ b/test_targets/webview-app-crawl/Android.bp
@@ -0,0 +1,23 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package {
+    default_applicable_licenses: ["Android-Apache-2.0"],
+}
+
+csuite_test {
+    name: "webview-app-crawl",
+    test_plan_include: "plan.xml",
+    test_config_template: "ui-automator-mode.xml",
+}
diff --git a/test_targets/webview-app-crawl/OWNERS b/test_targets/webview-app-crawl/OWNERS
new file mode 100644
index 0000000..af3a7c8
--- /dev/null
+++ b/test_targets/webview-app-crawl/OWNERS
@@ -0,0 +1,2 @@
+amitku@google.com
+rmhasan@google.com
\ No newline at end of file
diff --git a/test_targets/webview-app-crawl/plan.xml b/test_targets/webview-app-crawl/plan.xml
new file mode 100644
index 0000000..83c3e05
--- /dev/null
+++ b/test_targets/webview-app-crawl/plan.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2023 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+<configuration description="WebView C-Suite Crawler Test Plan">
+  <target_preparer class="com.android.webview.tests.WebviewInstallerToolPreparer"/>
+  <target_preparer class="com.android.csuite.core.AppCrawlTesterHostPreparer"/>
+</configuration>
diff --git a/test_targets/webview-app-crawl/ui-automator-mode.xml b/test_targets/webview-app-crawl/ui-automator-mode.xml
new file mode 100644
index 0000000..ec44190
--- /dev/null
+++ b/test_targets/webview-app-crawl/ui-automator-mode.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2023 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+<configuration description="Crawl's an app after installing WebView">
+    <target_preparer class="com.android.compatibility.targetprep.CheckGmsPreparer"/>
+    <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller" />
+    <target_preparer class="com.android.tradefed.targetprep.RunCommandTargetPreparer">
+        <option name="run-command" value="input keyevent KEYCODE_WAKEUP"/>
+        <option name="run-command" value="input keyevent KEYCODE_MENU"/>
+        <option name="run-command" value="input keyevent KEYCODE_HOME"/>
+    </target_preparer>
+    <test class="com.android.tradefed.testtype.HostTest" >
+        <option name="set-option" value="package-name:{package}"/>
+        <option name="set-option" value="install-apk:app\://{package}"/>
+        <option name="set-option" value="ui-automator-mode:true"/>
+        <option name="set-option" value="install-arg:-g"/>
+        <option name="class" value="com.android.webview.tests.WebviewAppCrawlTest" />
+    </test>
+</configuration>