Merge "Break dependency on GTest"

5 files changed, 264 insertions, 43 deletions

diff --git a/test_runner/Android.bp b/test_runner/Android.bp
index 19063c24b..8bba94abc 100644
--- a/test_runner/Android.bp
+++ b/test_runner/Android.bp
@@ -28,6 +28,7 @@ java_test_host {
     test_config: "AndroidNetTest.xml",
     libs: [
         "tradefed",
+        "framework-annotations-lib",
     ],
     required: [
         "cronet_net_tester_app",
@@ -56,6 +57,7 @@ java_test_host {
     test_config: "AndroidTest.xml",
     libs: [
         "tradefed",
+        "framework-annotations-lib",
     ],
     required: [
         "cronet_tester_app",
diff --git a/test_runner/AndroidNetTest.xml b/test_runner/AndroidNetTest.xml
index 5d9822350..dda64d3eb 100644
--- a/test_runner/AndroidNetTest.xml
+++ b/test_runner/AndroidNetTest.xml
@@ -26,12 +26,6 @@
         <option name="teardown-command" value="setenforce 1" />
         <option name="throw-if-cmd-fail" value="true" />
     </target_preparer>
-    <!-- This creates the file which gtest redirects its stdout output to -->
-    <target_preparer class="com.android.tradefed.targetprep.RunCommandTargetPreparer">
-        <option name="run-command" value="touch /data/local/tmp/cronet_gtest_output.txt"/>
-        <option name="teardown-command" value="rm /data/local/tmp/cronet_gtest_output.txt"/>
-        <option name="throw-if-cmd-fail" value="true"/>
-    </target_preparer>
     <!-- Uploading the APK to device and installing it -->
     <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">
         <option name="cleanup-apks" value="true"/>
diff --git a/test_runner/AndroidTest.xml b/test_runner/AndroidTest.xml
index c3ed8ebb0..8ba6d882b 100644
--- a/test_runner/AndroidTest.xml
+++ b/test_runner/AndroidTest.xml
@@ -24,12 +24,6 @@
         <option name="teardown-command" value="setenforce 1" />
         <option name="throw-if-cmd-fail" value="true" />
     </target_preparer>
-    <!-- This creates the file which gtest redirects its stdout output to -->
-    <target_preparer class="com.android.tradefed.targetprep.RunCommandTargetPreparer">
-        <option name="run-command" value="touch /data/local/tmp/cronet_gtest_output.txt"/>
-        <option name="teardown-command" value="rm /data/local/tmp/cronet_gtest_output.txt"/>
-        <option name="throw-if-cmd-fail" value="true"/>
-    </target_preparer>
     <!-- Uploading the APK to device and installing it -->
     <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">
         <option name="cleanup-apks" value="true"/>
diff --git a/test_runner/src/com.android.tests.chromium.host/ChromiumHostDrivenTest.java b/test_runner/src/com.android.tests.chromium.host/ChromiumHostDrivenTest.java
index 9de25cd63..a5c5f153f 100644
--- a/test_runner/src/com.android.tests.chromium.host/ChromiumHostDrivenTest.java
+++ b/test_runner/src/com.android.tests.chromium.host/ChromiumHostDrivenTest.java
@@ -26,94 +26,253 @@ import static com.android.tests.chromium.host.InstrumentationFlags.RUN_IN_SUBTHR
 import static com.android.tests.chromium.host.InstrumentationFlags.STDOUT_FILE_KEY;
 import static com.android.tests.chromium.host.InstrumentationFlags.TEST_RUNNER;
 
+import android.annotation.NonNull;
+
 import com.android.ddmlib.MultiLineReceiver;
 import com.android.tradefed.config.Option;
 import com.android.tradefed.device.CollectingOutputReceiver;
 import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
 import com.android.tradefed.invoker.TestInformation;
 import com.android.tradefed.log.LogUtil;
 import com.android.tradefed.result.ITestInvocationListener;
-import com.android.tradefed.testtype.GTest;
 import com.android.tradefed.testtype.GTestListTestParser;
 import com.android.tradefed.testtype.GTestResultParser;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.ITestCollector;
+import com.android.tradefed.testtype.ITestFilterReceiver;
+import com.android.tradefed.util.FileUtil;
 
+import com.google.common.base.Joiner;
 import com.google.common.base.Strings;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.time.Duration;
+import java.util.LinkedHashSet;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
-public class ChromiumHostDrivenTest extends GTest {
-    // This counter is used to retry
-    private static final int retries = 1;
-    private static final Duration testsTimeout = Duration.ofMinutes(30);
-    // This contains the gtest logs that is printed to stdout.
-    private static final String GTEST_OUTPUT_PATH = "/data/local/tmp/cronet_gtest_output.txt";
+/**
+ * A host-side test-runner capable of running Chromium unit-tests.
+ */
+public class ChromiumHostDrivenTest implements IRemoteTest, IDeviceTest, ITestCollector,
+        ITestFilterReceiver {
+
     private static final String CLEAR_CLANG_COVERAGE_FILES =
             "find /data/misc/trace -name '*.profraw' -delete";
+    private static final Duration TESTS_TIMEOUT = Duration.ofMinutes(30);
+    private static final String GTEST_FLAG_PRINT_TIME = "--gtest_print_time";
+    private static final String GTEST_FLAG_FILTER = "--gtest_filter";
+    private static final String GTEST_FLAG_LIST_TESTS = "--gtest_list_tests";
+    private static final String GTEST_FLAG_FILE = "--gtest_flagfile";
+    private final Set<String> includeFilters = new LinkedHashSet<>();
+    private final Set<String> excludeFilters = new LinkedHashSet<>();
+    private boolean collectTestsOnly = false;
+    private ITestDevice device = null;
+
     @Option(
             name = "dump-native-coverage",
             description = "Force APK under test to dump native test coverage upon exit"
     )
-    private boolean mCoverage = false;
+    private boolean isCoverageEnabled = false;
     @Option(
             name = "library-to-load",
             description = "Name of the .so file under test"
     )
     private String libraryToLoad = "";
 
-    private String createRunAllTestsCommand() throws DeviceNotAvailableException {
+
+    /**
+     * Creates a temporary file on the host machine then push it to the device in a temporary
+     * location It is necessary to create a temp file for output for each instrumentation run and
+     * not module invocation. This is preferred over using
+     * {@link com.android.tradefed.targetprep.RunCommandTargetPreparer}
+     * because RunCommandTargetPreparer is only run once before the test invocation which leads to
+     * incorrect parsing as the retries will all use the same file for test result outputs.
+     */
+    @NonNull
+    private String createTempResultFileOnDevice() throws DeviceNotAvailableException {
+        File resultFile = null;
+        String deviceFileDestination;
+        try {
+            resultFile = FileUtil.createTempFile("gtest_results", ".txt");
+            deviceFileDestination = String.format("/data/local/tmp/%s", resultFile.getName());
+            getDevice().pushFile(resultFile, deviceFileDestination);
+            FileUtil.deleteFile(resultFile);
+        } catch (IOException e) {
+            throw new FailedChromiumGTestException(
+                    "Failed to create temp file for result on the device.", e);
+        } finally {
+            FileUtil.deleteFile(resultFile);
+        }
+        return deviceFileDestination;
+    }
+
+    /**
+     * This creates the gtest filter string which indicates which test should be run.
+     * Sometimes the gtest filter is long (> 500 character) which results in creating
+     * a temporary flag file and have gtest result the filter from the flag file.
+     *
+     * @return A gtest argument for flag file or --gtest_filter directly.
+     */
+    @NonNull
+    private String getGTestFilters() throws DeviceNotAvailableException {
+        StringBuilder filter = new StringBuilder();
+        if (!includeFilters.isEmpty() || !excludeFilters.isEmpty()) {
+            filter.append(GTEST_FLAG_FILTER);
+            filter.append('=');
+            Joiner joiner = Joiner.on(":").skipNulls();
+            if (!includeFilters.isEmpty()) {
+                joiner.appendTo(filter, includeFilters);
+            }
+            if (!excludeFilters.isEmpty()) {
+                filter.append("-");
+                joiner.appendTo(filter, excludeFilters);
+            }
+        }
+        String filterFlag = filter.toString();
+        // Handle long args
+        if (filterFlag.length() > 500) {
+            String tmpFlag = createFlagFileOnDevice(filterFlag);
+            return String.format("%s=%s", GTEST_FLAG_FILE, tmpFlag);
+        }
+        return filterFlag;
+    }
+
+    /**
+     * Helper method for getGTestFilters which creates a temporary flag file and push it to device.
+     *
+     * If it fails to create a file then it will directly use the filter in the adb command.
+     *
+     * @param filter the string to append to the flag file.
+     * @return path to the flag file on device or null if it could not be created.
+     */
+    @NonNull
+    private String createFlagFileOnDevice(@NonNull String filter)
+            throws DeviceNotAvailableException {
+        File tmpFlagFile = null;
+        String devicePath;
+        try {
+            tmpFlagFile = FileUtil.createTempFile("flagfile", ".txt");
+            FileUtil.writeToFile(filter, tmpFlagFile);
+            devicePath = String.format("/data/local/tmp/%s", tmpFlagFile.getName());
+            getDevice().pushFile(tmpFlagFile, devicePath);
+        } catch (IOException e) {
+            throw new FailedChromiumGTestException(
+                    "Failed to create temp file for gtest filter flag on the device.", e);
+        } finally {
+            FileUtil.deleteFile(tmpFlagFile);
+        }
+        return devicePath;
+    }
+
+    @NonNull
+    private String getAllGTestFlags() throws DeviceNotAvailableException {
+        String flags = String.format("%s %s", GTEST_FLAG_PRINT_TIME, getGTestFilters());
+        if (isCollectTestsOnly()) {
+            flags = String.format("%s %s", flags, GTEST_FLAG_LIST_TESTS);
+        }
+        return flags;
+    }
+
+    /**
+     * The flags all exist in Chromium's instrumentation APK
+     * {@link org.chromium.build.gtest_apk.NativeTestInstrumentationTestRunner} and
+     * {@link org.chromium.native_test.NativeTest}.
+     *
+     * The following is a brief explanation for each flag
+     * <ul>
+     * <li> NATIVE_TEST_ACTIVITY_KEY: Indicates the name of the activity which should be
+     * started by the instrumentation APK. This activity is responsible for executing gtests.
+     * <li> RUN_IN_SUBTHREAD_KEY: Whether to run the tests in the main-thread or a sub-thread.
+     * <li> EXTRA_SHARD_NANO_TIMEOUT_KEY: Shard timeout (Equal to the test timeout and not
+     * important as we only use a single shard).
+     * <li> LIBRARY_TO_LOAD_ACTIVITY_KEY: Name of the native library which has the code under
+     * test. System.LoadLibrary will be invoked on the value of this flag
+     * <li> STDOUT_FILE_KEY: Path to the file where stdout/stderr will be redirected to.</li>
+     * <li> COMMAND_LINE_FLAGS_KEY: Command line flags delegated to the gtest executor. This is
+     * mostly used for gtest flags
+     * <li> DUMP_COVERAGE_KEY: Flag used to indicate that the apk should not exit before dumping
+     * native coverage.
+     * </ul>
+     *
+     * @param resultFilePath path to a temporary file on the device which the gtest result will be
+     *                       directed to
+     * @return an instrumentation command that can be executed using adb shell am instrument.
+     */
+    @NonNull
+    private String createRunAllTestsCommand(@NonNull String resultFilePath)
+            throws DeviceNotAvailableException {
         InstrumentationCommandBuilder builder = new InstrumentationCommandBuilder(TEST_RUNNER)
                 .addArgument(NATIVE_TEST_ACTIVITY_KEY, NATIVE_UNIT_TEST_ACTIVITY_KEY)
                 .addArgument(RUN_IN_SUBTHREAD_KEY, "1")
-                .addArgument(EXTRA_SHARD_NANO_TIMEOUT_KEY, String.valueOf(testsTimeout.toNanos()))
+                .addArgument(EXTRA_SHARD_NANO_TIMEOUT_KEY, String.valueOf(TESTS_TIMEOUT.toNanos()))
                 .addArgument(LIBRARY_TO_LOAD_ACTIVITY_KEY, libraryToLoad)
-                .addArgument(STDOUT_FILE_KEY, GTEST_OUTPUT_PATH)
+                .addArgument(STDOUT_FILE_KEY, resultFilePath)
                 .addArgument(COMMAND_LINE_FLAGS_KEY,
-                        String.format("'%s'", getAllGTestFlags("")));
-        if (mCoverage) {
+                        String.format("'%s'", getAllGTestFlags()));
+        if (isCoverageEnabled) {
             builder.addArgument(DUMP_COVERAGE_KEY, "true");
         }
         return builder.build();
     }
 
-    private void printHostLogs(String cmd) {
+    /**
+     * Those logs can be found in host_log_%s.txt which is bundled with test execution.
+     *
+     * @param cmd Command used to instrumentation, this has all the flags which can help debugging
+     *            unusual behaviour.
+     */
+    private void printHostLogs(@NonNull String cmd) {
         LogUtil.CLog.i(String.format("[Cronet] Library to be loaded: %s\n", libraryToLoad));
         LogUtil.CLog.i(String.format("[Cronet] Command used to run gtests: adb shell %s\n", cmd));
-        LogUtil.CLog.i(String.format("[Cronet] Native-Coverage = %b", mCoverage));
+        LogUtil.CLog.i(String.format("[Cronet] Native-Coverage = %b", isCoverageEnabled));
     }
 
+    /**
+     * This is automatically invoked by the {@link com.android.tradefed.testtype.HostTest}.
+     *
+     * @param testInfo The {@link TestInformation} object containing useful information to run
+     *                 tests.
+     * @param listener the {@link ITestInvocationListener} of test results
+     */
     @Override
     public void run(TestInformation testInfo, ITestInvocationListener listener)
             throws DeviceNotAvailableException {
         if (Strings.isNullOrEmpty(libraryToLoad)) {
             throw new IllegalStateException("No library provided to be loaded.");
         }
-        String cmd = createRunAllTestsCommand();
+        String resultFilePath = createTempResultFileOnDevice();
+        String cmd = createRunAllTestsCommand(resultFilePath);
         printHostLogs(cmd);
         getDevice().executeShellCommand(CLEAR_CLANG_COVERAGE_FILES);
         ITestInvocationListener listenerWithTime = new TestListenerWithTime(
                 System.currentTimeMillis(), listener);
         getDevice().executeShellCommand(cmd, new CollectingOutputReceiver(),
-                testsTimeout.toMinutes(), TimeUnit.MINUTES, /* retryAttempts */ 1);
-        try {
-            parseAndReport(getDevice().pullFile(GTEST_OUTPUT_PATH), listenerWithTime);
-        } catch (IOException e) {
-            throw new FailedReportingException("Failed to parse and report test results",
-                    e.getCause());
-        }
+                /* maxTimeBeforeTimeOut */ TESTS_TIMEOUT.toMinutes(),
+                /* timeUnit */ TimeUnit.MINUTES,
+                /* retryAttempts */ 1);
+        parseAndReport(resultFilePath, listenerWithTime);
     }
 
-    private void parseAndReport(File testResultsOutput, ITestInvocationListener listener)
-            throws IOException {
-        if (testResultsOutput == null) {
-            throw new IOException(
-                    String.format("Failed to retrieve %s from device", GTEST_OUTPUT_PATH));
+    private void parseAndReport(@NonNull String resultFilePath,
+            @NonNull ITestInvocationListener listener) throws DeviceNotAvailableException {
+        File resultFile = device.pullFile(resultFilePath);
+        if (resultFile == null) {
+            throw new FailedChromiumGTestException(
+                    "Failed to retrieve gtest results file from device.");
         }
         // Loading all the lines is fine since this is done on the host-machine.
-        String[] lines = Files.readAllLines(testResultsOutput.toPath()).toArray(String[]::new);
+        String[] lines;
+        try {
+            lines = Files.readAllLines(resultFile.toPath()).toArray(String[]::new);
+        } catch (IOException e) {
+            throw new FailedChromiumGTestException(
+                    "Failed to read gtest results file on host machine.", e);
+        }
         MultiLineReceiver parser;
         // the parser automatically reports the test result back to the infra through the listener.
         if (isCollectTestsOnly()) {
@@ -124,4 +283,72 @@ public class ChromiumHostDrivenTest extends GTest {
         parser.processNewLines(lines);
         parser.done();
     }
+
+    // ------- Everything below is called by HostTest and should not be invoked manually -----
+    public boolean isCollectTestsOnly() {
+        return collectTestsOnly;
+    }
+
+    @Override
+    public void setCollectTestsOnly(boolean shouldCollectTest) {
+        collectTestsOnly = shouldCollectTest;
+    }
+
+    public String cleanFilter(String filter) {
+        return filter.replace('#', '.');
+    }
+
+    @Override
+    public void addIncludeFilter(String filter) {
+        includeFilters.add(cleanFilter(filter));
+    }
+
+    @Override
+    public void addAllIncludeFilters(Set<String> filters) {
+        for (String filter : filters) {
+            includeFilters.add(cleanFilter(filter));
+        }
+    }
+
+    @Override
+    public void addExcludeFilter(String filter) {
+        excludeFilters.add(cleanFilter(filter));
+    }
+
+    @Override
+    public void addAllExcludeFilters(Set<String> filters) {
+        for (String filter : filters) {
+            excludeFilters.add(cleanFilter(filter));
+        }
+    }
+
+    @Override
+    public void clearIncludeFilters() {
+        includeFilters.clear();
+    }
+
+    @Override
+    public Set<String> getIncludeFilters() {
+        return includeFilters;
+    }
+
+    @Override
+    public Set<String> getExcludeFilters() {
+        return excludeFilters;
+    }
+
+    @Override
+    public void clearExcludeFilters() {
+        excludeFilters.clear();
+    }
+
+    @Override
+    public ITestDevice getDevice() {
+        return device;
+    }
+
+    @Override
+    public void setDevice(ITestDevice device) {
+        this.device = device;
+    }
 }
\ No newline at end of file
diff --git a/test_runner/src/com.android.tests.chromium.host/FailedReportingException.java b/test_runner/src/com.android.tests.chromium.host/FailedChromiumGTestException.java
index f9f0fb319..50ce1634a 100644
--- a/test_runner/src/com.android.tests.chromium.host/FailedReportingException.java
+++ b/test_runner/src/com.android.tests.chromium.host/FailedChromiumGTestException.java
@@ -16,8 +16,12 @@
 
 package com.android.tests.chromium.host;
 
-public class FailedReportingException extends RuntimeException {
-    public FailedReportingException(String message, Throwable cause) {
+public class FailedChromiumGTestException extends RuntimeException {
+    public FailedChromiumGTestException(String message) {
+        super(message);
+    }
+
+    public FailedChromiumGTestException(String message, Throwable cause) {
         super(message, cause);
     }
 }