Merge "Added a ClassMetricRule." into sc-dev am: f544bd973b

Original change: https://googleplex-android-review.googlesource.com/c/platform/platform_testing/+/17490126

Change-Id: Iff17b6dc6e238b6a3d203b0208330f0d9b318272
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

6 files changed, 302 insertions, 10 deletions

diff --git a/libraries/device-collectors/src/main/java/android/device/collectors/BaseMetricListener.java b/libraries/device-collectors/src/main/java/android/device/collectors/BaseMetricListener.java
index fcd596354..0663325ea 100644
--- a/libraries/device-collectors/src/main/java/android/device/collectors/BaseMetricListener.java
+++ b/libraries/device-collectors/src/main/java/android/device/collectors/BaseMetricListener.java
@@ -101,6 +101,10 @@ public class BaseMetricListener extends InstrumentationRunListener {
     private int mCollectIterationInterval = 1;
     private int mSkipMetricUntilIteration = 0;
 
+    // Whether to report the results as instrumentation results. Used by metric collector rules,
+    // which do not have the information to invoke InstrumentationRunFinished() to report metrics.
+    private boolean mReportAsInstrumentationResults = false;
+
     public BaseMetricListener() {
         mIncludeFilters = new ArrayList<>();
         mExcludeFilters = new ArrayList<>();
@@ -190,9 +194,13 @@ public class BaseMetricListener extends InstrumentationRunListener {
             }
             if (mTestData.hasMetrics()) {
                 // Only send the status progress if there are metrics
+                if (mReportAsInstrumentationResults) {
+                    getInstrumentation().addResults(mTestData.createBundleFromMetrics());
+                } else {
                 SendToInstrumentation.sendBundle(getInstrumentation(),
                         mTestData.createBundleFromMetrics());
             }
+            }
         }
         super.testFinished(description);
     }
@@ -368,6 +376,11 @@ public class BaseMetricListener extends InstrumentationRunListener {
         }
     }
 
+    /** Sets whether metrics should be reported directly to instrumentation results. */
+    public final void setReportAsInstrumentationResults(boolean enabled) {
+        mReportAsInstrumentationResults = enabled;
+    }
+
     /**
      * Returns the name of the current class to be used as a logging tag.
      */
diff --git a/libraries/device-collectors/src/test/java/android/device/collectors/BaseMetricListenerInstrumentedTest.java b/libraries/device-collectors/src/test/java/android/device/collectors/BaseMetricListenerInstrumentedTest.java
index 5e73d15ad..c7fe14502 100644
--- a/libraries/device-collectors/src/test/java/android/device/collectors/BaseMetricListenerInstrumentedTest.java
+++ b/libraries/device-collectors/src/test/java/android/device/collectors/BaseMetricListenerInstrumentedTest.java
@@ -725,4 +725,34 @@ public class BaseMetricListenerInstrumentedTest {
         assertEquals(RUN_END_VALUE, resultBundle.getString(RUN_END_KEY));
         assertEquals(2, resultBundle.size());
     }
+
+    /** Test that the report as instrumentation result option works. */
+    @MetricOption(group = "testGroup")
+    @Test
+    public void testReportAsInstrumentationResultsIfEnabled() throws Exception {
+        mListener.setReportAsInstrumentationResults(true);
+
+        Description runDescription = Description.createSuiteDescription("run");
+        mListener.testRunStarted(runDescription);
+        Description testDescription = Description.createTestDescription("class", "method");
+        mListener.testStarted(testDescription);
+        mListener.testFinished(testDescription);
+        mListener.testRunFinished(new Result());
+        // AJUR runner is then gonna call instrumentationRunFinished
+        Bundle resultBundle = new Bundle();
+        mListener.instrumentationRunFinished(System.out, resultBundle, new Result());
+
+        // Check that results are reported via Instrumentation.addResults().
+        ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class);
+        Mockito.verify(mMockInstrumentation, Mockito.times(1)).addResults(capture.capture());
+        Bundle addedResult = capture.getValue();
+        assertTrue(addedResult.containsKey(TEST_END_KEY));
+        assertEquals(TEST_END_VALUE + "method", addedResult.getString(TEST_END_KEY));
+
+        // Rather than Instrumentation.sendStatus().
+        Mockito.verify(mMockInstrumentation, Mockito.never())
+                .sendStatus(
+                        Mockito.eq(SendToInstrumentation.INST_STATUS_IN_PROGRESS),
+                        Mockito.any(Bundle.class));
+    }
 }
diff --git a/libraries/health/rules/src/android/platform/test/rule/ClassMetricRule.java b/libraries/health/rules/src/android/platform/test/rule/ClassMetricRule.java
new file mode 100644
index 000000000..2a2ff5082
--- /dev/null
+++ b/libraries/health/rules/src/android/platform/test/rule/ClassMetricRule.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.platform.test.rule;
+
+import android.app.Instrumentation;
+import android.device.collectors.BaseMetricListener;
+import android.os.Bundle;
+import androidx.annotation.VisibleForTesting;
+import androidx.test.InstrumentationRegistry;
+
+/**
+ * A rule that collects class-level metrics using a supplied list of metric collectors.
+ *
+ * <p>The metric collectors are passed in using the "class-metric-collectors" option, and the rule
+ * works by invoking the correct test-level callbacks on them at the corresponding stages of the
+ * test lifecycle. The metric collectors must be subclasses of {@link BaseMetricListener}, and can
+ * be passed in by their fully qualified class name, or simple class name if they are under the
+ * {@code android.device.collectors} package (but not subpackages).
+ *
+ * <p>Multiple metric collectors are supported as comma-separated values, The order they are
+ * triggered follows this example: for {@code -e class-metric-collectors Collector1,Collector2}, the
+ * evaluation order would be {@code Collector1#testStarted()}, {@code Collector2#testStarted()}, the
+ * entire test class, {@code Collector1#testFinished()}, {@code Collector1#testFinished()}.
+ *
+ * <p>For {@code Microbenchmark}s, this rule can be dynamically injected either inside or outside
+ * hardcoded rules (see {@code Microbenchmark})'s JavaDoc).
+ *
+ * <p>Note that metrics collected from this rule are reported as run metrics. Therefore, there is
+ * the risk of metric key collision if a run contains multiple classes that report metrics under the
+ * same key. At the moment, it's the responsibility of the metric collector to prevent collision
+ * across test classes.
+ *
+ * <p>Exceptions from metric listeners are silently logged. This behavior is in accordance with the
+ * approach taken by {@link BaseMetricListener}.
+ */
+public class ClassMetricRule extends TestMetricRule {
+    @VisibleForTesting static final String METRIC_COLLECTORS_OPTION = "class-metric-collectors";
+
+    public ClassMetricRule() {
+        this(InstrumentationRegistry.getArguments(), InstrumentationRegistry.getInstrumentation());
+    }
+
+    @VisibleForTesting
+    ClassMetricRule(Bundle args, Instrumentation instrumentation) {
+        super(
+                args,
+                instrumentation,
+                METRIC_COLLECTORS_OPTION,
+                ClassMetricRule.class.getSimpleName());
+        for (BaseMetricListener listener : mMetricListeners) {
+            listener.setReportAsInstrumentationResults(true);
+        }
+    }
+}
diff --git a/libraries/health/rules/src/android/platform/test/rule/TestMetricRule.java b/libraries/health/rules/src/android/platform/test/rule/TestMetricRule.java
index e27389fd3..2098a4f75 100644
--- a/libraries/health/rules/src/android/platform/test/rule/TestMetricRule.java
+++ b/libraries/health/rules/src/android/platform/test/rule/TestMetricRule.java
@@ -15,6 +15,7 @@
  */
 package android.platform.test.rule;
 
+import android.app.Instrumentation;
 import android.device.collectors.BaseMetricListener;
 import android.os.Bundle;
 import android.util.Log;
@@ -50,12 +51,12 @@ import java.util.List;
  * approach taken by {@link BaseMetricListener}.
  */
 public class TestMetricRule extends TestWatcher {
-    private static final String LOG_TAG = TestMetricRule.class.getSimpleName();
-
     @VisibleForTesting static final String METRIC_COLLECTORS_OPTION = "test-metric-collectors";
     @VisibleForTesting static final String METRIC_COLLECTORS_PACKAGE = "android.device.collectors";
 
-    private List<BaseMetricListener> mMetricListeners = new ArrayList<>();
+    protected List<BaseMetricListener> mMetricListeners = new ArrayList<>();
+
+    private final String mLogTag;
 
     public TestMetricRule() {
         this(InstrumentationRegistry.getArguments());
@@ -63,8 +64,25 @@ public class TestMetricRule extends TestWatcher {
 
     @VisibleForTesting
     TestMetricRule(Bundle args) {
+        this(
+                args,
+                InstrumentationRegistry.getInstrumentation(),
+                METRIC_COLLECTORS_OPTION,
+                TestMetricRule.class.getSimpleName());
+    }
+
+    /**
+     * A constructor that allows subclasses to change out various components used at initialization
+     * time.
+     */
+    protected TestMetricRule(
+            Bundle args,
+            Instrumentation instrumentation,
+            String collectorsOptionName,
+            String logTag) {
+        mLogTag = logTag;
         List<String> listenerNames =
-                Arrays.asList(args.getString(METRIC_COLLECTORS_OPTION, "").split(","));
+                Arrays.asList(args.getString(collectorsOptionName, "").split(","));
         for (String listenerName : listenerNames) {
             if (listenerName.isEmpty()) {
                 continue;
@@ -73,7 +91,7 @@ public class TestMetricRule extends TestWatcher {
             // We could use a regex here, but this is simpler and should work just as well.
             if (listenerName.contains(".")) {
                 Log.i(
-                        LOG_TAG,
+                        mLogTag,
                         String.format(
                                 "Attempting to dynamically load metric collector with fully "
                                         + "qualified name %s.",
@@ -91,7 +109,7 @@ public class TestMetricRule extends TestWatcher {
             } else {
                 String fullName = String.format("%s.%s", METRIC_COLLECTORS_PACKAGE, listenerName);
                 Log.i(
-                        LOG_TAG,
+                        mLogTag,
                         String.format(
                                 "Attempting to dynamically load metric collector with simple class "
                                         + "name %s (fully qualified name: %s).",
@@ -111,7 +129,7 @@ public class TestMetricRule extends TestWatcher {
         }
         // Initialize each listener.
         for (BaseMetricListener listener : mMetricListeners) {
-            listener.setInstrumentation(InstrumentationRegistry.getInstrumentation());
+            listener.setInstrumentation(instrumentation);
         }
     }
 
@@ -125,7 +143,7 @@ public class TestMetricRule extends TestWatcher {
                 listener.testStarted(description);
             } catch (Exception e) {
                 Log.e(
-                        LOG_TAG,
+                        mLogTag,
                         String.format(
                                 "Exception from listener %s during starting().",
                                 listener.getClass().getCanonicalName()),
@@ -141,7 +159,7 @@ public class TestMetricRule extends TestWatcher {
                 listener.testFinished(description);
             } catch (Exception e) {
                 Log.e(
-                        LOG_TAG,
+                        mLogTag,
                         String.format(
                                 "Exception from listener %s during finished().",
                                 listener.getClass().getCanonicalName()),
@@ -161,7 +179,7 @@ public class TestMetricRule extends TestWatcher {
                 listener.testFailure(failure);
             } catch (Exception e) {
                 Log.e(
-                        LOG_TAG,
+                        mLogTag,
                         String.format(
                                 "Exception from listener %s during failed().",
                                 listener.getClass().getCanonicalName()),
diff --git a/libraries/health/rules/tests/src/android/platform/test/rule/ClassMetricRuleTest.java b/libraries/health/rules/tests/src/android/platform/test/rule/ClassMetricRuleTest.java
new file mode 100644
index 000000000..ec93a37f8
--- /dev/null
+++ b/libraries/health/rules/tests/src/android/platform/test/rule/ClassMetricRuleTest.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.platform.test.rule;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+import android.app.Instrumentation;
+import android.device.collectors.BaseMetricListener;
+import android.device.collectors.DataRecord;
+import android.os.Bundle;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.Description;
+import org.junit.runner.Result;
+import org.junit.runners.model.Statement;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+
+import java.util.List;
+
+/**
+ * Tests for {@link ClassMetricRule}.
+ *
+ * <p>This test will focus on testing that collectors are loaded with the correct argument, and that
+ * they are reporting their results as run metrics. All the other logic has been tested in {@link
+ * TestMetricRuleTest}.
+ */
+public class ClassMetricRuleTest {
+
+    private static final Description DESCRIPTION =
+            Description.createTestDescription("class", "method");
+
+    private static final Statement TEST_STATEMENT =
+            new Statement() {
+                @Override
+                public void evaluate() {}
+            };
+
+    @Mock private Instrumentation mMockInstrumentation;
+
+    @Captor private ArgumentCaptor<Bundle> addResultsCaptor;
+
+    @Before
+    public void setUp() {
+        initMocks(this);
+    }
+
+    @Test
+    public void testRunsSpecifiedCollectorsAndReportRunMetrics() throws Throwable {
+        ClassMetricRule rule =
+                createWithMetricCollectorNames(
+                        "android.platform.test.rule.ClassMetricRuleTest$TestableCollector2",
+                        "android.platform.test.rule.ClassMetricRuleTest$TestableCollector1");
+        rule.apply(TEST_STATEMENT, DESCRIPTION).evaluate();
+
+        // We have two metric collectors, hence results are reported two times.
+        verify(mMockInstrumentation, times(2)).addResults(addResultsCaptor.capture());
+        List<Bundle> results = addResultsCaptor.getAllValues();
+        boolean hasCollector1 = false, hasCollector2 = false;
+        for (Bundle result : results) {
+            if (result.containsKey("TestableCollector1-test")) {
+                hasCollector1 = true;
+            } else if (result.containsKey("TestableCollector2-test")) {
+                hasCollector2 = true;
+            }
+        }
+        assertTrue(hasCollector1);
+        assertTrue(hasCollector2);
+    }
+
+    @Test
+    public void testUsesTestCallbackRatherThanRunCallback() throws Throwable {
+        ClassMetricRule rule =
+                createWithMetricCollectorNames(
+                        "android.platform.test.rule.ClassMetricRuleTest$TestableCollector1");
+        rule.apply(TEST_STATEMENT, DESCRIPTION).evaluate();
+
+        // We have one metric collector, hence results are reported a single time.
+        verify(mMockInstrumentation, times(1)).addResults(addResultsCaptor.capture());
+        Bundle result = addResultsCaptor.getValue();
+        assertTrue(result.containsKey("TestableCollector1-test"));
+        assertFalse(result.containsKey("TestableCollector1-run"));
+    }
+
+    private ClassMetricRule createWithMetricCollectorNames(String... names) {
+        Bundle args = new Bundle();
+        args.putString(ClassMetricRule.METRIC_COLLECTORS_OPTION, String.join(",", names));
+
+        return new ClassMetricRule(args, mMockInstrumentation);
+    }
+
+    public static class BaseTestableCollector extends BaseMetricListener {
+        private final String mName;
+
+        public BaseTestableCollector(String name) {
+            mName = name;
+        }
+
+        @Override
+        public void onTestEnd(DataRecord testData, Description description) {
+            testData.addStringMetric(mName + "-test", "value");
+        }
+
+        // This method should never be used by the rule.
+        @Override
+        public void onTestRunEnd(DataRecord runData, Result result) {
+            runData.addStringMetric(mName + "-run", "value");
+        }
+    }
+
+    public static class TestableCollector1 extends BaseTestableCollector {
+        public TestableCollector1() {
+            super(TestableCollector1.class.getSimpleName());
+        }
+    }
+
+    public static class TestableCollector2 extends BaseTestableCollector {
+        public TestableCollector2() {
+            super(TestableCollector2.class.getSimpleName());
+        }
+    }
+}
diff --git a/libraries/health/rules/tests/src/android/platform/test/rule/TestMetricRuleTest.java b/libraries/health/rules/tests/src/android/platform/test/rule/TestMetricRuleTest.java
index 4f16e922f..506cc7e68 100644
--- a/libraries/health/rules/tests/src/android/platform/test/rule/TestMetricRuleTest.java
+++ b/libraries/health/rules/tests/src/android/platform/test/rule/TestMetricRuleTest.java
@@ -171,6 +171,29 @@ public class TestMetricRuleTest {
         TestMetricRule rule = createWithMetricCollectorNames(simpleName);
     }
 
+    @Test
+    public void testInitWithDifferentOptionName() throws Throwable {
+        String optionName = "another-" + TestMetricRule.METRIC_COLLECTORS_OPTION;
+
+        Bundle args = new Bundle();
+        args.putString(
+                optionName, "android.platform.test.rule.TestMetricRuleTest$TestableCollector1");
+        TestMetricRule rule =
+                new TestMetricRule(args, new Instrumentation(), optionName, "log tag");
+
+        rule.apply(PASSING_STATEMENT, DESCRIPTION).evaluate();
+        assertThat(sLogs)
+                .containsExactly(
+                        "TestableCollector1#setInstrumentation",
+                        "TestableCollector1#setupAdditionalArgs",
+                        "TestableCollector1#onSetUp",
+                        String.format("Test %s: TestableCollector1#onTestStart", DESCRIPTION),
+                        "Test execution",
+                        String.format("Test %s: TestableCollector1#onTestEnd", DESCRIPTION),
+                        "TestableCollector1#onCleanUp")
+                .inOrder();
+    }
+
     private TestMetricRule createWithMetricCollectorNames(String... names) {
         Bundle args = new Bundle();
         args.putString(TestMetricRule.METRIC_COLLECTORS_OPTION, String.join(",", names));