diff options
author | Harry Zhang <harrytczhang@google.com> | 2021-08-19 21:10:56 +0000 |
---|---|---|
committer | Harry Zhang <harrytczhang@google.com> | 2021-09-30 23:33:31 +0000 |
commit | 118cd5eb5618b149d9c382940b3fd2078294a149 (patch) | |
tree | 2a3bb6153b4f6ff6894428f4c74ea5e12644a516 | |
parent | 91cb7474d600cf8b1c357c7ff01113cc1f49682b (diff) | |
download | platform_testing-118cd5eb5618b149d9c382940b3fd2078294a149.tar.gz |
Enable runtime test-level rule injection in microbenchmarks.
Original change: https://android-review.googlesource.com/c/platform/platform_testing/+/1798452
Bug: 123281375
Test: atest MicrobenchmarkRunnerTests
Change-Id: I3ec2e773bb42be96b730f39051319f257acad7cb
(cherry picked from commit f7b90f410d786c1f338cb5ac0b480bc1b85c488f)
Merged-In: Iffc26c6c4dfb40325bff5946d96a8577bcb19905
Merged-In: Ie725d34c7dcdbbeb0be4e772adf61e98f10ba103
Merged-In: I3ec2e773bb42be96b730f39051319f257acad7cb
2 files changed, 114 insertions, 5 deletions
diff --git a/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java b/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java index e82fd3e18..b6c6f134f 100644 --- a/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java +++ b/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java @@ -23,6 +23,7 @@ import android.os.BatteryManager; import android.os.Bundle; import android.os.SystemClock; import android.platform.test.composer.Iterate; +import android.platform.test.rule.DynamicRuleChain; import android.platform.test.rule.TracePointRule; import android.util.Log; import androidx.annotation.VisibleForTesting; @@ -32,6 +33,7 @@ import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -71,6 +73,10 @@ public class Microbenchmark extends BlockJUnit4ClassRunner { }; @VisibleForTesting static final String MIN_BATTERY_LEVEL_OPTION = "min-battery"; @VisibleForTesting static final String MAX_BATTERY_DRAIN_OPTION = "max-battery-drain"; + // Use these options to inject rules at runtime via the command line. For details, please see + // documentation for DynamicRuleChain. + @VisibleForTesting static final String DYNAMIC_OUTER_RULES_OPTION = "outer-rules"; + @VisibleForTesting static final String DYNAMIC_INNER_RULES_OPTION = "inner-rules"; // Options for aligning with the battery charge (coulomb) counter for power tests. We want to // start microbenchmarks just after the coulomb counter has decremented to account for the @@ -275,7 +281,11 @@ public class Microbenchmark extends BlockJUnit4ClassRunner { /** Re-implement the private rules wrapper from {@link BlockJUnit4ClassRunner} in JUnit 4.12. */ private Statement withRules(FrameworkMethod method, Object target, Statement statement) { Statement result = statement; - List<TestRule> testRules = getTestRules(target); + List<TestRule> testRules = new ArrayList<>(); + // Inner dynamic rules should be included first because RunRules applies rules inside-out. + testRules.add(new DynamicRuleChain(DYNAMIC_INNER_RULES_OPTION, mArguments)); + testRules.addAll(getTestRules(target)); + testRules.add(new DynamicRuleChain(DYNAMIC_OUTER_RULES_OPTION, mArguments)); // Apply legacy MethodRules, if they don't overlap with TestRules. for (org.junit.rules.MethodRule each : rules(target)) { if (!testRules.contains(each)) { @@ -283,10 +293,7 @@ public class Microbenchmark extends BlockJUnit4ClassRunner { } } // Apply modern, method-level TestRules in outer statements. - result = - testRules.isEmpty() - ? statement - : new RunRules(result, testRules, describeChild(method)); + result = new RunRules(result, testRules, describeChild(method)); return result; } diff --git a/libraries/health/runners/microbenchmark/tests/src/android/platform/test/microbenchmark/MicrobenchmarkTest.java b/libraries/health/runners/microbenchmark/tests/src/android/platform/test/microbenchmark/MicrobenchmarkTest.java index 8d4cd822a..0b44d1aac 100644 --- a/libraries/health/runners/microbenchmark/tests/src/android/platform/test/microbenchmark/MicrobenchmarkTest.java +++ b/libraries/health/runners/microbenchmark/tests/src/android/platform/test/microbenchmark/MicrobenchmarkTest.java @@ -25,10 +25,12 @@ import static org.mockito.Mockito.verify; import android.os.Bundle; import android.os.SystemClock; import android.platform.test.microbenchmark.Microbenchmark.TerminateEarlyException; +import android.platform.test.rule.TestWatcher; import android.platform.test.rule.TracePointRule; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; import org.junit.runner.Description; @@ -51,6 +53,15 @@ import java.util.List; */ @RunWith(JUnit4.class) public final class MicrobenchmarkTest { + // Static logs are needed to validate dynamic rules, which are instantiated reflectively and + // cannot access non-static variables. + private static List<String> sLogs = new ArrayList<>(); + + @Before + public void setUp() { + sLogs.clear(); + } + /** * Tests that iterations are respected for microbenchmark tests. */ @@ -408,6 +419,57 @@ public final class MicrobenchmarkTest { } /** + * Test successive iteration will be executed when the terminate on test fail option is + * disabled. + */ + @Test + public void testDynamicRuleInjection() throws InitializationError { + Bundle args = new Bundle(); + args.putString("iterations", "2"); + args.putString("rename-iterations", "false"); + args.putString("terminate-on-test-fail", "false"); + args.putString(Microbenchmark.DYNAMIC_INNER_RULES_OPTION, LoggingRule1.class.getName()); + args.putString(Microbenchmark.DYNAMIC_OUTER_RULES_OPTION, LoggingRule2.class.getName()); + LoggingMicrobenchmark loggingRunner = + new LoggingMicrobenchmark(LoggingTestWithRule.class, args); + loggingRunner.setOperationLog(sLogs); + new JUnitCore().run(loggingRunner); + assertThat(sLogs) + .containsExactly( + "logging rule 2 starting", + "hardcoded rule starting", + "logging rule 1 starting", + "before", + "tight before", + "begin: testMethod(" + + "android.platform.test.microbenchmark.MicrobenchmarkTest" + + "$LoggingTestWithRule)", + "test", + "end", + "tight after", + "after", + "logging rule 1 finished", + "hardcoded rule finished", + "logging rule 2 finished", + "logging rule 2 starting", + "hardcoded rule starting", + "logging rule 1 starting", + "before", + "tight before", + "begin: testMethod(" + + "android.platform.test.microbenchmark.MicrobenchmarkTest" + + "$LoggingTestWithRule)", + "test", + "end", + "tight after", + "after", + "logging rule 1 finished", + "hardcoded rule finished", + "logging rule 2 finished") + .inOrder(); + } + + /** * An extensions of the {@link Microbenchmark} runner that logs the start and end of collecting * traces. It also passes the operation log to the provided test {@code Class}, if it is a * {@link LoggingTest}. This is used for ensuring the proper order for evaluating test {@link @@ -503,10 +565,50 @@ public final class MicrobenchmarkTest { } } + public static class LoggingTestWithRule extends LoggingTest { + @Rule + public TestRule hardCodedRule = + new TestWatcher() { + @Override + public void starting(Description description) { + sLogs.add("hardcoded rule starting"); + } + + @Override + public void finished(Description description) { + sLogs.add("hardcoded rule finished"); + } + }; + } + public static class LoggingFailedTest extends LoggingTest { @Test public void testMethod() { throw new RuntimeException("I failed."); } } + + public static class LoggingRule1 extends TestWatcher { + @Override + public void starting(Description description) { + sLogs.add("logging rule 1 starting"); + } + + @Override + public void finished(Description description) { + sLogs.add("logging rule 1 finished"); + } + } + + public static class LoggingRule2 extends TestWatcher { + @Override + public void starting(Description description) { + sLogs.add("logging rule 2 starting"); + } + + @Override + public void finished(Description description) { + sLogs.add("logging rule 2 finished"); + } + } } |