diff options
author | TreeHugger Robot <treehugger-gerrit@google.com> | 2020-07-17 17:55:22 +0000 |
---|---|---|
committer | Android (Google) Code Review <android-gerrit@google.com> | 2020-07-17 17:55:22 +0000 |
commit | 335f8d2905e16e80aaae75711bc5e214378b9b60 (patch) | |
tree | 54a3de2e7f0bc69fabe4b303681badea780e334b | |
parent | 5033a0df336111af1818f7d490d6409d6e521873 (diff) | |
parent | 4a8c67c7346b57174950281da59d3cd56a3a6175 (diff) | |
download | platform_testing-335f8d2905e16e80aaae75711bc5e214378b9b60.tar.gz |
Merge "Add @NoMetricBefore and @NoMetricAfter annotations." into rvc-dev
3 files changed, 177 insertions, 5 deletions
diff --git a/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java b/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java index 445999b29..78a03e8d8 100644 --- a/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java +++ b/libraries/health/runners/microbenchmark/src/android/platform/test/microbenchmark/Microbenchmark.java @@ -21,7 +21,6 @@ import android.platform.test.rule.TracePointRule; import androidx.annotation.VisibleForTesting; import androidx.test.InstrumentationRegistry; - import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -31,6 +30,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.junit.internal.AssumptionViolatedException; +import org.junit.internal.runners.model.EachTestNotifier; +import org.junit.internal.runners.model.ReflectiveCallable; +import org.junit.internal.runners.statements.RunAfters; import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runner.notification.RunNotifier; @@ -38,6 +41,7 @@ import org.junit.runners.BlockJUnit4ClassRunner; import org.junit.runners.model.FrameworkMethod; import org.junit.runners.model.InitializationError; import org.junit.runners.model.Statement; +import org.junit.rules.RunRules; /** * The {@code Microbenchmark} runner allows you to run test methods repeatedly and with {@link @@ -51,6 +55,11 @@ public class Microbenchmark extends BlockJUnit4ClassRunner { // A constant to indicate that the iteration number is not set. @VisibleForTesting static final int ITERATION_NOT_SET = -1; public static final String RENAME_ITERATION_OPTION = "rename-iterations"; + private static final Statement EMPTY = + new Statement() { + @Override + public void evaluate() throws Throwable {} + }; private final String mIterationSep; private final Bundle mArguments; @@ -153,6 +162,25 @@ public class Microbenchmark extends BlockJUnit4ClassRunner { public @interface TightMethodRule {} /** + * A temporary annotation that acts like the {@code @Before} but is excluded from metric + * collection. + * + * <p>This should be removed as soon as possible. Do not use this unless explicitly instructed + * to do so. You'll regret it! + * + * <p>Note that all {@code TestOption}s must be instantiated as {@code @ClassRule}s to work + * inside these annotations. + */ + @Retention(RetentionPolicy.RUNTIME) + @Target({ElementType.FIELD, ElementType.METHOD}) + public @interface NoMetricBefore {} + + /** A temporary annotation, same as the above, but for replacing {@code @After} methods. */ + @Retention(RetentionPolicy.RUNTIME) + @Target({ElementType.FIELD, ElementType.METHOD}) + public @interface NoMetricAfter {} + + /** * Rename the child class name to add iterations if the renaming iteration option is enabled. * * <p>Renaming the class here is chosen over renaming the method name because @@ -173,17 +201,100 @@ public class Microbenchmark extends BlockJUnit4ClassRunner { String.valueOf(mIterations.get(original))), original.getMethodName()); } + /** Re-implement the private rules wrapper from {@link BlockJUnit4ClassRunner} in JUnit 4.12. */ + private Statement withRules(FrameworkMethod method, Object target, Statement statement) { + Statement result = statement; + List<TestRule> testRules = getTestRules(target); + // Apply legacy MethodRules, if they don't overlap with TestRules. + for (org.junit.rules.MethodRule each : rules(target)) { + if (!testRules.contains(each)) { + result = each.apply(result, method, target); + } + } + // Apply modern, method-level TestRules in outer statements. + result = + testRules.isEmpty() + ? statement + : new RunRules(result, testRules, describeChild(method)); + return result; + } + /** - * Keep track of the number of iterations for a particular method and - * set the current iteration count for changing the current description. + * Combine the {@code #runChild}, {@code #methodBlock}, and final {@code #runLeaf} methods to + * implement the specific {@code Microbenchmark} test behavior. In particular, (1) keep track of + * the number of iterations for a particular method description, and (2) run {@code + * NoMetricBefore} and {@code NoMetricAfter} methods outside of the {@code RunListener} test + * wrapping methods. */ @Override protected void runChild(final FrameworkMethod method, RunNotifier notifier) { + // Update the number of iterations this method has been run. if (mRenameIterations) { Description original = super.describeChild(method); mIterations.computeIfPresent(original, (k, v) -> v + 1); mIterations.computeIfAbsent(original, k -> 1); } - super.runChild(method, notifier); + + Description description = describeChild(method); + if (isIgnored(method)) { + notifier.fireTestIgnored(description); + } else { + EachTestNotifier eachNotifier = new EachTestNotifier(notifier, description); + + Object test; + try { + // Fail fast if the test is not successfully created. + test = + new ReflectiveCallable() { + @Override + protected Object runReflectiveCall() throws Throwable { + return createTest(); + } + }.run(); + + // Run {@code NoMetricBefore} methods first. Fail fast if they fail. + for (FrameworkMethod noMetricBefore : + getTestClass().getAnnotatedMethods(NoMetricBefore.class)) { + noMetricBefore.invokeExplosively(test); + } + } catch (Throwable e) { + eachNotifier.fireTestStarted(); + eachNotifier.addFailure(e); + eachNotifier.fireTestFinished(); + return; + } + + Statement statement = methodInvoker(method, test); + statement = possiblyExpectingExceptions(method, test, statement); + statement = withPotentialTimeout(method, test, statement); + statement = withBefores(method, test, statement); + statement = withAfters(method, test, statement); + statement = withRules(method, test, statement); + + // Fire test events from inside to exclude "no metric" methods. + eachNotifier.fireTestStarted(); + try { + statement.evaluate(); + } catch (AssumptionViolatedException e) { + eachNotifier.addFailedAssumption(e); + } catch (Throwable e) { + eachNotifier.addFailure(e); + } finally { + eachNotifier.fireTestFinished(); + } + + try { + // Run {@code NoMetricAfter} methods last, reporting all errors. + List<FrameworkMethod> afters = + getTestClass().getAnnotatedMethods(NoMetricAfter.class); + if (!afters.isEmpty()) { + new RunAfters(EMPTY, afters, test).evaluate(); + } + } catch (AssumptionViolatedException e) { + eachNotifier.addFailedAssumption(e); + } catch (Throwable e) { + eachNotifier.addFailure(e); + } + } } } diff --git a/tests/health/scenarios/src/android/platform/test/scenario/sample/SampleTest.java b/tests/health/scenarios/src/android/platform/test/scenario/sample/SampleTest.java index 01b904def..03e8a0725 100644 --- a/tests/health/scenarios/src/android/platform/test/scenario/sample/SampleTest.java +++ b/tests/health/scenarios/src/android/platform/test/scenario/sample/SampleTest.java @@ -17,6 +17,7 @@ package android.platform.test.scenario.sample; import android.util.Log; +import android.platform.test.option.BooleanOption; import android.platform.test.rule.TestWatcher; import android.platform.test.scenario.annotation.Scenario; @@ -57,31 +58,65 @@ public class SampleTest { .around(new PrintRule("@Rule #2")) .around(new PrintRule("@Rule #3")); + @ClassRule + public static BooleanOption failBeforeClass = + new BooleanOption("fail-before-class").setRequired(false).setDefault(false); + + @Rule + public BooleanOption failBefore = + new BooleanOption("fail-before").setRequired(false).setDefault(false); + + @Rule + public BooleanOption failTest = + new BooleanOption("fail-test").setRequired(false).setDefault(false); + + @Rule + public BooleanOption failAfter = + new BooleanOption("fail-after").setRequired(false).setDefault(false); + + @ClassRule + public static BooleanOption failAfterClass = + new BooleanOption("fail-after-class").setRequired(false).setDefault(false); + @BeforeClass public static void beforeClassMethod() { + failIfRequested(failBeforeClass, "@BeforeClass"); Log.d(LOG_TAG, "@BeforeClass"); } @Before public void beforeMethod() { + failIfRequested(failBefore, "@Before"); Log.d(LOG_TAG, "@Before"); } @Test public void testMethod() { + failIfRequested(failTest, "@Test"); Log.d(LOG_TAG, "@Test"); } @After public void afterMethod() { + failIfRequested(failAfter, "@After"); Log.d(LOG_TAG, "@After"); } @AfterClass public static void afterClassMethod() { + failIfRequested(failAfterClass, "@AfterClass"); Log.d(LOG_TAG, "@AfterClass"); } + /** Log and throw a failure if the provided {@code option} is set. */ + public static void failIfRequested(BooleanOption option, String location) { + if (option.get()) { + String message = String.format("Failed %s", location); + Log.d(LOG_TAG, message); + throw new RuntimeException(message); + } + } + /** A {@link TestWatcher} that prints the methods it executes. */ private static class PrintRule extends TestWatcher { diff --git a/tests/health/scenarios/tests/src/android/platform/test/scenario/sample/SampleMicrobenchmark.java b/tests/health/scenarios/tests/src/android/platform/test/scenario/sample/SampleMicrobenchmark.java index 0f93256c7..60ab12862 100644 --- a/tests/health/scenarios/tests/src/android/platform/test/scenario/sample/SampleMicrobenchmark.java +++ b/tests/health/scenarios/tests/src/android/platform/test/scenario/sample/SampleMicrobenchmark.java @@ -17,7 +17,12 @@ package android.platform.test.scenario.sample; import android.platform.test.microbenchmark.Microbenchmark; +import android.platform.test.microbenchmark.Microbenchmark.NoMetricAfter; +import android.platform.test.microbenchmark.Microbenchmark.NoMetricBefore; +import android.platform.test.option.BooleanOption; +import android.util.Log; +import org.junit.ClassRule; import org.junit.runner.RunWith; /** @@ -26,4 +31,25 @@ import org.junit.runner.RunWith; * <p>Run this test with the listener alongside, {@link PrintListener}, to see how they interact. */ @RunWith(Microbenchmark.class) -public class SampleMicrobenchmark extends SampleTest {} +public class SampleMicrobenchmark extends SampleTest { + + @ClassRule + public static BooleanOption failNoMetricBefore = + new BooleanOption("fail-no-metric-before").setRequired(false).setDefault(false); + + @ClassRule + public static BooleanOption failNoMetricAfter = + new BooleanOption("fail-no-metric-after").setRequired(false).setDefault(false); + + @NoMetricBefore + public void noMetricBefore() { + SampleTest.failIfRequested(failNoMetricBefore, "@NoMetricBefore"); + Log.d(SampleTest.LOG_TAG, "@NoMetricBefore"); + } + + @NoMetricAfter + public void noMetricAfter() { + SampleTest.failIfRequested(failNoMetricAfter, "@NoMetricAfter"); + Log.d(SampleTest.LOG_TAG, "@NoMetricAfter"); + } +} |