junit: Use experimental mutator in regression tests

2 files changed, 151 insertions, 0 deletions

diff --git a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
index 8795bbce..addecffe 100644
--- a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
+++ b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
@@ -242,3 +242,28 @@ java_test(
         "@maven//:org_junit_platform_junit_platform_testkit",
     ],
 )
+
+[
+    java_test(
+        name = "MutatorTest" + JAZZER_FUZZ,
+        srcs = ["MutatorTest.java"],
+        env = {
+            "JAZZER_FUZZ": JAZZER_FUZZ,
+        },
+        test_class = "com.code_intelligence.jazzer.junit.MutatorTest",
+        runtime_deps = [
+            "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar",
+            "@maven//:org_junit_jupiter_junit_jupiter_engine",
+        ],
+        deps = [
+            "//src/main/java/com/code_intelligence/jazzer/api:hooks",
+            "@maven//:junit_junit",
+            "@maven//:org_junit_platform_junit_platform_engine",
+            "@maven//:org_junit_platform_junit_platform_testkit",
+        ],
+    )
+    for JAZZER_FUZZ in [
+        "",
+        "_fuzzing",
+    ]
+]
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java b/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java
new file mode 100644
index 00000000..863c57f4
--- /dev/null
+++ b/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java
@@ -0,0 +1,126 @@
+// Copyright 2022 Code Intelligence GmbH
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.code_intelligence.jazzer.junit;
+
+import static org.junit.Assume.assumeFalse;
+import static org.junit.Assume.assumeTrue;
+import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
+import static org.junit.platform.testkit.engine.EventConditions.container;
+import static org.junit.platform.testkit.engine.EventConditions.displayName;
+import static org.junit.platform.testkit.engine.EventConditions.event;
+import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully;
+import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure;
+import static org.junit.platform.testkit.engine.EventConditions.test;
+import static org.junit.platform.testkit.engine.EventConditions.type;
+import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings;
+import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED;
+import static org.junit.platform.testkit.engine.EventType.FINISHED;
+import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED;
+import static org.junit.platform.testkit.engine.EventType.STARTED;
+import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.platform.testkit.engine.EngineExecutionResults;
+import org.junit.platform.testkit.engine.EngineTestKit;
+import org.junit.rules.TemporaryFolder;
+
+public class MutatorTest {
+  private static final String ENGINE = "engine:junit-jupiter";
+  private static final String CLASS_NAME = "com.example.MutatorFuzzTest";
+  private static final String CLAZZ = "class:" + CLASS_NAME;
+  private static final String LIFECYCLE_FUZZ = "test-template:mutatorFuzz(java.util.List)";
+  private static final String INVOCATION1 = "test-template-invocation:#1";
+  private static final String INVOCATION2 = "test-template-invocation:#2";
+
+  @Rule public TemporaryFolder temp = new TemporaryFolder();
+  private Path baseDir;
+
+  @Before
+  public void setup() throws IOException {
+    baseDir = temp.getRoot().toPath();
+    Path inputsDirectory = baseDir.resolve(
+        Paths.get("src", "test", "resources", "com", "example", "MutatorFuzzTestInputs"));
+    Files.createDirectories(inputsDirectory);
+    Files.write(inputsDirectory.resolve("invalid"), "invalid input".getBytes());
+  }
+
+  private EngineExecutionResults executeTests() {
+    System.setProperty("jazzer.experimental_mutator", "true");
+    return EngineTestKit.engine("junit-jupiter")
+        .selectors(selectClass(CLASS_NAME))
+        .configurationParameter("jazzer.instrument", "com.example.**")
+        .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString())
+        .execute();
+  }
+
+  @Test
+  public void fuzzingEnabled() {
+    assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty());
+
+    EngineExecutionResults results = executeTests();
+
+    results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)),
+        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
+        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)),
+            finishedSuccessfully()),
+        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
+        event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
+
+    results.testEvents().assertEventsMatchExactly(
+        event(
+            type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+            displayName("Fuzzing...")),
+        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+            displayName("Fuzzing..."), finishedWithFailure(instanceOf(AssertionError.class))));
+  }
+
+  @Test
+  public void fuzzingDisabled() {
+    assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty());
+
+    EngineExecutionResults results = executeTests();
+
+    results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)),
+        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
+        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+        event(type(REPORTING_ENTRY_PUBLISHED),
+            container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))),
+        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
+        event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
+
+    results.testEvents().assertEventsMatchExactly(
+        event(type(DYNAMIC_TEST_REGISTERED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1))),
+        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+            displayName("<empty input>")),
+        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION1)),
+            displayName("<empty input>"), finishedSuccessfully()),
+        event(type(DYNAMIC_TEST_REGISTERED),
+            test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2))),
+        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2)),
+            displayName("invalid")),
+        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION2)),
+            displayName("invalid"), finishedSuccessfully()));
+  }
+}