Honor explicitly stated corpus directory

The JUnit integration always created corpus files in .cifuzz-corpus
subdirectories, even if an explicit corpus directory was stated via
command line. Now explicit corpus parameters are honored.

5 files changed, 275 insertions, 1 deletions

diff --git a/examples/junit/src/test/java/com/example/CorpusDirectoryFuzzTest.java b/examples/junit/src/test/java/com/example/CorpusDirectoryFuzzTest.java
new file mode 100644
index 00000000..465c94cf
--- /dev/null
+++ b/examples/junit/src/test/java/com/example/CorpusDirectoryFuzzTest.java
@@ -0,0 +1,45 @@
+// Copyright 2023 Code Intelligence GmbH
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.example;
+
+import com.code_intelligence.jazzer.api.FuzzedDataProvider;
+import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium;
+import com.code_intelligence.jazzer.junit.FuzzTest;
+
+public class CorpusDirectoryFuzzTest {
+  private static int invocations = 0;
+
+  @FuzzTest(maxDuration = "5s")
+  public void corpusDirectoryFuzz(FuzzedDataProvider data) {
+    // Throw on the third invocation to generate corpus entries.
+    if (data.remainingBytes() == 0) {
+      return;
+    }
+    // Add a few branch statements to generate different coverage.
+    switch (invocations) {
+      case 0:
+        invocations++;
+        break;
+      case 1:
+        invocations++;
+        break;
+      case 2:
+        invocations++;
+        break;
+      case 3:
+        throw new FuzzerSecurityIssueMedium();
+    }
+  }
+}
diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java
index e20f511a..49252e84 100644
--- a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java
+++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java
@@ -41,6 +41,7 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.api.extension.ExtensionContext.Namespace;
@@ -94,7 +95,17 @@ class FuzzTestExecutor {
     ArrayList<String> libFuzzerArgs = new ArrayList<>();
     libFuzzerArgs.add(argv0);
 
-    // Store the generated corpus in a per-class directory under the project root, just like cifuzz:
+    // Add passed in corpus directories (and files) at the beginning of the arguments list.
+    // libFuzzer uses the first directory to store discovered inputs, whereas all others are
+    // only used to provide additional seeds and aren't written into.
+    List<String> corpusDirs = originalLibFuzzerArgs.stream()
+                                  .filter(arg -> !arg.startsWith("-"))
+                                  .collect(Collectors.toList());
+    originalLibFuzzerArgs.removeAll(corpusDirs);
+    libFuzzerArgs.addAll(corpusDirs);
+
+    // Use the specified corpus dir, if given, otherwise store the generated corpus in a per-class
+    // directory under the project root, just like cifuzz:
     // https://github.com/CodeIntelligenceTesting/cifuzz/blob/bf410dcfbafbae2a73cf6c5fbed031cdfe234f2f/internal/cmd/run/run.go#L381
     // The path is specified relative to the current working directory, which with JUnit is the
     // project directory.
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
index dfcb2ec6..a9a5e2ea 100644
--- a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
+++ b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel
@@ -171,6 +171,40 @@ java_test(
 
 [
     java_test(
+        name = "CorpusDirectoryTest" + JAZZER_FUZZ,
+        srcs = ["CorpusDirectoryTest.java"],
+        args = [
+            # Add a test resource root containing the seed corpus directory in a Maven layout to
+            # the classpath rather than seeds in a resource directory packaged in a JAR, as
+            # would happen if we added the directory to java_test's resources.
+            "--main_advice_classpath=$(rootpath test_resources_root)",
+        ],
+        data = ["test_resources_root"],
+        env = {
+            "JAZZER_FUZZ": JAZZER_FUZZ,
+        },
+        test_class = "com.code_intelligence.jazzer.junit.CorpusDirectoryTest",
+        runtime_deps = [
+            "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar",
+            "@maven//:org_junit_jupiter_junit_jupiter_engine",
+        ],
+        deps = [
+            "//src/main/java/com/code_intelligence/jazzer/api:hooks",
+            "@maven//:com_google_truth_extensions_truth_java8_extension",
+            "@maven//:com_google_truth_truth",
+            "@maven//:junit_junit",
+            "@maven//:org_junit_platform_junit_platform_engine",
+            "@maven//:org_junit_platform_junit_platform_testkit",
+        ],
+    )
+    for JAZZER_FUZZ in [
+        "",
+        "_fuzzing",
+    ]
+]
+
+[
+    java_test(
         name = "AutofuzzTest" + JAZZER_FUZZ,
         srcs = ["AutofuzzTest.java"],
         env = {
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/CorpusDirectoryTest.java b/src/test/java/com/code_intelligence/jazzer/junit/CorpusDirectoryTest.java
new file mode 100644
index 00000000..372718ef
--- /dev/null
+++ b/src/test/java/com/code_intelligence/jazzer/junit/CorpusDirectoryTest.java
@@ -0,0 +1,183 @@
+// Copyright 2023 Code Intelligence GmbH
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.code_intelligence.jazzer.junit;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth8.assertThat;
+import static org.junit.Assume.assumeFalse;
+import static org.junit.Assume.assumeTrue;
+import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
+import static org.junit.platform.testkit.engine.EventConditions.container;
+import static org.junit.platform.testkit.engine.EventConditions.displayName;
+import static org.junit.platform.testkit.engine.EventConditions.event;
+import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully;
+import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure;
+import static org.junit.platform.testkit.engine.EventConditions.test;
+import static org.junit.platform.testkit.engine.EventConditions.type;
+import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings;
+import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED;
+import static org.junit.platform.testkit.engine.EventType.FINISHED;
+import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED;
+import static org.junit.platform.testkit.engine.EventType.STARTED;
+import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf;
+
+import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.stream.Stream;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.platform.testkit.engine.EngineExecutionResults;
+import org.junit.platform.testkit.engine.EngineTestKit;
+import org.junit.rules.TemporaryFolder;
+
+public class CorpusDirectoryTest {
+  private static final String ENGINE = "engine:junit-jupiter";
+  private static final String CLAZZ = "class:com.example.CorpusDirectoryFuzzTest";
+  private static final String INPUTS_FUZZ =
+      "test-template:corpusDirectoryFuzz(com.code_intelligence.jazzer.api.FuzzedDataProvider)";
+  private static final String INVOCATION = "test-template-invocation:#";
+
+  @Rule public TemporaryFolder temp = new TemporaryFolder();
+  Path baseDir;
+
+  @Before
+  public void setup() {
+    baseDir = temp.getRoot().toPath();
+  }
+
+  @Test
+  public void fuzzingEnabled() throws IOException {
+    assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty());
+
+    // Create a fake test resource directory structure with an inputs directory to verify that
+    // Jazzer uses it and emits a crash file into it.
+    Path artifactsDirectory = baseDir.resolve(Paths.get("src", "test", "resources", "com",
+        "example", "CorpusDirectoryFuzzTestInputs", "corpusDirectoryFuzz"));
+    Files.createDirectories(artifactsDirectory);
+
+    // An explicitly stated corpus directory should be used to save new corpus entries.
+    Path explicitGeneratedCorpus = baseDir.resolve(Paths.get("corpus"));
+    Files.createDirectories(explicitGeneratedCorpus);
+
+    // The default generated corpus directory should only be used if no explicit corpus directory
+    // is given.
+    Path defaultGeneratedCorpus = baseDir.resolve(
+        Paths.get(".cifuzz-corpus", "com.example.CorpusDirectoryFuzzTest", "corpusDirectoryFuzz"));
+
+    EngineExecutionResults results =
+        EngineTestKit.engine("junit-jupiter")
+            .selectors(selectClass("com.example.CorpusDirectoryFuzzTest"))
+            .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString())
+            // Add corpus directory as initial libFuzzer parameter.
+            .configurationParameter("jazzer.internal.arg.0", "fake_test_argv0")
+            .configurationParameter(
+                "jazzer.internal.arg.1", explicitGeneratedCorpus.toAbsolutePath().toString())
+            .execute();
+
+    results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)),
+        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
+        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
+        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ)),
+            finishedSuccessfully()),
+        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
+        event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
+
+    results.testEvents().assertEventsMatchExactly(
+        event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
+        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))),
+        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)),
+            displayName("<empty input>"), finishedSuccessfully()),
+        event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
+        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))),
+        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)),
+            displayName("seed"), finishedSuccessfully()),
+        event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
+        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)),
+            displayName("Fuzzing...")),
+        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)),
+            displayName("Fuzzing..."),
+            finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class))));
+
+    // Crash file should be emitted into the artifacts directory and not into corpus directory.
+    assertCrashFileExistsIn(artifactsDirectory);
+    assertNoCrashFileExistsIn(baseDir);
+    assertNoCrashFileExistsIn(explicitGeneratedCorpus);
+    assertNoCrashFileExistsIn(defaultGeneratedCorpus);
+
+    // Verify that corpus files are written to given corpus directory and not generated one.
+    assertThat(Files.list(explicitGeneratedCorpus)).isNotEmpty();
+    assertThat(Files.list(defaultGeneratedCorpus)).isEmpty();
+  }
+
+  @Test
+  public void fuzzingDisabled() throws IOException {
+    assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty());
+
+    Path corpusDirectory = baseDir.resolve(Paths.get("corpus"));
+    Files.createDirectories(corpusDirectory);
+    Files.createFile(corpusDirectory.resolve("corpus_entry"));
+
+    EngineExecutionResults results =
+        EngineTestKit.engine("junit-jupiter")
+            .selectors(selectClass("com.example.CorpusDirectoryFuzzTest"))
+            .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString())
+            // Add corpus directory as initial libFuzzer parameter.
+            .configurationParameter("jazzer.internal.arg.0", "fake_test_argv0")
+            .configurationParameter(
+                "jazzer.internal.arg.1", corpusDirectory.toAbsolutePath().toString())
+            .execute();
+
+    results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)),
+        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))),
+        event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
+        event(type(REPORTING_ENTRY_PUBLISHED),
+            container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
+        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
+        event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()),
+        event(type(FINISHED), container(ENGINE), finishedSuccessfully()));
+
+    // Verify that corpus_entry is not picked up and corpus directory is ignored in regression mode.
+    results.testEvents().assertEventsMatchExactly(
+        event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
+        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))),
+        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)),
+            displayName("<empty input>"), finishedSuccessfully()),
+        event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))),
+        event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))),
+        event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)),
+            displayName("seed"), finishedSuccessfully()));
+  }
+
+  private static void assertCrashFileExistsIn(Path artifactsDirectory) throws IOException {
+    try (Stream<Path> crashFiles =
+             Files.list(artifactsDirectory)
+                 .filter(path -> path.getFileName().toString().startsWith("crash-"))) {
+      assertThat(crashFiles).isNotEmpty();
+    }
+  }
+
+  private static void assertNoCrashFileExistsIn(Path generatedCorpus) throws IOException {
+    try (Stream<Path> crashFiles =
+             Files.list(generatedCorpus)
+                 .filter(path -> path.getFileName().toString().startsWith("crash-"))) {
+      assertThat(crashFiles).isEmpty();
+    }
+  }
+}
diff --git a/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/CorpusDirectoryFuzzTestInputs/corpusDirectoryFuzz/seed b/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/CorpusDirectoryFuzzTestInputs/corpusDirectoryFuzz/seed
new file mode 100644
index 00000000..e31de1f3
--- /dev/null
+++ b/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/CorpusDirectoryFuzzTestInputs/corpusDirectoryFuzz/seed
@@ -0,0 +1 @@
+seed