junit: Record parameterized test arguments as seeds

The seeds are serialized to files in a temporary directory that is
passed to libFuzzer as an additional seed directory.

8 files changed, 278 insertions, 4 deletions

diff --git a/examples/junit/src/test/java/com/example/BUILD.bazel b/examples/junit/src/test/java/com/example/BUILD.bazel
index 056562ac..841db001 100644
--- a/examples/junit/src/test/java/com/example/BUILD.bazel
+++ b/examples/junit/src/test/java/com/example/BUILD.bazel
@@ -15,6 +15,7 @@ java_binary(
         "//examples/junit/src/main/java/com/example:parser",
         "//examples/junit/src/test/resources:example_seed_corpora",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
+        "@maven//:org_junit_jupiter_junit_jupiter_params",
         "@maven//:org_mockito_mockito_core",
     ],
 )
@@ -198,6 +199,50 @@ java_fuzz_target_test(
     ],
 )
 
+java_fuzz_target_test(
+    name = "JavaSeedFuzzTest",
+    srcs = ["JavaSeedFuzzTest.java"],
+    allowed_findings = ["java.lang.Error"],
+    env = {"JAZZER_FUZZ": "1"},
+    fuzzer_args = [
+        "--instrumentation_includes=com.example.**",
+        "--custom_hook_includes=com.example.**",
+        "--experimental_mutator",
+    ],
+    target_class = "com.example.JavaSeedFuzzTest",
+    verify_crash_reproducer = False,
+    runtime_deps = [
+        ":junit_runtime",
+    ],
+    deps = [
+        "//src/main/java/com/code_intelligence/jazzer/junit:fuzz_test",
+        "//src/main/java/com/code_intelligence/jazzer/mutation/annotation",
+        "@maven//:org_junit_jupiter_junit_jupiter_api",
+        "@maven//:org_junit_jupiter_junit_jupiter_params",
+    ],
+)
+
+java_fuzz_target_test(
+    name = "JavaBinarySeedFuzzTest",
+    srcs = ["JavaBinarySeedFuzzTest.java"],
+    allowed_findings = ["java.lang.Error"],
+    env = {"JAZZER_FUZZ": "1"},
+    fuzzer_args = [
+        "--instrumentation_includes=com.example.**",
+        "--custom_hook_includes=com.example.**",
+    ],
+    target_class = "com.example.JavaBinarySeedFuzzTest",
+    verify_crash_reproducer = False,
+    runtime_deps = [
+        ":junit_runtime",
+    ],
+    deps = [
+        "//src/main/java/com/code_intelligence/jazzer/junit:fuzz_test",
+        "@maven//:org_junit_jupiter_junit_jupiter_api",
+        "@maven//:org_junit_jupiter_junit_jupiter_params",
+    ],
+)
+
 java_library(
     name = "junit_runtime",
     runtime_deps = [
diff --git a/examples/junit/src/test/java/com/example/JavaBinarySeedFuzzTest.java b/examples/junit/src/test/java/com/example/JavaBinarySeedFuzzTest.java
new file mode 100644
index 00000000..70b35352
--- /dev/null
+++ b/examples/junit/src/test/java/com/example/JavaBinarySeedFuzzTest.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2023 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.example;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+import com.code_intelligence.jazzer.junit.FuzzTest;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+import org.junit.jupiter.params.converter.ArgumentConversionException;
+import org.junit.jupiter.params.converter.ConvertWith;
+import org.junit.jupiter.params.converter.SimpleArgumentConverter;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class JavaBinarySeedFuzzTest {
+  // Generated via:
+  // printf 'tH15_1S-4_53Cr3T.fl4G' | openssl dgst -binary -sha256 | openssl base64 -A
+  // Luckily the fuzzer can't read comments ;-)
+  private static final byte[] FLAG_SHA256 =
+      Base64.getDecoder().decode("q0vPdz5oeJIW3k2U4VJ+aWDufzzZbKAcevc9cNoUTSM=");
+
+  static class Utf8BytesConverter extends SimpleArgumentConverter {
+    @Override
+    protected Object convert(Object source, Class<?> targetType)
+        throws ArgumentConversionException {
+      assertEquals(byte[].class, targetType);
+      assertTrue(source instanceof byte[] || source instanceof String);
+      if (source instanceof byte[]) {
+        return source;
+      }
+      return ((String) source).getBytes(UTF_8);
+    }
+  }
+
+  @ValueSource(strings = {"red herring", "tH15_1S-4_53Cr3T.fl4Ga"})
+  @FuzzTest
+  void fuzzTheFlag(@ConvertWith(Utf8BytesConverter.class) byte[] bytes)
+      throws NoSuchAlgorithmException {
+    assumeTrue(bytes.length > 0);
+    MessageDigest digest = MessageDigest.getInstance("SHA-256");
+    digest.update(bytes, 0, bytes.length - 1);
+    byte[] hash = digest.digest();
+    byte secret = bytes[bytes.length - 1];
+    if (MessageDigest.isEqual(hash, FLAG_SHA256) && secret == 's') {
+      throw new Error("Fl4g 4nd s3cr3et f0und!");
+    }
+  }
+}
diff --git a/examples/junit/src/test/java/com/example/JavaSeedFuzzTest.java b/examples/junit/src/test/java/com/example/JavaSeedFuzzTest.java
new file mode 100644
index 00000000..4f63e9a1
--- /dev/null
+++ b/examples/junit/src/test/java/com/example/JavaSeedFuzzTest.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2023 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.example;
+
+import static java.util.Arrays.asList;
+import static org.junit.jupiter.params.provider.Arguments.arguments;
+
+import com.code_intelligence.jazzer.junit.FuzzTest;
+import com.code_intelligence.jazzer.mutation.annotation.NotNull;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+import java.util.List;
+import java.util.stream.Stream;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+class JavaSeedFuzzTest {
+  // Generated via:
+  // printf 'tH15_1S-4_53Cr3T.fl4G' | openssl dgst -binary -sha256 | openssl base64 -A
+  // Luckily the fuzzer can't read comments ;-)
+  private static final byte[] FLAG_SHA256 =
+      Base64.getDecoder().decode("q0vPdz5oeJIW3k2U4VJ+aWDufzzZbKAcevc9cNoUTSM=");
+
+  static Stream<Arguments> fuzzTheFlag() {
+    return Stream.of(arguments(asList("red", "herring"), 0),
+        // This argument passes the hash check, but does not trigger the finding right away. This
+        // is meant to verify that the seed ends up in the corpus, serving as the base for future
+        // mutations rather than just being executed once.
+        arguments(asList("tH15_1S", "-4_53Cr3T", ".fl4G"), 42));
+  }
+
+  @MethodSource
+  @FuzzTest
+  void fuzzTheFlag(@NotNull List<@NotNull String> flagParts, int secret)
+      throws NoSuchAlgorithmException {
+    byte[] hash = MessageDigest.getInstance("SHA-256").digest(
+        String.join("", flagParts).getBytes(StandardCharsets.UTF_8));
+    if (MessageDigest.isEqual(hash, FLAG_SHA256) && secret == 1337) {
+      throw new Error("Fl4g 4nd s3cr3et f0und!");
+    }
+  }
+}
diff --git a/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel b/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel
index e4eb0fcc..3eb8959f 100644
--- a/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel
+++ b/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel
@@ -43,6 +43,7 @@ java_library(
         ":utils",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
         "@maven//:org_junit_jupiter_junit_jupiter_params",
+        "@maven//:org_junit_platform_junit_platform_commons",
     ],
 )
 
@@ -68,6 +69,8 @@ java_jni_library(
         "//src/main/java/com/code_intelligence/jazzer/mutation",
         "//src/main/java/com/code_intelligence/jazzer/utils",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
+        "@maven//:org_junit_jupiter_junit_jupiter_params",
+        "@maven//:org_junit_platform_junit_platform_commons",
     ],
 )
 
diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java
index f4eca021..e20f511a 100644
--- a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java
+++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java
@@ -41,19 +41,25 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Stream;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.api.extension.ExtensionContext.Namespace;
 import org.junit.jupiter.api.extension.ReflectiveInvocationContext;
+import org.junit.jupiter.params.provider.ArgumentsSource;
+import org.junit.platform.commons.support.AnnotationSupport;
 
 class FuzzTestExecutor {
   private static final AtomicBoolean hasBeenPrepared = new AtomicBoolean();
   private static final AtomicBoolean agentInstalled = new AtomicBoolean(false);
 
   private final List<String> libFuzzerArgs;
+  private final Path javaSeedsDir;
   private final boolean isRunFromCommandLine;
 
-  private FuzzTestExecutor(List<String> libFuzzerArgs, boolean isRunFromCommandLine) {
+  private FuzzTestExecutor(
+      List<String> libFuzzerArgs, Path javaSeedsDir, boolean isRunFromCommandLine) {
     this.libFuzzerArgs = libFuzzerArgs;
+    this.javaSeedsDir = javaSeedsDir;
     this.isRunFromCommandLine = isRunFromCommandLine;
   }
 
@@ -67,6 +73,17 @@ class FuzzTestExecutor {
     Class<?> fuzzTestClass = context.getRequiredTestClass();
     Method fuzzTestMethod = context.getRequiredTestMethod();
 
+    List<ArgumentsSource> allSources = AnnotationSupport.findRepeatableAnnotations(
+        context.getRequiredTestMethod(), ArgumentsSource.class);
+    // Non-empty as it always contains FuzzingArgumentsProvider.
+    ArgumentsSource lastSource = allSources.get(allSources.size() - 1);
+    // Ensure that our ArgumentsProviders run last so that we can record all the seeds generated by
+    // user-provided ones.
+    if (lastSource.value().getPackage() != FuzzTestExecutor.class.getPackage()) {
+      throw new IllegalArgumentException("@FuzzTest must be the last annotation on a fuzz test,"
+          + " but it came after the (meta-)annotation " + lastSource);
+    }
+
     Path baseDir =
         Paths.get(context.getConfigurationParameter("jazzer.internal.basedir").orElse(""))
             .toAbsolutePath();
@@ -126,6 +143,8 @@ class FuzzTestExecutor {
     // From the second positional argument on, files and directories are used as seeds but not
     // modified.
     inputsDirectory.ifPresent(dir -> libFuzzerArgs.add(dir.toAbsolutePath().toString()));
+    Path javaSeedsDir = Files.createTempDirectory("jazzer-java-seeds");
+    libFuzzerArgs.add(javaSeedsDir.toAbsolutePath().toString());
     libFuzzerArgs.add(String.format("-artifact_prefix=%s%c",
         findingsDirectory.orElse(baseDir).toAbsolutePath(), File.separatorChar));
 
@@ -142,7 +161,7 @@ class FuzzTestExecutor {
     // Prefer original libFuzzerArgs set via command line by appending them last.
     libFuzzerArgs.addAll(originalLibFuzzerArgs);
 
-    return new FuzzTestExecutor(libFuzzerArgs, Utils.runFromCommandLine(context));
+    return new FuzzTestExecutor(libFuzzerArgs, javaSeedsDir, Utils.runFromCommandLine(context));
   }
 
   /**
@@ -181,6 +200,11 @@ class FuzzTestExecutor {
         .get(FuzzTestExecutor.class, FuzzTestExecutor.class);
   }
 
+  public void addSeed(byte[] bytes) throws IOException {
+    Path seed = Files.createTempFile(javaSeedsDir, "seed", null);
+    Files.write(seed, bytes);
+  }
+
   @SuppressWarnings("OptionalGetWithoutIsPresent")
   public Optional<Throwable> execute(
       ReflectiveInvocationContext<Method> invocationContext, SeedSerializer seedSerializer) {
@@ -216,6 +240,7 @@ class FuzzTestExecutor {
     }
 
     int exitCode = FuzzTargetRunner.startLibFuzzer(libFuzzerArgs);
+    deleteJavaSeedsDir();
     Throwable finding = atomicFinding.get();
     if (finding != null) {
       return Optional.of(finding);
@@ -226,4 +251,21 @@ class FuzzTestExecutor {
       return Optional.empty();
     }
   }
+
+  private void deleteJavaSeedsDir() {
+    // The directory only consists of files, which we need to delete before deleting the directory
+    // itself.
+    try (Stream<Path> entries = Files.list(javaSeedsDir)) {
+      entries.forEach(FuzzTestExecutor::deleteIgnoringErrors);
+    } catch (IOException ignored) {
+    }
+    deleteIgnoringErrors(javaSeedsDir);
+  }
+
+  private static void deleteIgnoringErrors(Path path) {
+    try {
+      Files.deleteIfExists(path);
+    } catch (IOException ignored) {
+    }
+  }
 }
diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java
index 72882de6..0e8ceec9 100644
--- a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java
+++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java
@@ -14,8 +14,10 @@
 
 package com.code_intelligence.jazzer.junit;
 
+import java.io.IOException;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
+import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicReference;
 import org.junit.jupiter.api.extension.ConditionEvaluationResult;
@@ -24,6 +26,7 @@ import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.api.extension.ExtensionContext.Namespace;
 import org.junit.jupiter.api.extension.InvocationInterceptor;
 import org.junit.jupiter.api.extension.ReflectiveInvocationContext;
+import org.junit.platform.commons.support.AnnotationSupport;
 
 class FuzzTestExtensions implements ExecutionCondition, InvocationInterceptor {
   private static final String JAZZER_INTERNAL =
@@ -36,11 +39,23 @@ class FuzzTestExtensions implements ExecutionCondition, InvocationInterceptor {
   public void interceptTestTemplateMethod(Invocation<Void> invocation,
       ReflectiveInvocationContext<Method> invocationContext, ExtensionContext extensionContext)
       throws Throwable {
+    FuzzTest fuzzTest =
+        AnnotationSupport.findAnnotation(invocationContext.getExecutable(), FuzzTest.class).get();
+    FuzzTestExecutor.configureAndInstallAgent(extensionContext, fuzzTest.maxDuration());
     // Skip the invocation of the test method with the special arguments provided by
     // FuzzTestArgumentsProvider and start fuzzing instead.
     if (Utils.isMarkedInvocation(invocationContext)) {
       startFuzzing(invocation, invocationContext, extensionContext);
     } else {
+      // Blocked by https://github.com/junit-team/junit5/issues/3282:
+      // TODO: The seeds from the input directory are duplicated here as there is no way to
+      //  recognize them.
+      // TODO: Error out if there is a non-Jazzer ArgumentsProvider and the SeedSerializer does not
+      //  support write.
+      if (Utils.isFuzzing(extensionContext)) {
+        // JUnit verifies that the arguments for this invocation are valid.
+        recordSeedForFuzzing(invocationContext.getArguments(), extensionContext);
+      }
       runWithHooks(invocation);
     }
   }
@@ -95,6 +110,16 @@ class FuzzTestExtensions implements ExecutionCondition, InvocationInterceptor {
     }
   }
 
+  private void recordSeedForFuzzing(List<Object> arguments, ExtensionContext extensionContext)
+      throws IOException {
+    SeedSerializer seedSerializer = getOrCreateSeedSerializer(extensionContext);
+    try {
+      FuzzTestExecutor.fromContext(extensionContext)
+          .addSeed(seedSerializer.write(arguments.toArray()));
+    } catch (UnsupportedOperationException ignored) {
+    }
+  }
+
   @Override
   public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext extensionContext) {
     if (!Utils.isFuzzing(extensionContext)) {
diff --git a/src/main/java/com/code_intelligence/jazzer/junit/SeedSerializer.java b/src/main/java/com/code_intelligence/jazzer/junit/SeedSerializer.java
index 463c6856..0cebef2f 100644
--- a/src/main/java/com/code_intelligence/jazzer/junit/SeedSerializer.java
+++ b/src/main/java/com/code_intelligence/jazzer/junit/SeedSerializer.java
@@ -22,6 +22,7 @@ import com.code_intelligence.jazzer.driver.FuzzedDataProviderImpl;
 import com.code_intelligence.jazzer.driver.Opt;
 import com.code_intelligence.jazzer.mutation.ArgumentsMutator;
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.lang.reflect.Method;
 import java.util.Optional;
 
@@ -31,6 +32,10 @@ interface SeedSerializer {
     return true;
   }
 
+  // Implementations can assume that the argument array contains valid arguments for the method that
+  // this instance has been constructed for.
+  byte[] write(Object[] args) throws UnsupportedOperationException;
+
   /**
    * Creates specialized {@link SeedSerializer} instances for the following method parameters:
    * <ul>
@@ -64,6 +69,11 @@ final class ByteArraySeedSerializer implements SeedSerializer {
   public Object[] read(byte[] bytes) {
     return new Object[] {bytes};
   }
+
+  @Override
+  public byte[] write(Object[] args) {
+    return (byte[]) args[0];
+  }
 }
 
 final class FuzzedDataProviderSeedSerializer implements SeedSerializer {
@@ -71,6 +81,13 @@ final class FuzzedDataProviderSeedSerializer implements SeedSerializer {
   public Object[] read(byte[] bytes) {
     return new Object[] {FuzzedDataProviderImpl.withJavaData(bytes)};
   }
+
+  @Override
+  public byte[] write(Object[] args) throws UnsupportedOperationException {
+    // While we could get the underlying bytes, it's not possible to provide Java seeds for fuzz
+    // tests with a FuzzedDataProvider parameter.
+    throw new UnsupportedOperationException();
+  }
 }
 
 final class ArgumentsMutatorSeedSerializer implements SeedSerializer {
@@ -91,6 +108,13 @@ final class ArgumentsMutatorSeedSerializer implements SeedSerializer {
   public boolean allReadsValid() {
     return allReadsValid;
   }
+
+  @Override
+  public byte[] write(Object[] args) {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    mutator.writeAny(out, args);
+    return out.toByteArray();
+  }
 }
 
 final class AutofuzzSeedSerializer implements SeedSerializer {
@@ -112,4 +136,9 @@ final class AutofuzzSeedSerializer implements SeedSerializer {
       return meta.consumeArguments(data, method, null);
     }
   }
+
+  @Override
+  public byte[] write(Object[] args) throws UnsupportedOperationException {
+    throw new UnsupportedOperationException();
+  }
 }
diff --git a/src/main/java/com/code_intelligence/jazzer/mutation/ArgumentsMutator.java b/src/main/java/com/code_intelligence/jazzer/mutation/ArgumentsMutator.java
index e7345ca8..fabda057 100644
--- a/src/main/java/com/code_intelligence/jazzer/mutation/ArgumentsMutator.java
+++ b/src/main/java/com/code_intelligence/jazzer/mutation/ArgumentsMutator.java
@@ -16,7 +16,6 @@
 
 package com.code_intelligence.jazzer.mutation;
 
-import static com.code_intelligence.jazzer.mutation.mutator.Mutators.newFactory;
 import static com.code_intelligence.jazzer.mutation.mutator.Mutators.validateAnnotationUsage;
 import static com.code_intelligence.jazzer.mutation.support.InputStreamSupport.extendWithReadExactly;
 import static com.code_intelligence.jazzer.mutation.support.Preconditions.require;
@@ -165,8 +164,15 @@ public final class ArgumentsMutator {
    */
   public void write(OutputStream data) {
     failIfArgumentsExposed();
+    writeAny(data, arguments);
+  }
+
+  /**
+   * @throws UncheckedIOException if the underlying OutputStream throws
+   */
+  public void writeAny(OutputStream data, Object[] args) throws UncheckedIOException {
     try {
-      productMutator.writeExclusive(arguments, data);
+      productMutator.writeExclusive(args, data);
     } catch (IOException e) {
       throw new UncheckedIOException(e);
     }