diff options
Diffstat (limited to 'src/main/java/com/code_intelligence/jazzer/junit')
10 files changed, 1504 insertions, 0 deletions
diff --git a/src/main/java/com/code_intelligence/jazzer/junit/AgentConfigurator.java b/src/main/java/com/code_intelligence/jazzer/junit/AgentConfigurator.java new file mode 100644 index 00000000..1f286a31 --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/junit/AgentConfigurator.java @@ -0,0 +1,72 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.code_intelligence.jazzer.junit.Utils.getClassPathBasedInstrumentationFilter; +import static com.code_intelligence.jazzer.junit.Utils.getLegacyInstrumentationFilter; + +import java.io.File; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.extension.ExtensionContext; + +class AgentConfigurator { + private static final AtomicBoolean hasBeenConfigured = new AtomicBoolean(); + + static void forRegressionTest(ExtensionContext extensionContext) { + if (!hasBeenConfigured.compareAndSet(false, true)) { + return; + } + + applyCommonConfiguration(); + + // Add logic to the hook instrumentation that allows us to enable and disable hooks at runtime. + System.setProperty("jazzer.internal.conditional_hooks", "true"); + // Apply all hooks, but no coverage or compare instrumentation. + System.setProperty("jazzer.instrumentation_excludes", "**"); + extensionContext.getConfigurationParameter("jazzer.instrument") + .ifPresent(s + -> System.setProperty( + "jazzer.custom_hook_includes", String.join(File.pathSeparator, s.split(",")))); + } + + static void forFuzzing(ExtensionContext executionRequest) { + if (!hasBeenConfigured.compareAndSet(false, true)) { + throw new IllegalStateException("Only a single fuzz test should be executed per fuzzing run"); + } + + applyCommonConfiguration(); + + String instrumentationFilter = + executionRequest.getConfigurationParameter("jazzer.instrument") + .orElseGet( + () + -> getClassPathBasedInstrumentationFilter(System.getProperty("java.class.path")) + .orElseGet(() + -> getLegacyInstrumentationFilter( + executionRequest.getRequiredTestClass()))); + String filter = String.join(File.pathSeparator, instrumentationFilter.split(",")); + System.setProperty("jazzer.custom_hook_includes", filter); + System.setProperty("jazzer.instrumentation_includes", filter); + } + + private static void applyCommonConfiguration() { + // Do not hook common IDE and JUnit classes and their dependencies. + System.setProperty("jazzer.custom_hook_excludes", + String.join(File.pathSeparator, "com.google.testing.junit.**", "com.intellij.**", + "org.jetbrains.**", "io.github.classgraph.**", "junit.framework.**", "net.bytebuddy.**", + "org.apiguardian.**", "org.assertj.core.**", "org.hamcrest.**", "org.junit.**", + "org.opentest4j.**", "org.mockito.**", "org.apache.maven.**", "org.gradle.**")); + } +} diff --git a/src/main/java/com/code_intelligence/jazzer/junit/AgentConfiguringArgumentsProvider.java b/src/main/java/com/code_intelligence/jazzer/junit/AgentConfiguringArgumentsProvider.java new file mode 100644 index 00000000..e65f028b --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/junit/AgentConfiguringArgumentsProvider.java @@ -0,0 +1,43 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.junit; + +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.support.AnnotationConsumer; + +public class AgentConfiguringArgumentsProvider + implements ArgumentsProvider, AnnotationConsumer<FuzzTest> { + private FuzzTest fuzzTest; + + @Override + public void accept(FuzzTest fuzzTest) { + this.fuzzTest = fuzzTest; + } + + @Override + public Stream<? extends Arguments> provideArguments(ExtensionContext extensionContext) + throws Exception { + // FIXME(fmeum): Calling this here feels like a hack. There should be a lifecycle hook that runs + // before the argument discovery for a ParameterizedTest is kicked off, but I haven't found + // one. + FuzzTestExecutor.configureAndInstallAgent(extensionContext, fuzzTest.maxDuration()); + return Stream.empty(); + } +} diff --git a/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel b/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel new file mode 100644 index 00000000..3eb8959f --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel @@ -0,0 +1,99 @@ +load("@fmeum_rules_jni//jni:defs.bzl", "java_jni_library") + +java_library( + name = "junit", + visibility = ["//deploy:__pkg__"], + runtime_deps = [ + ":fuzz_test", + ], +) + +java_library( + name = "agent_configurator", + srcs = [ + "AgentConfigurator.java", + ], + deps = [ + ":utils", + "@maven//:org_junit_jupiter_junit_jupiter_api", + ], +) + +java_library( + name = "fuzz_test", + srcs = [ + "AgentConfiguringArgumentsProvider.java", + "FuzzTest.java", + "FuzzTestExtensions.java", + "FuzzingArgumentsProvider.java", + "SeedArgumentsProvider.java", + ], + visibility = [ + "//examples/junit/src/test/java/com/example:__pkg__", + ], + runtime_deps = [ + # The JUnit launcher that is part of the Jazzer driver needs this on the classpath + # to run an @FuzzTest with JUnit. This will also result in a transitive dependency + # in the generated pom file. + "@maven//:org_junit_platform_junit_platform_launcher", + ], + deps = [ + ":fuzz_test_executor", + ":seed_serializer", + ":utils", + "@maven//:org_junit_jupiter_junit_jupiter_api", + "@maven//:org_junit_jupiter_junit_jupiter_params", + "@maven//:org_junit_platform_junit_platform_commons", + ], +) + +java_jni_library( + name = "fuzz_test_executor", + srcs = [ + "FuzzTestExecutor.java", + ], + native_libs = [ + "//src/main/native/com/code_intelligence/jazzer/driver:jazzer_driver", + ], + deps = [ + ":agent_configurator", + ":seed_serializer", + ":utils", + "//src/main/java/com/code_intelligence/jazzer/agent:agent_installer", + "//src/main/java/com/code_intelligence/jazzer/api", + "//src/main/java/com/code_intelligence/jazzer/autofuzz", + "//src/main/java/com/code_intelligence/jazzer/driver:fuzz_target_holder", + "//src/main/java/com/code_intelligence/jazzer/driver:fuzz_target_runner", + "//src/main/java/com/code_intelligence/jazzer/driver:opt", + "//src/main/java/com/code_intelligence/jazzer/driver/junit:exit_code_exception", + "//src/main/java/com/code_intelligence/jazzer/mutation", + "//src/main/java/com/code_intelligence/jazzer/utils", + "@maven//:org_junit_jupiter_junit_jupiter_api", + "@maven//:org_junit_jupiter_junit_jupiter_params", + "@maven//:org_junit_platform_junit_platform_commons", + ], +) + +java_library( + name = "seed_serializer", + srcs = ["SeedSerializer.java"], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api", + "//src/main/java/com/code_intelligence/jazzer/autofuzz", + "//src/main/java/com/code_intelligence/jazzer/driver:fuzzed_data_provider_impl", + "//src/main/java/com/code_intelligence/jazzer/driver:opt", + "//src/main/java/com/code_intelligence/jazzer/mutation", + ], +) + +java_library( + name = "utils", + srcs = ["Utils.java"], + visibility = ["//src/test/java/com/code_intelligence/jazzer/junit:__pkg__"], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/utils:unsafe_provider", + "//src/main/java/com/code_intelligence/jazzer/utils:unsafe_utils", + "@maven//:org_junit_jupiter_junit_jupiter_api", + "@maven//:org_junit_jupiter_junit_jupiter_params", + ], +) diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTest.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTest.java new file mode 100644 index 00000000..041db96d --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTest.java @@ -0,0 +1,122 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.parallel.ResourceAccessMode; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.junit.jupiter.api.parallel.Resources; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ArgumentsSource; + +/** + * A parameterized test with parameters generated automatically by the Java fuzzer <a + * href="https://github.com/CodeIntelligenceTesting/jazzer">Jazzer</a>. + * + * <h2>Test parameters</h2> + * + * <p>Methods annotated with {@link FuzzTest} can take either of the following types of parameters: + * + * <dl> + * <dt>{@code byte[]}</dt> + * <dd>Raw byte input mutated by the fuzzer. Use this signature when your fuzz test naturally + * handles raw bytes (e.g. when fuzzing a binary format parser). This is the most efficient, but + * also the least convenient way to write a fuzz test.</dd> + * + * <dt>{@link com.code_intelligence.jazzer.api.FuzzedDataProvider}</dt> + * <dd>Provides convenience methods that generate instances of commonly used Java types from the raw + * fuzzer input. This is generally the best way to write fuzz tests.</dd> + * + * <dt>any non-zero number of parameters of any type</dt> + * <dd>In this case, Jazzer will rely on reflection and class path scanning to instantiate concrete + * arguments. While convenient and a good way to get started, fuzz tests using this feature will + * generally be less efficient than fuzz tests using any of the other possible signatures. Due to + * the reliance on class path scanning, any change to the class path may also render previous + * findings unreproducible.</dd> + * </dl> + * + * <h2>Test modes</h2> + * + * A fuzz test can be run in two modes: fuzzing and regression testing. + * + * <h3>Fuzzing</h3> + * <p>When the environment variable {@code JAZZER_FUZZ} is set to any non-empty value, fuzz tests + * run in "fuzzing" mode. In this mode, the method annotated with {@link FuzzTest} are invoked + * repeatedly with inputs that Jazzer generates and mutates based on feedback obtained from + * instrumentation it applies to the test and every class loaded by it. + * + * <p>When an assertion in the test fails, an exception is thrown but not caught, or Jazzer's + * instrumentation detects a security issue (e.g. SQL injection or insecure deserialization), the + * fuzz test is reported as failed and the input is collected in the inputs directory for the test + * class (see "Regression testing" for details). + * + * <p>When no issue has been found after the configured {@link FuzzTest#maxDuration()}, the test + * passes. + * + * <p>Only a single fuzz test per test run will be executed in fuzzing mode. All other fuzz tests + * will be skipped. + * + * <h3>Regression testing</h3> + * <p>By default, a fuzz test is executed as a regular JUnit {@link ParameterizedTest} running on a + * fixed set of inputs. It can be run together with regular unit tests and used to verify that past + * findings remain fixed. In IDEs with JUnit 5 integration, it can also be used to conveniently + * debug individual findings. + * + * <p>Fuzz tests are always executed on the empty input as well as all input files contained in the + * resource directory called {@code <TestClassName>Inputs} in the current package. For example, + * all fuzz tests contained in the class {@code com.example.MyFuzzTests} would run on all files + * under {@code src/test/resources/com/example/MyFuzzTestsInputs}. + */ +@Target({ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@AgentConfiguringArgumentsProviderArgumentsSource +@ArgumentsSource(SeedArgumentsProvider.class) +@FuzzingArgumentsProviderArgumentsSource +@ExtendWith(FuzzTestExtensions.class) +// {0} is expanded to the basename of the seed by the ArgumentProvider. +@ParameterizedTest(name = "{0}") +@Tag("jazzer") +// Fuzz tests can't run in parallel with other fuzz tests since the last finding is kept in a global +// variable. +// Fuzz tests also can't run in parallel with other non-fuzz tests since method hooks are enabled +// conditionally based on a global variable. +@ResourceLock(value = Resources.GLOBAL, mode = ResourceAccessMode.READ_WRITE) +public @interface FuzzTest { + /** + * A duration string such as "1h 2m 30s" indicating for how long the fuzz test should be executed + * during fuzzing. + * + * <p>This option has no effect during regression testing. + */ + String maxDuration() default "5m"; +} + +// Internal use only. +// These wrappers are needed only because the container annotation for @ArgumentsSource, +// @ArgumentsSources, can't be applied to annotations. +@Target({ElementType.ANNOTATION_TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@ArgumentsSource(AgentConfiguringArgumentsProvider.class) +@interface AgentConfiguringArgumentsProviderArgumentsSource {} + +@Target({ElementType.ANNOTATION_TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@ArgumentsSource(FuzzingArgumentsProvider.class) +@interface FuzzingArgumentsProviderArgumentsSource {} diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java new file mode 100644 index 00000000..49252e84 --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java @@ -0,0 +1,282 @@ +// Copyright 2023 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.code_intelligence.jazzer.junit.Utils.durationStringToSeconds; +import static com.code_intelligence.jazzer.junit.Utils.generatedCorpusPath; +import static com.code_intelligence.jazzer.junit.Utils.inputsDirectoryResourcePath; +import static com.code_intelligence.jazzer.junit.Utils.inputsDirectorySourcePath; + +import com.code_intelligence.jazzer.agent.AgentInstaller; +import com.code_intelligence.jazzer.driver.FuzzTargetHolder; +import com.code_intelligence.jazzer.driver.FuzzTargetRunner; +import com.code_intelligence.jazzer.driver.Opt; +import com.code_intelligence.jazzer.driver.junit.ExitCodeException; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Executable; +import java.lang.reflect.Method; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; +import org.junit.jupiter.api.extension.ReflectiveInvocationContext; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.junit.platform.commons.support.AnnotationSupport; + +class FuzzTestExecutor { + private static final AtomicBoolean hasBeenPrepared = new AtomicBoolean(); + private static final AtomicBoolean agentInstalled = new AtomicBoolean(false); + + private final List<String> libFuzzerArgs; + private final Path javaSeedsDir; + private final boolean isRunFromCommandLine; + + private FuzzTestExecutor( + List<String> libFuzzerArgs, Path javaSeedsDir, boolean isRunFromCommandLine) { + this.libFuzzerArgs = libFuzzerArgs; + this.javaSeedsDir = javaSeedsDir; + this.isRunFromCommandLine = isRunFromCommandLine; + } + + public static FuzzTestExecutor prepare(ExtensionContext context, String maxDuration) + throws IOException { + if (!hasBeenPrepared.compareAndSet(false, true)) { + throw new IllegalStateException( + "FuzzTestExecutor#prepare can only be called once per test run"); + } + + Class<?> fuzzTestClass = context.getRequiredTestClass(); + Method fuzzTestMethod = context.getRequiredTestMethod(); + + List<ArgumentsSource> allSources = AnnotationSupport.findRepeatableAnnotations( + context.getRequiredTestMethod(), ArgumentsSource.class); + // Non-empty as it always contains FuzzingArgumentsProvider. + ArgumentsSource lastSource = allSources.get(allSources.size() - 1); + // Ensure that our ArgumentsProviders run last so that we can record all the seeds generated by + // user-provided ones. + if (lastSource.value().getPackage() != FuzzTestExecutor.class.getPackage()) { + throw new IllegalArgumentException("@FuzzTest must be the last annotation on a fuzz test," + + " but it came after the (meta-)annotation " + lastSource); + } + + Path baseDir = + Paths.get(context.getConfigurationParameter("jazzer.internal.basedir").orElse("")) + .toAbsolutePath(); + + List<String> originalLibFuzzerArgs = getLibFuzzerArgs(context); + String argv0 = originalLibFuzzerArgs.isEmpty() ? "fake_argv0" : originalLibFuzzerArgs.remove(0); + + ArrayList<String> libFuzzerArgs = new ArrayList<>(); + libFuzzerArgs.add(argv0); + + // Add passed in corpus directories (and files) at the beginning of the arguments list. + // libFuzzer uses the first directory to store discovered inputs, whereas all others are + // only used to provide additional seeds and aren't written into. + List<String> corpusDirs = originalLibFuzzerArgs.stream() + .filter(arg -> !arg.startsWith("-")) + .collect(Collectors.toList()); + originalLibFuzzerArgs.removeAll(corpusDirs); + libFuzzerArgs.addAll(corpusDirs); + + // Use the specified corpus dir, if given, otherwise store the generated corpus in a per-class + // directory under the project root, just like cifuzz: + // https://github.com/CodeIntelligenceTesting/cifuzz/blob/bf410dcfbafbae2a73cf6c5fbed031cdfe234f2f/internal/cmd/run/run.go#L381 + // The path is specified relative to the current working directory, which with JUnit is the + // project directory. + Path generatedCorpusDir = baseDir.resolve(generatedCorpusPath(fuzzTestClass, fuzzTestMethod)); + Files.createDirectories(generatedCorpusDir); + libFuzzerArgs.add(generatedCorpusDir.toAbsolutePath().toString()); + + // We can only emit findings into the source tree version of the inputs directory, not e.g. the + // copy under Maven's target directory. If it doesn't exist, collect the inputs in the current + // working directory, which is usually the project's source root. + Optional<Path> findingsDirectory = + inputsDirectorySourcePath(fuzzTestClass, fuzzTestMethod, baseDir); + if (!findingsDirectory.isPresent()) { + context.publishReportEntry(String.format( + "Collecting crashing inputs in the project root directory.\nIf you want to keep them " + + "organized by fuzz test and automatically run them as regression tests with " + + "JUnit Jupiter, create a test resource directory called '%s' in package '%s' " + + "and move the files there.", + inputsDirectoryResourcePath(fuzzTestClass, fuzzTestMethod), + fuzzTestClass.getPackage().getName())); + } + + // We prefer the inputs directory on the classpath, if it exists, as that is more reliable than + // heuristically looking into the source tree based on the current working directory. + Optional<Path> inputsDirectory; + URL inputsDirectoryUrl = + fuzzTestClass.getResource(inputsDirectoryResourcePath(fuzzTestClass, fuzzTestMethod)); + if (inputsDirectoryUrl != null && "file".equals(inputsDirectoryUrl.getProtocol())) { + // The inputs directory is a regular directory on disk (i.e., the test is not run from a + // JAR). + try { + // Using inputsDirectoryUrl.getFile() fails on Windows. + inputsDirectory = Optional.of(Paths.get(inputsDirectoryUrl.toURI())); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } else { + if (inputsDirectoryUrl != null && !findingsDirectory.isPresent()) { + context.publishReportEntry( + "When running Jazzer fuzz tests from a JAR rather than class files, the inputs " + + "directory isn't used unless it is located under src/test/resources/..."); + } + inputsDirectory = findingsDirectory; + } + + // From the second positional argument on, files and directories are used as seeds but not + // modified. + inputsDirectory.ifPresent(dir -> libFuzzerArgs.add(dir.toAbsolutePath().toString())); + Path javaSeedsDir = Files.createTempDirectory("jazzer-java-seeds"); + libFuzzerArgs.add(javaSeedsDir.toAbsolutePath().toString()); + libFuzzerArgs.add(String.format("-artifact_prefix=%s%c", + findingsDirectory.orElse(baseDir).toAbsolutePath(), File.separatorChar)); + + libFuzzerArgs.add("-max_total_time=" + durationStringToSeconds(maxDuration)); + // Disable libFuzzer's out of memory detection: It is only useful for native library fuzzing, + // which we don't support without our native driver, and leads to false positives where it picks + // up IntelliJ's memory usage. + libFuzzerArgs.add("-rss_limit_mb=0"); + if (Utils.permissivelyParseBoolean( + context.getConfigurationParameter("jazzer.valueprofile").orElse("false"))) { + libFuzzerArgs.add("-use_value_profile=1"); + } + + // Prefer original libFuzzerArgs set via command line by appending them last. + libFuzzerArgs.addAll(originalLibFuzzerArgs); + + return new FuzzTestExecutor(libFuzzerArgs, javaSeedsDir, Utils.runFromCommandLine(context)); + } + + /** + * Returns the list of arguments set on the command line. + */ + private static List<String> getLibFuzzerArgs(ExtensionContext extensionContext) { + List<String> args = new ArrayList<>(); + for (int i = 0;; i++) { + Optional<String> arg = extensionContext.getConfigurationParameter("jazzer.internal.arg." + i); + if (!arg.isPresent()) { + break; + } + args.add(arg.get()); + } + return args; + } + + static void configureAndInstallAgent(ExtensionContext extensionContext, String maxDuration) + throws IOException { + if (!agentInstalled.compareAndSet(false, true)) { + return; + } + if (Utils.isFuzzing(extensionContext)) { + FuzzTestExecutor executor = prepare(extensionContext, maxDuration); + extensionContext.getRoot().getStore(Namespace.GLOBAL).put(FuzzTestExecutor.class, executor); + AgentConfigurator.forFuzzing(extensionContext); + } else { + AgentConfigurator.forRegressionTest(extensionContext); + } + AgentInstaller.install(Opt.hooks); + } + + static FuzzTestExecutor fromContext(ExtensionContext extensionContext) { + return extensionContext.getRoot() + .getStore(Namespace.GLOBAL) + .get(FuzzTestExecutor.class, FuzzTestExecutor.class); + } + + public void addSeed(byte[] bytes) throws IOException { + Path seed = Files.createTempFile(javaSeedsDir, "seed", null); + Files.write(seed, bytes); + } + + @SuppressWarnings("OptionalGetWithoutIsPresent") + public Optional<Throwable> execute( + ReflectiveInvocationContext<Method> invocationContext, SeedSerializer seedSerializer) { + if (seedSerializer instanceof AutofuzzSeedSerializer) { + FuzzTargetHolder.fuzzTarget = FuzzTargetHolder.autofuzzFuzzTarget(() -> { + // Provide an empty throws declaration to prevent autofuzz from + // ignoring the defined test exceptions. All exceptions in tests + // should cause them to fail. + Map<Executable, Class<?>[]> throwsDeclarations = new HashMap<>(1); + throwsDeclarations.put(invocationContext.getExecutable(), new Class[0]); + + com.code_intelligence.jazzer.autofuzz.FuzzTarget.setTarget( + new Executable[] {invocationContext.getExecutable()}, + invocationContext.getTarget().get(), invocationContext.getExecutable().toString(), + Collections.emptySet(), throwsDeclarations); + return null; + }); + } else { + FuzzTargetHolder.fuzzTarget = + new FuzzTargetHolder.FuzzTarget(invocationContext.getExecutable(), + () -> invocationContext.getTarget().get(), Optional.empty()); + } + + // Only register a finding handler in case the fuzz test is executed by JUnit. + // It short-circuits the handling in FuzzTargetRunner and prevents settings + // like --keep_going. + AtomicReference<Throwable> atomicFinding = new AtomicReference<>(); + if (!isRunFromCommandLine) { + FuzzTargetRunner.registerFindingHandler(t -> { + atomicFinding.set(t); + return false; + }); + } + + int exitCode = FuzzTargetRunner.startLibFuzzer(libFuzzerArgs); + deleteJavaSeedsDir(); + Throwable finding = atomicFinding.get(); + if (finding != null) { + return Optional.of(finding); + } else if (exitCode != 0) { + return Optional.of( + new ExitCodeException("Jazzer exited with exit code " + exitCode, exitCode)); + } else { + return Optional.empty(); + } + } + + private void deleteJavaSeedsDir() { + // The directory only consists of files, which we need to delete before deleting the directory + // itself. + try (Stream<Path> entries = Files.list(javaSeedsDir)) { + entries.forEach(FuzzTestExecutor::deleteIgnoringErrors); + } catch (IOException ignored) { + } + deleteIgnoringErrors(javaSeedsDir); + } + + private static void deleteIgnoringErrors(Path path) { + try { + Files.deleteIfExists(path); + } catch (IOException ignored) { + } + } +} diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java new file mode 100644 index 00000000..0e8ceec9 --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java @@ -0,0 +1,170 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.extension.ConditionEvaluationResult; +import org.junit.jupiter.api.extension.ExecutionCondition; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; +import org.junit.jupiter.api.extension.InvocationInterceptor; +import org.junit.jupiter.api.extension.ReflectiveInvocationContext; +import org.junit.platform.commons.support.AnnotationSupport; + +class FuzzTestExtensions implements ExecutionCondition, InvocationInterceptor { + private static final String JAZZER_INTERNAL = + "com.code_intelligence.jazzer.runtime.JazzerInternal"; + private static final AtomicReference<Method> fuzzTestMethod = new AtomicReference<>(); + private static Field lastFindingField; + private static Field hooksEnabledField; + + @Override + public void interceptTestTemplateMethod(Invocation<Void> invocation, + ReflectiveInvocationContext<Method> invocationContext, ExtensionContext extensionContext) + throws Throwable { + FuzzTest fuzzTest = + AnnotationSupport.findAnnotation(invocationContext.getExecutable(), FuzzTest.class).get(); + FuzzTestExecutor.configureAndInstallAgent(extensionContext, fuzzTest.maxDuration()); + // Skip the invocation of the test method with the special arguments provided by + // FuzzTestArgumentsProvider and start fuzzing instead. + if (Utils.isMarkedInvocation(invocationContext)) { + startFuzzing(invocation, invocationContext, extensionContext); + } else { + // Blocked by https://github.com/junit-team/junit5/issues/3282: + // TODO: The seeds from the input directory are duplicated here as there is no way to + // recognize them. + // TODO: Error out if there is a non-Jazzer ArgumentsProvider and the SeedSerializer does not + // support write. + if (Utils.isFuzzing(extensionContext)) { + // JUnit verifies that the arguments for this invocation are valid. + recordSeedForFuzzing(invocationContext.getArguments(), extensionContext); + } + runWithHooks(invocation); + } + } + + /** + * Mimics the logic of Jazzer's FuzzTargetRunner, which reports findings in the following way: + * <ol> + * <li>If a hook used Jazzer#reportFindingFromHook to explicitly report a finding, the last such + * finding, as stored in JazzerInternal#lastFinding, is reported. <li>If the fuzz target method + * threw a Throwable, that is reported. <li>3. Otherwise, nothing is reported. + * </ol> + */ + private static void runWithHooks(Invocation<Void> invocation) throws Throwable { + Throwable thrown = null; + getLastFindingField().set(null, null); + // When running in regression test mode, the agent emits additional bytecode logic in front of + // method hook invocations that enables them only while a global variable managed by + // withHooksEnabled is true. + // + // Alternatives considered: + // * Using a dedicated class loader for @FuzzTests: First-class support for this isn't + // available in JUnit 5 (https://github.com/junit-team/junit5/issues/201), but + // third-party extensions have done it: + // https://github.com/spring-projects/spring-boot/blob/main/spring-boot-project/spring-boot-tools/spring-boot-test-support/src/main/java/org/springframework/boot/testsupport/classpath/ModifiedClassPathExtension.java + // However, as this involves launching a new test run as part of running a test, this + // introduces a number of inconsistencies if applied on the test method rather than test + // class level. For example, @BeforeAll methods will have to be run twice in different class + // loaders, which may not be safe if they are using global resources not separated by class + // loaders (e.g. files). + try (AutoCloseable ignored = withHooksEnabled()) { + invocation.proceed(); + } catch (Throwable t) { + thrown = t; + } + Throwable stored = (Throwable) getLastFindingField().get(null); + if (stored != null) { + throw stored; + } else if (thrown != null) { + throw thrown; + } + } + + private static void startFuzzing(Invocation<Void> invocation, + ReflectiveInvocationContext<Method> invocationContext, ExtensionContext extensionContext) + throws Throwable { + invocation.skip(); + Optional<Throwable> throwable = + FuzzTestExecutor.fromContext(extensionContext) + .execute(invocationContext, getOrCreateSeedSerializer(extensionContext)); + if (throwable.isPresent()) { + throw throwable.get(); + } + } + + private void recordSeedForFuzzing(List<Object> arguments, ExtensionContext extensionContext) + throws IOException { + SeedSerializer seedSerializer = getOrCreateSeedSerializer(extensionContext); + try { + FuzzTestExecutor.fromContext(extensionContext) + .addSeed(seedSerializer.write(arguments.toArray())); + } catch (UnsupportedOperationException ignored) { + } + } + + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext extensionContext) { + if (!Utils.isFuzzing(extensionContext)) { + return ConditionEvaluationResult.enabled( + "Regression tests are run instead of fuzzing since JAZZER_FUZZ has not been set to a non-empty value"); + } + // Only fuzz the first @FuzzTest that makes it here. + if (FuzzTestExtensions.fuzzTestMethod.compareAndSet( + null, extensionContext.getRequiredTestMethod()) + || extensionContext.getRequiredTestMethod().equals( + FuzzTestExtensions.fuzzTestMethod.get())) { + return ConditionEvaluationResult.enabled( + "Fuzzing " + extensionContext.getRequiredTestMethod()); + } + return ConditionEvaluationResult.disabled( + "Only one fuzz test can be run at a time, but multiple tests have been annotated with @FuzzTest"); + } + + private static SeedSerializer getOrCreateSeedSerializer(ExtensionContext extensionContext) { + Method method = extensionContext.getRequiredTestMethod(); + return extensionContext.getStore(Namespace.create(FuzzTestExtensions.class, method)) + .getOrComputeIfAbsent( + SeedSerializer.class, unused -> SeedSerializer.of(method), SeedSerializer.class); + } + + private static Field getLastFindingField() throws ClassNotFoundException, NoSuchFieldException { + if (lastFindingField == null) { + Class<?> jazzerInternal = Class.forName(JAZZER_INTERNAL); + lastFindingField = jazzerInternal.getField("lastFinding"); + } + return lastFindingField; + } + + private static Field getHooksEnabledField() throws ClassNotFoundException, NoSuchFieldException { + if (hooksEnabledField == null) { + Class<?> jazzerInternal = Class.forName(JAZZER_INTERNAL); + hooksEnabledField = jazzerInternal.getField("hooksEnabled"); + } + return hooksEnabledField; + } + + private static AutoCloseable withHooksEnabled() + throws NoSuchFieldException, ClassNotFoundException, IllegalAccessException { + Field hooksEnabledField = getHooksEnabledField(); + hooksEnabledField.setBoolean(null, true); + return () -> hooksEnabledField.setBoolean(null, false); + } +} diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzingArgumentsProvider.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzingArgumentsProvider.java new file mode 100644 index 00000000..61a8d3fe --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzingArgumentsProvider.java @@ -0,0 +1,42 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.junit; + +import static com.code_intelligence.jazzer.junit.Utils.isFuzzing; + +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; + +class FuzzingArgumentsProvider implements ArgumentsProvider { + @Override + public Stream<? extends Arguments> provideArguments(ExtensionContext extensionContext) { + if (!isFuzzing(extensionContext)) { + return Stream.empty(); + } + + // When fuzzing, supply a special set of arguments that our InvocationInterceptor uses as a + // sign to start fuzzing. + // FIXME: This is a hack that is needed only because there does not seem to be a way to + // communicate out of band that a certain invocation was triggered by a particular argument + // provider. We should get rid of this hack as soon as + // https://github.com/junit-team/junit5/issues/3282 has been addressed. + return Stream.of( + Utils.getMarkedArguments(extensionContext.getRequiredTestMethod(), "Fuzzing...")); + } +} diff --git a/src/main/java/com/code_intelligence/jazzer/junit/SeedArgumentsProvider.java b/src/main/java/com/code_intelligence/jazzer/junit/SeedArgumentsProvider.java new file mode 100644 index 00000000..687a1f7f --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/junit/SeedArgumentsProvider.java @@ -0,0 +1,225 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.code_intelligence.jazzer.junit.Utils.isFuzzing; +import static com.code_intelligence.jazzer.junit.Utils.runFromCommandLine; +import static org.junit.jupiter.api.Named.named; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiPredicate; +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; + +class SeedArgumentsProvider implements ArgumentsProvider { + @Override + public Stream<? extends Arguments> provideArguments(ExtensionContext extensionContext) + throws IOException { + if (runFromCommandLine(extensionContext)) { + // libFuzzer always runs on the file-based seeds first anyway and the additional visual + // indication provided by test invocations for seeds isn't effective on the command line, so + // we skip these invocations. + return Stream.empty(); + } + + Class<?> testClass = extensionContext.getRequiredTestClass(); + Method testMethod = extensionContext.getRequiredTestMethod(); + + Stream<Map.Entry<String, byte[]>> rawSeeds = + Stream.of(new SimpleImmutableEntry<>("<empty input>", new byte[0])); + rawSeeds = Stream.concat(rawSeeds, walkInputs(testClass, testMethod)); + + if (Utils.isCoverageAgentPresent() + && Files.isDirectory(Utils.generatedCorpusPath(testClass, testMethod))) { + rawSeeds = Stream.concat(rawSeeds, + walkInputsInPath(Utils.generatedCorpusPath(testClass, testMethod), Integer.MAX_VALUE)); + } + + SeedSerializer serializer = SeedSerializer.of(testMethod); + return rawSeeds + .map(entry -> { + Object[] args = serializer.read(entry.getValue()); + args[0] = named(entry.getKey(), args[0]); + return arguments(args); + }) + .onClose(() -> { + if (!isFuzzing(extensionContext)) { + extensionContext.publishReportEntry( + "No fuzzing has been performed, the fuzz test has only been executed on the fixed " + + "set of inputs in the seed corpus.\n" + + "To start fuzzing, run a test with the environment variable JAZZER_FUZZ set to a " + + "non-empty value."); + } + if (!serializer.allReadsValid()) { + extensionContext.publishReportEntry( + "Some files in the seed corpus do not match the fuzz target signature.\n" + + "This indicates that they were generated with a different signature and may cause " + + "issues reproducing previous findings."); + } + }); + } + + /** + * Used in regression mode to get test cases for the associated {@code testMethod} + * This will return a stream of files consisting of: + * <ul> + * <li>{@code resources/<classpath>/<testClass name>Inputs/*}</li> + * <li>{@code resources/<classpath>/<testClass name>Inputs/<testMethod name>/**}</li> + * </ul> + * Or the equivalent behavior on resources inside a jar file. + * <p> + * Note that the first {@code <testClass name>Inputs} path will not recursively search all + * directories but only gives files in that directory whereas the {@code <testMethod name>} + * directory is searched recursively. This allows for multiple tests to share inputs without + * needing to explicitly copy them into each test's directory. + * + * @param testClass the class of the test being run + * @param testMethod the test function being run + * @return a stream of findings files to use as inputs for the test function + */ + private Stream<Map.Entry<String, byte[]>> walkInputs(Class<?> testClass, Method testMethod) + throws IOException { + URL classInputsDirUrl = testClass.getResource(Utils.inputsDirectoryResourcePath(testClass)); + + if (classInputsDirUrl == null) { + return Stream.empty(); + } + URI classInputsDirUri; + try { + classInputsDirUri = classInputsDirUrl.toURI(); + } catch (URISyntaxException e) { + throw new IOException("Failed to open inputs resource directory: " + classInputsDirUrl, e); + } + if (classInputsDirUri.getScheme().equals("file")) { + // The test is executed from class files, which usually happens when run from inside an IDE. + Path classInputsPath = Paths.get(classInputsDirUri); + + return Stream.concat( + walkClassInputs(classInputsPath), walkTestInputs(classInputsPath, testMethod)); + + } else if (classInputsDirUri.getScheme().equals("jar")) { + FileSystem jar = FileSystems.newFileSystem(classInputsDirUri, new HashMap<>()); + // inputsDirUrl looks like this: + // file:/tmp/testdata/ExampleFuzzTest_deploy.jar!/com/code_intelligence/jazzer/junit/testdata/ExampleFuzzTestInputs + String pathInJar = + classInputsDirUrl.getFile().substring(classInputsDirUrl.getFile().indexOf('!') + 1); + + Path classPathInJar = jar.getPath(pathInJar); + + return Stream + .concat(walkClassInputs(classPathInJar), walkTestInputs(classPathInJar, testMethod)) + .onClose(() -> { + try { + jar.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } else { + throw new IOException( + "Unsupported protocol for inputs resource directory: " + classInputsDirUrl); + } + } + + /** + * Walks over the inputs for the method being tested, recurses into subdirectories + * @param classInputsPath the path of the class being tested, used as the base path where the test + * method's directory + * should be + * @param testMethod the method being tested + * @return a stream of all files under {@code <classInputsPath>/<testMethod name>} + * @throws IOException can be thrown by the underlying call to {@link Files#find} + */ + private static Stream<Map.Entry<String, byte[]>> walkTestInputs( + Path classInputsPath, Method testMethod) throws IOException { + Path testInputsPath = classInputsPath.resolve(testMethod.getName()); + try { + return walkInputsInPath(testInputsPath, Integer.MAX_VALUE); + } catch (NoSuchFileException e) { + return Stream.empty(); + } + } + + /** + * Walks over the inputs for the class being tested. Does not recurse into subdirectories + * @param path the path to search to files + * @return a stream of all files (without directories) within {@code path}. If {@code path} is not + * found, an empty + * stream is returned. + * @throws IOException can be thrown by the underlying call to {@link Files#find} + */ + private static Stream<Map.Entry<String, byte[]>> walkClassInputs(Path path) throws IOException { + try { + // using a depth of 1 will get all files within the given path but does not recurse into + // subdirectories + return walkInputsInPath(path, 1); + } catch (NoSuchFileException e) { + return Stream.empty(); + } + } + + /** + * Gets a sorted stream of all files (without directories) within under the given {@code path} + * @param path the path to walk + * @param depth the maximum depth of subdirectories to search from within {@code path}. 1 + * indicates it should return + * only the files directly in {@code path} and not search any of its subdirectories + * @return a stream of file name -> file contents as a raw byte array + * @throws IOException can be thrown by the call to {@link Files#find(Path, int, BiPredicate, + * FileVisitOption...)} + */ + private static Stream<Map.Entry<String, byte[]>> walkInputsInPath(Path path, int depth) + throws IOException { + // @ParameterTest automatically closes Streams and AutoCloseable instances. + // noinspection resource + return Files + .find(path, depth, + (fileOrDir, basicFileAttributes) + -> !basicFileAttributes.isDirectory(), + FileVisitOption.FOLLOW_LINKS) + // JUnit identifies individual runs of a `@ParameterizedTest` via their invocation number. + // In order to get reproducible behavior e.g. when trying to debug a particular input, all + // inputs thus have to be provided in deterministic order. + .sorted() + .map(file + -> new SimpleImmutableEntry<>( + file.getFileName().toString(), readAllBytesUnchecked(file))); + } + + private static byte[] readAllBytesUnchecked(Path path) { + try { + return Files.readAllBytes(path); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/src/main/java/com/code_intelligence/jazzer/junit/SeedSerializer.java b/src/main/java/com/code_intelligence/jazzer/junit/SeedSerializer.java new file mode 100644 index 00000000..0cebef2f --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/junit/SeedSerializer.java @@ -0,0 +1,144 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.junit; + +import com.code_intelligence.jazzer.api.FuzzedDataProvider; +import com.code_intelligence.jazzer.autofuzz.Meta; +import com.code_intelligence.jazzer.driver.FuzzedDataProviderImpl; +import com.code_intelligence.jazzer.driver.Opt; +import com.code_intelligence.jazzer.mutation.ArgumentsMutator; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Method; +import java.util.Optional; + +interface SeedSerializer { + Object[] read(byte[] bytes); + default boolean allReadsValid() { + return true; + } + + // Implementations can assume that the argument array contains valid arguments for the method that + // this instance has been constructed for. + byte[] write(Object[] args) throws UnsupportedOperationException; + + /** + * Creates specialized {@link SeedSerializer} instances for the following method parameters: + * <ul> + * <li>{@code byte[]} + * <li>{@code FuzzDataProvider} + * <li>Any other types will attempt to be created using either Autofuzz or the experimental + * mutator framework if {@link Opt}'s {@code experimentalMutator} is set. + * </ul> + */ + static SeedSerializer of(Method method) { + if (method.getParameterCount() == 0) { + throw new IllegalArgumentException( + "Methods annotated with @FuzzTest must take at least one parameter"); + } + if (method.getParameterCount() == 1 && method.getParameterTypes()[0] == byte[].class) { + return new ByteArraySeedSerializer(); + } else if (method.getParameterCount() == 1 + && method.getParameterTypes()[0] == FuzzedDataProvider.class) { + return new FuzzedDataProviderSeedSerializer(); + } else { + Optional<ArgumentsMutator> argumentsMutator = + Opt.experimentalMutator ? ArgumentsMutator.forMethod(method) : Optional.empty(); + return argumentsMutator.<SeedSerializer>map(ArgumentsMutatorSeedSerializer::new) + .orElseGet(() -> new AutofuzzSeedSerializer(method)); + } + } +} + +final class ByteArraySeedSerializer implements SeedSerializer { + @Override + public Object[] read(byte[] bytes) { + return new Object[] {bytes}; + } + + @Override + public byte[] write(Object[] args) { + return (byte[]) args[0]; + } +} + +final class FuzzedDataProviderSeedSerializer implements SeedSerializer { + @Override + public Object[] read(byte[] bytes) { + return new Object[] {FuzzedDataProviderImpl.withJavaData(bytes)}; + } + + @Override + public byte[] write(Object[] args) throws UnsupportedOperationException { + // While we could get the underlying bytes, it's not possible to provide Java seeds for fuzz + // tests with a FuzzedDataProvider parameter. + throw new UnsupportedOperationException(); + } +} + +final class ArgumentsMutatorSeedSerializer implements SeedSerializer { + private final ArgumentsMutator mutator; + private boolean allReadsValid; + + public ArgumentsMutatorSeedSerializer(ArgumentsMutator mutator) { + this.mutator = mutator; + } + + @Override + public Object[] read(byte[] bytes) { + allReadsValid &= mutator.read(new ByteArrayInputStream(bytes)); + return mutator.getArguments(); + } + + @Override + public boolean allReadsValid() { + return allReadsValid; + } + + @Override + public byte[] write(Object[] args) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + mutator.writeAny(out, args); + return out.toByteArray(); + } +} + +final class AutofuzzSeedSerializer implements SeedSerializer { + private final Meta meta; + private final Method method; + + public AutofuzzSeedSerializer(Method method) { + this.meta = new Meta(method.getDeclaringClass()); + this.method = method; + } + + @Override + public Object[] read(byte[] bytes) { + try (FuzzedDataProviderImpl data = FuzzedDataProviderImpl.withJavaData(bytes)) { + // The Autofuzz FuzzTarget uses data to construct an instance of the test class before + // it constructs the fuzz test arguments. We don't need the instance here, but still + // generate it as that mutates the FuzzedDataProvider state. + meta.consumeNonStatic(data, method.getDeclaringClass()); + return meta.consumeArguments(data, method, null); + } + } + + @Override + public byte[] write(Object[] args) throws UnsupportedOperationException { + throw new UnsupportedOperationException(); + } +} diff --git a/src/main/java/com/code_intelligence/jazzer/junit/Utils.java b/src/main/java/com/code_intelligence/jazzer/junit/Utils.java new file mode 100644 index 00000000..4ab81cd1 --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/junit/Utils.java @@ -0,0 +1,305 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static java.util.Arrays.stream; +import static java.util.Collections.newSetFromMap; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toList; +import static org.junit.jupiter.api.Named.named; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import com.code_intelligence.jazzer.utils.UnsafeProvider; +import com.code_intelligence.jazzer.utils.UnsafeUtils; +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodType; +import java.lang.management.ManagementFactory; +import java.lang.reflect.Array; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Proxy; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ReflectiveInvocationContext; +import org.junit.jupiter.params.provider.Arguments; + +class Utils { + /** + * Returns the resource path of the inputs directory for a given test class and method. The path + * will have the form + * {@code <class name>Inputs/<method name>} + */ + static String inputsDirectoryResourcePath(Class<?> testClass, Method testMethod) { + return testClass.getSimpleName() + "Inputs" + + "/" + testMethod.getName(); + } + + static String inputsDirectoryResourcePath(Class<?> testClass) { + return testClass.getSimpleName() + "Inputs"; + } + + /** + * Returns the file system path of the inputs corpus directory in the source tree, if it exists. + * The directory is created if it does not exist, but the test resource directory itself exists. + */ + static Optional<Path> inputsDirectorySourcePath( + Class<?> testClass, Method testMethod, Path baseDir) { + String inputsResourcePath = Utils.inputsDirectoryResourcePath(testClass, testMethod); + // Make the inputs resource path absolute. + if (!inputsResourcePath.startsWith("/")) { + String inputsPackage = testClass.getPackage().getName().replace('.', '/'); + inputsResourcePath = "/" + inputsPackage + "/" + inputsResourcePath; + } + + // Following the Maven directory layout, we look up the inputs directory under + // src/test/resources. This should be correct also for multi-module projects as JUnit is usually + // launched in the current module's root directory. + Path testResourcesDirectory = baseDir.resolve("src").resolve("test").resolve("resources"); + Path sourceInputsDirectory = testResourcesDirectory; + for (String segment : inputsResourcePath.split("/")) { + sourceInputsDirectory = sourceInputsDirectory.resolve(segment); + } + if (Files.isDirectory(sourceInputsDirectory)) { + return Optional.of(sourceInputsDirectory); + } + // If we can at least find the test resource directory, create the inputs directory. + if (!Files.isDirectory(testResourcesDirectory)) { + return Optional.empty(); + } + try { + return Optional.of(Files.createDirectories(sourceInputsDirectory)); + } catch (Exception e) { + return Optional.empty(); + } + } + + static Path generatedCorpusPath(Class<?> testClass, Method testMethod) { + return Paths.get(".cifuzz-corpus", testClass.getName(), testMethod.getName()); + } + + /** + * Returns a heuristic default value for jazzer.instrument based on the test class. + */ + static String getLegacyInstrumentationFilter(Class<?> testClass) { + // This is an extremely rough "implementation" of the public suffix list algorithm + // (https://publicsuffix.org/): It tries to guess the shortest prefix of the package name that + // isn't public. It doesn't use the actual list, but instead assumes that every root segment as + // well as "com.github" are public. Examples: + // - com.example.Test --> com.example.** + // - com.example.foobar.Test --> com.example.** + // - com.github.someones.repo.Test --> com.github.someones.** + String packageName = testClass.getPackage().getName(); + String[] packageSegments = packageName.split("\\."); + int numSegments = 2; + if (packageSegments.length > 2 && packageSegments[0].equals("com") + && packageSegments[1].equals("github")) { + numSegments = 3; + } + return Stream.concat(Arrays.stream(packageSegments).limit(numSegments), Stream.of("**")) + .collect(joining(".")); + } + + private static final Pattern CLASSPATH_SPLITTER = + Pattern.compile(Pattern.quote(File.pathSeparator)); + + /** + * Returns a heuristic default value for jazzer.instrument based on the files on the provided + * classpath. + */ + static Optional<String> getClassPathBasedInstrumentationFilter(String classPath) { + List<Path> includes = + CLASSPATH_SPLITTER.splitAsStream(classPath) + .map(Paths::get) + // We consider classpath entries that are directories rather than jar files to contain + // the classes of the current project rather than external dependencies. This is just a + // heuristic and breaks with build systems that package all classes in jar files, e.g. + // with Bazel. + .filter(Files::isDirectory) + .flatMap(root -> { + HashSet<Path> pkgs = new HashSet<>(); + try { + Files.walkFileTree(root, new SimpleFileVisitor<Path>() { + @Override + public FileVisitResult preVisitDirectory( + Path dir, BasicFileAttributes basicFileAttributes) throws IOException { + try (Stream<Path> entries = Files.list(dir)) { + // If a directory contains a .class file, we add an include filter matching it + // and all subdirectories. + // Special case: If there is a class defined at the root, only the unnamed + // package is included, so continue with the traversal of subdirectories + // to discover additional includes. + if (entries.filter(path -> path.toString().endsWith(".class")) + .anyMatch(Files::isRegularFile)) { + Path pkgPath = root.relativize(dir); + pkgs.add(pkgPath); + if (pkgPath.toString().isEmpty()) { + return FileVisitResult.CONTINUE; + } else { + return FileVisitResult.SKIP_SUBTREE; + } + } + } + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + // This is only a best-effort heuristic anyway, ignore this directory. + return Stream.of(); + } + return pkgs.stream(); + }) + .distinct() + .collect(toList()); + if (includes.isEmpty()) { + return Optional.empty(); + } + return Optional.of( + includes.stream() + .map(Path::toString) + // For classes without a package, only include the unnamed package. + .map(path -> path.isEmpty() ? "*" : path.replace(File.separator, ".") + ".**") + .sorted() + // jazzer.instrument uses ',' as the separator. + .collect(joining(","))); + } + + private static final Pattern COVERAGE_AGENT_ARG = + Pattern.compile("-javaagent:.*(?:intellij-coverage-agent|jacoco).*"); + static boolean isCoverageAgentPresent() { + return ManagementFactory.getRuntimeMXBean().getInputArguments().stream().anyMatch( + s -> COVERAGE_AGENT_ARG.matcher(s).matches()); + } + + private static final boolean IS_FUZZING_ENV = + System.getenv("JAZZER_FUZZ") != null && !System.getenv("JAZZER_FUZZ").isEmpty(); + static boolean isFuzzing(ExtensionContext extensionContext) { + return IS_FUZZING_ENV || runFromCommandLine(extensionContext); + } + + static boolean runFromCommandLine(ExtensionContext extensionContext) { + return extensionContext.getConfigurationParameter("jazzer.internal.commandLine") + .map(Boolean::parseBoolean) + .orElse(false); + } + + /** + * Returns true if and only if the value is equal to "true", "1", or "yes" case-insensitively. + */ + static boolean permissivelyParseBoolean(String value) { + return value.equalsIgnoreCase("true") || value.equals("1") || value.equalsIgnoreCase("yes"); + } + + /** + * Convert the string to ISO 8601 (https://en.wikipedia.org/wiki/ISO_8601#Durations). We do not + * allow for duration units longer than hours, so we can always prepend PT. + */ + static long durationStringToSeconds(String duration) { + String isoDuration = + "PT" + duration.replace("sec", "s").replace("min", "m").replace("hr", "h").replace(" ", ""); + return Duration.parse(isoDuration).getSeconds(); + } + + /** + * Creates {@link Arguments} for a single invocation of a parameterized test that can be + * identified as having been created in this way by {@link #isMarkedInvocation}. + * + * @param displayName the display name to assign to every argument + */ + static Arguments getMarkedArguments(Method method, String displayName) { + return arguments(stream(method.getParameterTypes()) + .map(Utils::getMarkedInstance) + // Wrap in named as toString may crash on marked instances. + .map(arg -> named(displayName, arg)) + .toArray(Object[] ::new)); + } + + /** + * @return {@code true} if and only if the arguments for this test method invocation were created + * with {@link #getMarkedArguments} + */ + static boolean isMarkedInvocation(ReflectiveInvocationContext<Method> invocationContext) { + if (invocationContext.getArguments().stream().anyMatch(Utils::isMarkedInstance)) { + if (invocationContext.getArguments().stream().allMatch(Utils::isMarkedInstance)) { + return true; + } + throw new IllegalStateException( + "Some, but not all arguments were marked in invocation of " + invocationContext); + } else { + return false; + } + } + + private static final ClassValue<Object> uniqueInstanceCache = new ClassValue<Object>() { + @Override + protected Object computeValue(Class<?> clazz) { + return makeMarkedInstance(clazz); + } + }; + private static final Set<Object> uniqueInstances = newSetFromMap(new IdentityHashMap<>()); + + // Visible for testing. + static <T> T getMarkedInstance(Class<T> clazz) { + // makeMarkedInstance creates new classes, which is expensive and can cause the JVM to run out + // of metaspace. We thus cache the marked instances per class. + Object instance = uniqueInstanceCache.get(clazz); + uniqueInstances.add(instance); + return (T) instance; + } + + // Visible for testing. + static boolean isMarkedInstance(Object instance) { + return uniqueInstances.contains(instance); + } + + private static Object makeMarkedInstance(Class<?> clazz) { + if (clazz == Class.class) { + return new Object() {}.getClass(); + } + if (clazz.isArray()) { + return Array.newInstance(clazz.getComponentType(), 0); + } + if (clazz.isInterface()) { + return Proxy.newProxyInstance( + Utils.class.getClassLoader(), new Class[] {clazz}, (o, method, objects) -> null); + } + + if (clazz.isPrimitive()) { + clazz = MethodType.methodType(clazz).wrap().returnType(); + } else if (Modifier.isAbstract(clazz.getModifiers())) { + clazz = UnsafeUtils.defineAnonymousConcreteSubclass(clazz); + } + + try { + return clazz.cast(UnsafeProvider.getUnsafe().allocateInstance(clazz)); + } catch (InstantiationException e) { + throw new IllegalStateException(e); + } + } +} |