diff options
Diffstat (limited to 'agent/src/main/java/com/code_intelligence')
44 files changed, 1989 insertions, 1172 deletions
diff --git a/agent/src/main/java/com/code_intelligence/jazzer/agent/Agent.kt b/agent/src/main/java/com/code_intelligence/jazzer/agent/Agent.kt index 33d02263..f9b026f1 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/agent/Agent.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/agent/Agent.kt @@ -16,38 +16,35 @@ package com.code_intelligence.jazzer.agent +import com.code_intelligence.jazzer.driver.Opt import com.code_intelligence.jazzer.instrumentor.CoverageRecorder +import com.code_intelligence.jazzer.instrumentor.Hooks import com.code_intelligence.jazzer.instrumentor.InstrumentationType -import com.code_intelligence.jazzer.instrumentor.loadHooks -import com.code_intelligence.jazzer.runtime.ManifestUtils +import com.code_intelligence.jazzer.runtime.NativeLibHooks +import com.code_intelligence.jazzer.runtime.TraceCmpHooks +import com.code_intelligence.jazzer.runtime.TraceDivHooks +import com.code_intelligence.jazzer.runtime.TraceIndirHooks import com.code_intelligence.jazzer.utils.ClassNameGlobber +import com.code_intelligence.jazzer.utils.ManifestUtils import java.io.File import java.lang.instrument.Instrumentation +import java.net.URI import java.nio.file.Paths import java.util.jar.JarFile import kotlin.io.path.ExperimentalPathApi import kotlin.io.path.exists import kotlin.io.path.isDirectory -val KNOWN_ARGUMENTS = listOf( - "instrumentation_includes", - "instrumentation_excludes", - "custom_hook_includes", - "custom_hook_excludes", - "trace", - "custom_hooks", - "id_sync_file", - "dump_classes_dir", -) - private object AgentJarFinder { - private val agentJarPath = AgentJarFinder::class.java.protectionDomain?.codeSource?.location?.toURI() - val agentJarFile = agentJarPath?.let { JarFile(File(it)) } + val agentJarFile = jarUriForClass(AgentJarFinder::class.java)?.let { JarFile(File(it)) } } -private val argumentDelimiter = if (System.getProperty("os.name").startsWith("Windows")) ";" else ":" +fun jarUriForClass(clazz: Class<*>): URI? { + return clazz.protectionDomain?.codeSource?.location?.toURI() +} @OptIn(ExperimentalPathApi::class) +@Suppress("UNUSED_PARAMETER") fun premain(agentArgs: String?, instrumentation: Instrumentation) { // Add the agent jar (i.e., the jar out of which we are currently executing) to the search path of the bootstrap // class loader to ensure that instrumented classes can find the CoverageMap class regardless of which ClassLoader @@ -57,37 +54,25 @@ fun premain(agentArgs: String?, instrumentation: Instrumentation) { } else { println("WARN: Failed to add agent JAR to bootstrap class loader search path") } - val argumentMap = (agentArgs ?: "") - .split(',') - .mapNotNull { - val splitArg = it.split('=', limit = 2) - when { - splitArg.size != 2 -> { - if (splitArg[0].isNotEmpty()) - println("WARN: Ignoring argument ${splitArg[0]} without value") - null - } - splitArg[0] !in KNOWN_ARGUMENTS -> { - println("WARN: Ignoring unknown argument ${splitArg[0]}") - null - } - else -> splitArg[0] to splitArg[1].split(argumentDelimiter) - } - }.toMap() - val manifestCustomHookNames = ManifestUtils.combineManifestValues(ManifestUtils.HOOK_CLASSES).flatMap { - it.split(':') + + val manifestCustomHookNames = + ManifestUtils.combineManifestValues(ManifestUtils.HOOK_CLASSES).flatMap { + it.split(':') + }.filter { it.isNotBlank() } + val allCustomHookNames = (manifestCustomHookNames + Opt.customHooks).toSet() + val disabledCustomHookNames = Opt.disabledHooks.toSet() + val customHookNames = allCustomHookNames - disabledCustomHookNames + val disabledCustomHooksToPrint = allCustomHookNames - customHookNames.toSet() + if (disabledCustomHooksToPrint.isNotEmpty()) { + println("INFO: Not using the following disabled hooks: ${disabledCustomHooksToPrint.joinToString(", ")}") } - val customHookNames = manifestCustomHookNames + (argumentMap["custom_hooks"] ?: emptyList()) - val classNameGlobber = ClassNameGlobber( - argumentMap["instrumentation_includes"] ?: emptyList(), - (argumentMap["instrumentation_excludes"] ?: emptyList()) + customHookNames - ) + + val classNameGlobber = ClassNameGlobber(Opt.instrumentationIncludes, Opt.instrumentationExcludes + customHookNames) CoverageRecorder.classNameGlobber = classNameGlobber - val dependencyClassNameGlobber = ClassNameGlobber( - argumentMap["custom_hook_includes"] ?: emptyList(), - (argumentMap["custom_hook_excludes"] ?: emptyList()) + customHookNames - ) - val instrumentationTypes = (argumentMap["trace"] ?: listOf("all")).flatMap { + val customHookClassNameGlobber = ClassNameGlobber(Opt.customHookIncludes, Opt.customHookExcludes + customHookNames) + // FIXME: Setting trace to the empty string explicitly results in all rather than no trace types + // being applied - this is unintuitive. + val instrumentationTypes = (Opt.trace.takeIf { it.isNotEmpty() } ?: listOf("all")).flatMap { when (it) { "cmp" -> setOf(InstrumentationType.CMP) "cov" -> setOf(InstrumentationType.COV) @@ -106,13 +91,13 @@ fun premain(agentArgs: String?, instrumentation: Instrumentation) { } } }.toSet() - val idSyncFile = argumentMap["id_sync_file"]?.let { - Paths.get(it.single()).also { path -> + val idSyncFile = Opt.idSyncFile.takeUnless { it.isEmpty() }?.let { + Paths.get(it).also { path -> println("INFO: Synchronizing coverage IDs in ${path.toAbsolutePath()}") } } - val dumpClassesDir = argumentMap["dump_classes_dir"]?.let { - Paths.get(it.single()).toAbsolutePath().also { path -> + val dumpClassesDir = Opt.dumpClassesDir.takeUnless { it.isEmpty() }?.let { + Paths.get(it).toAbsolutePath().also { path -> if (path.exists() && path.isDirectory()) { println("INFO: Dumping instrumented classes into $path") } else { @@ -120,41 +105,67 @@ fun premain(agentArgs: String?, instrumentation: Instrumentation) { } } } + val includedHookNames = instrumentationTypes + .mapNotNull { type -> + when (type) { + InstrumentationType.CMP -> TraceCmpHooks::class.java.name + InstrumentationType.DIV -> TraceDivHooks::class.java.name + InstrumentationType.INDIR -> TraceIndirHooks::class.java.name + InstrumentationType.NATIVE -> NativeLibHooks::class.java.name + else -> null + } + } + val coverageIdSynchronizer = if (idSyncFile != null) + FileSyncCoverageIdStrategy(idSyncFile) + else + MemSyncCoverageIdStrategy() + + val (includedHooks, customHooks) = Hooks.loadHooks(includedHookNames.toSet(), customHookNames.toSet()) + // If we don't append the JARs containing the custom hooks to the bootstrap class loader, + // third-party hooks not contained in the agent JAR will not be able to instrument Java standard + // library classes. These classes are loaded by the bootstrap / system class loader and would + // not be considered when resolving references to hook methods, leading to NoClassDefFoundError + // being thrown. + customHooks.hookClasses + .mapNotNull { jarUriForClass(it) } + .toSet() + .map { JarFile(File(it)) } + .forEach { instrumentation.appendToBootstrapClassLoaderSearch(it) } + val runtimeInstrumentor = RuntimeInstrumentor( instrumentation, classNameGlobber, - dependencyClassNameGlobber, + customHookClassNameGlobber, instrumentationTypes, - idSyncFile, + includedHooks.hooks, + customHooks.hooks, + customHooks.additionalHookClassNameGlobber, + coverageIdSynchronizer, dumpClassesDir, ) - instrumentation.apply { - addTransformer(runtimeInstrumentor) - } - val relevantClassesLoadedBeforeCustomHooks = instrumentation.allLoadedClasses - .map { it.name } - .filter { classNameGlobber.includes(it) || dependencyClassNameGlobber.includes(it) } - .toSet() - val customHooks = customHookNames.toSet().flatMap { hookClassName -> - try { - loadHooks(Class.forName(hookClassName)).also { - println("INFO: Loaded ${it.size} hooks from $hookClassName") - } - } catch (_: ClassNotFoundException) { - println("WARN: Failed to load hooks from $hookClassName") - emptySet() + // These classes are e.g. dependencies of the RuntimeInstrumentor or hooks and thus were loaded + // before the instrumentor was ready. Since we haven't enabled it yet, they can safely be + // "retransformed": They haven't been transformed yet. + val classesToRetransform = instrumentation.allLoadedClasses + .filter { + classNameGlobber.includes(it.name) || + customHookClassNameGlobber.includes(it.name) || + customHooks.additionalHookClassNameGlobber.includes(it.name) } - } - val relevantClassesLoadedAfterCustomHooks = instrumentation.allLoadedClasses - .map { it.name } - .filter { classNameGlobber.includes(it) || dependencyClassNameGlobber.includes(it) } - .toSet() - val nonHookClassesLoadedByHooks = relevantClassesLoadedAfterCustomHooks - relevantClassesLoadedBeforeCustomHooks - if (nonHookClassesLoadedByHooks.isNotEmpty()) { - println("WARN: Hooks were not applied to the following classes as they are dependencies of hooks:") - println("WARN: ${nonHookClassesLoadedByHooks.joinToString()}") - } + .filter { + instrumentation.isModifiableClass(it) + } + .toTypedArray() + + instrumentation.addTransformer(runtimeInstrumentor, true) - runtimeInstrumentor.registerCustomHooks(customHooks) + if (classesToRetransform.isNotEmpty()) { + if (instrumentation.isRetransformClassesSupported) { + instrumentation.retransformClasses(*classesToRetransform) + } else { + println("WARN: Instrumentation was not applied to the following classes as they are dependencies of hooks:") + println("WARN: ${classesToRetransform.joinToString()}") + } + } } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/agent/BUILD.bazel b/agent/src/main/java/com/code_intelligence/jazzer/agent/BUILD.bazel index 2d5eec5c..db6ae264 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/agent/BUILD.bazel +++ b/agent/src/main/java/com/code_intelligence/jazzer/agent/BUILD.bazel @@ -11,5 +11,6 @@ kt_jvm_library( deps = [ "//agent/src/main/java/com/code_intelligence/jazzer/instrumentor", "//agent/src/main/java/com/code_intelligence/jazzer/runtime", + "//driver/src/main/java/com/code_intelligence/jazzer/driver:opt", ], ) diff --git a/agent/src/main/java/com/code_intelligence/jazzer/agent/CoverageIdStrategy.kt b/agent/src/main/java/com/code_intelligence/jazzer/agent/CoverageIdStrategy.kt index fd2a1e7c..5d1d28e3 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/agent/CoverageIdStrategy.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/agent/CoverageIdStrategy.kt @@ -14,7 +14,8 @@ package com.code_intelligence.jazzer.agent -import java.nio.ByteBuffer +import com.code_intelligence.jazzer.utils.append +import com.code_intelligence.jazzer.utils.readFully import java.nio.channels.FileChannel import java.nio.channels.FileLock import java.nio.file.Path @@ -24,59 +25,42 @@ import java.util.UUID /** * Indicates a fatal failure to generate synchronized coverage IDs. */ -internal class CoverageIdException(cause: Throwable? = null) : +class CoverageIdException(cause: Throwable? = null) : RuntimeException("Failed to synchronize coverage IDs", cause) +/** + * [CoverageIdStrategy] provides an abstraction to switch between context specific coverage ID generation. + * + * Coverage (i.e., edge) IDs differ from other kinds of IDs, such as those generated for call sites or cmp + * instructions, in that they should be consecutive, collision-free, and lie in a known, small range. + * This precludes us from generating them simply as hashes of class names. + */ interface CoverageIdStrategy { - /** - * Obtain the first coverage ID to be used for the class [className]. - * The caller *must* also call [commitIdCount] once it has instrumented that class, even if instrumentation fails. - */ - @Throws(CoverageIdException::class) - fun obtainFirstId(className: String): Int /** - * Records the number of coverage IDs used to instrument the class specified in a previous call to [obtainFirstId]. - * If instrumenting the class should fail, this function must still be called. In this case, [idCount] is set to 0. + * [withIdForClass] provides the initial coverage ID of the given [className] as parameter to the + * [block] to execute. [block] has to return the number of additionally used IDs. */ @Throws(CoverageIdException::class) - fun commitIdCount(idCount: Int) + fun withIdForClass(className: String, block: (Int) -> Int) } /** - * An unsynchronized strategy for coverage ID generation that simply increments a global counter. + * A memory synced strategy for coverage ID generation. + * + * This strategy uses a synchronized block to guard access to a global edge ID counter. + * Even though concurrent fuzzing is not fully supported this strategy enables consistent coverage + * IDs in case of concurrent class loading. + * + * It only prevents races within one VM instance. */ -internal class TrivialCoverageIdStrategy : CoverageIdStrategy { +class MemSyncCoverageIdStrategy : CoverageIdStrategy { private var nextEdgeId = 0 - override fun obtainFirstId(className: String) = nextEdgeId - - override fun commitIdCount(idCount: Int) { - nextEdgeId += idCount - } -} - -/** - * Reads the [FileChannel] to the end as a UTF-8 string. - */ -private fun FileChannel.readFully(): String { - check(size() <= Int.MAX_VALUE) - val buffer = ByteBuffer.allocate(size().toInt()) - while (buffer.hasRemaining()) { - when (read(buffer)) { - 0 -> throw IllegalStateException("No bytes read") - -1 -> break - } + @Synchronized + override fun withIdForClass(className: String, block: (Int) -> Int) { + nextEdgeId += block(nextEdgeId) } - return String(buffer.array()) -} - -/** - * Appends [string] to the end of the [FileChannel]. - */ -private fun FileChannel.append(string: String) { - position(size()) - write(ByteBuffer.wrap(string.toByteArray())) } /** @@ -84,19 +68,30 @@ private fun FileChannel.append(string: String) { * specified [idSyncFile]. * This class takes care of synchronizing the access to the file between multiple processes as long as the general * contract of [CoverageIdStrategy] is followed. - * - * Rationale: Coverage (i.e., edge) IDs differ from other kinds of IDs, such as those generated for call sites or cmp - * instructions, in that they should be consecutive, collision-free, and lie in a known, small range. This precludes us - * from generating them simply as hashes of class names and explains why go through the arduous process of synchronizing - * them across multiple agents. */ -internal class SynchronizedCoverageIdStrategy(private val idSyncFile: Path) : CoverageIdStrategy { - val uuid: UUID = UUID.randomUUID() - var idFileLock: FileLock? = null +class FileSyncCoverageIdStrategy(private val idSyncFile: Path) : CoverageIdStrategy { + private val uuid: UUID = UUID.randomUUID() + private var idFileLock: FileLock? = null + + private var cachedFirstId: Int? = null + private var cachedClassName: String? = null + private var cachedIdCount: Int? = null - var cachedFirstId: Int? = null - var cachedClassName: String? = null - var cachedIdCount: Int? = null + /** + * This method is synchronized to prevent concurrent access to the internal file lock which would result in + * [java.nio.channels.OverlappingFileLockException]. Furthermore, every coverage ID obtained by [obtainFirstId] + * is always committed back again to the sync file by [commitIdCount]. + */ + @Synchronized + override fun withIdForClass(className: String, block: (Int) -> Int) { + var actualNumEdgeIds = 0 + try { + val firstId = obtainFirstId(className) + actualNumEdgeIds = block(firstId) + } finally { + commitIdCount(actualNumEdgeIds) + } + } /** * Obtains a coverage ID for [className] such that all cooperating agent processes will obtain the same ID. @@ -108,7 +103,7 @@ internal class SynchronizedCoverageIdStrategy(private val idSyncFile: Path) : Co * In this case, the lock on the file is returned immediately and the extracted first coverage ID is returned to * the caller. The caller is still expected to call [commitIdCount] so that desynchronization can be detected. */ - override fun obtainFirstId(className: String): Int { + private fun obtainFirstId(className: String): Int { try { check(idFileLock == null) { "Already holding a lock on the ID file" } val localIdFile = FileChannel.open( @@ -170,7 +165,11 @@ internal class SynchronizedCoverageIdStrategy(private val idSyncFile: Path) : Co } } - override fun commitIdCount(idCount: Int) { + /** + * Records the number of coverage IDs used to instrument the class specified in a previous call to [obtainFirstId]. + * If instrumenting the class should fail, this function must still be called. In this case, [idCount] is set to 0. + */ + private fun commitIdCount(idCount: Int) { val localIdFileLock = idFileLock try { check(cachedClassName != null) diff --git a/agent/src/main/java/com/code_intelligence/jazzer/agent/RuntimeInstrumentor.kt b/agent/src/main/java/com/code_intelligence/jazzer/agent/RuntimeInstrumentor.kt index e2283aa2..fe2efd54 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/agent/RuntimeInstrumentor.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/agent/RuntimeInstrumentor.kt @@ -18,11 +18,6 @@ import com.code_intelligence.jazzer.instrumentor.ClassInstrumentor import com.code_intelligence.jazzer.instrumentor.CoverageRecorder import com.code_intelligence.jazzer.instrumentor.Hook import com.code_intelligence.jazzer.instrumentor.InstrumentationType -import com.code_intelligence.jazzer.instrumentor.loadHooks -import com.code_intelligence.jazzer.runtime.NativeLibHooks -import com.code_intelligence.jazzer.runtime.TraceCmpHooks -import com.code_intelligence.jazzer.runtime.TraceDivHooks -import com.code_intelligence.jazzer.runtime.TraceIndirHooks import com.code_intelligence.jazzer.utils.ClassNameGlobber import java.lang.instrument.ClassFileTransformer import java.lang.instrument.Instrumentation @@ -32,37 +27,25 @@ import kotlin.math.roundToInt import kotlin.system.exitProcess import kotlin.time.measureTimedValue -internal class RuntimeInstrumentor( +class RuntimeInstrumentor( private val instrumentation: Instrumentation, - private val classesToInstrument: ClassNameGlobber, - private val dependencyClassesToInstrument: ClassNameGlobber, + private val classesToFullyInstrument: ClassNameGlobber, + private val classesToHookInstrument: ClassNameGlobber, private val instrumentationTypes: Set<InstrumentationType>, - idSyncFile: Path?, + private val includedHooks: List<Hook>, + private val customHooks: List<Hook>, + // Dedicated name globber for additional classes to hook stated in hook annotations is needed due to + // existing include and exclude pattern of classesToHookInstrument. All classes are included in hook + // instrumentation except the ones from default excludes, like JDK and Kotlin classes. But additional + // classes to hook, based on annotations, are allowed to reference normally ignored ones, like JDK + // and Kotlin internals. + // FIXME: Adding an additional class to hook will apply _all_ hooks to it and not only the one it's + // defined in. At some point we might want to track the list of classes per custom hook rather than globally. + private val additionalClassesToHookInstrument: ClassNameGlobber, + private val coverageIdSynchronizer: CoverageIdStrategy, private val dumpClassesDir: Path?, ) : ClassFileTransformer { - private val coverageIdSynchronizer = if (idSyncFile != null) - SynchronizedCoverageIdStrategy(idSyncFile) - else - TrivialCoverageIdStrategy() - - private val includedHooks = instrumentationTypes - .mapNotNull { type -> - when (type) { - InstrumentationType.CMP -> TraceCmpHooks::class.java - InstrumentationType.DIV -> TraceDivHooks::class.java - InstrumentationType.INDIR -> TraceIndirHooks::class.java - InstrumentationType.NATIVE -> NativeLibHooks::class.java - else -> null - } - } - .flatMap { loadHooks(it) } - private val customHooks = emptyList<Hook>().toMutableList() - - fun registerCustomHooks(hooks: List<Hook>) { - customHooks.addAll(hooks) - } - @OptIn(kotlin.time.ExperimentalTime::class) override fun transform( loader: ClassLoader?, @@ -86,15 +69,20 @@ internal class RuntimeInstrumentor( }.also { instrumentedByteCode -> // Only dump classes that were instrumented. if (instrumentedByteCode != null && dumpClassesDir != null) { - val relativePath = "$internalClassName.class" - val absolutePath = dumpClassesDir.resolve(relativePath) - val dumpFile = absolutePath.toFile() - dumpFile.parentFile.mkdirs() - dumpFile.writeBytes(instrumentedByteCode) + dumpToClassFile(internalClassName, instrumentedByteCode) + dumpToClassFile(internalClassName, classfileBuffer, basenameSuffix = ".original") } } } + private fun dumpToClassFile(internalClassName: String, bytecode: ByteArray, basenameSuffix: String = "") { + val relativePath = "$internalClassName$basenameSuffix.class" + val absolutePath = dumpClassesDir!!.resolve(relativePath) + val dumpFile = absolutePath.toFile() + dumpFile.parentFile.mkdirs() + dumpFile.writeBytes(bytecode) + } + override fun transform( module: Module?, loader: ClassLoader?, @@ -103,33 +91,42 @@ internal class RuntimeInstrumentor( protectionDomain: ProtectionDomain?, classfileBuffer: ByteArray ): ByteArray? { - if (module != null && !module.canRead(RuntimeInstrumentor::class.java.module)) { - // Make all other modules read our (unnamed) module, which allows them to access the classes needed by the - // instrumentations, e.g. CoverageMap. If a module can't be modified, it should not be instrumented as the - // injected bytecode might throw NoClassDefFoundError. - // https://mail.openjdk.java.net/pipermail/jigsaw-dev/2021-May/014663.html - if (!instrumentation.isModifiableModule(module)) { - val prettyClassName = internalClassName.replace('/', '.') - println("WARN: Failed to instrument $prettyClassName in unmodifiable module ${module.name}, skipping") - return null + return try { + if (module != null && !module.canRead(RuntimeInstrumentor::class.java.module)) { + // Make all other modules read our (unnamed) module, which allows them to access the classes needed by the + // instrumentations, e.g. CoverageMap. If a module can't be modified, it should not be instrumented as the + // injected bytecode might throw NoClassDefFoundError. + // https://mail.openjdk.java.net/pipermail/jigsaw-dev/2021-May/014663.html + if (!instrumentation.isModifiableModule(module)) { + val prettyClassName = internalClassName.replace('/', '.') + println("WARN: Failed to instrument $prettyClassName in unmodifiable module ${module.name}, skipping") + return null + } + instrumentation.redefineModule( + module, + /* extraReads */ setOf(RuntimeInstrumentor::class.java.module), + emptyMap(), + emptyMap(), + emptySet(), + emptyMap() + ) } - instrumentation.redefineModule( - module, - /* extraReads */ setOf(RuntimeInstrumentor::class.java.module), - emptyMap(), - emptyMap(), - emptySet(), - emptyMap() - ) + transform(loader, internalClassName, classBeingRedefined, protectionDomain, classfileBuffer) + } catch (t: Throwable) { + // Throwables raised from transform are silently dropped, making it extremely hard to detect instrumentation + // failures. The docs advise to use a top-level try-catch. + // https://docs.oracle.com/javase/9/docs/api/java/lang/instrument/ClassFileTransformer.html + t.printStackTrace() + throw t } - return transform(loader, internalClassName, classBeingRedefined, protectionDomain, classfileBuffer) } @OptIn(kotlin.time.ExperimentalTime::class) fun transformInternal(internalClassName: String, classfileBuffer: ByteArray): ByteArray? { val fullInstrumentation = when { - classesToInstrument.includes(internalClassName) -> true - dependencyClassesToInstrument.includes(internalClassName) -> false + classesToFullyInstrument.includes(internalClassName) -> true + classesToHookInstrument.includes(internalClassName) -> false + additionalClassesToHookInstrument.includes(internalClassName) -> false else -> return null } val prettyClassName = internalClassName.replace('/', '.') @@ -165,14 +162,16 @@ internal class RuntimeInstrumentor( // trigger the GEP callbacks for ByteBuffer. traceDataFlow(instrumentationTypes) hooks(includedHooks + customHooks) - val firstId = coverageIdSynchronizer.obtainFirstId(internalClassName) - var actualNumEdgeIds = 0 - try { - actualNumEdgeIds = coverage(firstId) - } finally { - coverageIdSynchronizer.commitIdCount(actualNumEdgeIds) + coverageIdSynchronizer.withIdForClass(internalClassName) { firstId -> + coverage(firstId).also { actualNumEdgeIds -> + CoverageRecorder.recordInstrumentedClass( + internalClassName, + bytecode, + firstId, + actualNumEdgeIds + ) + } } - CoverageRecorder.recordInstrumentedClass(internalClassName, bytecode, firstId, firstId + actualNumEdgeIds) } else { hooks(customHooks) } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/api/BUILD.bazel b/agent/src/main/java/com/code_intelligence/jazzer/api/BUILD.bazel index e573e757..b26bb846 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/api/BUILD.bazel +++ b/agent/src/main/java/com/code_intelligence/jazzer/api/BUILD.bazel @@ -23,6 +23,7 @@ java_library( "Jazzer.java", "MethodHook.java", "MethodHooks.java", + "//agent/src/main/java/jaz", ], visibility = ["//visibility:public"], ) diff --git a/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueCritical.java b/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueCritical.java index 4402a7f3..fbde853b 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueCritical.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueCritical.java @@ -17,7 +17,7 @@ package com.code_intelligence.jazzer.api; /** * Thrown to indicate that a fuzz target has detected a critical severity security issue rather than * a normal bug. - * + * <p> * There is only a semantical but no functional difference between throwing exceptions of this type * or any other. However, automated fuzzing platforms can use the extra information to handle the * detected issues appropriately. diff --git a/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueHigh.java b/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueHigh.java index 4d323e56..05837b0e 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueHigh.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueHigh.java @@ -17,7 +17,7 @@ package com.code_intelligence.jazzer.api; /** * Thrown to indicate that a fuzz target has detected a high severity security issue rather than a * normal bug. - * + * <p> * There is only a semantical but no functional difference between throwing exceptions of this type * or any other. However, automated fuzzing platforms can use the extra information to handle the * detected issues appropriately. diff --git a/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueMedium.java b/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueMedium.java index f0de4ce7..be7c8c8f 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueMedium.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/api/FuzzerSecurityIssueMedium.java @@ -17,7 +17,7 @@ package com.code_intelligence.jazzer.api; /** * Thrown to indicate that a fuzz target has detected a medium severity security issue rather than a * normal bug. - * + * <p> * There is only a semantical but no functional difference between throwing exceptions of this type * or any other. However, automated fuzzing platforms can use the extra information to handle the * detected issues appropriately. diff --git a/agent/src/main/java/com/code_intelligence/jazzer/api/HookType.java b/agent/src/main/java/com/code_intelligence/jazzer/api/HookType.java index 1c564a78..8ed4337f 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/api/HookType.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/api/HookType.java @@ -17,6 +17,7 @@ package com.code_intelligence.jazzer.api; /** * The type of a {@link MethodHook}. */ +// Note: The order of entries is important and is used during instrumentation. public enum HookType { BEFORE, REPLACE, diff --git a/agent/src/main/java/com/code_intelligence/jazzer/api/Jazzer.java b/agent/src/main/java/com/code_intelligence/jazzer/api/Jazzer.java index e45f7600..97adf578 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/api/Jazzer.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/api/Jazzer.java @@ -18,32 +18,69 @@ import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; import java.lang.reflect.InvocationTargetException; +import java.security.SecureRandom; /** * Helper class with static methods that interact with Jazzer at runtime. */ final public class Jazzer { - private static Class<?> jazzerInternal = null; - - private static MethodHandle traceStrcmp = null; - private static MethodHandle traceStrstr = null; - private static MethodHandle traceMemcmp = null; - - private static MethodHandle consume = null; - private static MethodHandle autofuzzFunction1 = null; - private static MethodHandle autofuzzFunction2 = null; - private static MethodHandle autofuzzFunction3 = null; - private static MethodHandle autofuzzFunction4 = null; - private static MethodHandle autofuzzFunction5 = null; - private static MethodHandle autofuzzConsumer1 = null; - private static MethodHandle autofuzzConsumer2 = null; - private static MethodHandle autofuzzConsumer3 = null; - private static MethodHandle autofuzzConsumer4 = null; - private static MethodHandle autofuzzConsumer5 = null; + /** + * A 32-bit random number that hooks can use to make pseudo-random choices + * between multiple possible mutations they could guide the fuzzer towards. + * Hooks <b>must not</b> base the decision whether or not to report a finding + * on this number as this will make findings non-reproducible. + * <p> + * This is the same number that libFuzzer uses as a seed internally, which + * makes it possible to deterministically reproduce a previous fuzzing run by + * supplying the seed value printed by libFuzzer as the value of the + * {@code -seed}. + */ + public static final int SEED = getLibFuzzerSeed(); + + private static final Class<?> JAZZER_INTERNAL; + + private static final MethodHandle ON_FUZZ_TARGET_READY; + + private static final MethodHandle TRACE_STRCMP; + private static final MethodHandle TRACE_STRSTR; + private static final MethodHandle TRACE_MEMCMP; + private static final MethodHandle TRACE_PC_INDIR; + + private static final MethodHandle CONSUME; + private static final MethodHandle AUTOFUZZ_FUNCTION_1; + private static final MethodHandle AUTOFUZZ_FUNCTION_2; + private static final MethodHandle AUTOFUZZ_FUNCTION_3; + private static final MethodHandle AUTOFUZZ_FUNCTION_4; + private static final MethodHandle AUTOFUZZ_FUNCTION_5; + private static final MethodHandle AUTOFUZZ_CONSUMER_1; + private static final MethodHandle AUTOFUZZ_CONSUMER_2; + private static final MethodHandle AUTOFUZZ_CONSUMER_3; + private static final MethodHandle AUTOFUZZ_CONSUMER_4; + private static final MethodHandle AUTOFUZZ_CONSUMER_5; static { + Class<?> jazzerInternal = null; + MethodHandle onFuzzTargetReady = null; + MethodHandle traceStrcmp = null; + MethodHandle traceStrstr = null; + MethodHandle traceMemcmp = null; + MethodHandle tracePcIndir = null; + MethodHandle consume = null; + MethodHandle autofuzzFunction1 = null; + MethodHandle autofuzzFunction2 = null; + MethodHandle autofuzzFunction3 = null; + MethodHandle autofuzzFunction4 = null; + MethodHandle autofuzzFunction5 = null; + MethodHandle autofuzzConsumer1 = null; + MethodHandle autofuzzConsumer2 = null; + MethodHandle autofuzzConsumer3 = null; + MethodHandle autofuzzConsumer4 = null; + MethodHandle autofuzzConsumer5 = null; try { jazzerInternal = Class.forName("com.code_intelligence.jazzer.runtime.JazzerInternal"); + MethodType onFuzzTargetReadyType = MethodType.methodType(void.class, Runnable.class); + onFuzzTargetReady = MethodHandles.publicLookup().findStatic( + jazzerInternal, "registerOnFuzzTargetReadyCallback", onFuzzTargetReadyType); Class<?> traceDataFlowNativeCallbacks = Class.forName("com.code_intelligence.jazzer.runtime.TraceDataFlowNativeCallbacks"); @@ -60,6 +97,9 @@ final public class Jazzer { MethodType.methodType(void.class, byte[].class, byte[].class, int.class, int.class); traceMemcmp = MethodHandles.publicLookup().findStatic( traceDataFlowNativeCallbacks, "traceMemcmp", traceMemcmpType); + MethodType tracePcIndirType = MethodType.methodType(void.class, int.class, int.class); + tracePcIndir = MethodHandles.publicLookup().findStatic( + traceDataFlowNativeCallbacks, "tracePcIndir", tracePcIndirType); Class<?> metaClass = Class.forName("com.code_intelligence.jazzer.autofuzz.Meta"); MethodType consumeType = @@ -96,6 +136,23 @@ final public class Jazzer { e.printStackTrace(); System.exit(1); } + JAZZER_INTERNAL = jazzerInternal; + ON_FUZZ_TARGET_READY = onFuzzTargetReady; + TRACE_STRCMP = traceStrcmp; + TRACE_STRSTR = traceStrstr; + TRACE_MEMCMP = traceMemcmp; + TRACE_PC_INDIR = tracePcIndir; + CONSUME = consume; + AUTOFUZZ_FUNCTION_1 = autofuzzFunction1; + AUTOFUZZ_FUNCTION_2 = autofuzzFunction2; + AUTOFUZZ_FUNCTION_3 = autofuzzFunction3; + AUTOFUZZ_FUNCTION_4 = autofuzzFunction4; + AUTOFUZZ_FUNCTION_5 = autofuzzFunction5; + AUTOFUZZ_CONSUMER_1 = autofuzzConsumer1; + AUTOFUZZ_CONSUMER_2 = autofuzzConsumer2; + AUTOFUZZ_CONSUMER_3 = autofuzzConsumer3; + AUTOFUZZ_CONSUMER_4 = autofuzzConsumer4; + AUTOFUZZ_CONSUMER_5 = autofuzzConsumer5; } private Jazzer() {} @@ -103,7 +160,7 @@ final public class Jazzer { /** * Attempts to invoke {@code func} with arguments created automatically from the fuzzer input * using only public methods available on the classpath. - * + * <p> * <b>Note:</b> This function is inherently heuristic and may fail to execute {@code func} in * meaningful ways for a number of reasons. * @@ -120,7 +177,7 @@ final public class Jazzer { @SuppressWarnings("unchecked") public static <T1, R> R autofuzz(FuzzedDataProvider data, Function1<T1, R> func) { try { - return (R) autofuzzFunction1.invoke(data, func); + return (R) AUTOFUZZ_FUNCTION_1.invoke(data, func); } catch (AutofuzzInvocationException e) { rethrowUnchecked(e.getCause()); } catch (Throwable t) { @@ -133,7 +190,7 @@ final public class Jazzer { /** * Attempts to invoke {@code func} with arguments created automatically from the fuzzer input * using only public methods available on the classpath. - * + * <p> * <b>Note:</b> This function is inherently heuristic and may fail to execute {@code func} in * meaningful ways for a number of reasons. * @@ -150,7 +207,7 @@ final public class Jazzer { @SuppressWarnings("unchecked") public static <T1, T2, R> R autofuzz(FuzzedDataProvider data, Function2<T1, T2, R> func) { try { - return (R) autofuzzFunction2.invoke(data, func); + return (R) AUTOFUZZ_FUNCTION_2.invoke(data, func); } catch (AutofuzzInvocationException e) { rethrowUnchecked(e.getCause()); } catch (Throwable t) { @@ -163,7 +220,7 @@ final public class Jazzer { /** * Attempts to invoke {@code func} with arguments created automatically from the fuzzer input * using only public methods available on the classpath. - * + * <p> * <b>Note:</b> This function is inherently heuristic and may fail to execute {@code func} in * meaningful ways for a number of reasons. * @@ -180,7 +237,7 @@ final public class Jazzer { @SuppressWarnings("unchecked") public static <T1, T2, T3, R> R autofuzz(FuzzedDataProvider data, Function3<T1, T2, T3, R> func) { try { - return (R) autofuzzFunction3.invoke(data, func); + return (R) AUTOFUZZ_FUNCTION_3.invoke(data, func); } catch (AutofuzzInvocationException e) { rethrowUnchecked(e.getCause()); } catch (Throwable t) { @@ -193,7 +250,7 @@ final public class Jazzer { /** * Attempts to invoke {@code func} with arguments created automatically from the fuzzer input * using only public methods available on the classpath. - * + * <p> * <b>Note:</b> This function is inherently heuristic and may fail to execute {@code func} in * meaningful ways for a number of reasons. * @@ -211,7 +268,7 @@ final public class Jazzer { public static <T1, T2, T3, T4, R> R autofuzz( FuzzedDataProvider data, Function4<T1, T2, T3, T4, R> func) { try { - return (R) autofuzzFunction4.invoke(data, func); + return (R) AUTOFUZZ_FUNCTION_4.invoke(data, func); } catch (AutofuzzInvocationException e) { rethrowUnchecked(e.getCause()); } catch (Throwable t) { @@ -224,7 +281,7 @@ final public class Jazzer { /** * Attempts to invoke {@code func} with arguments created automatically from the fuzzer input * using only public methods available on the classpath. - * + * <p> * <b>Note:</b> This function is inherently heuristic and may fail to execute {@code func} in * meaningful ways for a number of reasons. * @@ -242,7 +299,7 @@ final public class Jazzer { public static <T1, T2, T3, T4, T5, R> R autofuzz( FuzzedDataProvider data, Function5<T1, T2, T3, T4, T5, R> func) { try { - return (R) autofuzzFunction5.invoke(data, func); + return (R) AUTOFUZZ_FUNCTION_5.invoke(data, func); } catch (AutofuzzInvocationException e) { rethrowUnchecked(e.getCause()); } catch (Throwable t) { @@ -255,7 +312,7 @@ final public class Jazzer { /** * Attempts to invoke {@code func} with arguments created automatically from the fuzzer input * using only public methods available on the classpath. - * + * <p> * <b>Note:</b> This function is inherently heuristic and may fail to execute {@code func} in * meaningful ways for a number of reasons. * @@ -269,7 +326,7 @@ final public class Jazzer { */ public static <T1> void autofuzz(FuzzedDataProvider data, Consumer1<T1> func) { try { - autofuzzConsumer1.invoke(data, func); + AUTOFUZZ_CONSUMER_1.invoke(data, func); } catch (AutofuzzInvocationException e) { rethrowUnchecked(e.getCause()); } catch (Throwable t) { @@ -280,7 +337,7 @@ final public class Jazzer { /** * Attempts to invoke {@code func} with arguments created automatically from the fuzzer input * using only public methods available on the classpath. - * + * <p> * <b>Note:</b> This function is inherently heuristic and may fail to execute {@code func} in * meaningful ways for a number of reasons. * @@ -294,7 +351,7 @@ final public class Jazzer { */ public static <T1, T2> void autofuzz(FuzzedDataProvider data, Consumer2<T1, T2> func) { try { - autofuzzConsumer2.invoke(data, func); + AUTOFUZZ_CONSUMER_2.invoke(data, func); } catch (AutofuzzInvocationException e) { rethrowUnchecked(e.getCause()); } catch (Throwable t) { @@ -305,7 +362,7 @@ final public class Jazzer { /** * Attempts to invoke {@code func} with arguments created automatically from the fuzzer input * using only public methods available on the classpath. - * + * <p> * <b>Note:</b> This function is inherently heuristic and may fail to execute {@code func} in * meaningful ways for a number of reasons. * @@ -319,7 +376,7 @@ final public class Jazzer { */ public static <T1, T2, T3> void autofuzz(FuzzedDataProvider data, Consumer3<T1, T2, T3> func) { try { - autofuzzConsumer3.invoke(data, func); + AUTOFUZZ_CONSUMER_3.invoke(data, func); } catch (AutofuzzInvocationException e) { rethrowUnchecked(e.getCause()); } catch (Throwable t) { @@ -330,7 +387,7 @@ final public class Jazzer { /** * Attempts to invoke {@code func} with arguments created automatically from the fuzzer input * using only public methods available on the classpath. - * + * <p> * <b>Note:</b> This function is inherently heuristic and may fail to execute {@code func} in * meaningful ways for a number of reasons. * @@ -345,7 +402,7 @@ final public class Jazzer { public static <T1, T2, T3, T4> void autofuzz( FuzzedDataProvider data, Consumer4<T1, T2, T3, T4> func) { try { - autofuzzConsumer4.invoke(data, func); + AUTOFUZZ_CONSUMER_4.invoke(data, func); } catch (AutofuzzInvocationException e) { rethrowUnchecked(e.getCause()); } catch (Throwable t) { @@ -356,7 +413,7 @@ final public class Jazzer { /** * Attempts to invoke {@code func} with arguments created automatically from the fuzzer input * using only public methods available on the classpath. - * + * <p> * <b>Note:</b> This function is inherently heuristic and may fail to execute {@code func} in * meaningful ways for a number of reasons. * @@ -371,7 +428,7 @@ final public class Jazzer { public static <T1, T2, T3, T4, T5> void autofuzz( FuzzedDataProvider data, Consumer5<T1, T2, T3, T4, T5> func) { try { - autofuzzConsumer5.invoke(data, func); + AUTOFUZZ_CONSUMER_5.invoke(data, func); } catch (AutofuzzInvocationException e) { rethrowUnchecked(e.getCause()); } catch (Throwable t) { @@ -382,7 +439,7 @@ final public class Jazzer { /** * Attempts to construct an instance of {@code type} from the fuzzer input using only public * methods available on the classpath. - * + * <p> * <b>Note:</b> This function is inherently heuristic and may fail to return meaningful values for * a variety of reasons. * @@ -394,7 +451,7 @@ final public class Jazzer { @SuppressWarnings("unchecked") public static <T> T consume(FuzzedDataProvider data, Class<T> type) { try { - return (T) consume.invokeExact(data, type); + return (T) CONSUME.invokeExact(data, type); } catch (AutofuzzConstructionException ignored) { return null; } catch (Throwable t) { @@ -407,7 +464,7 @@ final public class Jazzer { /** * Instructs the fuzzer to guide its mutations towards making {@code current} equal to {@code * target}. - * + * <p> * If the relation between the raw fuzzer input and the value of {@code current} is relatively * complex, running the fuzzer with the argument {@code -use_value_profile=1} may be necessary to * achieve equality. @@ -417,8 +474,11 @@ final public class Jazzer { * @param id a (probabilistically) unique identifier for this particular compare hint */ public static void guideTowardsEquality(String current, String target, int id) { + if (TRACE_STRCMP == null) { + return; + } try { - traceStrcmp.invokeExact(current, target, 1, id); + TRACE_STRCMP.invokeExact(current, target, 1, id); } catch (Throwable e) { e.printStackTrace(); } @@ -427,7 +487,7 @@ final public class Jazzer { /** * Instructs the fuzzer to guide its mutations towards making {@code current} equal to {@code * target}. - * + * <p> * If the relation between the raw fuzzer input and the value of {@code current} is relatively * complex, running the fuzzer with the argument {@code -use_value_profile=1} may be necessary to * achieve equality. @@ -437,8 +497,11 @@ final public class Jazzer { * @param id a (probabilistically) unique identifier for this particular compare hint */ public static void guideTowardsEquality(byte[] current, byte[] target, int id) { + if (TRACE_MEMCMP == null) { + return; + } try { - traceMemcmp.invokeExact(current, target, 1, id); + TRACE_MEMCMP.invokeExact(current, target, 1, id); } catch (Throwable e) { e.printStackTrace(); } @@ -447,7 +510,7 @@ final public class Jazzer { /** * Instructs the fuzzer to guide its mutations towards making {@code haystack} contain {@code * needle} as a substring. - * + * <p> * If the relation between the raw fuzzer input and the value of {@code haystack} is relatively * complex, running the fuzzer with the argument {@code -use_value_profile=1} may be necessary to * satisfy the substring check. @@ -458,28 +521,81 @@ final public class Jazzer { * @param id a (probabilistically) unique identifier for this particular compare hint */ public static void guideTowardsContainment(String haystack, String needle, int id) { + if (TRACE_STRSTR == null) { + return; + } try { - traceStrstr.invokeExact(haystack, needle, id); + TRACE_STRSTR.invokeExact(haystack, needle, id); } catch (Throwable e) { e.printStackTrace(); } } /** - * Make Jazzer report the provided {@link Throwable} as a finding. + * Instructs the fuzzer to attain as many possible values for the absolute value of {@code state} + * as possible. + * <p> + * Call this function from a fuzz target or a hook to help the fuzzer track partial progress + * (e.g. by passing the length of a common prefix of two lists that should become equal) or + * explore different values of state that is not directly related to code coverage (see the + * MazeFuzzer example). + * <p> + * <b>Note:</b> This hint only takes effect if the fuzzer is run with the argument + * {@code -use_value_profile=1}. * + * @param state a numeric encoding of a state that should be varied by the fuzzer + * @param id a (probabilistically) unique identifier for this particular state hint + */ + public static void exploreState(byte state, int id) { + if (TRACE_PC_INDIR == null) { + return; + } + // We only use the lower 7 bits of state, which allows for 128 different state values tracked + // per id. The particular amount of 7 bits of state is also used in libFuzzer's + // TracePC::HandleCmp: + // https://github.com/llvm/llvm-project/blob/c12d49c4e286fa108d4d69f1c6d2b8d691993ffd/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp#L390 + // This value should be large enough for most use cases (e.g. tracking the length of a prefix in + // a comparison) while being small enough that the bitmap isn't filled up too quickly + // (65536 bits/ 128 bits per id = 512 ids). + + // We use tracePcIndir as a way to set a bit in libFuzzer's value profile bitmap. In + // TracePC::HandleCallerCallee, which is what this function ultimately calls through to, the + // lower 12 bits of each argument are combined into a 24-bit index into the bitmap, which is + // then reduced modulo a 16-bit prime. To keep the modulo bias small, we should fill as many + // of the relevant bits as possible. However, there are the following restrictions: + // 1. Since we use the return address trampoline to set the caller address indirectly, its + // upper 3 bits are fixed, which leaves a total of 21 variable bits on x86_64. + // 2. On arm64 macOS, where every instruction is aligned to 4 bytes, the lower 2 bits of the + // caller address will always be zero, further reducing the number of variable bits in the + // caller parameter to 7. + // https://github.com/llvm/llvm-project/blob/c12d49c4e286fa108d4d69f1c6d2b8d691993ffd/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp#L121 + // Even taking these restrictions into consideration, we pass state in the lowest bits of the + // caller address, which is used to form the lowest bits of the bitmap index. This should result + // in the best caching behavior as state is expected to change quickly in consecutive runs and + // in this way all its bitmap entries would be located close to each other in memory. + int lowerBits = (state & 0x7f) | (id << 7); + int upperBits = id >>> 5; + try { + TRACE_PC_INDIR.invokeExact(upperBits, lowerBits); + } catch (Throwable e) { + e.printStackTrace(); + } + } + + /** + * Make Jazzer report the provided {@link Throwable} as a finding. + * <p> * <b>Note:</b> This method must only be called from a method hook. In a * fuzz target, simply throw an exception to trigger a finding. * @param finding the finding that Jazzer should report */ public static void reportFindingFromHook(Throwable finding) { try { - jazzerInternal.getMethod("reportFindingFromHook", Throwable.class).invoke(null, finding); + JAZZER_INTERNAL.getMethod("reportFindingFromHook", Throwable.class).invoke(null, finding); } catch (NullPointerException | IllegalAccessException | NoSuchMethodException e) { - // We can only reach this point if the runtime is not in the classpath, but it must be if - // hooks work and this function should only be called from them. - System.err.println("ERROR: Jazzer.reportFindingFromHook must be called from a method hook"); - System.exit(1); + // We can only reach this point if the runtime is not on the classpath, e.g. in case of a + // reproducer. Just throw the finding. + rethrowUnchecked(finding); } catch (InvocationTargetException e) { // reportFindingFromHook throws a HardToCatchThrowable, which will bubble up wrapped in an // InvocationTargetException that should not be stopped here. @@ -491,6 +607,34 @@ final public class Jazzer { } } + /** + * Register a callback to be executed right before the fuzz target is executed for the first time. + * <p> + * This can be used to disable hooks until after Jazzer has been fully initializing, e.g. to + * prevent Jazzer internals from triggering hooks on Java standard library classes. + * + * @param callback the callback to execute + */ + public static void onFuzzTargetReady(Runnable callback) { + try { + ON_FUZZ_TARGET_READY.invokeExact(callback); + } catch (Throwable e) { + e.printStackTrace(); + } + } + + private static int getLibFuzzerSeed() { + // The Jazzer driver sets this property based on the value of libFuzzer's -seed command-line + // option, which allows for fully reproducible fuzzing runs if set. If not running in the + // context of the driver, fall back to a random number instead. + String rawSeed = System.getProperty("jazzer.seed"); + if (rawSeed == null) { + return new SecureRandom().nextInt(); + } + // If jazzer.seed is set, we expect it to be a valid integer. + return Integer.parseUnsignedInt(rawSeed); + } + // Rethrows a (possibly checked) exception while avoiding a throws declaration. @SuppressWarnings("unchecked") private static <T extends Throwable> void rethrowUnchecked(Throwable t) throws T { diff --git a/agent/src/main/java/com/code_intelligence/jazzer/api/MethodHook.java b/agent/src/main/java/com/code_intelligence/jazzer/api/MethodHook.java index 0d17a4a0..3a1c5f39 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/api/MethodHook.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/api/MethodHook.java @@ -23,11 +23,12 @@ import java.lang.annotation.Target; import java.lang.invoke.MethodType; /** - * Registers this method as a hook that should run after the method - * specified by the annotation parameters has returned. + * Registers the annotated method as a hook that should run before, instead or + * after the method specified by the annotation parameters. * <p> - * This method will be called after every call to the target method and has - * access to its return value. The target method is specified by + * Depending on {@link #type()} this method will be called after, instead or + * before every call to the target method and has + * access to its parameters and return value. The target method is specified by * {@link #targetClassName()} and {@link #targetMethod()}. In case of an * overloaded method, {@link #targetMethodDescriptor()} can be used to restrict * the application of the hook to a particular overload. @@ -87,7 +88,7 @@ import java.lang.invoke.MethodType; * <p> * Return value: the value that should take the role of the value the target * method would have returned - * + * <p> * <dt><span class="strong">{@link HookType#AFTER}</span> * <dd> * <pre>{@code @@ -114,6 +115,13 @@ import java.lang.invoke.MethodType; * will be wrapped into their corresponding wrapper type (e.g. {@link Boolean}). * If the original method has return type {@code void}, this value will be * {@code null}. + * <p> + * Multiple {@link HookType#BEFORE} and {@link HookType#AFTER} hooks are + * allowed to reference the same target method. Exclusively one + * {@link HookType#REPLACE} hook may reference a target method, no other types + * allowed. Attention must be paid to not guide the Fuzzer in different + * directions via {@link Jazzer}'s {@code guideTowardsXY} methods in the + * different hooks. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD) @@ -142,6 +150,11 @@ public @interface MethodHook { * The name of the class that contains the method that should be hooked, * as returned by {@link Class#getName()}. * <p> + * If an interface or abstract class is specified, also calls to all + * implementations and subclasses available on the classpath during startup + * are hooked, respectively. Interfaces and subclasses are not taken into + * account for concrete classes. + * <p> * Examples: * <p><ul> * <li>{@link String}: {@code "java.lang.String"} @@ -180,4 +193,15 @@ public @interface MethodHook { * @return the descriptor of the method to be hooked */ String targetMethodDescriptor() default ""; + + /** + * Array of additional classes to hook. + * <p> + * Hooks are applied on call sites. This means that classes calling the one + * defined in this annotation need to be instrumented to actually execute + * the hook. This property can be used to hook normally ignored classes. + * + * @return fully qualified class names to hook + */ + String[] additionalClassesToHook() default {}; } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/autofuzz/FuzzTarget.java b/agent/src/main/java/com/code_intelligence/jazzer/autofuzz/FuzzTarget.java index 8c344621..3b0d046b 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/autofuzz/FuzzTarget.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/autofuzz/FuzzTarget.java @@ -20,11 +20,18 @@ import com.code_intelligence.jazzer.api.FuzzedDataProvider; import com.code_intelligence.jazzer.utils.SimpleGlobMatcher; import com.code_intelligence.jazzer.utils.Utils; import java.io.Closeable; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.lang.reflect.Constructor; import java.lang.reflect.Executable; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -32,7 +39,12 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -public class FuzzTarget { +public final class FuzzTarget { + private static final String AUTOFUZZ_REPRODUCER_TEMPLATE = "public class Crash_%s {\n" + + " public static void main(String[] args) throws Throwable {\n" + + " %s;\n" + + " }\n" + + "}"; private static final long MAX_EXECUTIONS_WITHOUT_INVOCATION = 100; private static String methodReference; @@ -40,7 +52,6 @@ public class FuzzTarget { private static Map<Executable, Class<?>[]> throwsDeclarations; private static Set<SimpleGlobMatcher> ignoredExceptionMatchers; private static long executionsSinceLastInvocation = 0; - private static AutofuzzCodegenVisitor codegenVisitor; public static void fuzzerInitialize(String[] args) { if (args.length == 0 || !args[0].contains("::")) { @@ -73,19 +84,28 @@ public class FuzzTarget { descriptor = null; } - Class<?> targetClass; - try { - // Explicitly invoking static initializers to trigger some coverage in the code. - targetClass = Class.forName(className, true, ClassLoader.getSystemClassLoader()); - } catch (ClassNotFoundException e) { - System.err.printf( - "Failed to find class %s for autofuzz, please ensure it is contained in the classpath " - + "specified with --cp and specify the full package name%n", - className); - e.printStackTrace(); - System.exit(1); - return; - } + Class<?> targetClass = null; + String targetClassName = className; + do { + try { + // Explicitly invoking static initializers to trigger some coverage in the code. + targetClass = Class.forName(targetClassName, true, ClassLoader.getSystemClassLoader()); + } catch (ClassNotFoundException e) { + int classSeparatorIndex = targetClassName.lastIndexOf("."); + if (classSeparatorIndex == -1) { + System.err.printf( + "Failed to find class %s for autofuzz, please ensure it is contained in the classpath " + + "specified with --cp and specify the full package name%n", + className); + e.printStackTrace(); + System.exit(1); + return; + } + StringBuilder classNameBuilder = new StringBuilder(targetClassName); + classNameBuilder.setCharAt(classSeparatorIndex, '$'); + targetClassName = classNameBuilder.toString(); + } + } while (targetClass == null); boolean isConstructor = methodName.equals("new"); if (isConstructor) { @@ -96,8 +116,13 @@ public class FuzzTarget { || Utils.getReadableDescriptor(constructor).equals(descriptor)) .toArray(Executable[] ::new); } else { + // We use getDeclaredMethods and filter for the public access modifier instead of using + // getMethods as we want to exclude methods inherited from superclasses or interfaces, which + // can lead to unexpected results when autofuzzing. If desired, these can be autofuzzed + // explicitly instead. targetExecutables = - Arrays.stream(targetClass.getMethods()) + Arrays.stream(targetClass.getDeclaredMethods()) + .filter(method -> Modifier.isPublic(method.getModifiers())) .filter(method -> method.getName().equals(methodName) && (descriptor == null @@ -179,9 +204,36 @@ public class FuzzTarget { } public static void fuzzerTestOneInput(FuzzedDataProvider data) throws Throwable { + AutofuzzCodegenVisitor codegenVisitor = null; if (Meta.isDebug()) { codegenVisitor = new AutofuzzCodegenVisitor(); } + fuzzerTestOneInput(data, codegenVisitor); + if (codegenVisitor != null) { + System.err.println(codegenVisitor.generate()); + } + } + + public static void dumpReproducer(FuzzedDataProvider data, String reproducerPath, String sha) { + AutofuzzCodegenVisitor codegenVisitor = new AutofuzzCodegenVisitor(); + try { + fuzzerTestOneInput(data, codegenVisitor); + } catch (Throwable ignored) { + } + String javaSource = String.format(AUTOFUZZ_REPRODUCER_TEMPLATE, sha, codegenVisitor.generate()); + Path javaPath = Paths.get(reproducerPath, String.format("Crash_%s.java", sha)); + try { + Files.write(javaPath, javaSource.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE); + } catch (IOException e) { + System.err.printf("ERROR: Failed to write Java reproducer to %s%n", javaPath); + e.printStackTrace(); + } + System.out.printf( + "reproducer_path='%s'; Java reproducer written to %s%n", reproducerPath, javaPath); + } + + private static void fuzzerTestOneInput( + FuzzedDataProvider data, AutofuzzCodegenVisitor codegenVisitor) throws Throwable { Executable targetExecutable; if (FuzzTarget.targetExecutables.length == 1) { targetExecutable = FuzzTarget.targetExecutables[0]; @@ -196,9 +248,6 @@ public class FuzzTarget { returnValue = Meta.autofuzz(data, (Constructor<?>) targetExecutable, codegenVisitor); } executionsSinceLastInvocation = 0; - if (codegenVisitor != null) { - System.err.println(codegenVisitor.generate()); - } } catch (AutofuzzConstructionException e) { if (Meta.isDebug()) { e.printStackTrace(); @@ -245,8 +294,8 @@ public class FuzzTarget { } } - // Removes all stack trace elements that live in the Java standard library, internal JDK classes - // or the autofuzz package from the bottom of all stack frames. + // Removes all stack trace elements that live in the Java reflection packages or the autofuzz + // package from the bottom of all stack frames. private static void cleanStackTraces(Throwable t) { Throwable cause = t; while (cause != null) { @@ -255,8 +304,9 @@ public class FuzzTarget { for (firstInterestingPos = elements.length - 1; firstInterestingPos > 0; firstInterestingPos--) { String className = elements[firstInterestingPos].getClassName(); - if (!className.startsWith("com.code_intelligence.jazzer.autofuzz") - && !className.startsWith("java.") && !className.startsWith("jdk.")) { + if (!className.startsWith("com.code_intelligence.jazzer.autofuzz.") + && !className.startsWith("java.lang.reflect.") + && !className.startsWith("jdk.internal.reflect.")) { break; } } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/autofuzz/Meta.java b/agent/src/main/java/com/code_intelligence/jazzer/autofuzz/Meta.java index 96980530..3d48017f 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/autofuzz/Meta.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/autofuzz/Meta.java @@ -36,9 +36,14 @@ import java.io.InputStream; import java.lang.reflect.Array; import java.lang.reflect.Constructor; import java.lang.reflect.Executable; +import java.lang.reflect.GenericArrayType; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.lang.reflect.TypeVariable; +import java.lang.reflect.WildcardType; import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -46,11 +51,13 @@ import net.jodah.typetools.TypeResolver; import net.jodah.typetools.TypeResolver.Unknown; public class Meta { - static WeakHashMap<Class<?>, List<Class<?>>> implementingClassesCache = new WeakHashMap<>(); - static WeakHashMap<Class<?>, List<Class<?>>> nestedBuilderClassesCache = new WeakHashMap<>(); - static WeakHashMap<Class<?>, List<Method>> originalObjectCreationMethodsCache = + static final WeakHashMap<Class<?>, List<Class<?>>> implementingClassesCache = new WeakHashMap<>(); + static final WeakHashMap<Class<?>, List<Class<?>>> nestedBuilderClassesCache = + new WeakHashMap<>(); + static final WeakHashMap<Class<?>, List<Method>> originalObjectCreationMethodsCache = + new WeakHashMap<>(); + static final WeakHashMap<Class<?>, List<Method>> cascadingBuilderMethodsCache = new WeakHashMap<>(); - static WeakHashMap<Class<?>, List<Method>> cascadingBuilderMethodsCache = new WeakHashMap<>(); public static Object autofuzz(FuzzedDataProvider data, Method method) { return autofuzz(data, method, null); @@ -64,22 +71,29 @@ public class Meta { visitor.pushGroup( String.format("%s.", method.getDeclaringClass().getCanonicalName()), "", ""); } - result = autofuzz(data, method, null, visitor); - if (visitor != null) { - visitor.popGroup(); + try { + result = autofuzz(data, method, null, visitor); + } finally { + if (visitor != null) { + visitor.popGroup(); + } } } else { if (visitor != null) { // This group will always have two elements: The thisObject and the method call. - visitor.pushGroup("", ".", ""); + // Since the this object can be a complex expression, wrap it in paranthesis. + visitor.pushGroup("(", ").", ""); } Object thisObject = consume(data, method.getDeclaringClass(), visitor); if (thisObject == null) { throw new AutofuzzConstructionException(); } - result = autofuzz(data, method, thisObject, visitor); - if (visitor != null) { - visitor.popGroup(); + try { + result = autofuzz(data, method, thisObject, visitor); + } finally { + if (visitor != null) { + visitor.popGroup(); + } } } return result; @@ -210,7 +224,13 @@ public class Meta { return consume(data, type, null); } - static Object consume(FuzzedDataProvider data, Class<?> type, AutofuzzCodegenVisitor visitor) { + // Invariant: The Java source code representation of the returned object visited by visitor must + // represent an object of the same type as genericType. For example, a null value returned for + // the genericType Class<java.lang.String> should lead to the generated code + // "(java.lang.String) null", not just "null". This makes it possible to safely use consume in + // recursive argument constructions. + static Object consume(FuzzedDataProvider data, Type genericType, AutofuzzCodegenVisitor visitor) { + Class<?> type = getRawType(genericType); if (type == byte.class || type == Byte.class) { byte result = data.consumeByte(); if (visitor != null) @@ -252,13 +272,18 @@ public class Meta { visitor.addCharLiteral(result); return result; } - // Return null for non-primitive and non-boxed types in ~5% of the cases. + // Sometimes, but rarely return null for non-primitive and non-boxed types. // TODO: We might want to return null for boxed types sometimes, but this is complicated by the // fact that TypeUtils can't distinguish between a primitive type and its wrapper and may // thus easily cause false-positive NullPointerExceptions. - if (!type.isPrimitive() && data.consumeByte((byte) 0, (byte) 19) == 0) { - if (visitor != null) - visitor.pushElement("null"); + if (!type.isPrimitive() && data.consumeByte() == 0) { + if (visitor != null) { + if (type == Object.class) { + visitor.pushElement("null"); + } else { + visitor.pushElement(String.format("(%s) null", type.getCanonicalName())); + } + } return null; } if (type == String.class || type == CharSequence.class) { @@ -344,6 +369,60 @@ public class Meta { ", ", "new java.io.ByteArrayInputStream(new byte[]{", "})"))); } return new ByteArrayInputStream(array); + } else if (type == Map.class) { + ParameterizedType mapType = (ParameterizedType) genericType; + if (mapType.getActualTypeArguments().length != 2) { + throw new AutofuzzError( + "Expected Map generic type to have two type parameters: " + mapType); + } + Type keyType = mapType.getActualTypeArguments()[0]; + Type valueType = mapType.getActualTypeArguments()[1]; + if (visitor != null) { + // Do not use Collectors.toMap() since it cannot handle null values. + // Also annotate the type of the entry stream since it might be empty, in which case type + // inference on the accumulator could fail. + visitor.pushGroup( + String.format("java.util.stream.Stream.<java.util.AbstractMap.SimpleEntry<%s, %s>>of(", + keyType.getTypeName(), valueType.getTypeName()), + ", ", + ").collect(java.util.HashMap::new, (map, e) -> map.put(e.getKey(), e.getValue()), java.util.HashMap::putAll)"); + } + int remainingBytesBeforeFirstEntryCreation = data.remainingBytes(); + if (visitor != null) { + visitor.pushGroup("new java.util.AbstractMap.SimpleEntry<>(", ", ", ")"); + } + Object firstKey = consume(data, keyType, visitor); + Object firstValue = consume(data, valueType, visitor); + if (visitor != null) { + visitor.popGroup(); + } + int remainingBytesAfterFirstEntryCreation = data.remainingBytes(); + int sizeOfElementEstimate = + remainingBytesBeforeFirstEntryCreation - remainingBytesAfterFirstEntryCreation; + int mapSize = consumeArrayLength(data, sizeOfElementEstimate); + Map<Object, Object> map = new HashMap<>(mapSize); + for (int i = 0; i < mapSize; i++) { + if (i == 0) { + map.put(firstKey, firstValue); + } else { + if (visitor != null) { + visitor.pushGroup("new java.util.AbstractMap.SimpleEntry<>(", ", ", ")"); + } + map.put(consume(data, keyType, visitor), consume(data, valueType, visitor)); + if (visitor != null) { + visitor.popGroup(); + } + } + } + if (visitor != null) { + if (mapSize == 0) { + // We implicitly pushed the first entry with the call to consume above, but it is not + // part of the array. + visitor.popElement(); + } + visitor.popGroup(); + } + return map; } else if (type.isEnum()) { Enum<?> enumValue = (Enum<?>) data.pickValue(type.getEnumConstants()); if (visitor != null) { @@ -578,7 +657,7 @@ public class Meta { FuzzedDataProvider data, Executable executable, AutofuzzCodegenVisitor visitor) { Object[] result; try { - result = Arrays.stream(executable.getParameterTypes()) + result = Arrays.stream(executable.getGenericParameterTypes()) .map((type) -> consume(data, type, visitor)) .toArray(); return result; @@ -616,4 +695,22 @@ public class Meta { } return result; } + + private static Class<?> getRawType(Type genericType) { + if (genericType instanceof Class<?>) { + return (Class<?>) genericType; + } else if (genericType instanceof ParameterizedType) { + return getRawType(((ParameterizedType) genericType).getRawType()); + } else if (genericType instanceof WildcardType) { + // TODO: Improve this. + return Object.class; + } else if (genericType instanceof TypeVariable<?>) { + throw new AutofuzzError("Did not expect genericType to be a TypeVariable: " + genericType); + } else if (genericType instanceof GenericArrayType) { + // TODO: Improve this; + return Object[].class; + } else { + throw new AutofuzzError("Got unexpected class implementing Type: " + genericType); + } + } } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/generated/BUILD.bazel b/agent/src/main/java/com/code_intelligence/jazzer/generated/BUILD.bazel deleted file mode 100644 index fceda64c..00000000 --- a/agent/src/main/java/com/code_intelligence/jazzer/generated/BUILD.bazel +++ /dev/null @@ -1,40 +0,0 @@ -java_binary( - name = "NoThrowDoclet", - srcs = ["NoThrowDoclet.java"], - create_executable = False, - tags = ["manual"], -) - -# To regenerate the list of methods, ensure that your local JDK is as recent as possible and contains `lib/src.zip`. -# This will be the case if you are using the release binaries of the OpenJDK or if the `openjdk-<version>-source` -# package is installed. -# Then, execute -# agent/src/main/java/com/code_intelligence/jazzer/generated/update_java_no_throw_methods_list.sh -# from the Bazel root and copy the file into -# org.jacoco.core/src/org/jacoco/core/internal/flow/java_no_throw_methods_list.dat -# in the CodeIntelligenceTesting/jacoco repository. -genrule( - name = "java_no_throw_methods_list", - srcs = [ - "@local_jdk//:lib/src.zip", - ], - outs = [ - "java_no_throw_methods_list.dat.generated", - ], - cmd = """ - TMP=$$(mktemp -d) && \ - unzip $(execpath @local_jdk//:lib/src.zip) -d $$TMP && \ - $(execpath @local_jdk//:bin/javadoc) \ - -doclet com.code_intelligence.jazzer.generated.NoThrowDoclet \ - -docletpath $(execpath :NoThrowDoclet_deploy.jar) \ - --module java.base \ - --source-path $$TMP/java.base \ - --out $@ && \ - sort -o $@ $@ && \ - rm -rf $$TMP""", - tags = ["manual"], - tools = [ - ":NoThrowDoclet_deploy.jar", - "@local_jdk//:bin/javadoc", - ], -) diff --git a/agent/src/main/java/com/code_intelligence/jazzer/generated/NoThrowDoclet.java b/agent/src/main/java/com/code_intelligence/jazzer/generated/NoThrowDoclet.java deleted file mode 100644 index 1b52a228..00000000 --- a/agent/src/main/java/com/code_intelligence/jazzer/generated/NoThrowDoclet.java +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2021 Code Intelligence GmbH -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.code_intelligence.jazzer.generated; - -import com.sun.source.doctree.DocCommentTree; -import com.sun.source.doctree.DocTree; -import com.sun.source.doctree.ThrowsTree; -import com.sun.source.util.DocTrees; -import java.io.BufferedWriter; -import java.io.FileWriter; -import java.io.IOException; -import java.util.List; -import java.util.Locale; -import java.util.Set; -import java.util.stream.Collectors; -import javax.lang.model.SourceVersion; -import javax.lang.model.element.ElementKind; -import javax.lang.model.element.ExecutableElement; -import javax.lang.model.element.Modifier; -import javax.lang.model.element.ModuleElement; -import javax.lang.model.element.PackageElement; -import javax.lang.model.element.TypeElement; -import javax.lang.model.element.VariableElement; -import javax.lang.model.type.ArrayType; -import javax.lang.model.type.DeclaredType; -import javax.lang.model.type.TypeMirror; -import javax.lang.model.util.ElementFilter; -import jdk.javadoc.doclet.Doclet; -import jdk.javadoc.doclet.DocletEnvironment; -import jdk.javadoc.doclet.Reporter; - -/** - * A Doclet that extracts a list of all method signatures in {@code java.*} that are declared not to - * throw any exceptions, including {@link RuntimeException} but excluding {@link - * VirtualMachineError}. - * - * Crucially, whereas the throws declaration of a method does not contain subclasses of {@link - * RuntimeException}, the {@code @throws} Javadoc tag does. - */ -public class NoThrowDoclet implements Doclet { - private BufferedWriter out; - - @Override - public void init(Locale locale, Reporter reporter) {} - - @Override - public String getName() { - return getClass().getSimpleName(); - } - - @Override - public Set<? extends Option> getSupportedOptions() { - return Set.of(new Option() { - @Override - public int getArgumentCount() { - return 1; - } - - @Override - public String getDescription() { - return "Output file (.kt)"; - } - - @Override - public Kind getKind() { - return Kind.STANDARD; - } - - @Override - public List<String> getNames() { - return List.of("--out"); - } - - @Override - public String getParameters() { - return "<output file (.kt)>"; - } - - @Override - public boolean process(String option, List<String> args) { - try { - out = new BufferedWriter(new FileWriter(args.get(0))); - } catch (IOException e) { - e.printStackTrace(); - return false; - } - return true; - } - }); - } - - @Override - public SourceVersion getSupportedSourceVersion() { - return null; - } - - private String toDescriptor(TypeMirror type) { - switch (type.getKind()) { - case BOOLEAN: - return "Z"; - case BYTE: - return "B"; - case CHAR: - return "C"; - case DOUBLE: - return "D"; - case FLOAT: - return "F"; - case INT: - return "I"; - case LONG: - return "J"; - case SHORT: - return "S"; - case VOID: - return "V"; - case ARRAY: - return "[" + toDescriptor(((ArrayType) type).getComponentType()); - case DECLARED: - return "L" + getFullyQualifiedName((DeclaredType) type) + ";"; - case TYPEVAR: - return "Ljava/lang/Object;"; - } - throw new IllegalArgumentException( - "Unexpected kind " + type.getKind() + ": " + type.toString()); - } - - private String getFullyQualifiedName(DeclaredType declaredType) { - TypeElement element = (TypeElement) declaredType.asElement(); - return element.getQualifiedName().toString().replace('.', '/'); - } - - private void handleExecutableElement(DocTrees trees, ExecutableElement executable) - throws IOException { - if (!executable.getModifiers().contains(Modifier.PUBLIC)) - return; - - DocCommentTree tree = trees.getDocCommentTree(executable); - if (tree != null) { - for (DocTree tag : tree.getBlockTags()) { - if (tag instanceof ThrowsTree) { - return; - } - } - } - - String methodName = executable.getSimpleName().toString(); - String className = - ((TypeElement) executable.getEnclosingElement()).getQualifiedName().toString(); - String internalClassName = className.replace('.', '/'); - - String paramTypeDescriptors = executable.getParameters() - .stream() - .map(VariableElement::asType) - .map(this::toDescriptor) - .collect(Collectors.joining("")); - String returnTypeDescriptor = toDescriptor(executable.getReturnType()); - String methodDescriptor = String.format("(%s)%s", paramTypeDescriptors, returnTypeDescriptor); - - out.write(String.format("%s#%s#%s%n", internalClassName, methodName, methodDescriptor)); - } - - public void handleTypeElement(DocTrees trees, TypeElement typeElement) throws IOException { - List<ExecutableElement> executables = - ElementFilter.constructorsIn(typeElement.getEnclosedElements()); - executables.addAll(ElementFilter.methodsIn(typeElement.getEnclosedElements())); - for (ExecutableElement executableElement : executables) { - handleExecutableElement(trees, executableElement); - } - } - - @Override - public boolean run(DocletEnvironment docletEnvironment) { - try { - DocTrees trees = docletEnvironment.getDocTrees(); - for (ModuleElement moduleElement : - ElementFilter.modulesIn(docletEnvironment.getSpecifiedElements())) { - for (PackageElement packageElement : - ElementFilter.packagesIn(moduleElement.getEnclosedElements())) { - if (packageElement.getQualifiedName().toString().startsWith("java.")) { - for (TypeElement typeElement : - ElementFilter.typesIn(packageElement.getEnclosedElements())) { - ElementKind kind = typeElement.getKind(); - if (kind == ElementKind.CLASS || kind == ElementKind.ENUM - || kind == ElementKind.INTERFACE) { - handleTypeElement(trees, typeElement); - } - } - } - } - } - } catch (IOException e) { - e.printStackTrace(); - return false; - } - try { - out.close(); - } catch (IOException e) { - e.printStackTrace(); - return false; - } - return true; - } -}
\ No newline at end of file diff --git a/agent/src/main/java/com/code_intelligence/jazzer/generated/update_java_no_throw_methods_list.sh b/agent/src/main/java/com/code_intelligence/jazzer/generated/update_java_no_throw_methods_list.sh deleted file mode 100755 index 1463c602..00000000 --- a/agent/src/main/java/com/code_intelligence/jazzer/generated/update_java_no_throw_methods_list.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env sh -# Copyright 2021 Code Intelligence GmbH -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -bazel build //agent/src/main/java/com/code_intelligence/jazzer/generated:java_no_throw_methods_list -cp bazel-bin/agent/src/main/java/com/code_intelligence/jazzer/generated/java_no_throw_methods_list.dat.generated agent/src/main/java/com/code_intelligence/jazzer/generated/java_no_throw_methods_list.dat diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/BUILD.bazel b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/BUILD.bazel index 50d10705..db93dcae 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/BUILD.bazel +++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/BUILD.bazel @@ -12,34 +12,24 @@ kt_jvm_library( "Hook.kt", "HookInstrumentor.kt", "HookMethodVisitor.kt", + "Hooks.kt", "Instrumentor.kt", + "StaticMethodStrategy.java", "TraceDataFlowInstrumentor.kt", ], visibility = [ + "//agent/src/jmh/java/com/code_intelligence/jazzer/instrumentor:__pkg__", "//agent/src/main/java/com/code_intelligence/jazzer/agent:__pkg__", "//agent/src/test/java/com/code_intelligence/jazzer/instrumentor:__pkg__", + "//driver/src/main/java/com/code_intelligence/jazzer/driver:__pkg__", ], deps = [ - ":shaded_deps", "//agent/src/main/java/com/code_intelligence/jazzer/runtime", "//agent/src/main/java/com/code_intelligence/jazzer/utils", "@com_github_classgraph_classgraph//:classgraph", "@com_github_jetbrains_kotlin//:kotlin-reflect", - ], -) - -jar_jar( - name = "shaded_deps", - input_jar = "unshaded_deps_deploy.jar", - rules = "shade_rules", -) - -java_binary( - name = "unshaded_deps", - create_executable = False, - runtime_deps = [ "@jazzer_jacoco//:jacoco_internal", - "@jazzer_ow2_asm//:asm", - "@jazzer_ow2_asm//:asm_commons", + "@org_ow2_asm_asm//jar", + "@org_ow2_asm_asm_commons//jar", ], ) diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/ClassInstrumentor.kt b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/ClassInstrumentor.kt index f6728a1a..4c3eabcb 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/ClassInstrumentor.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/ClassInstrumentor.kt @@ -14,6 +14,8 @@ package com.code_intelligence.jazzer.instrumentor +import com.code_intelligence.jazzer.runtime.CoverageMap + fun extractClassFileMajorVersion(classfileBuffer: ByteArray): Int { return ((classfileBuffer[6].toInt() and 0xff) shl 8) or (classfileBuffer[7].toInt() and 0xff) } @@ -24,7 +26,11 @@ class ClassInstrumentor constructor(bytecode: ByteArray) { private set fun coverage(initialEdgeId: Int): Int { - val edgeCoverageInstrumentor = EdgeCoverageInstrumentor(initialEdgeId) + val edgeCoverageInstrumentor = EdgeCoverageInstrumentor( + defaultEdgeCoverageStrategy, + defaultCoverageMap, + initialEdgeId, + ) instrumentedBytecode = edgeCoverageInstrumentor.instrument(instrumentedBytecode) return edgeCoverageInstrumentor.numEdges } @@ -41,13 +47,7 @@ class ClassInstrumentor constructor(bytecode: ByteArray) { } companion object { - init { - try { - // Calls JNI_OnLoad_jazzer_initialize in the driver, which registers the native methods. - System.loadLibrary("jazzer_initialize") - } catch (_: UnsatisfiedLinkError) { - // Make it possible to use (parts of) the agent without the driver. - } - } + val defaultEdgeCoverageStrategy = StaticMethodStrategy() + val defaultCoverageMap = CoverageMap::class.java } } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/CoverageRecorder.kt b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/CoverageRecorder.kt index 65956189..098cf389 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/CoverageRecorder.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/CoverageRecorder.kt @@ -15,18 +15,17 @@ package com.code_intelligence.jazzer.instrumentor import com.code_intelligence.jazzer.runtime.CoverageMap -import com.code_intelligence.jazzer.third_party.jacoco.core.analysis.CoverageBuilder -import com.code_intelligence.jazzer.third_party.jacoco.core.data.ExecutionData -import com.code_intelligence.jazzer.third_party.jacoco.core.data.ExecutionDataReader -import com.code_intelligence.jazzer.third_party.jacoco.core.data.ExecutionDataStore -import com.code_intelligence.jazzer.third_party.jacoco.core.data.ExecutionDataWriter -import com.code_intelligence.jazzer.third_party.jacoco.core.data.SessionInfo -import com.code_intelligence.jazzer.third_party.jacoco.core.data.SessionInfoStore -import com.code_intelligence.jazzer.third_party.jacoco.core.internal.data.CRC64 +import com.code_intelligence.jazzer.third_party.org.jacoco.core.analysis.CoverageBuilder +import com.code_intelligence.jazzer.third_party.org.jacoco.core.data.ExecutionData +import com.code_intelligence.jazzer.third_party.org.jacoco.core.data.ExecutionDataStore +import com.code_intelligence.jazzer.third_party.org.jacoco.core.data.ExecutionDataWriter +import com.code_intelligence.jazzer.third_party.org.jacoco.core.data.SessionInfo +import com.code_intelligence.jazzer.third_party.org.jacoco.core.internal.data.CRC64 import com.code_intelligence.jazzer.utils.ClassNameGlobber import io.github.classgraph.ClassGraph -import java.io.ByteArrayInputStream -import java.io.ByteArrayOutputStream +import java.io.File +import java.io.FileOutputStream +import java.io.OutputStream import java.time.Instant import java.util.UUID @@ -52,26 +51,26 @@ object CoverageRecorder { } /** - * Manually records coverage IDs based on the current state of [CoverageMap.mem]. + * Manually records coverage IDs based on the current state of [CoverageMap]. * Should be called after static initializers have run. */ @JvmStatic fun updateCoveredIdsWithCoverageMap() { - val mem = CoverageMap.mem - val size = mem.capacity() - additionalCoverage.addAll((0 until size).filter { mem[it] > 0 }) + additionalCoverage.addAll(CoverageMap.getCoveredIds()) } + /** + * [dumpCoverageReport] dumps a human-readable coverage report of files using any [coveredIds] to [dumpFileName]. + */ @JvmStatic - fun replayCoveredIds() { - val mem = CoverageMap.mem - for (coverageId in additionalCoverage) { - mem.put(coverageId, 1) + fun dumpCoverageReport(coveredIds: IntArray, dumpFileName: String) { + File(dumpFileName).bufferedWriter().use { writer -> + writer.write(computeFileCoverage(coveredIds)) } } - @JvmStatic - fun computeFileCoverage(coveredIds: IntArray): String { + private fun computeFileCoverage(coveredIds: IntArray): String { + fun Double.format(digits: Int) = "%.${digits}f".format(this) val coverage = analyzeCoverage(coveredIds.toSet()) ?: return "No classes were instrumented" return coverage.sourceFiles.joinToString( "\n", @@ -109,21 +108,42 @@ object CoverageRecorder { } } - private fun Double.format(digits: Int) = "%.${digits}f".format(this) + /** + * [dumpJacocoCoverage] dumps the JaCoCo coverage of files using any [coveredIds] to [dumpFileName]. + * JaCoCo only exports coverage for files containing at least one coverage data point. The dump + * can be used by the JaCoCo report command to create reports also including not covered files. + */ + @JvmStatic + fun dumpJacocoCoverage(coveredIds: IntArray, dumpFileName: String) { + FileOutputStream(dumpFileName).use { outStream -> + dumpJacocoCoverage(coveredIds, outStream) + } + } + + /** + * [dumpJacocoCoverage] dumps the JaCoCo coverage of files using any [coveredIds] to [outStream]. + */ + @JvmStatic + fun dumpJacocoCoverage(coveredIds: IntArray, outStream: OutputStream) { + // Return if no class has been instrumented. + val startTimestamp = startTimestamp ?: return - fun dumpJacocoCoverage(coveredIds: Set<Int>): ByteArray? { // Update the list of covered IDs with the coverage information for the current run. updateCoveredIdsWithCoverageMap() val dumpTimestamp = Instant.now() - val outStream = ByteArrayOutputStream() val outWriter = ExecutionDataWriter(outStream) - // Return null if no class has been instrumented. - val startTimestamp = startTimestamp ?: return null outWriter.visitSessionInfo( SessionInfo(UUID.randomUUID().toString(), startTimestamp.epochSecond, dumpTimestamp.epochSecond) ) + analyzeJacocoCoverage(coveredIds.toSet()).accept(outWriter) + } + /** + * Build up a JaCoCo [ExecutionDataStore] based on [coveredIds] containing the internally gathered coverage information. + */ + private fun analyzeJacocoCoverage(coveredIds: Set<Int>): ExecutionDataStore { + val executionDataStore = ExecutionDataStore() val sortedCoveredIds = (additionalCoverage + coveredIds).sorted().toIntArray() for ((internalClassName, info) in instrumentedClassInfo) { // Determine the subarray of coverage IDs in sortedCoveredIds that contains the IDs generated while @@ -153,32 +173,27 @@ object CoverageRecorder { .forEach { classLocalEdgeId -> probes[classLocalEdgeId] = true } - outWriter.visitClassExecution(ExecutionData(info.classId, internalClassName, probes)) + executionDataStore.visitClassExecution(ExecutionData(info.classId, internalClassName, probes)) } - return outStream.toByteArray() + return executionDataStore } + /** + * Create a [CoverageBuilder] containing all classes matching the include/exclude pattern and their coverage statistics. + */ fun analyzeCoverage(coveredIds: Set<Int>): CoverageBuilder? { return try { val coverage = CoverageBuilder() analyzeAllUncoveredClasses(coverage) - val rawExecutionData = dumpJacocoCoverage(coveredIds) ?: return null - val executionDataStore = ExecutionDataStore() - val sessionInfoStore = SessionInfoStore() - ByteArrayInputStream(rawExecutionData).use { stream -> - ExecutionDataReader(stream).run { - setExecutionDataVisitor(executionDataStore) - setSessionInfoVisitor(sessionInfoStore) - read() - } - } + val executionDataStore = analyzeJacocoCoverage(coveredIds) for ((internalClassName, info) in instrumentedClassInfo) { - EdgeCoverageInstrumentor(0).analyze( - executionDataStore, - coverage, - info.bytecode, - internalClassName - ) + EdgeCoverageInstrumentor(ClassInstrumentor.defaultEdgeCoverageStrategy, ClassInstrumentor.defaultCoverageMap, 0) + .analyze( + executionDataStore, + coverage, + info.bytecode, + internalClassName + ) } coverage } catch (e: Exception) { @@ -198,7 +213,6 @@ object CoverageRecorder { .asSequence() .map { it.replace('/', '.') } .toSet() - val emptyExecutionDataStore = ExecutionDataStore() ClassGraph() .enableClassInfo() .ignoreClassVisibility() @@ -209,13 +223,16 @@ object CoverageRecorder { "jaz", ) .scan().use { result -> + // ExecutionDataStore is used to look up existing coverage during analysis of the class files, + // no entries are added during that. Passing in an empty store is fine for uncovered files. + val emptyExecutionDataStore = ExecutionDataStore() result.allClasses .asSequence() .filter { classInfo -> classNameGlobber.includes(classInfo.name) } .filterNot { classInfo -> classInfo.name in coveredClassNames } .forEach { classInfo -> classInfo.resource.use { resource -> - EdgeCoverageInstrumentor(0).analyze( + EdgeCoverageInstrumentor(ClassInstrumentor.defaultEdgeCoverageStrategy, ClassInstrumentor.defaultCoverageMap, 0).analyze( emptyExecutionDataStore, coverage, resource.load(), diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/EdgeCoverageInstrumentor.kt b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/EdgeCoverageInstrumentor.kt index ba5b7ee9..8fb3dc2b 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/EdgeCoverageInstrumentor.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/EdgeCoverageInstrumentor.kt @@ -14,37 +14,92 @@ package com.code_intelligence.jazzer.instrumentor -import com.code_intelligence.jazzer.runtime.CoverageMap -import com.code_intelligence.jazzer.third_party.jacoco.core.analysis.Analyzer -import com.code_intelligence.jazzer.third_party.jacoco.core.analysis.ICoverageVisitor -import com.code_intelligence.jazzer.third_party.jacoco.core.data.ExecutionDataStore -import com.code_intelligence.jazzer.third_party.jacoco.core.internal.flow.ClassProbesAdapter -import com.code_intelligence.jazzer.third_party.jacoco.core.internal.flow.ClassProbesVisitor -import com.code_intelligence.jazzer.third_party.jacoco.core.internal.flow.IClassProbesAdapterFactory -import com.code_intelligence.jazzer.third_party.jacoco.core.internal.flow.JavaNoThrowMethods -import com.code_intelligence.jazzer.third_party.jacoco.core.internal.instr.ClassInstrumenter -import com.code_intelligence.jazzer.third_party.jacoco.core.internal.instr.IProbeArrayStrategy -import com.code_intelligence.jazzer.third_party.jacoco.core.internal.instr.IProbeInserterFactory -import com.code_intelligence.jazzer.third_party.jacoco.core.internal.instr.InstrSupport -import com.code_intelligence.jazzer.third_party.jacoco.core.internal.instr.ProbeInserter -import com.code_intelligence.jazzer.third_party.objectweb.asm.ClassReader -import com.code_intelligence.jazzer.third_party.objectweb.asm.ClassVisitor -import com.code_intelligence.jazzer.third_party.objectweb.asm.ClassWriter -import com.code_intelligence.jazzer.third_party.objectweb.asm.MethodVisitor -import com.code_intelligence.jazzer.third_party.objectweb.asm.Opcodes +import com.code_intelligence.jazzer.third_party.org.jacoco.core.analysis.Analyzer +import com.code_intelligence.jazzer.third_party.org.jacoco.core.analysis.ICoverageVisitor +import com.code_intelligence.jazzer.third_party.org.jacoco.core.data.ExecutionDataStore +import com.code_intelligence.jazzer.third_party.org.jacoco.core.internal.flow.ClassProbesAdapter +import com.code_intelligence.jazzer.third_party.org.jacoco.core.internal.flow.ClassProbesVisitor +import com.code_intelligence.jazzer.third_party.org.jacoco.core.internal.flow.IClassProbesAdapterFactory +import com.code_intelligence.jazzer.third_party.org.jacoco.core.internal.instr.ClassInstrumenter +import com.code_intelligence.jazzer.third_party.org.jacoco.core.internal.instr.IProbeArrayStrategy +import com.code_intelligence.jazzer.third_party.org.jacoco.core.internal.instr.IProbeInserterFactory +import com.code_intelligence.jazzer.third_party.org.jacoco.core.internal.instr.InstrSupport +import com.code_intelligence.jazzer.third_party.org.jacoco.core.internal.instr.ProbeInserter +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassVisitor +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.MethodVisitor +import java.lang.invoke.MethodHandle +import java.lang.invoke.MethodHandles.publicLookup +import java.lang.invoke.MethodType.methodType import kotlin.math.max +/** + * A particular way to instrument bytecode for edge coverage using a coverage map class available to + * hold the collected coverage data at runtime. + */ +interface EdgeCoverageStrategy { + + /** + * Inject bytecode instrumentation on a control flow edge with ID [edgeId], with access to the + * local variable [variable] that is populated at the beginning of each method by the + * instrumentation injected in [loadLocalVariable]. + */ + fun instrumentControlFlowEdge( + mv: MethodVisitor, + edgeId: Int, + variable: Int, + coverageMapInternalClassName: String + ) + + /** + * The maximal number of stack elements used by [instrumentControlFlowEdge]. + */ + val instrumentControlFlowEdgeStackSize: Int + + /** + * The type of the local variable used by the instrumentation in the format used by + * [MethodVisitor.visitFrame]'s `local` parameter, or `null` if the instrumentation does not use + * one. + * @see https://asm.ow2.io/javadoc/org/objectweb/asm/MethodVisitor.html#visitFrame(int,int,java.lang.Object%5B%5D,int,java.lang.Object%5B%5D) + */ + val localVariableType: Any? + + /** + * Inject bytecode that loads the coverage counters of the coverage map class described by + * [coverageMapInternalClassName] into the local variable [variable]. + */ + fun loadLocalVariable(mv: MethodVisitor, variable: Int, coverageMapInternalClassName: String) + + /** + * The maximal number of stack elements used by [loadLocalVariable]. + */ + val loadLocalVariableStackSize: Int +} + +// An instance of EdgeCoverageInstrumentor should only be used to instrument a single class as it +// internally tracks the edge IDs, which have to be globally unique. class EdgeCoverageInstrumentor( + private val strategy: EdgeCoverageStrategy, + /** + * The class must have the following public static member + * - method enlargeIfNeeded(int nextEdgeId): Called before a new edge ID is emitted. + */ + coverageMapClass: Class<*>, private val initialEdgeId: Int, - private val coverageMapClass: Class<*> = CoverageMap::class.java ) : Instrumentor { private var nextEdgeId = initialEdgeId + private val coverageMapInternalClassName = coverageMapClass.name.replace('.', '/') - init { - if (isTesting) { - JavaNoThrowMethods.isTesting = true - } - } + private val enlargeIfNeeded: MethodHandle = + publicLookup().findStatic( + coverageMapClass, + "enlargeIfNeeded", + methodType( + Void::class.javaPrimitiveType, + Int::class.javaPrimitiveType + ) + ) override fun instrument(bytecode: ByteArray): ByteArray { val reader = InstrSupport.classReaderFor(bytecode) @@ -67,93 +122,14 @@ class EdgeCoverageInstrumentor( val numEdges get() = nextEdgeId - initialEdgeId - private val isTesting - get() = coverageMapClass != CoverageMap::class.java - private fun nextEdgeId(): Int { - if (nextEdgeId >= CoverageMap.mem.capacity()) { - if (!isTesting) { - CoverageMap.enlargeCoverageMap() - } - } + enlargeIfNeeded.invokeExact(nextEdgeId) return nextEdgeId++ } /** - * The maximal number of stack elements used by [loadCoverageMap]. - */ - private val loadCoverageMapStackSize = 1 - - /** - * Inject bytecode that loads the coverage map into local variable [variable]. - */ - private fun loadCoverageMap(mv: MethodVisitor, variable: Int) { - mv.apply { - visitFieldInsn( - Opcodes.GETSTATIC, - coverageMapInternalClassName, - "mem", - "Ljava/nio/ByteBuffer;" - ) - // Stack: mem (maxStack: 1) - visitVarInsn(Opcodes.ASTORE, variable) - } - } - - /** - * The maximal number of stack elements used by [instrumentControlFlowEdge]. - */ - private val instrumentControlFlowEdgeStackSize = 5 - - /** - * Inject bytecode instrumentation on a control flow edge with ID [edgeId]. The coverage map can be loaded from - * local variable [variable]. - */ - private fun instrumentControlFlowEdge(mv: MethodVisitor, edgeId: Int, variable: Int) { - mv.apply { - visitVarInsn(Opcodes.ALOAD, variable) - // Stack: mem - push(edgeId) - // Stack: mem | edgeId - visitInsn(Opcodes.DUP2) - // Stack: mem | edgeId | mem | edgeId - visitMethodInsn(Opcodes.INVOKEVIRTUAL, "java/nio/ByteBuffer", "get", "(I)B", false) - // Increment the counter, but ensure that it never stays at 0 after an overflow by incrementing it again in - // that case. - // This approach performs better than saturating the counter at 255 (see Section 3.3 of - // https://www.usenix.org/system/files/woot20-paper-fioraldi.pdf) - // Stack: mem | edgeId | counter (sign-extended to int) - push(0xff) - // Stack: mem | edgeId | counter (sign-extended to int) | 0x000000ff - visitInsn(Opcodes.IAND) - // Stack: mem | edgeId | counter (zero-extended to int) - push(1) - // Stack: mem | edgeId | counter | 1 - visitInsn(Opcodes.IADD) - // Stack: mem | edgeId | counter + 1 - visitInsn(Opcodes.DUP) - // Stack: mem | edgeId | counter + 1 | counter + 1 - push(8) - // Stack: mem | edgeId | counter + 1 | counter + 1 | 8 (maxStack: +5) - visitInsn(Opcodes.ISHR) - // Stack: mem | edgeId | counter + 1 | 1 if the increment overflowed to 0, 0 otherwise - visitInsn(Opcodes.IADD) - // Stack: mem | edgeId | counter + 2 if the increment overflowed, counter + 1 otherwise - visitMethodInsn(Opcodes.INVOKEVIRTUAL, "java/nio/ByteBuffer", "put", "(IB)Ljava/nio/ByteBuffer;", false) - // Stack: mem - visitInsn(Opcodes.POP) - if (isTesting) { - visitMethodInsn(Opcodes.INVOKESTATIC, coverageMapInternalClassName, "updated", "()V", false) - } - } - } - -// The remainder of this file interfaces with classes in org.jacoco.core.internal. Changes to this part should not be -// necessary unless JaCoCo is updated or the way we instrument for coverage changes fundamentally. - - /** - * A [ProbeInserter] that injects the bytecode instrumentation returned by [instrumentControlFlowEdge] and modifies - * the stack size and number of local variables accordingly. + * A [ProbeInserter] that injects bytecode instrumentation at every control flow edge and + * modifies the stack size and number of local variables accordingly. */ private inner class EdgeCoverageProbeInserter( access: Int, @@ -163,13 +139,16 @@ class EdgeCoverageInstrumentor( arrayStrategy: IProbeArrayStrategy, ) : ProbeInserter(access, name, desc, mv, arrayStrategy) { override fun insertProbe(id: Int) { - instrumentControlFlowEdge(mv, id, variable) + strategy.instrumentControlFlowEdge(mv, id, variable, coverageMapInternalClassName) } override fun visitMaxs(maxStack: Int, maxLocals: Int) { - val newMaxStack = max(maxStack + instrumentControlFlowEdgeStackSize, loadCoverageMapStackSize) - mv.visitMaxs(newMaxStack, maxLocals + 1) + val newMaxStack = max(maxStack + strategy.instrumentControlFlowEdgeStackSize, strategy.loadLocalVariableStackSize) + val newMaxLocals = maxLocals + if (strategy.localVariableType != null) 1 else 0 + mv.visitMaxs(newMaxStack, newMaxLocals) } + + override fun getLocalVariableType() = strategy.localVariableType } private val edgeCoverageProbeInserterFactory = @@ -177,9 +156,16 @@ class EdgeCoverageInstrumentor( EdgeCoverageProbeInserter(access, name, desc, mv, arrayStrategy) } - private inner class EdgeCoverageClassProbesAdapter(cv: ClassProbesVisitor, trackFrames: Boolean) : - ClassProbesAdapter(cv, trackFrames) { + private inner class EdgeCoverageClassProbesAdapter(private val cpv: ClassProbesVisitor, trackFrames: Boolean) : + ClassProbesAdapter(cpv, trackFrames) { override fun nextId(): Int = nextEdgeId() + + override fun visitEnd() { + cpv.visitTotalProbeCount(numEdges) + // Avoid calling super.visitEnd() as that invokes cpv.visitTotalProbeCount with an + // incorrect value of `count`. + cv.visitEnd() + } } private val edgeCoverageClassProbesAdapterFactory = IClassProbesAdapterFactory { probesVisitor, trackFrames -> @@ -188,14 +174,14 @@ class EdgeCoverageInstrumentor( private val edgeCoverageProbeArrayStrategy = object : IProbeArrayStrategy { override fun storeInstance(mv: MethodVisitor, clinit: Boolean, variable: Int): Int { - loadCoverageMap(mv, variable) - return loadCoverageMapStackSize + strategy.loadLocalVariable(mv, variable, coverageMapInternalClassName) + return strategy.loadLocalVariableStackSize } override fun addMembers(cv: ClassVisitor, probeCount: Int) {} } +} - private fun MethodVisitor.push(value: Int) { - InstrSupport.push(this, value) - } +fun MethodVisitor.push(value: Int) { + InstrSupport.push(this, value) } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/Hook.kt b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/Hook.kt index 92106e14..ff68ad94 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/Hook.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/Hook.kt @@ -18,46 +18,65 @@ package com.code_intelligence.jazzer.instrumentor import com.code_intelligence.jazzer.api.HookType import com.code_intelligence.jazzer.api.MethodHook -import com.code_intelligence.jazzer.api.MethodHooks import com.code_intelligence.jazzer.utils.descriptor import java.lang.invoke.MethodHandle import java.lang.reflect.Method import java.lang.reflect.Modifier -class Hook private constructor(hookMethod: Method, annotation: MethodHook) { - // Allowing arbitrary exterior whitespace in the target class name allows for an easy workaround - // for mangled hooks due to shading applied to hooks. - private val targetClassName = annotation.targetClassName.trim() - val targetMethodName = annotation.targetMethod - val targetMethodDescriptor = annotation.targetMethodDescriptor.takeIf { it.isNotEmpty() } - val hookType = annotation.type - - val targetInternalClassName = targetClassName.replace('.', '/') - private val targetReturnTypeDescriptor = targetMethodDescriptor?.let { extractReturnTypeDescriptor(it) } - private val targetWrappedReturnTypeDescriptor = targetReturnTypeDescriptor?.let { getWrapperTypeDescriptor(it) } - - private val hookClassName: String = hookMethod.declaringClass.name - val hookInternalClassName = hookClassName.replace('.', '/') - val hookMethodName: String = hookMethod.name - val hookMethodDescriptor = hookMethod.descriptor +class Hook private constructor( + private val targetClassName: String, + val hookType: HookType, + val targetMethodName: String, + val targetMethodDescriptor: String?, + val additionalClassesToHook: List<String>, + val targetInternalClassName: String, + private val targetReturnTypeDescriptor: String?, + private val targetWrappedReturnTypeDescriptor: String?, + private val hookClassName: String, + val hookInternalClassName: String, + val hookMethodName: String, + val hookMethodDescriptor: String +) { override fun toString(): String { - return "$hookType $targetClassName.$targetMethodName: $hookClassName.$hookMethodName" + return "$hookType $targetClassName.$targetMethodName: $hookClassName.$hookMethodName $additionalClassesToHook" } companion object { + fun createAndVerifyHook(hookMethod: Method, hookData: MethodHook, className: String): Hook { + return createHook(hookMethod, hookData, className).also { + verify(hookMethod, it) + } + } - fun verifyAndGetHook(hookMethod: Method, hookData: MethodHook): Hook { - // Verify the annotation type and extract information for debug statements. - val potentialHook = Hook(hookMethod, hookData) + private fun createHook(hookMethod: Method, annotation: MethodHook, targetClassName: String): Hook { + val targetReturnTypeDescriptor = annotation.targetMethodDescriptor + .takeIf { it.isNotBlank() }?.let { extractReturnTypeDescriptor(it) } + val hookClassName: String = hookMethod.declaringClass.name + return Hook( + targetClassName = targetClassName, + hookType = annotation.type, + targetMethodName = annotation.targetMethod, + targetMethodDescriptor = annotation.targetMethodDescriptor.takeIf { it.isNotBlank() }, + additionalClassesToHook = annotation.additionalClassesToHook.asList(), + targetInternalClassName = targetClassName.replace('.', '/'), + targetReturnTypeDescriptor = targetReturnTypeDescriptor, + targetWrappedReturnTypeDescriptor = targetReturnTypeDescriptor?.let { getWrapperTypeDescriptor(it) }, + hookClassName = hookClassName, + hookInternalClassName = hookClassName.replace('.', '/'), + hookMethodName = hookMethod.name, + hookMethodDescriptor = hookMethod.descriptor + ) + } + private fun verify(hookMethod: Method, potentialHook: Hook) { // Verify the hook method's modifiers (public static). require(Modifier.isPublic(hookMethod.modifiers)) { "$potentialHook: hook method must be public" } require(Modifier.isStatic(hookMethod.modifiers)) { "$potentialHook: hook method must be static" } // Verify the hook method's parameter count. val numParameters = hookMethod.parameters.size - when (hookData.type) { + when (potentialHook.hookType) { HookType.BEFORE, HookType.REPLACE -> require(numParameters == 4) { "$potentialHook: incorrect number of parameters (expected 4)" } HookType.AFTER -> require(numParameters == 5) { "$potentialHook: incorrect number of parameters (expected 5)" } } @@ -70,17 +89,18 @@ class Hook private constructor(hookMethod: Method, annotation: MethodHook) { require(parameterTypes[3] == Int::class.javaPrimitiveType) { "$potentialHook: fourth parameter must have type int" } // Verify the hook method's return type if possible. - when (hookData.type) { + when (potentialHook.hookType) { HookType.BEFORE, HookType.AFTER -> require(hookMethod.returnType == Void.TYPE) { "$potentialHook: return type must be void" } HookType.REPLACE -> if (potentialHook.targetReturnTypeDescriptor != null) { - val returnTypeDescriptor = hookMethod.returnType.descriptor - if (potentialHook.targetReturnTypeDescriptor == "V") { - require(returnTypeDescriptor == "V") { "$potentialHook: return type must be void to match targetMethodDescriptor" } + if (potentialHook.targetMethodName == "<init>") { + require(hookMethod.returnType.name == potentialHook.targetClassName) { "$potentialHook: return type must be ${potentialHook.targetClassName} to match target constructor" } + } else if (potentialHook.targetReturnTypeDescriptor == "V") { + require(hookMethod.returnType.descriptor == "V") { "$potentialHook: return type must be void" } } else { require( - returnTypeDescriptor in listOf( + hookMethod.returnType.descriptor in listOf( java.lang.Object::class.java.descriptor, potentialHook.targetReturnTypeDescriptor, potentialHook.targetWrappedReturnTypeDescriptor @@ -92,28 +112,22 @@ class Hook private constructor(hookMethod: Method, annotation: MethodHook) { } } - // AfterMethodHook only: Verify the type of the last parameter if known. - if (hookData.type == HookType.AFTER && potentialHook.targetReturnTypeDescriptor != null) { - require( - parameterTypes[4] == java.lang.Object::class.java || - parameterTypes[4].descriptor == potentialHook.targetWrappedReturnTypeDescriptor - ) { - "$potentialHook: fifth parameter must have type Object or match the descriptor ${potentialHook.targetWrappedReturnTypeDescriptor}" + // AfterMethodHook only: Verify the type of the last parameter if known. Even if not + // known, it must not be a primitive value. + if (potentialHook.hookType == HookType.AFTER) { + if (potentialHook.targetReturnTypeDescriptor != null) { + require( + parameterTypes[4] == java.lang.Object::class.java || + parameterTypes[4].descriptor == potentialHook.targetWrappedReturnTypeDescriptor + ) { + "$potentialHook: fifth parameter must have type Object or match the descriptor ${potentialHook.targetWrappedReturnTypeDescriptor}" + } + } else { + require(!parameterTypes[4].isPrimitive) { + "$potentialHook: fifth parameter must not be a primitive type, use a boxed type instead" + } } } - - return potentialHook - } - } -} - -fun loadHooks(hookClass: Class<*>): List<Hook> { - val hooks = mutableListOf<Hook>() - for (method in hookClass.methods) { - method.getAnnotation(MethodHook::class.java)?.let { hooks.add(Hook.verifyAndGetHook(method, it)) } - method.getAnnotation(MethodHooks::class.java)?.let { - it.value.forEach { hookAnnotation -> hooks.add(Hook.verifyAndGetHook(method, hookAnnotation)) } } } - return hooks } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/HookInstrumentor.kt b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/HookInstrumentor.kt index ac5f1780..6db76605 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/HookInstrumentor.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/HookInstrumentor.kt @@ -14,10 +14,10 @@ package com.code_intelligence.jazzer.instrumentor -import com.code_intelligence.jazzer.third_party.objectweb.asm.ClassReader -import com.code_intelligence.jazzer.third_party.objectweb.asm.ClassVisitor -import com.code_intelligence.jazzer.third_party.objectweb.asm.ClassWriter -import com.code_intelligence.jazzer.third_party.objectweb.asm.MethodVisitor +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassVisitor +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.MethodVisitor internal class HookInstrumentor(private val hooks: Iterable<Hook>, private val java6Mode: Boolean) : Instrumentor { diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/HookMethodVisitor.kt b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/HookMethodVisitor.kt index 7c23c703..1694be58 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/HookMethodVisitor.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/HookMethodVisitor.kt @@ -15,11 +15,12 @@ package com.code_intelligence.jazzer.instrumentor import com.code_intelligence.jazzer.api.HookType -import com.code_intelligence.jazzer.third_party.objectweb.asm.Handle -import com.code_intelligence.jazzer.third_party.objectweb.asm.MethodVisitor -import com.code_intelligence.jazzer.third_party.objectweb.asm.Opcodes -import com.code_intelligence.jazzer.third_party.objectweb.asm.Type -import com.code_intelligence.jazzer.third_party.objectweb.asm.commons.LocalVariablesSorter +import org.objectweb.asm.Handle +import org.objectweb.asm.MethodVisitor +import org.objectweb.asm.Opcodes +import org.objectweb.asm.Type +import org.objectweb.asm.commons.LocalVariablesSorter +import java.util.concurrent.atomic.AtomicBoolean internal fun makeHookMethodVisitor( access: Int, @@ -41,6 +42,10 @@ private class HookMethodVisitor( private val random: DeterministicRandom, ) : MethodVisitor(Instrumentor.ASM_API_VERSION, methodVisitor) { + companion object { + private val showUnsupportedHookWarning = AtomicBoolean(true) + } + val lvs = object : LocalVariablesSorter(Instrumentor.ASM_API_VERSION, access, descriptor, this) { override fun updateNewLocals(newLocals: Array<Any>) { // The local variables involved in calling hooks do not need to outlive the current @@ -51,7 +56,7 @@ private class HookMethodVisitor( } } - private val hooks = hooks.associateBy { hook -> + private val hooks = hooks.groupBy { hook -> var hookKey = "${hook.hookType}#${hook.targetInternalClassName}#${hook.targetMethodName}" if (hook.targetMethodDescriptor != null) hookKey += "#${hook.targetMethodDescriptor}" @@ -69,63 +74,23 @@ private class HookMethodVisitor( mv.visitMethodInsn(opcode, owner, methodName, methodDescriptor, isInterface) return } - handleMethodInsn(HookType.BEFORE, opcode, owner, methodName, methodDescriptor, isInterface) - } - - /** - * Emits the bytecode for a method call instruction for the next applicable hook type in order (BEFORE, REPLACE, - * AFTER). Since the instrumented code is indistinguishable from an uninstrumented call instruction, it can be - * safely nested. Combining REPLACE hooks with other hooks is however not supported as these hooks already subsume - * the functionality of BEFORE and AFTER hooks. - */ - private fun visitNextHookTypeOrCall( - hookType: HookType, - appliedHook: Boolean, - opcode: Int, - owner: String, - methodName: String, - methodDescriptor: String, - isInterface: Boolean, - ) = when (hookType) { - HookType.BEFORE -> { - val nextHookType = if (appliedHook) { - // After a BEFORE hook has been applied, we can safely apply an AFTER hook by replacing the actual - // call instruction with the full bytecode injected for the AFTER hook. - HookType.AFTER - } else { - // If no BEFORE hook is registered, look for a REPLACE hook next. - HookType.REPLACE - } - handleMethodInsn(nextHookType, opcode, owner, methodName, methodDescriptor, isInterface) - } - HookType.REPLACE -> { - // REPLACE hooks can't (and don't need to) be mixed with other hooks. We only cycle through them if we - // couldn't find a matching REPLACE hook, in which case we try an AFTER hook next. - require(!appliedHook) - handleMethodInsn(HookType.AFTER, opcode, owner, methodName, methodDescriptor, isInterface) - } - // An AFTER hook is always the last in the chain. Whether a hook has been applied or not, always emit the - // actual call instruction. - HookType.AFTER -> mv.visitMethodInsn(opcode, owner, methodName, methodDescriptor, isInterface) + handleMethodInsn(opcode, owner, methodName, methodDescriptor, isInterface) } fun handleMethodInsn( - hookType: HookType, opcode: Int, owner: String, methodName: String, methodDescriptor: String, isInterface: Boolean, ) { - val hook = findMatchingHook(hookType, owner, methodName, methodDescriptor) - if (hook == null) { - visitNextHookTypeOrCall(hookType, false, opcode, owner, methodName, methodDescriptor, isInterface) + val matchingHooks = findMatchingHooks(owner, methodName, methodDescriptor) + + if (matchingHooks.isEmpty()) { + mv.visitMethodInsn(opcode, owner, methodName, methodDescriptor, isInterface) return } - // The hookId is used to identify a call site. - val hookId = random.nextInt() - val paramDescriptors = extractParameterTypeDescriptors(methodDescriptor) val localObjArr = storeMethodArguments(paramDescriptors) // If the method we're hooking is not static there is now a reference to @@ -142,138 +107,158 @@ private class HookMethodVisitor( // We now removed all values for the original method call from the operand stack // and saved them to local variables. - // Start to build the arguments for the hook method. - if (methodName == "<init>") { - // Special case for constructors: - // We cannot create a MethodHandle for a constructor, so we push null instead. - mv.visitInsn(Opcodes.ACONST_NULL) // push nullref - // Only pass the this object if it has been initialized by the time the hook is invoked. - if (hook.hookType == HookType.AFTER) { - mv.visitVarInsn(Opcodes.ALOAD, localOwnerObj) - } else { - mv.visitInsn(Opcodes.ACONST_NULL) // push nullref - } - } else { - // Push a MethodHandle representing the hooked method. - val handleOpcode = when (opcode) { - Opcodes.INVOKEVIRTUAL -> Opcodes.H_INVOKEVIRTUAL - Opcodes.INVOKEINTERFACE -> Opcodes.H_INVOKEINTERFACE - Opcodes.INVOKESTATIC -> Opcodes.H_INVOKESTATIC - Opcodes.INVOKESPECIAL -> Opcodes.H_INVOKESPECIAL - else -> -1 - } - if (java6Mode) { - // MethodHandle constants (type 15) are not supported in Java 6 class files (major version 50). + val returnTypeDescriptor = extractReturnTypeDescriptor(methodDescriptor) + // Create a local variable to store the return value + val localReturnObj = lvs.newLocal(Type.getType(getWrapperTypeDescriptor(returnTypeDescriptor))) + + matchingHooks.forEachIndexed { index, hook -> + // The hookId is used to identify a call site. + val hookId = random.nextInt() + + // Start to build the arguments for the hook method. + if (methodName == "<init>") { + // Constructor is invoked on an uninitialized object, and that's still on the stack. + // In case of REPLACE pop it from the stack and replace it afterwards with the returned + // one from the hook. + if (hook.hookType == HookType.REPLACE) { + mv.visitInsn(Opcodes.POP) + } + // Special case for constructors: + // We cannot create a MethodHandle for a constructor, so we push null instead. mv.visitInsn(Opcodes.ACONST_NULL) // push nullref + // Only pass the this object if it has been initialized by the time the hook is invoked. + if (hook.hookType == HookType.AFTER) { + mv.visitVarInsn(Opcodes.ALOAD, localOwnerObj) + } else { + mv.visitInsn(Opcodes.ACONST_NULL) // push nullref + } } else { - mv.visitLdcInsn( - Handle( - handleOpcode, - owner, - methodName, - methodDescriptor, - isInterface - ) - ) // push MethodHandle - } - // Stack layout: ... | MethodHandle (objectref) - // Push the owner object again - mv.visitVarInsn(Opcodes.ALOAD, localOwnerObj) - } - // Stack layout: ... | MethodHandle (objectref) | owner (objectref) - // Push a reference to our object array with the saved arguments - mv.visitVarInsn(Opcodes.ALOAD, localObjArr) - // Stack layout: ... | MethodHandle (objectref) | owner (objectref) | object array (arrayref) - // Push the hook id - mv.visitLdcInsn(hookId) - // Stack layout: ... | MethodHandle (objectref) | owner (objectref) | object array (arrayref) | hookId (int) - // How we proceed depends on the type of hook we want to implement - when (hook.hookType) { - HookType.BEFORE -> { - // Call the hook method - mv.visitMethodInsn( - Opcodes.INVOKESTATIC, - hook.hookInternalClassName, - hook.hookMethodName, - hook.hookMethodDescriptor, - false - ) - // Stack layout: ... - // Push the values for the original method call onto the stack again - if (opcode != Opcodes.INVOKESTATIC) { - mv.visitVarInsn(Opcodes.ALOAD, localOwnerObj) // push owner object + // Push a MethodHandle representing the hooked method. + val handleOpcode = when (opcode) { + Opcodes.INVOKEVIRTUAL -> Opcodes.H_INVOKEVIRTUAL + Opcodes.INVOKEINTERFACE -> Opcodes.H_INVOKEINTERFACE + Opcodes.INVOKESTATIC -> Opcodes.H_INVOKESTATIC + Opcodes.INVOKESPECIAL -> Opcodes.H_INVOKESPECIAL + else -> -1 } - loadMethodArguments(paramDescriptors, localObjArr) // push all method arguments - // Stack layout: ... | [owner (objectref)] | arg1 (primitive/objectref) | arg2 (primitive/objectref) | ... - // Call the original method or the next hook in order. - visitNextHookTypeOrCall(hookType, true, opcode, owner, methodName, methodDescriptor, isInterface) + if (java6Mode) { + // MethodHandle constants (type 15) are not supported in Java 6 class files (major version 50). + mv.visitInsn(Opcodes.ACONST_NULL) // push nullref + } else { + mv.visitLdcInsn( + Handle( + handleOpcode, + owner, + methodName, + methodDescriptor, + isInterface + ) + ) // push MethodHandle + } + // Stack layout: ... | MethodHandle (objectref) + // Push the owner object again + mv.visitVarInsn(Opcodes.ALOAD, localOwnerObj) } - HookType.REPLACE -> { - // Call the hook method - mv.visitMethodInsn( - Opcodes.INVOKESTATIC, - hook.hookInternalClassName, - hook.hookMethodName, - hook.hookMethodDescriptor, - false - ) - // Stack layout: ... | [return value (primitive/objectref)] - // Check if we need to process the return value - val returnTypeDescriptor = extractReturnTypeDescriptor(methodDescriptor) - if (returnTypeDescriptor != "V") { - val hookMethodReturnType = extractReturnTypeDescriptor(hook.hookMethodDescriptor) - // if the hook method's return type is primitive we don't need to unwrap or cast it - if (!isPrimitiveType(hookMethodReturnType)) { - // Check if the returned object type is different than the one that should be returned - // If a primitive should be returned we check it's wrapper type - val expectedType = getWrapperTypeDescriptor(returnTypeDescriptor) - if (expectedType != hookMethodReturnType) { - // Cast object - mv.visitTypeInsn(Opcodes.CHECKCAST, extractInternalClassName(expectedType)) + // Stack layout: ... | MethodHandle (objectref) | owner (objectref) + // Push a reference to our object array with the saved arguments + mv.visitVarInsn(Opcodes.ALOAD, localObjArr) + // Stack layout: ... | MethodHandle (objectref) | owner (objectref) | object array (arrayref) + // Push the hook id + mv.visitLdcInsn(hookId) + // Stack layout: ... | MethodHandle (objectref) | owner (objectref) | object array (arrayref) | hookId (int) + // How we proceed depends on the type of hook we want to implement + when (hook.hookType) { + HookType.BEFORE -> { + // Call the hook method + mv.visitMethodInsn( + Opcodes.INVOKESTATIC, + hook.hookInternalClassName, + hook.hookMethodName, + hook.hookMethodDescriptor, + false + ) + + // Call the original method if this is the last BEFORE hook. If not, the original method will be + // called by the next AFTER hook. + if (index == matchingHooks.lastIndex) { + // Stack layout: ... + // Push the values for the original method call onto the stack again + if (opcode != Opcodes.INVOKESTATIC) { + mv.visitVarInsn(Opcodes.ALOAD, localOwnerObj) // push owner object } - // Check if we need to unwrap the returned object - unwrapTypeIfPrimitive(returnTypeDescriptor) + loadMethodArguments(paramDescriptors, localObjArr) // push all method arguments + // Stack layout: ... | [owner (objectref)] | arg1 (primitive/objectref) | arg2 (primitive/objectref) | ... + mv.visitMethodInsn(opcode, owner, methodName, methodDescriptor, isInterface) } } - } - HookType.AFTER -> { - // Push the values for the original method call again onto the stack - if (opcode != Opcodes.INVOKESTATIC) { - mv.visitVarInsn(Opcodes.ALOAD, localOwnerObj) // push owner object - } - loadMethodArguments(paramDescriptors, localObjArr) // push all method arguments - // Stack layout: ... | MethodHandle (objectref) | owner (objectref) | object array (arrayref) | hookId (int) - // | [owner (objectref)] | arg1 (primitive/objectref) | arg2 (primitive/objectref) | ... - // Call the original method or the next hook in order - visitNextHookTypeOrCall(hookType, true, opcode, owner, methodName, methodDescriptor, isInterface) - val returnTypeDescriptor = extractReturnTypeDescriptor(methodDescriptor) - if (returnTypeDescriptor == "V") { - // If the method didn't return anything, we push a nullref as placeholder - mv.visitInsn(Opcodes.ACONST_NULL) // push nullref + HookType.REPLACE -> { + // Call the hook method + mv.visitMethodInsn( + Opcodes.INVOKESTATIC, + hook.hookInternalClassName, + hook.hookMethodName, + hook.hookMethodDescriptor, + false + ) + // Stack layout: ... | [return value (primitive/objectref)] + // Check if we need to process the return value + if (returnTypeDescriptor != "V") { + val hookMethodReturnType = extractReturnTypeDescriptor(hook.hookMethodDescriptor) + // if the hook method's return type is primitive we don't need to unwrap or cast it + if (!isPrimitiveType(hookMethodReturnType)) { + // Check if the returned object type is different than the one that should be returned + // If a primitive should be returned we check it's wrapper type + val expectedType = getWrapperTypeDescriptor(returnTypeDescriptor) + if (expectedType != hookMethodReturnType) { + // Cast object + mv.visitTypeInsn(Opcodes.CHECKCAST, extractInternalClassName(expectedType)) + } + // Check if we need to unwrap the returned object + unwrapTypeIfPrimitive(returnTypeDescriptor) + } + } } - // Wrap return value if it is a primitive type - wrapTypeIfPrimitive(returnTypeDescriptor) - // Stack layout: ... | MethodHandle (objectref) | owner (objectref) | object array (arrayref) | hookId (int) - // | return value (objectref) - // Store the result value in a local variable (but keep it on the stack) - val localReturnObj = lvs.newLocal(Type.getType(getWrapperTypeDescriptor(returnTypeDescriptor))) - mv.visitVarInsn(Opcodes.ASTORE, localReturnObj) // consume objectref - mv.visitVarInsn(Opcodes.ALOAD, localReturnObj) // push objectref - // Call the hook method - mv.visitMethodInsn( - Opcodes.INVOKESTATIC, - hook.hookInternalClassName, - hook.hookMethodName, - hook.hookMethodDescriptor, - false - ) - // Stack layout: ... - if (returnTypeDescriptor != "V") { - // Push the return value again + HookType.AFTER -> { + // Call the original method before the first AFTER hook + if (index == 0 || matchingHooks[index - 1].hookType != HookType.AFTER) { + // Push the values for the original method call again onto the stack + if (opcode != Opcodes.INVOKESTATIC) { + mv.visitVarInsn(Opcodes.ALOAD, localOwnerObj) // push owner object + } + loadMethodArguments(paramDescriptors, localObjArr) // push all method arguments + // Stack layout: ... | MethodHandle (objectref) | owner (objectref) | object array (arrayref) | hookId (int) + // | [owner (objectref)] | arg1 (primitive/objectref) | arg2 (primitive/objectref) | ... + mv.visitMethodInsn(opcode, owner, methodName, methodDescriptor, isInterface) + if (returnTypeDescriptor == "V") { + // If the method didn't return anything, we push a nullref as placeholder + mv.visitInsn(Opcodes.ACONST_NULL) // push nullref + } + // Wrap return value if it is a primitive type + wrapTypeIfPrimitive(returnTypeDescriptor) + mv.visitVarInsn(Opcodes.ASTORE, localReturnObj) // consume objectref + } mv.visitVarInsn(Opcodes.ALOAD, localReturnObj) // push objectref - // Unwrap it, if it was a primitive value - unwrapTypeIfPrimitive(returnTypeDescriptor) - // Stack layout: ... | return value (primitive/objectref) + + // Stack layout: ... | MethodHandle (objectref) | owner (objectref) | object array (arrayref) | hookId (int) + // | return value (objectref) + // Store the result value in a local variable (but keep it on the stack) + // Call the hook method + mv.visitMethodInsn( + Opcodes.INVOKESTATIC, + hook.hookInternalClassName, + hook.hookMethodName, + hook.hookMethodDescriptor, + false + ) + // Stack layout: ... + // Push the return value on the stack after the last AFTER hook if the original method returns a value + if (index == matchingHooks.size - 1 && returnTypeDescriptor != "V") { + // Push the return value again + mv.visitVarInsn(Opcodes.ALOAD, localReturnObj) // push objectref + // Unwrap it, if it was a primitive value + unwrapTypeIfPrimitive(returnTypeDescriptor) + // Stack layout: ... | return value (primitive/objectref) + } } } } @@ -286,10 +271,38 @@ private class HookMethodVisitor( Opcodes.INVOKESPECIAL ) - private fun findMatchingHook(hookType: HookType, owner: String, name: String, descriptor: String): Hook? { - val withoutDescriptorKey = "$hookType#$owner#$name" - val withDescriptorKey = "$withoutDescriptorKey#$descriptor" - return hooks[withDescriptorKey] ?: hooks[withoutDescriptorKey] + private fun findMatchingHooks(owner: String, name: String, descriptor: String): List<Hook> { + val result = HookType.values().flatMap { hookType -> + val withoutDescriptorKey = "$hookType#$owner#$name" + val withDescriptorKey = "$withoutDescriptorKey#$descriptor" + hooks[withDescriptorKey].orEmpty() + hooks[withoutDescriptorKey].orEmpty() + }.sortedBy { it.hookType } + val replaceHookCount = result.count { it.hookType == HookType.REPLACE } + check( + replaceHookCount == 0 || + (replaceHookCount == 1 && result.size == 1) + ) { + "For a given method, You can either have a single REPLACE hook or BEFORE/AFTER hooks. Found:\n $result" + } + + return result + .filter { !isReplaceHookInJava6mode(it) } + .sortedByDescending { it.toString() } + } + + private fun isReplaceHookInJava6mode(hook: Hook): Boolean { + if (java6Mode && hook.hookType == HookType.REPLACE) { + if (showUnsupportedHookWarning.getAndSet(false)) { + println( + """WARN: Some hooks could not be applied to class files built for Java 7 or lower. + |WARN: Ensure that the fuzz target and its dependencies are compiled with + |WARN: -target 8 or higher to identify as many bugs as possible. + """.trimMargin() + ) + } + return true + } + return false } // Stores all arguments for a method call in a local object array. @@ -350,7 +363,7 @@ private class HookMethodVisitor( } // Removes a primitive value from the top of the operand stack - // and pushes it enclosed in it's wrapper type (e.g. removes int, pushes Integer). + // and pushes it enclosed in its wrapper type (e.g. removes int, pushes Integer). // This is done by calling .valueOf(...) on the wrapper class. private fun wrapTypeIfPrimitive(unwrappedTypeDescriptor: String) { if (!isPrimitiveType(unwrappedTypeDescriptor) || unwrappedTypeDescriptor == "V") return diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/Hooks.kt b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/Hooks.kt new file mode 100644 index 00000000..66a21ee7 --- /dev/null +++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/Hooks.kt @@ -0,0 +1,114 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor + +import com.code_intelligence.jazzer.api.MethodHook +import com.code_intelligence.jazzer.api.MethodHooks +import com.code_intelligence.jazzer.utils.ClassNameGlobber +import com.code_intelligence.jazzer.utils.descriptor +import io.github.classgraph.ClassGraph +import io.github.classgraph.ScanResult +import java.lang.reflect.Method + +data class Hooks( + val hooks: List<Hook>, + val hookClasses: Set<Class<*>>, + val additionalHookClassNameGlobber: ClassNameGlobber +) { + + companion object { + fun loadHooks(vararg hookClassNames: Set<String>): List<Hooks> { + return ClassGraph() + .enableClassInfo() + .enableSystemJarsAndModules() + .rejectPackages("jaz.*", "com.code_intelligence.jazzer.*") + .scan() + .use { scanResult -> + // Capture scanResult in HooksLoader field to not pass it through + // all internal hook loading methods. + val loader = HooksLoader(scanResult) + hookClassNames.map(loader::load) + } + } + + private class HooksLoader(private val scanResult: ScanResult) { + fun load(hookClassNames: Set<String>): Hooks { + val hooksWithHookClasses = hookClassNames.flatMap(::loadHooks) + val hooks = hooksWithHookClasses.map { it.first } + val hookClasses = hooksWithHookClasses.map { it.second }.toSet() + val additionalHookClassNameGlobber = ClassNameGlobber( + hooks.flatMap(Hook::additionalClassesToHook), + emptyList() + ) + return Hooks(hooks, hookClasses, additionalHookClassNameGlobber) + } + + private fun loadHooks(hookClassName: String): List<Pair<Hook, Class<*>>> { + return try { + // Custom hook classes outside the agent jar can not be found by bootstrap + // class loader, so use the system class loader as that will be the main application + // class loader and can access jars on the classpath. + // We let the static initializers of hook classes execute so that hooks can run + // code before the fuzz target class has been loaded (e.g., register themselves + // for the onFuzzTargetReady callback). + val hookClass = Class.forName(hookClassName, true, ClassLoader.getSystemClassLoader()) + loadHooks(hookClass).also { + println("INFO: Loaded ${it.size} hooks from $hookClassName") + }.map { + it to hookClass + } + } catch (e: ClassNotFoundException) { + println("WARN: Failed to load hooks from $hookClassName: ${e.printStackTrace()}") + emptyList() + } + } + + private fun loadHooks(hookClass: Class<*>): List<Hook> { + val hooks = mutableListOf<Hook>() + for (method in hookClass.methods.sortedBy { it.descriptor }) { + method.getAnnotation(MethodHook::class.java)?.let { + hooks.addAll(verifyAndGetHooks(method, it)) + } + method.getAnnotation(MethodHooks::class.java)?.let { + it.value.forEach { hookAnnotation -> + hooks.addAll(verifyAndGetHooks(method, hookAnnotation)) + } + } + } + return hooks + } + + private fun verifyAndGetHooks(hookMethod: Method, hookData: MethodHook): List<Hook> { + return lookupClassesToHook(hookData.targetClassName) + .map { className -> + Hook.createAndVerifyHook(hookMethod, hookData, className) + } + } + + private fun lookupClassesToHook(annotationTargetClassName: String): List<String> { + // Allowing arbitrary exterior whitespace in the target class name allows for an easy workaround + // for mangled hooks due to shading applied to hooks. + val targetClassName = annotationTargetClassName.trim() + val targetClassInfo = scanResult.getClassInfo(targetClassName) ?: return listOf(targetClassName) + val additionalTargetClasses = when { + targetClassInfo.isInterface -> scanResult.getClassesImplementing(targetClassName) + targetClassInfo.isAbstract -> scanResult.getSubclasses(targetClassName) + else -> emptyList() + } + return (listOf(targetClassName) + additionalTargetClasses.map { it.name }).sorted() + } + } + } +} diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/Instrumentor.kt b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/Instrumentor.kt index 86ad45a3..78793842 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/Instrumentor.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/Instrumentor.kt @@ -14,8 +14,8 @@ package com.code_intelligence.jazzer.instrumentor -import com.code_intelligence.jazzer.third_party.objectweb.asm.Opcodes -import com.code_intelligence.jazzer.third_party.objectweb.asm.tree.MethodNode +import org.objectweb.asm.Opcodes +import org.objectweb.asm.tree.MethodNode enum class InstrumentationType { CMP, diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/StaticMethodStrategy.java b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/StaticMethodStrategy.java new file mode 100644 index 00000000..0512ec2a --- /dev/null +++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/StaticMethodStrategy.java @@ -0,0 +1,48 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import com.code_intelligence.jazzer.third_party.org.jacoco.core.internal.instr.InstrSupport; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +public class StaticMethodStrategy implements EdgeCoverageStrategy { + @Override + public void instrumentControlFlowEdge( + MethodVisitor mv, int edgeId, int variable, String coverageMapInternalClassName) { + InstrSupport.push(mv, edgeId); + mv.visitMethodInsn( + Opcodes.INVOKESTATIC, coverageMapInternalClassName, "recordCoverage", "(I)V", false); + } + + @Override + public int getInstrumentControlFlowEdgeStackSize() { + return 1; + } + + @Override + public Object getLocalVariableType() { + return null; + } + + @Override + public void loadLocalVariable( + MethodVisitor mv, int variable, String coverageMapInternalClassName) {} + + @Override + public int getLoadLocalVariableStackSize() { + return 0; + } +} diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/TraceDataFlowInstrumentor.kt b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/TraceDataFlowInstrumentor.kt index e6d3176e..65f11e52 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/TraceDataFlowInstrumentor.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/TraceDataFlowInstrumentor.kt @@ -15,19 +15,19 @@ package com.code_intelligence.jazzer.instrumentor import com.code_intelligence.jazzer.runtime.TraceDataFlowNativeCallbacks -import com.code_intelligence.jazzer.third_party.objectweb.asm.ClassReader -import com.code_intelligence.jazzer.third_party.objectweb.asm.ClassWriter -import com.code_intelligence.jazzer.third_party.objectweb.asm.Opcodes -import com.code_intelligence.jazzer.third_party.objectweb.asm.tree.AbstractInsnNode -import com.code_intelligence.jazzer.third_party.objectweb.asm.tree.ClassNode -import com.code_intelligence.jazzer.third_party.objectweb.asm.tree.InsnList -import com.code_intelligence.jazzer.third_party.objectweb.asm.tree.InsnNode -import com.code_intelligence.jazzer.third_party.objectweb.asm.tree.IntInsnNode -import com.code_intelligence.jazzer.third_party.objectweb.asm.tree.LdcInsnNode -import com.code_intelligence.jazzer.third_party.objectweb.asm.tree.LookupSwitchInsnNode -import com.code_intelligence.jazzer.third_party.objectweb.asm.tree.MethodInsnNode -import com.code_intelligence.jazzer.third_party.objectweb.asm.tree.MethodNode -import com.code_intelligence.jazzer.third_party.objectweb.asm.tree.TableSwitchInsnNode +import org.objectweb.asm.ClassReader +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.Opcodes +import org.objectweb.asm.tree.AbstractInsnNode +import org.objectweb.asm.tree.ClassNode +import org.objectweb.asm.tree.InsnList +import org.objectweb.asm.tree.InsnNode +import org.objectweb.asm.tree.IntInsnNode +import org.objectweb.asm.tree.LdcInsnNode +import org.objectweb.asm.tree.LookupSwitchInsnNode +import org.objectweb.asm.tree.MethodInsnNode +import org.objectweb.asm.tree.MethodNode +import org.objectweb.asm.tree.TableSwitchInsnNode internal class TraceDataFlowInstrumentor(private val types: Set<InstrumentationType>, callbackClass: Class<*> = TraceDataFlowNativeCallbacks::class.java) : Instrumentor { @@ -133,7 +133,7 @@ internal class TraceDataFlowInstrumentor(private val types: Set<InstrumentationT } private fun InsnList.pushFakePc() { - add(LdcInsnNode(random.nextInt(4096))) + add(LdcInsnNode(random.nextInt(512))) } private fun longCmpInstrumentation() = InsnList().apply { diff --git a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/shade_rules b/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/shade_rules deleted file mode 100644 index c2092b3b..00000000 --- a/agent/src/main/java/com/code_intelligence/jazzer/instrumentor/shade_rules +++ /dev/null @@ -1 +0,0 @@ -rule org.** com.code_intelligence.jazzer.third_party.@1
\ No newline at end of file diff --git a/agent/src/main/java/com/code_intelligence/jazzer/replay/BUILD.bazel b/agent/src/main/java/com/code_intelligence/jazzer/replay/BUILD.bazel index df28adb4..08bd7653 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/replay/BUILD.bazel +++ b/agent/src/main/java/com/code_intelligence/jazzer/replay/BUILD.bazel @@ -3,8 +3,7 @@ load("@fmeum_rules_jni//jni:defs.bzl", "java_jni_library") java_jni_library( name = "replay", srcs = ["Replayer.java"], - native_libs = ["//agent/src/main/native/com/code_intelligence/jazzer/replay"], - visibility = ["//agent/src/main/native/com/code_intelligence/jazzer/replay:__pkg__"], + native_libs = ["//driver/src/main/native/com/code_intelligence/jazzer/driver:fuzzed_data_provider_standalone"], deps = [ "//agent/src/main/java/com/code_intelligence/jazzer/api", "//agent/src/main/java/com/code_intelligence/jazzer/runtime:fuzzed_data_provider", diff --git a/agent/src/main/java/com/code_intelligence/jazzer/replay/Replayer.java b/agent/src/main/java/com/code_intelligence/jazzer/replay/Replayer.java index fc6bfc4f..0a250d1a 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/replay/Replayer.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/replay/Replayer.java @@ -29,8 +29,10 @@ public class Replayer { public static final int STATUS_OTHER_ERROR = 1; static { + System.setProperty("jazzer.is_replayer", "true"); try { - RulesJni.loadLibrary("replay", Replayer.class); + RulesJni.loadLibrary( + "fuzzed_data_provider_standalone", "/com/code_intelligence/jazzer/driver"); } catch (Throwable t) { t.printStackTrace(); System.exit(STATUS_OTHER_ERROR); @@ -104,7 +106,9 @@ public class Replayer { try { Method fuzzerTestOneInput = fuzzTarget.getMethod("fuzzerTestOneInput", FuzzedDataProvider.class); - fuzzerTestOneInput.invoke(null, makeFuzzedDataProvider(input)); + try (FuzzedDataProviderImpl fuzzedDataProvider = FuzzedDataProviderImpl.withJavaData(input)) { + fuzzerTestOneInput.invoke(null, fuzzedDataProvider); + } return; } catch (Exception e) { handleInvokeException(e, fuzzTarget); @@ -149,11 +153,4 @@ public class Replayer { } } } - - private static FuzzedDataProvider makeFuzzedDataProvider(byte[] input) { - feedFuzzedDataProvider(input); - return new FuzzedDataProviderImpl(); - } - - private static native void feedFuzzedDataProvider(byte[] input); } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/runtime/BUILD.bazel b/agent/src/main/java/com/code_intelligence/jazzer/runtime/BUILD.bazel index 095b0bf8..0d8162d5 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/runtime/BUILD.bazel +++ b/agent/src/main/java/com/code_intelligence/jazzer/runtime/BUILD.bazel @@ -1,47 +1,87 @@ -load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") +load("@fmeum_rules_jni//jni:defs.bzl", "java_jni_library") -java_library( +java_jni_library( name = "fuzzed_data_provider", srcs = [ "FuzzedDataProviderImpl.java", ], - visibility = ["//agent/src/main/java/com/code_intelligence/jazzer/replay:__pkg__"], + visibility = [ + "//agent/src/main/java/com/code_intelligence/jazzer/replay:__pkg__", + "//agent/src/test/java/com/code_intelligence/jazzer/runtime:__pkg__", + "//driver/src/main/java/com/code_intelligence/jazzer/driver:__pkg__", + "//driver/src/main/native/com/code_intelligence/jazzer/driver:__pkg__", + ], deps = [ + ":unsafe_provider", "//agent/src/main/java/com/code_intelligence/jazzer/api", ], ) -java_library( +java_jni_library( + name = "coverage_map", + srcs = ["CoverageMap.java"], + visibility = [ + "//agent/src/jmh/java/com/code_intelligence/jazzer/instrumentor:__pkg__", + "//agent/src/main/java/com/code_intelligence/jazzer/instrumentor:__pkg__", + "//driver/src/main/java/com/code_intelligence/jazzer/driver:__pkg__", + "//driver/src/main/native/com/code_intelligence/jazzer/driver:__pkg__", + "//driver/src/test:__subpackages__", + ], + deps = [ + ":unsafe_provider", + ], +) + +java_jni_library( name = "signal_handler", srcs = ["SignalHandler.java"], - javacopts = [ - "-XDenableSunApiLintControl", + native_libs = ["//agent/src/main/native/com/code_intelligence/jazzer/runtime:jazzer_signal_handler"], + visibility = [ + "//agent/src/main/native/com/code_intelligence/jazzer/runtime:__pkg__", + "//driver/src/main/java/com/code_intelligence/jazzer/driver:__pkg__", ], ) -kt_jvm_library( +java_jni_library( + name = "trace_data_flow_native_callbacks", + srcs = ["TraceDataFlowNativeCallbacks.java"], + visibility = [ + "//driver/src/main/native/com/code_intelligence/jazzer/driver:__pkg__", + ], + deps = [ + "//agent/src/main/java/com/code_intelligence/jazzer/utils", + ], +) + +java_library( + name = "unsafe_provider", + srcs = ["UnsafeProvider.java"], + visibility = [ + "//driver/src:__subpackages__", + "//sanitizers/src/main/java:__subpackages__", + ], +) + +java_library( name = "runtime", srcs = [ - "CoverageMap.java", - "ExceptionUtils.kt", "HardToCatchError.java", "JazzerInternal.java", - "ManifestUtils.kt", "NativeLibHooks.java", "RecordingFuzzedDataProvider.java", - "SignalHandler.java", "TraceCmpHooks.java", - "TraceDataFlowNativeCallbacks.java", "TraceDivHooks.java", "TraceIndirHooks.java", ], visibility = ["//visibility:public"], runtime_deps = [ + ":signal_handler", "//agent/src/main/java/com/code_intelligence/jazzer/autofuzz", ], deps = [ + ":coverage_map", ":fuzzed_data_provider", - ":signal_handler", + ":trace_data_flow_native_callbacks", "//agent/src/main/java/com/code_intelligence/jazzer/api", "//agent/src/main/java/com/code_intelligence/jazzer/utils", ], diff --git a/agent/src/main/java/com/code_intelligence/jazzer/runtime/CoverageMap.java b/agent/src/main/java/com/code_intelligence/jazzer/runtime/CoverageMap.java index af2424a2..4069d25a 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/runtime/CoverageMap.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/runtime/CoverageMap.java @@ -14,20 +14,116 @@ package com.code_intelligence.jazzer.runtime; -import java.nio.ByteBuffer; +import com.github.fmeum.rules_jni.RulesJni; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import sun.misc.Unsafe; /** - * Represents the Java view on a libFuzzer 8 bit counter coverage map. - * By using a direct ByteBuffer, the counter array is shared directly with - * native code. + * Represents the Java view on a libFuzzer 8 bit counter coverage map. By using a direct ByteBuffer, + * the counters are shared directly with native code. */ final public class CoverageMap { - public static ByteBuffer mem = ByteBuffer.allocateDirect(0); + static { + RulesJni.loadLibrary("jazzer_driver", "/com/code_intelligence/jazzer/driver"); + } + + private static final String ENV_MAX_NUM_COUNTERS = "JAZZER_MAX_NUM_COUNTERS"; + + private static final int MAX_NUM_COUNTERS = System.getenv(ENV_MAX_NUM_COUNTERS) != null + ? Integer.parseInt(System.getenv(ENV_MAX_NUM_COUNTERS)) + : 1 << 20; - public static void enlargeCoverageMap() { - registerNewCoverageCounters(); - System.out.println("INFO: New number of inline 8-bit counters: " + mem.capacity()); + private static final Unsafe UNSAFE = UnsafeProvider.getUnsafe(); + + static { + if (UNSAFE == null) { + System.out.println("ERROR: Failed to get Unsafe instance for CoverageMap.%n" + + " Please file a bug at:%n" + + " https://github.com/CodeIntelligenceTesting/jazzer/issues/new"); + System.exit(1); + } } - private static native void registerNewCoverageCounters(); + /** + * The collection of coverage counters directly interacted with by classes that are instrumented + * for coverage. The instrumentation assumes that this is always one contiguous block of memory, + * so it is allocated once at maximum size. Using a larger number here increases the memory usage + * of all fuzz targets, but has otherwise no impact on performance. + */ + public static final long countersAddress = UNSAFE.allocateMemory(MAX_NUM_COUNTERS); + + static { + UNSAFE.setMemory(countersAddress, MAX_NUM_COUNTERS, (byte) 0); + initialize(countersAddress); + } + + private static final int INITIAL_NUM_COUNTERS = 1 << 9; + + static { + registerNewCounters(0, INITIAL_NUM_COUNTERS); + } + + /** + * The number of coverage counters that are currently registered with libFuzzer. This number grows + * dynamically as classes are instrumented and should be kept as low as possible as libFuzzer has + * to iterate over the whole map for every execution. + */ + private static int currentNumCounters = INITIAL_NUM_COUNTERS; + + // Called via reflection. + @SuppressWarnings("unused") + public static void enlargeIfNeeded(int nextId) { + int newNumCounters = currentNumCounters; + while (nextId >= newNumCounters) { + newNumCounters = 2 * newNumCounters; + if (newNumCounters > MAX_NUM_COUNTERS) { + System.out.printf("ERROR: Maximum number (%s) of coverage counters exceeded. Try to%n" + + " limit the scope of a single fuzz target as much as possible to keep the%n" + + " fuzzer fast.%n" + + " If that is not possible, the maximum number of counters can be increased%n" + + " via the %s environment variable.", + MAX_NUM_COUNTERS, ENV_MAX_NUM_COUNTERS); + System.exit(1); + } + } + if (newNumCounters > currentNumCounters) { + registerNewCounters(currentNumCounters, newNumCounters); + currentNumCounters = newNumCounters; + System.out.println("INFO: New number of coverage counters: " + currentNumCounters); + } + } + + // Called by the coverage instrumentation. + @SuppressWarnings("unused") + public static void recordCoverage(final int id) { + final long address = countersAddress + id; + final byte counter = UNSAFE.getByte(address); + UNSAFE.putByte(address, (byte) (counter == -1 ? 1 : counter + 1)); + } + + public static Set<Integer> getCoveredIds() { + Set<Integer> coveredIds = new HashSet<>(); + for (int id = 0; id < currentNumCounters; id++) { + if (UNSAFE.getByte(countersAddress + id) > 0) { + coveredIds.add(id); + } + } + return Collections.unmodifiableSet(coveredIds); + } + + public static void replayCoveredIds(Set<Integer> coveredIds) { + for (int id : coveredIds) { + UNSAFE.putByte(countersAddress + id, (byte) 1); + } + } + + // Returns the IDs of all blocks that have been covered in at least one run (not just the current + // one). + public static native int[] getEverCoveredIds(); + + private static native void initialize(long countersAddress); + + private static native void registerNewCounters(int oldNumCounters, int newNumCounters); } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/runtime/FuzzedDataProviderImpl.java b/agent/src/main/java/com/code_intelligence/jazzer/runtime/FuzzedDataProviderImpl.java index fe4d8ac7..b7aad33e 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/runtime/FuzzedDataProviderImpl.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/runtime/FuzzedDataProviderImpl.java @@ -15,9 +15,119 @@ package com.code_intelligence.jazzer.runtime; import com.code_intelligence.jazzer.api.FuzzedDataProvider; - -public class FuzzedDataProviderImpl implements FuzzedDataProvider { - public FuzzedDataProviderImpl() {} +import com.github.fmeum.rules_jni.RulesJni; +import sun.misc.Unsafe; + +public class FuzzedDataProviderImpl implements FuzzedDataProvider, AutoCloseable { + static { + // The replayer loads a standalone version of the FuzzedDataProvider. + if (System.getProperty("jazzer.is_replayer") == null) { + RulesJni.loadLibrary("jazzer_driver", "/com/code_intelligence/jazzer/driver"); + } + nativeInit(); + } + + private static native void nativeInit(); + + private final boolean ownsNativeData; + private long originalDataPtr; + private int originalRemainingBytes; + + // Accessed in fuzzed_data_provider.cpp. + private long dataPtr; + private int remainingBytes; + + private FuzzedDataProviderImpl(long dataPtr, int remainingBytes, boolean ownsNativeData) { + this.ownsNativeData = ownsNativeData; + this.originalDataPtr = dataPtr; + this.dataPtr = dataPtr; + this.originalRemainingBytes = remainingBytes; + this.remainingBytes = remainingBytes; + } + + /** + * Creates a {@link FuzzedDataProvider} that consumes bytes from an already existing native array. + * + * <ul> + * <li>{@link #close()} <b>must</b> be called on instances created with this method to free the + * native copy of the Java + * {@code byte} array. + * <li>{@link #setNativeData(long, int)} <b>must not</b> be called on instances created with this + * method. + * + * @param data the raw bytes used as input + * @return a {@link FuzzedDataProvider} backed by {@code data} + */ + public static FuzzedDataProviderImpl withJavaData(byte[] data) { + return new FuzzedDataProviderImpl(allocateNativeCopy(data), data.length, true); + } + + /** + * Creates a {@link FuzzedDataProvider} that consumes bytes from an already existing native array. + * + * <p>The backing array can be set at any time using {@link #setNativeData(long, int)} and is + * initially empty. + * + * @return a {@link FuzzedDataProvider} backed by an empty array. + */ + public static FuzzedDataProviderImpl withNativeData() { + return new FuzzedDataProviderImpl(0, 0, false); + } + + /** + * Replaces the current native backing array. + * + * <p><b>Must not</b> be called on instances created with {@link #withJavaData(byte[])}. + * + * @param dataPtr a native pointer to the new backing array + * @param dataLength the length of the new backing array + */ + public void setNativeData(long dataPtr, int dataLength) { + this.originalDataPtr = dataPtr; + this.dataPtr = dataPtr; + this.originalRemainingBytes = dataLength; + this.remainingBytes = dataLength; + } + + /** + * Resets the FuzzedDataProvider state to read from the beginning to the end of its current + * backing item. + */ + public void reset() { + dataPtr = originalDataPtr; + remainingBytes = originalRemainingBytes; + } + + /** + * Releases native memory allocated for this instance (if any). + * + * <p>While the instance should not be used after this method returns, no usage of {@link + * FuzzedDataProvider} methods can result in memory corruption. + */ + @Override + public void close() { + if (originalDataPtr == 0) { + return; + } + if (ownsNativeData) { + UNSAFE.freeMemory(originalDataPtr); + } + // Prevent double-frees and use-after-frees by effectively making all methods no-ops after + // close() has been called. + originalDataPtr = 0; + originalRemainingBytes = 0; + dataPtr = 0; + remainingBytes = 0; + } + + private static final Unsafe UNSAFE = UnsafeProvider.getUnsafe(); + private static final long BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); + + private static long allocateNativeCopy(byte[] data) { + long nativeCopy = UNSAFE.allocateMemory(data.length); + UNSAFE.copyMemory(data, BYTE_ARRAY_OFFSET, null, nativeCopy, data.length); + return nativeCopy; + } @Override public native boolean consumeBoolean(); @@ -25,23 +135,51 @@ public class FuzzedDataProviderImpl implements FuzzedDataProvider { @Override public native byte consumeByte(); - @Override public native byte consumeByte(byte min, byte max); + @Override + public byte consumeByte(byte min, byte max) { + if (min > max) { + throw new IllegalArgumentException( + String.format("min must be <= max (got min: %d, max: %d)", min, max)); + } + return consumeByteUnchecked(min, max); + } @Override public native short consumeShort(); - @Override public native short consumeShort(short min, short max); + @Override + public short consumeShort(short min, short max) { + if (min > max) { + throw new IllegalArgumentException( + String.format("min must be <= max (got min: %d, max: %d)", min, max)); + } + return consumeShortUnchecked(min, max); + } @Override public native short[] consumeShorts(int maxLength); @Override public native int consumeInt(); - @Override public native int consumeInt(int min, int max); + @Override + public int consumeInt(int min, int max) { + if (min > max) { + throw new IllegalArgumentException( + String.format("min must be <= max (got min: %d, max: %d)", min, max)); + } + return consumeIntUnchecked(min, max); + } @Override public native int[] consumeInts(int maxLength); @Override public native long consumeLong(); - @Override public native long consumeLong(long min, long max); + @Override + public long consumeLong(long min, long max) { + if (min > max) { + throw new IllegalArgumentException( + String.format("min must be <= max (got min: %d, max: %d)", min, max)); + } + return consumeLongUnchecked(min, max); + } @Override public native long[] consumeLongs(int maxLength); @@ -49,13 +187,27 @@ public class FuzzedDataProviderImpl implements FuzzedDataProvider { @Override public native float consumeRegularFloat(); - @Override public native float consumeRegularFloat(float min, float max); + @Override + public float consumeRegularFloat(float min, float max) { + if (min > max) { + throw new IllegalArgumentException( + String.format("min must be <= max (got min: %f, max: %f)", min, max)); + } + return consumeRegularFloatUnchecked(min, max); + } @Override public native float consumeProbabilityFloat(); @Override public native double consumeDouble(); - @Override public native double consumeRegularDouble(double min, double max); + @Override + public double consumeRegularDouble(double min, double max) { + if (min > max) { + throw new IllegalArgumentException( + String.format("min must be <= max (got min: %f, max: %f)", min, max)); + } + return consumeRegularDoubleUnchecked(min, max); + } @Override public native double consumeRegularDouble(); @@ -63,7 +215,14 @@ public class FuzzedDataProviderImpl implements FuzzedDataProvider { @Override public native char consumeChar(); - @Override public native char consumeChar(char min, char max); + @Override + public char consumeChar(char min, char max) { + if (min > max) { + throw new IllegalArgumentException( + String.format("min must be <= max (got min: %c, max: %c)", min, max)); + } + return consumeCharUnchecked(min, max); + } @Override public native char consumeCharNoSurrogates(); @@ -80,4 +239,12 @@ public class FuzzedDataProviderImpl implements FuzzedDataProvider { @Override public native byte[] consumeRemainingAsBytes(); @Override public native int remainingBytes(); + + private native byte consumeByteUnchecked(byte min, byte max); + private native short consumeShortUnchecked(short min, short max); + private native char consumeCharUnchecked(char min, char max); + private native int consumeIntUnchecked(int min, int max); + private native long consumeLongUnchecked(long min, long max); + private native float consumeRegularFloatUnchecked(float min, float max); + private native double consumeRegularDoubleUnchecked(double min, double max); } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/runtime/JazzerInternal.java b/agent/src/main/java/com/code_intelligence/jazzer/runtime/JazzerInternal.java index 8bc1b38c..79c851ad 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/runtime/JazzerInternal.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/runtime/JazzerInternal.java @@ -14,9 +14,12 @@ package com.code_intelligence.jazzer.runtime; +import java.util.ArrayList; + final public class JazzerInternal { - // Accessed from native code. - private static Throwable lastFinding; + private static final ArrayList<Runnable> ON_FUZZ_TARGET_READY_CALLBACKS = new ArrayList<>(); + + public static Throwable lastFinding; // Accessed from api.Jazzer via reflection. public static void reportFindingFromHook(Throwable finding) { @@ -26,4 +29,13 @@ final public class JazzerInternal { // target returns even if this Error is swallowed. throw new HardToCatchError(); } + + public static void registerOnFuzzTargetReadyCallback(Runnable callback) { + ON_FUZZ_TARGET_READY_CALLBACKS.add(callback); + } + + public static void onFuzzTargetReady(String fuzzTargetClass) { + ON_FUZZ_TARGET_READY_CALLBACKS.forEach(Runnable::run); + ON_FUZZ_TARGET_READY_CALLBACKS.clear(); + } } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/runtime/RecordingFuzzedDataProvider.java b/agent/src/main/java/com/code_intelligence/jazzer/runtime/RecordingFuzzedDataProvider.java index 976e024c..4eb80222 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/runtime/RecordingFuzzedDataProvider.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/runtime/RecordingFuzzedDataProvider.java @@ -18,49 +18,33 @@ import com.code_intelligence.jazzer.api.FuzzedDataProvider; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.Method; -import java.lang.reflect.Proxy; import java.util.ArrayList; import java.util.Base64; // Wraps the native FuzzedDataProviderImpl and serializes all its return values // into a Base64-encoded string. -final class RecordingFuzzedDataProvider implements InvocationHandler { - private final FuzzedDataProvider target = new FuzzedDataProviderImpl(); +public final class RecordingFuzzedDataProvider implements FuzzedDataProvider { + private final FuzzedDataProvider target; private final ArrayList<Object> recordedReplies = new ArrayList<>(); - private RecordingFuzzedDataProvider() {} + private RecordingFuzzedDataProvider(FuzzedDataProvider target) { + this.target = target; + } - // Called from native code. - public static FuzzedDataProvider makeFuzzedDataProviderProxy() { - return (FuzzedDataProvider) Proxy.newProxyInstance( - RecordingFuzzedDataProvider.class.getClassLoader(), new Class[] {FuzzedDataProvider.class}, - new RecordingFuzzedDataProvider()); + public static FuzzedDataProvider makeFuzzedDataProviderProxy(FuzzedDataProvider target) { + return new RecordingFuzzedDataProvider(target); } - // Called from native code. public static String serializeFuzzedDataProviderProxy(FuzzedDataProvider proxy) throws IOException { - return ((RecordingFuzzedDataProvider) Proxy.getInvocationHandler(proxy)).serialize(); + return ((RecordingFuzzedDataProvider) proxy).serialize(); } - private Object recordAndReturn(Object object) { + private <T> T recordAndReturn(T object) { recordedReplies.add(object); return object; } - @Override - public Object invoke(Object object, Method method, Object[] args) throws Throwable { - if (method.isDefault()) { - // Default methods in FuzzedDataProvider are implemented in Java and - // don't need to be recorded. - return method.invoke(target, args); - } else { - return recordAndReturn(method.invoke(target, args)); - } - } - private String serialize() throws IOException { byte[] rawOut; try (ByteArrayOutputStream byteStream = new ByteArrayOutputStream()) { @@ -71,4 +55,159 @@ final class RecordingFuzzedDataProvider implements InvocationHandler { } return Base64.getEncoder().encodeToString(rawOut); } + + @Override + public boolean consumeBoolean() { + return recordAndReturn(target.consumeBoolean()); + } + + @Override + public boolean[] consumeBooleans(int maxLength) { + return recordAndReturn(target.consumeBooleans(maxLength)); + } + + @Override + public byte consumeByte() { + return recordAndReturn(target.consumeByte()); + } + + @Override + public byte consumeByte(byte min, byte max) { + return recordAndReturn(target.consumeByte(min, max)); + } + + @Override + public byte[] consumeBytes(int maxLength) { + return recordAndReturn(target.consumeBytes(maxLength)); + } + + @Override + public byte[] consumeRemainingAsBytes() { + return recordAndReturn(target.consumeRemainingAsBytes()); + } + + @Override + public short consumeShort() { + return recordAndReturn(target.consumeShort()); + } + + @Override + public short consumeShort(short min, short max) { + return recordAndReturn(target.consumeShort(min, max)); + } + + @Override + public short[] consumeShorts(int maxLength) { + return recordAndReturn(target.consumeShorts(maxLength)); + } + + @Override + public int consumeInt() { + return recordAndReturn(target.consumeInt()); + } + + @Override + public int consumeInt(int min, int max) { + return recordAndReturn(target.consumeInt(min, max)); + } + + @Override + public int[] consumeInts(int maxLength) { + return recordAndReturn(target.consumeInts(maxLength)); + } + + @Override + public long consumeLong() { + return recordAndReturn(target.consumeLong()); + } + + @Override + public long consumeLong(long min, long max) { + return recordAndReturn(target.consumeLong(min, max)); + } + + @Override + public long[] consumeLongs(int maxLength) { + return recordAndReturn(target.consumeLongs(maxLength)); + } + + @Override + public float consumeFloat() { + return recordAndReturn(target.consumeFloat()); + } + + @Override + public float consumeRegularFloat() { + return recordAndReturn(target.consumeRegularFloat()); + } + + @Override + public float consumeRegularFloat(float min, float max) { + return recordAndReturn(target.consumeRegularFloat(min, max)); + } + + @Override + public float consumeProbabilityFloat() { + return recordAndReturn(target.consumeProbabilityFloat()); + } + + @Override + public double consumeDouble() { + return recordAndReturn(target.consumeDouble()); + } + + @Override + public double consumeRegularDouble() { + return recordAndReturn(target.consumeRegularDouble()); + } + + @Override + public double consumeRegularDouble(double min, double max) { + return recordAndReturn(target.consumeRegularDouble(min, max)); + } + + @Override + public double consumeProbabilityDouble() { + return recordAndReturn(target.consumeProbabilityDouble()); + } + + @Override + public char consumeChar() { + return recordAndReturn(target.consumeChar()); + } + + @Override + public char consumeChar(char min, char max) { + return recordAndReturn(target.consumeChar(min, max)); + } + + @Override + public char consumeCharNoSurrogates() { + return recordAndReturn(target.consumeCharNoSurrogates()); + } + + @Override + public String consumeString(int maxLength) { + return recordAndReturn(target.consumeString(maxLength)); + } + + @Override + public String consumeRemainingAsString() { + return recordAndReturn(target.consumeRemainingAsString()); + } + + @Override + public String consumeAsciiString(int maxLength) { + return recordAndReturn(target.consumeAsciiString(maxLength)); + } + + @Override + public String consumeRemainingAsAsciiString() { + return recordAndReturn(target.consumeRemainingAsAsciiString()); + } + + @Override + public int remainingBytes() { + return recordAndReturn(target.remainingBytes()); + } } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/runtime/SignalHandler.java b/agent/src/main/java/com/code_intelligence/jazzer/runtime/SignalHandler.java index 0a42aa94..49ee80c8 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/runtime/SignalHandler.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/runtime/SignalHandler.java @@ -14,13 +14,18 @@ package com.code_intelligence.jazzer.runtime; +import com.github.fmeum.rules_jni.RulesJni; import sun.misc.Signal; -@SuppressWarnings({"unused", "sunapi"}) -final class SignalHandler { - public static native void handleInterrupt(); - - public static void setupSignalHandlers() { +public final class SignalHandler { + static { + RulesJni.loadLibrary("jazzer_signal_handler", SignalHandler.class); Signal.handle(new Signal("INT"), sig -> handleInterrupt()); } + + public static void initialize() { + // Implicitly runs the static initializer. + } + + private static native void handleInterrupt(); } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/runtime/TraceCmpHooks.java b/agent/src/main/java/com/code_intelligence/jazzer/runtime/TraceCmpHooks.java index 352da8ea..37e8eaeb 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/runtime/TraceCmpHooks.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/runtime/TraceCmpHooks.java @@ -18,6 +18,7 @@ import com.code_intelligence.jazzer.api.HookType; import com.code_intelligence.jazzer.api.MethodHook; import java.lang.invoke.MethodHandle; import java.util.Arrays; +import java.util.ConcurrentModificationException; import java.util.Map; import java.util.TreeMap; @@ -80,6 +81,18 @@ final public class TraceCmpHooks { } } + @MethodHook(type = HookType.AFTER, targetClassName = "java.lang.Object", targetMethod = "equals") + @MethodHook( + type = HookType.AFTER, targetClassName = "java.lang.CharSequence", targetMethod = "equals") + @MethodHook(type = HookType.AFTER, targetClassName = "java.lang.Number", targetMethod = "equals") + public static void + genericEquals( + MethodHandle method, Object thisObject, Object[] arguments, int hookId, Boolean returnValue) { + if (!returnValue && arguments[0] != null && thisObject.getClass() == arguments[0].getClass()) { + TraceDataFlowNativeCallbacks.traceGenericCmp(thisObject, arguments[0], hookId); + } + } + @MethodHook( type = HookType.AFTER, targetClassName = "java.lang.String", targetMethod = "compareTo") @MethodHook(type = HookType.AFTER, targetClassName = "java.lang.String", @@ -193,9 +206,9 @@ final public class TraceCmpHooks { replace( MethodHandle method, Object thisObject, Object[] arguments, int hookId, String returnValue) { String original = (String) thisObject; - String target = arguments[0].toString(); // Report only if the replacement was not successful. if (original.equals(returnValue)) { + String target = arguments[0].toString(); TraceDataFlowNativeCallbacks.traceStrstr(original, target, hookId); } } @@ -205,11 +218,11 @@ final public class TraceCmpHooks { public static void arraysEquals( MethodHandle method, Object thisObject, Object[] arguments, int hookId, Boolean returnValue) { + if (returnValue) + return; byte[] first = (byte[]) arguments[0]; byte[] second = (byte[]) arguments[1]; - if (!returnValue) { - TraceDataFlowNativeCallbacks.traceMemcmp(first, second, 1, hookId); - } + TraceDataFlowNativeCallbacks.traceMemcmp(first, second, 1, hookId); } @MethodHook(type = HookType.AFTER, targetClassName = "java.util.Arrays", targetMethod = "equals", @@ -217,13 +230,13 @@ final public class TraceCmpHooks { public static void arraysEqualsRange( MethodHandle method, Object thisObject, Object[] arguments, int hookId, Boolean returnValue) { + if (returnValue) + return; byte[] first = Arrays.copyOfRange((byte[]) arguments[0], (int) arguments[1], (int) arguments[2]); byte[] second = Arrays.copyOfRange((byte[]) arguments[3], (int) arguments[4], (int) arguments[5]); - if (!returnValue) { - TraceDataFlowNativeCallbacks.traceMemcmp(first, second, 1, hookId); - } + TraceDataFlowNativeCallbacks.traceMemcmp(first, second, 1, hookId); } @MethodHook(type = HookType.AFTER, targetClassName = "java.util.Arrays", targetMethod = "compare", @@ -233,11 +246,11 @@ final public class TraceCmpHooks { public static void arraysCompare( MethodHandle method, Object thisObject, Object[] arguments, int hookId, Integer returnValue) { + if (returnValue == 0) + return; byte[] first = (byte[]) arguments[0]; byte[] second = (byte[]) arguments[1]; - if (returnValue != 0) { - TraceDataFlowNativeCallbacks.traceMemcmp(first, second, returnValue, hookId); - } + TraceDataFlowNativeCallbacks.traceMemcmp(first, second, returnValue, hookId); } @MethodHook(type = HookType.AFTER, targetClassName = "java.util.Arrays", targetMethod = "compare", @@ -247,34 +260,22 @@ final public class TraceCmpHooks { public static void arraysCompareRange( MethodHandle method, Object thisObject, Object[] arguments, int hookId, Integer returnValue) { + if (returnValue == 0) + return; byte[] first = Arrays.copyOfRange((byte[]) arguments[0], (int) arguments[1], (int) arguments[2]); byte[] second = Arrays.copyOfRange((byte[]) arguments[3], (int) arguments[4], (int) arguments[5]); - if (returnValue != 0) { - TraceDataFlowNativeCallbacks.traceMemcmp(first, second, returnValue, hookId); - } + TraceDataFlowNativeCallbacks.traceMemcmp(first, second, returnValue, hookId); } // The maximal number of elements of a non-TreeMap Map that will be sorted and searched for the // key closest to the current lookup key in the mapGet hook. private static final int MAX_NUM_KEYS_TO_ENUMERATE = 100; - @MethodHook(type = HookType.AFTER, targetClassName = "com.google.common.collect.ImmutableMap", - targetMethod = "get") - @MethodHook( - type = HookType.AFTER, targetClassName = "java.util.AbstractMap", targetMethod = "get") - @MethodHook(type = HookType.AFTER, targetClassName = "java.util.EnumMap", targetMethod = "get") - @MethodHook(type = HookType.AFTER, targetClassName = "java.util.HashMap", targetMethod = "get") - @MethodHook( - type = HookType.AFTER, targetClassName = "java.util.LinkedHashMap", targetMethod = "get") + @SuppressWarnings({"rawtypes", "unchecked"}) @MethodHook(type = HookType.AFTER, targetClassName = "java.util.Map", targetMethod = "get") - @MethodHook(type = HookType.AFTER, targetClassName = "java.util.SortedMap", targetMethod = "get") - @MethodHook(type = HookType.AFTER, targetClassName = "java.util.TreeMap", targetMethod = "get") - @MethodHook(type = HookType.AFTER, targetClassName = "java.util.concurrent.ConcurrentMap", - targetMethod = "get") - public static void - mapGet( + public static void mapGet( MethodHandle method, Object thisObject, Object[] arguments, int hookId, Object returnValue) { if (returnValue != null) return; @@ -291,31 +292,47 @@ final public class TraceCmpHooks { // https://github.com/llvm/llvm-project/blob/318942de229beb3b2587df09e776a50327b5cef0/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp#L564 Object lowerBoundKey = null; Object upperBoundKey = null; - if (map instanceof TreeMap) { - final TreeMap treeMap = (TreeMap) map; - lowerBoundKey = treeMap.floorKey(currentKey); - upperBoundKey = treeMap.ceilingKey(currentKey); - } else if (currentKey instanceof Comparable) { - final Comparable comparableKey = (Comparable) currentKey; - // Find two keys that bracket currentKey. - // Note: This is not deterministic if map.size() > MAX_NUM_KEYS_TO_ENUMERATE. - int enumeratedKeys = 0; - for (Object validKey : map.keySet()) { - if (validKey == null) - continue; - // If the key sorts lower than the non-existing key, but higher than the current lower - // bound, update the lower bound and vice versa for the upper bound. - if (comparableKey.compareTo(validKey) > 0 - && (lowerBoundKey == null || ((Comparable) validKey).compareTo(lowerBoundKey) > 0)) { - lowerBoundKey = validKey; + try { + if (map instanceof TreeMap) { + final TreeMap treeMap = (TreeMap) map; + try { + lowerBoundKey = treeMap.floorKey(currentKey); + upperBoundKey = treeMap.ceilingKey(currentKey); + } catch (ClassCastException ignored) { + // Can be thrown by floorKey and ceilingKey if currentKey is of a type that can't be + // compared to the maps keys. } - if (comparableKey.compareTo(validKey) < 0 - && (upperBoundKey == null || ((Comparable) validKey).compareTo(upperBoundKey) < 0)) { - upperBoundKey = validKey; + } else if (currentKey instanceof Comparable) { + final Comparable comparableCurrentKey = (Comparable) currentKey; + // Find two keys that bracket currentKey. + // Note: This is not deterministic if map.size() > MAX_NUM_KEYS_TO_ENUMERATE. + int enumeratedKeys = 0; + for (Object validKey : map.keySet()) { + if (!(validKey instanceof Comparable)) + continue; + final Comparable comparableValidKey = (Comparable) validKey; + // If the key sorts lower than the non-existing key, but higher than the current lower + // bound, update the lower bound and vice versa for the upper bound. + try { + if (comparableValidKey.compareTo(comparableCurrentKey) < 0 + && (lowerBoundKey == null || comparableValidKey.compareTo(lowerBoundKey) > 0)) { + lowerBoundKey = validKey; + } + if (comparableValidKey.compareTo(comparableCurrentKey) > 0 + && (upperBoundKey == null || comparableValidKey.compareTo(upperBoundKey) < 0)) { + upperBoundKey = validKey; + } + } catch (ClassCastException ignored) { + // Can be thrown by floorKey and ceilingKey if currentKey is of a type that can't be + // compared to the maps keys. + } + if (enumeratedKeys++ > MAX_NUM_KEYS_TO_ENUMERATE) + break; } - if (enumeratedKeys++ > MAX_NUM_KEYS_TO_ENUMERATE) - break; } + } catch (ConcurrentModificationException ignored) { + // map was modified by another thread, skip this invocation + return; } // Modify the hook ID so that compares against distinct valid keys are traced separately. if (lowerBoundKey != null) { diff --git a/agent/src/main/java/com/code_intelligence/jazzer/runtime/TraceDataFlowNativeCallbacks.java b/agent/src/main/java/com/code_intelligence/jazzer/runtime/TraceDataFlowNativeCallbacks.java index 456d0cb9..821ade0d 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/runtime/TraceDataFlowNativeCallbacks.java +++ b/agent/src/main/java/com/code_intelligence/jazzer/runtime/TraceDataFlowNativeCallbacks.java @@ -15,49 +15,32 @@ package com.code_intelligence.jazzer.runtime; import com.code_intelligence.jazzer.utils.Utils; +import com.github.fmeum.rules_jni.RulesJni; import java.lang.reflect.Executable; +import java.nio.charset.Charset; @SuppressWarnings("unused") final public class TraceDataFlowNativeCallbacks { - /* trace-cmp */ - // Calls: void __sanitizer_cov_trace_cmp4(uint32_t Arg1, uint32_t Arg2); - public static native void traceCmpInt(int arg1, int arg2, int pc); - - // Calls: void __sanitizer_cov_trace_const_cmp4(uint32_t Arg1, uint32_t Arg2); - public static native void traceConstCmpInt(int arg1, int arg2, int pc); - - // Calls: void __sanitizer_cov_trace_cmp4(uint32_t Arg1, uint32_t Arg2); - public static native void traceCmpLong(long arg1, long arg2, int pc); + static { + RulesJni.loadLibrary("jazzer_driver", "/com/code_intelligence/jazzer/driver"); + } - // Calls: void __sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases); - public static native void traceSwitch(long val, long[] cases, int pc); + // Note that we are not encoding as modified UTF-8 here: The FuzzedDataProvider transparently + // converts CESU8 into modified UTF-8 by coding null bytes on two bytes. Since the fuzzer is more + // likely to insert literal null bytes, having both the fuzzer input and the reported string + // comparisons be CESU8 should perform even better than the current implementation using modified + // UTF-8. + private static final Charset FUZZED_DATA_CHARSET = Charset.forName("CESU8"); - // Calls: void __sanitizer_weak_hook_memcmp(void *caller_pc, const void *b1, const void *b2, - // size_t n, int result); public static native void traceMemcmp(byte[] b1, byte[] b2, int result, int pc); - // Calls: void __sanitizer_weak_hook_strcmp(void *called_pc, const char *s1, const char *s2, int - // result); - public static native void traceStrcmp(String s1, String s2, int result, int pc); - - // Calls: void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1, const char *s2, char - // *result); - public static native void traceStrstr(String s1, String s2, int pc); - - /* trace-div */ - // Calls: void __sanitizer_cov_trace_div4(uint32_t Val); - public static native void traceDivInt(int val, int pc); - - // Calls: void __sanitizer_cov_trace_div8(uint64_t Val); - public static native void traceDivLong(long val, int pc); - - /* trace-gep */ - // Calls: void __sanitizer_cov_trace_gep(uintptr_t Idx); - public static native void traceGep(long val, int pc); + public static void traceStrcmp(String s1, String s2, int result, int pc) { + traceMemcmp(encodeForLibFuzzer(s1), encodeForLibFuzzer(s2), result, pc); + } - /* indirect-calls */ - // Calls: void __sanitizer_cov_trace_pc_indir(uintptr_t Callee); - private static native void tracePcIndir(int callee, int caller); + public static void traceStrstr(String s1, String s2, int pc) { + traceStrstr0(encodeForLibFuzzer(s2), pc); + } public static void traceReflectiveCall(Executable callee, int pc) { String className = callee.getDeclaringClass().getCanonicalName(); @@ -75,17 +58,45 @@ final public class TraceDataFlowNativeCallbacks { // The caller has to ensure that arg1 and arg2 have the same class. public static void traceGenericCmp(Object arg1, Object arg2, int pc) { - if (arg1 instanceof String) { - traceStrcmp((String) arg1, (String) arg2, 1, pc); - } else if (arg1 instanceof Integer || arg1 instanceof Short || arg1 instanceof Byte - || arg1 instanceof Character) { + if (arg1 instanceof CharSequence) { + traceStrcmp(arg1.toString(), arg2.toString(), 1, pc); + } else if (arg1 instanceof Integer) { traceCmpInt((int) arg1, (int) arg2, pc); } else if (arg1 instanceof Long) { traceCmpLong((long) arg1, (long) arg2, pc); + } else if (arg1 instanceof Short) { + traceCmpInt((short) arg1, (short) arg2, pc); + } else if (arg1 instanceof Byte) { + traceCmpInt((byte) arg1, (byte) arg2, pc); + } else if (arg1 instanceof Character) { + traceCmpInt((char) arg1, (char) arg2, pc); + } else if (arg1 instanceof Number) { + traceCmpLong(((Number) arg1).longValue(), ((Number) arg2).longValue(), pc); } else if (arg1 instanceof byte[]) { traceMemcmp((byte[]) arg1, (byte[]) arg2, 1, pc); } } + /* trace-cmp */ + public static native void traceCmpInt(int arg1, int arg2, int pc); + public static native void traceConstCmpInt(int arg1, int arg2, int pc); + public static native void traceCmpLong(long arg1, long arg2, int pc); + public static native void traceSwitch(long val, long[] cases, int pc); + /* trace-div */ + public static native void traceDivInt(int val, int pc); + public static native void traceDivLong(long val, int pc); + /* trace-gep */ + public static native void traceGep(long val, int pc); + /* indirect-calls */ + public static native void tracePcIndir(int callee, int caller); + public static native void handleLibraryLoad(); + + private static byte[] encodeForLibFuzzer(String str) { + // libFuzzer string hooks only ever consume the first 64 bytes, so we can definitely cut the + // string off after 64 characters. + return str.substring(0, Math.min(str.length(), 64)).getBytes(FUZZED_DATA_CHARSET); + } + + private static native void traceStrstr0(byte[] needle, int pc); } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/runtime/UnsafeProvider.java b/agent/src/main/java/com/code_intelligence/jazzer/runtime/UnsafeProvider.java new file mode 100644 index 00000000..81f2a208 --- /dev/null +++ b/agent/src/main/java/com/code_intelligence/jazzer/runtime/UnsafeProvider.java @@ -0,0 +1,50 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.runtime; + +import java.lang.reflect.Field; +import sun.misc.Unsafe; + +public final class UnsafeProvider { + private static final Unsafe UNSAFE = getUnsafeInternal(); + + public static Unsafe getUnsafe() { + return UNSAFE; + } + + private static Unsafe getUnsafeInternal() { + try { + // The Java agent is loaded by the bootstrap class loader and should thus + // pass the security checks in getUnsafe. + return Unsafe.getUnsafe(); + } catch (Throwable unused) { + // If not running as an agent, use the classical reflection trick to get an Unsafe instance, + // taking into account that the private field may have a name other than "theUnsafe": + // https://android.googlesource.com/platform/libcore/+/gingerbread/luni/src/main/java/sun/misc/Unsafe.java#32 + try { + for (Field f : Unsafe.class.getDeclaredFields()) { + if (f.getType() == Unsafe.class) { + f.setAccessible(true); + return (Unsafe) f.get(null); + } + } + return null; + } catch (Throwable t) { + t.printStackTrace(); + return null; + } + } + } +} diff --git a/agent/src/main/java/com/code_intelligence/jazzer/utils/BUILD.bazel b/agent/src/main/java/com/code_intelligence/jazzer/utils/BUILD.bazel index 5e301efc..10e3477c 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/utils/BUILD.bazel +++ b/agent/src/main/java/com/code_intelligence/jazzer/utils/BUILD.bazel @@ -4,7 +4,12 @@ kt_jvm_library( name = "utils", srcs = [ "ClassNameGlobber.kt", + "ExceptionUtils.kt", + "ManifestUtils.kt", "Utils.kt", ], visibility = ["//visibility:public"], + deps = [ + "//agent/src/main/java/com/code_intelligence/jazzer/api", + ], ) diff --git a/agent/src/main/java/com/code_intelligence/jazzer/utils/ClassNameGlobber.kt b/agent/src/main/java/com/code_intelligence/jazzer/utils/ClassNameGlobber.kt index 1f09afe3..44249c81 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/utils/ClassNameGlobber.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/utils/ClassNameGlobber.kt @@ -14,28 +14,41 @@ package com.code_intelligence.jazzer.utils -import java.lang.IllegalArgumentException - private val BASE_INCLUDED_CLASS_NAME_GLOBS = listOf( "**", // everything ) +// We use both a strong indicator for running as a Bazel test together with an indicator for a +// Bazel coverage run to rule out false positives. +private val IS_BAZEL_COVERAGE_RUN = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR") != null && + System.getenv("COVERAGE_DIR") != null + +private val ADDITIONAL_EXCLUDED_NAME_GLOBS_FOR_BAZEL_COVERAGE = listOf( + "com.google.testing.coverage.**", + "org.jacoco.**", +) + private val BASE_EXCLUDED_CLASS_NAME_GLOBS = listOf( + // JDK internals "\\[**", // array types - "com.code_intelligence.jazzer.**", - "com.sun.**", // package for Proxy objects "java.**", "javax.**", - "jaz.Ter", // safe companion of the honeypot class used by sanitizers - "jaz.Zer", // honeypot class used by sanitizers "jdk.**", - "kotlin.**", "sun.**", -) + "com.sun.**", // package for Proxy objects + // Azul JDK internals + "com.azul.tooling.**", + // Kotlin internals + "kotlin.**", + // Jazzer internals + "com.code_intelligence.jazzer.**", + "jaz.Ter", // safe companion of the honeypot class used by sanitizers + "jaz.Zer", // honeypot class used by sanitizers +) + if (IS_BAZEL_COVERAGE_RUN) ADDITIONAL_EXCLUDED_NAME_GLOBS_FOR_BAZEL_COVERAGE else listOf() class ClassNameGlobber(includes: List<String>, excludes: List<String>) { // If no include globs are provided, start with all classes. - private val includeMatchers = (if (includes.isEmpty()) BASE_INCLUDED_CLASS_NAME_GLOBS else includes) + private val includeMatchers = includes.ifEmpty { BASE_INCLUDED_CLASS_NAME_GLOBS } .map(::SimpleGlobMatcher) // If no include globs are provided, additionally exclude stdlib classes as well as our own classes. diff --git a/agent/src/main/java/com/code_intelligence/jazzer/runtime/ExceptionUtils.kt b/agent/src/main/java/com/code_intelligence/jazzer/utils/ExceptionUtils.kt index 31a61740..30f6fb30 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/runtime/ExceptionUtils.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/utils/ExceptionUtils.kt @@ -14,7 +14,7 @@ @file:JvmName("ExceptionUtils") -package com.code_intelligence.jazzer.runtime +package com.code_intelligence.jazzer.utils import com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow import java.lang.management.ManagementFactory @@ -163,4 +163,10 @@ fun dumpAllStackTraces() { } System.err.println() } + System.err.println("Garbage collector stats:") + System.err.println( + ManagementFactory.getGarbageCollectorMXBeans().joinToString("\n", "\n", "\n") { + "${it.name}: ${it.collectionCount} collections took ${it.collectionTime}ms" + } + ) } diff --git a/agent/src/main/java/com/code_intelligence/jazzer/runtime/ManifestUtils.kt b/agent/src/main/java/com/code_intelligence/jazzer/utils/ManifestUtils.kt index d88c3e18..e7165e55 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/runtime/ManifestUtils.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/utils/ManifestUtils.kt @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package com.code_intelligence.jazzer.runtime +package com.code_intelligence.jazzer.utils import java.util.jar.Manifest object ManifestUtils { - const val FUZZ_TARGET_CLASS = "Jazzer-Fuzz-Target-Class" + private const val FUZZ_TARGET_CLASS = "Jazzer-Fuzz-Target-Class" const val HOOK_CLASSES = "Jazzer-Hook-Classes" fun combineManifestValues(attribute: String): List<String> { diff --git a/agent/src/main/java/com/code_intelligence/jazzer/utils/Utils.kt b/agent/src/main/java/com/code_intelligence/jazzer/utils/Utils.kt index af8cce9b..1b399baf 100644 --- a/agent/src/main/java/com/code_intelligence/jazzer/utils/Utils.kt +++ b/agent/src/main/java/com/code_intelligence/jazzer/utils/Utils.kt @@ -17,6 +17,8 @@ package com.code_intelligence.jazzer.utils import java.lang.reflect.Executable import java.lang.reflect.Method +import java.nio.ByteBuffer +import java.nio.channels.FileChannel val Class<*>.descriptor: String get() = when { @@ -80,3 +82,26 @@ fun simpleFastHash(vararg strings: String): Int { } return hash } + +/** + * Reads the [FileChannel] to the end as a UTF-8 string. + */ +fun FileChannel.readFully(): String { + check(size() <= Int.MAX_VALUE) + val buffer = ByteBuffer.allocate(size().toInt()) + while (buffer.hasRemaining()) { + when (read(buffer)) { + 0 -> throw IllegalStateException("No bytes read") + -1 -> break + } + } + return String(buffer.array()) +} + +/** + * Appends [string] to the end of the [FileChannel]. + */ +fun FileChannel.append(string: String) { + position(size()) + write(ByteBuffer.wrap(string.toByteArray())) +} |