diff options
Diffstat (limited to 'agent/src/main/java/com/code_intelligence/jazzer/agent')
4 files changed, 0 insertions, 568 deletions
diff --git a/agent/src/main/java/com/code_intelligence/jazzer/agent/Agent.kt b/agent/src/main/java/com/code_intelligence/jazzer/agent/Agent.kt deleted file mode 100644 index f9b026f1..00000000 --- a/agent/src/main/java/com/code_intelligence/jazzer/agent/Agent.kt +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2021 Code Intelligence GmbH -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -@file:JvmName("Agent") - -package com.code_intelligence.jazzer.agent - -import com.code_intelligence.jazzer.driver.Opt -import com.code_intelligence.jazzer.instrumentor.CoverageRecorder -import com.code_intelligence.jazzer.instrumentor.Hooks -import com.code_intelligence.jazzer.instrumentor.InstrumentationType -import com.code_intelligence.jazzer.runtime.NativeLibHooks -import com.code_intelligence.jazzer.runtime.TraceCmpHooks -import com.code_intelligence.jazzer.runtime.TraceDivHooks -import com.code_intelligence.jazzer.runtime.TraceIndirHooks -import com.code_intelligence.jazzer.utils.ClassNameGlobber -import com.code_intelligence.jazzer.utils.ManifestUtils -import java.io.File -import java.lang.instrument.Instrumentation -import java.net.URI -import java.nio.file.Paths -import java.util.jar.JarFile -import kotlin.io.path.ExperimentalPathApi -import kotlin.io.path.exists -import kotlin.io.path.isDirectory - -private object AgentJarFinder { - val agentJarFile = jarUriForClass(AgentJarFinder::class.java)?.let { JarFile(File(it)) } -} - -fun jarUriForClass(clazz: Class<*>): URI? { - return clazz.protectionDomain?.codeSource?.location?.toURI() -} - -@OptIn(ExperimentalPathApi::class) -@Suppress("UNUSED_PARAMETER") -fun premain(agentArgs: String?, instrumentation: Instrumentation) { - // Add the agent jar (i.e., the jar out of which we are currently executing) to the search path of the bootstrap - // class loader to ensure that instrumented classes can find the CoverageMap class regardless of which ClassLoader - // they are using. - if (AgentJarFinder.agentJarFile != null) { - instrumentation.appendToBootstrapClassLoaderSearch(AgentJarFinder.agentJarFile) - } else { - println("WARN: Failed to add agent JAR to bootstrap class loader search path") - } - - val manifestCustomHookNames = - ManifestUtils.combineManifestValues(ManifestUtils.HOOK_CLASSES).flatMap { - it.split(':') - }.filter { it.isNotBlank() } - val allCustomHookNames = (manifestCustomHookNames + Opt.customHooks).toSet() - val disabledCustomHookNames = Opt.disabledHooks.toSet() - val customHookNames = allCustomHookNames - disabledCustomHookNames - val disabledCustomHooksToPrint = allCustomHookNames - customHookNames.toSet() - if (disabledCustomHooksToPrint.isNotEmpty()) { - println("INFO: Not using the following disabled hooks: ${disabledCustomHooksToPrint.joinToString(", ")}") - } - - val classNameGlobber = ClassNameGlobber(Opt.instrumentationIncludes, Opt.instrumentationExcludes + customHookNames) - CoverageRecorder.classNameGlobber = classNameGlobber - val customHookClassNameGlobber = ClassNameGlobber(Opt.customHookIncludes, Opt.customHookExcludes + customHookNames) - // FIXME: Setting trace to the empty string explicitly results in all rather than no trace types - // being applied - this is unintuitive. - val instrumentationTypes = (Opt.trace.takeIf { it.isNotEmpty() } ?: listOf("all")).flatMap { - when (it) { - "cmp" -> setOf(InstrumentationType.CMP) - "cov" -> setOf(InstrumentationType.COV) - "div" -> setOf(InstrumentationType.DIV) - "gep" -> setOf(InstrumentationType.GEP) - "indir" -> setOf(InstrumentationType.INDIR) - "native" -> setOf(InstrumentationType.NATIVE) - // Disable GEP instrumentation by default as it appears to negatively affect fuzzing - // performance. Our current GEP instrumentation only reports constant indices, but even - // when we instead reported non-constant indices, they tended to completely fill up the - // table of recent compares and value profile map. - "all" -> InstrumentationType.values().toSet() - InstrumentationType.GEP - else -> { - println("WARN: Skipping unknown instrumentation type $it") - emptySet() - } - } - }.toSet() - val idSyncFile = Opt.idSyncFile.takeUnless { it.isEmpty() }?.let { - Paths.get(it).also { path -> - println("INFO: Synchronizing coverage IDs in ${path.toAbsolutePath()}") - } - } - val dumpClassesDir = Opt.dumpClassesDir.takeUnless { it.isEmpty() }?.let { - Paths.get(it).toAbsolutePath().also { path -> - if (path.exists() && path.isDirectory()) { - println("INFO: Dumping instrumented classes into $path") - } else { - println("ERROR: Cannot dump instrumented classes into $path; does not exist or not a directory") - } - } - } - val includedHookNames = instrumentationTypes - .mapNotNull { type -> - when (type) { - InstrumentationType.CMP -> TraceCmpHooks::class.java.name - InstrumentationType.DIV -> TraceDivHooks::class.java.name - InstrumentationType.INDIR -> TraceIndirHooks::class.java.name - InstrumentationType.NATIVE -> NativeLibHooks::class.java.name - else -> null - } - } - val coverageIdSynchronizer = if (idSyncFile != null) - FileSyncCoverageIdStrategy(idSyncFile) - else - MemSyncCoverageIdStrategy() - - val (includedHooks, customHooks) = Hooks.loadHooks(includedHookNames.toSet(), customHookNames.toSet()) - // If we don't append the JARs containing the custom hooks to the bootstrap class loader, - // third-party hooks not contained in the agent JAR will not be able to instrument Java standard - // library classes. These classes are loaded by the bootstrap / system class loader and would - // not be considered when resolving references to hook methods, leading to NoClassDefFoundError - // being thrown. - customHooks.hookClasses - .mapNotNull { jarUriForClass(it) } - .toSet() - .map { JarFile(File(it)) } - .forEach { instrumentation.appendToBootstrapClassLoaderSearch(it) } - - val runtimeInstrumentor = RuntimeInstrumentor( - instrumentation, - classNameGlobber, - customHookClassNameGlobber, - instrumentationTypes, - includedHooks.hooks, - customHooks.hooks, - customHooks.additionalHookClassNameGlobber, - coverageIdSynchronizer, - dumpClassesDir, - ) - - // These classes are e.g. dependencies of the RuntimeInstrumentor or hooks and thus were loaded - // before the instrumentor was ready. Since we haven't enabled it yet, they can safely be - // "retransformed": They haven't been transformed yet. - val classesToRetransform = instrumentation.allLoadedClasses - .filter { - classNameGlobber.includes(it.name) || - customHookClassNameGlobber.includes(it.name) || - customHooks.additionalHookClassNameGlobber.includes(it.name) - } - .filter { - instrumentation.isModifiableClass(it) - } - .toTypedArray() - - instrumentation.addTransformer(runtimeInstrumentor, true) - - if (classesToRetransform.isNotEmpty()) { - if (instrumentation.isRetransformClassesSupported) { - instrumentation.retransformClasses(*classesToRetransform) - } else { - println("WARN: Instrumentation was not applied to the following classes as they are dependencies of hooks:") - println("WARN: ${classesToRetransform.joinToString()}") - } - } -} diff --git a/agent/src/main/java/com/code_intelligence/jazzer/agent/BUILD.bazel b/agent/src/main/java/com/code_intelligence/jazzer/agent/BUILD.bazel deleted file mode 100644 index db6ae264..00000000 --- a/agent/src/main/java/com/code_intelligence/jazzer/agent/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") - -kt_jvm_library( - name = "agent_lib", - srcs = [ - "Agent.kt", - "CoverageIdStrategy.kt", - "RuntimeInstrumentor.kt", - ], - visibility = ["//visibility:public"], - deps = [ - "//agent/src/main/java/com/code_intelligence/jazzer/instrumentor", - "//agent/src/main/java/com/code_intelligence/jazzer/runtime", - "//driver/src/main/java/com/code_intelligence/jazzer/driver:opt", - ], -) diff --git a/agent/src/main/java/com/code_intelligence/jazzer/agent/CoverageIdStrategy.kt b/agent/src/main/java/com/code_intelligence/jazzer/agent/CoverageIdStrategy.kt deleted file mode 100644 index 5d1d28e3..00000000 --- a/agent/src/main/java/com/code_intelligence/jazzer/agent/CoverageIdStrategy.kt +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2021 Code Intelligence GmbH -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.code_intelligence.jazzer.agent - -import com.code_intelligence.jazzer.utils.append -import com.code_intelligence.jazzer.utils.readFully -import java.nio.channels.FileChannel -import java.nio.channels.FileLock -import java.nio.file.Path -import java.nio.file.StandardOpenOption -import java.util.UUID - -/** - * Indicates a fatal failure to generate synchronized coverage IDs. - */ -class CoverageIdException(cause: Throwable? = null) : - RuntimeException("Failed to synchronize coverage IDs", cause) - -/** - * [CoverageIdStrategy] provides an abstraction to switch between context specific coverage ID generation. - * - * Coverage (i.e., edge) IDs differ from other kinds of IDs, such as those generated for call sites or cmp - * instructions, in that they should be consecutive, collision-free, and lie in a known, small range. - * This precludes us from generating them simply as hashes of class names. - */ -interface CoverageIdStrategy { - - /** - * [withIdForClass] provides the initial coverage ID of the given [className] as parameter to the - * [block] to execute. [block] has to return the number of additionally used IDs. - */ - @Throws(CoverageIdException::class) - fun withIdForClass(className: String, block: (Int) -> Int) -} - -/** - * A memory synced strategy for coverage ID generation. - * - * This strategy uses a synchronized block to guard access to a global edge ID counter. - * Even though concurrent fuzzing is not fully supported this strategy enables consistent coverage - * IDs in case of concurrent class loading. - * - * It only prevents races within one VM instance. - */ -class MemSyncCoverageIdStrategy : CoverageIdStrategy { - private var nextEdgeId = 0 - - @Synchronized - override fun withIdForClass(className: String, block: (Int) -> Int) { - nextEdgeId += block(nextEdgeId) - } -} - -/** - * A strategy for coverage ID generation that synchronizes the IDs assigned to a class with other processes via the - * specified [idSyncFile]. - * This class takes care of synchronizing the access to the file between multiple processes as long as the general - * contract of [CoverageIdStrategy] is followed. - */ -class FileSyncCoverageIdStrategy(private val idSyncFile: Path) : CoverageIdStrategy { - private val uuid: UUID = UUID.randomUUID() - private var idFileLock: FileLock? = null - - private var cachedFirstId: Int? = null - private var cachedClassName: String? = null - private var cachedIdCount: Int? = null - - /** - * This method is synchronized to prevent concurrent access to the internal file lock which would result in - * [java.nio.channels.OverlappingFileLockException]. Furthermore, every coverage ID obtained by [obtainFirstId] - * is always committed back again to the sync file by [commitIdCount]. - */ - @Synchronized - override fun withIdForClass(className: String, block: (Int) -> Int) { - var actualNumEdgeIds = 0 - try { - val firstId = obtainFirstId(className) - actualNumEdgeIds = block(firstId) - } finally { - commitIdCount(actualNumEdgeIds) - } - } - - /** - * Obtains a coverage ID for [className] such that all cooperating agent processes will obtain the same ID. - * There are two cases to consider: - * - This agent process is the first to encounter [className], i.e., it does not find a record for that class in - * [idSyncFile]. In this case, a lock on the file is held until the class has been instrumented and a record with - * the required number of coverage IDs has been added. - * - Another agent process has already encountered [className], i.e., there is a record that class in [idSyncFile]. - * In this case, the lock on the file is returned immediately and the extracted first coverage ID is returned to - * the caller. The caller is still expected to call [commitIdCount] so that desynchronization can be detected. - */ - private fun obtainFirstId(className: String): Int { - try { - check(idFileLock == null) { "Already holding a lock on the ID file" } - val localIdFile = FileChannel.open( - idSyncFile, - StandardOpenOption.WRITE, - StandardOpenOption.READ - ) - // Wait until we have obtained the lock on the sync file. We hold the lock from this point until we have - // finished reading and writing (if necessary) to the file. - val localIdFileLock = localIdFile.lock() - check(localIdFileLock.isValid && !localIdFileLock.isShared) - // Parse the sync file, which consists of lines of the form - // <class name>:<first ID>:<num IDs> - val idInfo = localIdFileLock.channel().readFully() - .lineSequence() - .filterNot { it.isBlank() } - .map { line -> - val parts = line.split(':') - check(parts.size == 4) { - "Expected ID file line to be of the form '<class name>:<first ID>:<num IDs>:<uuid>', got '$line'" - } - val lineClassName = parts[0] - val lineFirstId = parts[1].toInt() - check(lineFirstId >= 0) { "Negative first ID in line: $line" } - val lineIdCount = parts[2].toInt() - check(lineIdCount >= 0) { "Negative ID count in line: $line" } - Triple(lineClassName, lineFirstId, lineIdCount) - }.toList() - cachedClassName = className - val idInfoForClass = idInfo.filter { it.first == className } - return when (idInfoForClass.size) { - 0 -> { - // We are the first to encounter this class and thus need to hold the lock until the class has been - // instrumented and we know the required number of coverage IDs. - idFileLock = localIdFileLock - // Compute the next free ID as the maximum over the sums of first ID and ID count, starting at 0 if - // this is the first ID to be assigned. In fact, since this is the only way new lines are added to - // the file, the maximum is always attained by the last line. - val nextFreeId = idInfo.asSequence().map { it.second + it.third }.lastOrNull() ?: 0 - cachedFirstId = nextFreeId - nextFreeId - } - 1 -> { - // This class has already been instrumented elsewhere, so we just return the first ID and ID count - // reported from there and release the lock right away. The caller is still expected to call - // commitIdCount. - localIdFile.close() - cachedIdCount = idInfoForClass.single().third - idInfoForClass.single().second - } - else -> { - localIdFile.close() - System.err.println(idInfo.joinToString("\n") { "${it.first}:${it.second}:${it.third}" }) - throw IllegalStateException("Multiple entries for $className in ID file") - } - } - } catch (e: Exception) { - throw CoverageIdException(e) - } - } - - /** - * Records the number of coverage IDs used to instrument the class specified in a previous call to [obtainFirstId]. - * If instrumenting the class should fail, this function must still be called. In this case, [idCount] is set to 0. - */ - private fun commitIdCount(idCount: Int) { - val localIdFileLock = idFileLock - try { - check(cachedClassName != null) - if (localIdFileLock == null) { - // We released the lock already in obtainFirstId since the class had already been instrumented - // elsewhere. As we know the expected number of IDs for the current class in this case, check for - // deviations. - check(cachedIdCount != null) - check(idCount == cachedIdCount) { - "$cachedClassName has $idCount edges, but $cachedIdCount edges reserved in ID file" - } - } else { - // We are the first to instrument this class and should record the number of IDs in the sync file. - check(cachedFirstId != null) - localIdFileLock.channel().append("$cachedClassName:$cachedFirstId:$idCount:$uuid\n") - localIdFileLock.channel().force(true) - } - idFileLock = null - cachedFirstId = null - cachedIdCount = null - cachedClassName = null - } catch (e: Exception) { - throw CoverageIdException(e) - } finally { - localIdFileLock?.channel()?.close() - } - } -} diff --git a/agent/src/main/java/com/code_intelligence/jazzer/agent/RuntimeInstrumentor.kt b/agent/src/main/java/com/code_intelligence/jazzer/agent/RuntimeInstrumentor.kt deleted file mode 100644 index fe2efd54..00000000 --- a/agent/src/main/java/com/code_intelligence/jazzer/agent/RuntimeInstrumentor.kt +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2021 Code Intelligence GmbH -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.code_intelligence.jazzer.agent - -import com.code_intelligence.jazzer.instrumentor.ClassInstrumentor -import com.code_intelligence.jazzer.instrumentor.CoverageRecorder -import com.code_intelligence.jazzer.instrumentor.Hook -import com.code_intelligence.jazzer.instrumentor.InstrumentationType -import com.code_intelligence.jazzer.utils.ClassNameGlobber -import java.lang.instrument.ClassFileTransformer -import java.lang.instrument.Instrumentation -import java.nio.file.Path -import java.security.ProtectionDomain -import kotlin.math.roundToInt -import kotlin.system.exitProcess -import kotlin.time.measureTimedValue - -class RuntimeInstrumentor( - private val instrumentation: Instrumentation, - private val classesToFullyInstrument: ClassNameGlobber, - private val classesToHookInstrument: ClassNameGlobber, - private val instrumentationTypes: Set<InstrumentationType>, - private val includedHooks: List<Hook>, - private val customHooks: List<Hook>, - // Dedicated name globber for additional classes to hook stated in hook annotations is needed due to - // existing include and exclude pattern of classesToHookInstrument. All classes are included in hook - // instrumentation except the ones from default excludes, like JDK and Kotlin classes. But additional - // classes to hook, based on annotations, are allowed to reference normally ignored ones, like JDK - // and Kotlin internals. - // FIXME: Adding an additional class to hook will apply _all_ hooks to it and not only the one it's - // defined in. At some point we might want to track the list of classes per custom hook rather than globally. - private val additionalClassesToHookInstrument: ClassNameGlobber, - private val coverageIdSynchronizer: CoverageIdStrategy, - private val dumpClassesDir: Path?, -) : ClassFileTransformer { - - @OptIn(kotlin.time.ExperimentalTime::class) - override fun transform( - loader: ClassLoader?, - internalClassName: String, - classBeingRedefined: Class<*>?, - protectionDomain: ProtectionDomain?, - classfileBuffer: ByteArray, - ): ByteArray? { - return try { - // Bail out early if we would instrument ourselves. This prevents ClassCircularityErrors as we might need to - // load additional Jazzer classes until we reach the full exclusion logic. - if (internalClassName.startsWith("com/code_intelligence/jazzer/")) - return null - transformInternal(internalClassName, classfileBuffer) - } catch (t: Throwable) { - // Throwables raised from transform are silently dropped, making it extremely hard to detect instrumentation - // failures. The docs advise to use a top-level try-catch. - // https://docs.oracle.com/javase/9/docs/api/java/lang/instrument/ClassFileTransformer.html - t.printStackTrace() - throw t - }.also { instrumentedByteCode -> - // Only dump classes that were instrumented. - if (instrumentedByteCode != null && dumpClassesDir != null) { - dumpToClassFile(internalClassName, instrumentedByteCode) - dumpToClassFile(internalClassName, classfileBuffer, basenameSuffix = ".original") - } - } - } - - private fun dumpToClassFile(internalClassName: String, bytecode: ByteArray, basenameSuffix: String = "") { - val relativePath = "$internalClassName$basenameSuffix.class" - val absolutePath = dumpClassesDir!!.resolve(relativePath) - val dumpFile = absolutePath.toFile() - dumpFile.parentFile.mkdirs() - dumpFile.writeBytes(bytecode) - } - - override fun transform( - module: Module?, - loader: ClassLoader?, - internalClassName: String, - classBeingRedefined: Class<*>?, - protectionDomain: ProtectionDomain?, - classfileBuffer: ByteArray - ): ByteArray? { - return try { - if (module != null && !module.canRead(RuntimeInstrumentor::class.java.module)) { - // Make all other modules read our (unnamed) module, which allows them to access the classes needed by the - // instrumentations, e.g. CoverageMap. If a module can't be modified, it should not be instrumented as the - // injected bytecode might throw NoClassDefFoundError. - // https://mail.openjdk.java.net/pipermail/jigsaw-dev/2021-May/014663.html - if (!instrumentation.isModifiableModule(module)) { - val prettyClassName = internalClassName.replace('/', '.') - println("WARN: Failed to instrument $prettyClassName in unmodifiable module ${module.name}, skipping") - return null - } - instrumentation.redefineModule( - module, - /* extraReads */ setOf(RuntimeInstrumentor::class.java.module), - emptyMap(), - emptyMap(), - emptySet(), - emptyMap() - ) - } - transform(loader, internalClassName, classBeingRedefined, protectionDomain, classfileBuffer) - } catch (t: Throwable) { - // Throwables raised from transform are silently dropped, making it extremely hard to detect instrumentation - // failures. The docs advise to use a top-level try-catch. - // https://docs.oracle.com/javase/9/docs/api/java/lang/instrument/ClassFileTransformer.html - t.printStackTrace() - throw t - } - } - - @OptIn(kotlin.time.ExperimentalTime::class) - fun transformInternal(internalClassName: String, classfileBuffer: ByteArray): ByteArray? { - val fullInstrumentation = when { - classesToFullyInstrument.includes(internalClassName) -> true - classesToHookInstrument.includes(internalClassName) -> false - additionalClassesToHookInstrument.includes(internalClassName) -> false - else -> return null - } - val prettyClassName = internalClassName.replace('/', '.') - val (instrumentedBytecode, duration) = measureTimedValue { - try { - instrument(internalClassName, classfileBuffer, fullInstrumentation) - } catch (e: CoverageIdException) { - System.err.println("ERROR: Coverage IDs are out of sync") - e.printStackTrace() - exitProcess(1) - } catch (e: Exception) { - println("WARN: Failed to instrument $prettyClassName, skipping") - e.printStackTrace() - return null - } - } - val durationInMs = duration.inWholeMilliseconds - val sizeIncrease = ((100.0 * (instrumentedBytecode.size - classfileBuffer.size)) / classfileBuffer.size).roundToInt() - if (fullInstrumentation) { - println("INFO: Instrumented $prettyClassName (took $durationInMs ms, size +$sizeIncrease%)") - } else { - println("INFO: Instrumented $prettyClassName with custom hooks only (took $durationInMs ms, size +$sizeIncrease%)") - } - return instrumentedBytecode - } - - private fun instrument(internalClassName: String, bytecode: ByteArray, fullInstrumentation: Boolean): ByteArray { - return ClassInstrumentor(bytecode).run { - if (fullInstrumentation) { - // Hook instrumentation must be performed after data flow tracing as the injected - // bytecode would trigger the GEP callbacks for byte[]. Coverage instrumentation - // must be performed after hook instrumentation as the injected bytecode would - // trigger the GEP callbacks for ByteBuffer. - traceDataFlow(instrumentationTypes) - hooks(includedHooks + customHooks) - coverageIdSynchronizer.withIdForClass(internalClassName) { firstId -> - coverage(firstId).also { actualNumEdgeIds -> - CoverageRecorder.recordInstrumentedClass( - internalClassName, - bytecode, - firstId, - actualNumEdgeIds - ) - } - } - } else { - hooks(customHooks) - } - instrumentedBytecode - } - } -} |