1 files changed, 178 insertions, 0 deletions
diff --git a/agent/src/jmh/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationBenchmark.java b/agent/src/jmh/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationBenchmark.java
new file mode 100644
index 00000000..f388c4cc
--- /dev/null
+++ b/agent/src/jmh/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationBenchmark.java
@@ -0,0 +1,178 @@
+// Copyright 2022 Code Intelligence GmbH
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.code_intelligence.jazzer.instrumentor;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+
+/**
+ * This benchmark compares the throughput of a typical fuzz target when instrumented with different
+ * edge coverage instrumentation strategies and coverage map implementations.
+ *
+ * The benchmark currently uses the OWASP json-sanitizer as its target, which has the following
+ * desirable properties for a benchmark:
+ * - It is a reasonably sized project that does not consist of many different classes.
+ * - It is very heavy on computation with a high density of branching.
+ * - It is entirely CPU bound with no IO and does not call expensive methods from the standard
+ *   library.
+ * With these properties, results obtained from this benchmark should provide reasonable lower
+ * bounds on the relative slowdown introduced by the various approaches to instrumentations.
+ */
+@State(Scope.Benchmark)
+public class CoverageInstrumentationBenchmark {
+  private static final String TARGET_CLASSNAME = "com.google.json.JsonSanitizer";
+  private static final String TARGET_PACKAGE =
+      TARGET_CLASSNAME.substring(0, TARGET_CLASSNAME.lastIndexOf('.'));
+  private static final String TARGET_METHOD = "sanitize";
+  private static final MethodType TARGET_TYPE = MethodType.methodType(String.class, String.class);
+
+  // This is part of the benchmark's state and not a constant to prevent constant folding.
+  String TARGET_ARG =
+      "{\"foo\":1123987,\"bar\":[true, false],\"baz\":{\"foo\":\"132ä3\",\"bar\":1.123e-005}}";
+
+  MethodHandle uninstrumented_sanitize;
+  MethodHandle local_DirectByteBuffer_NeverZero_sanitize;
+  MethodHandle staticMethod_DirectByteBuffer_NeverZero_sanitize;
+  MethodHandle staticMethod_DirectByteBuffer2_NeverZero_sanitize;
+  MethodHandle staticMethod_Unsafe_NeverZero_sanitize;
+  MethodHandle staticMethod_Unsafe_NeverZero2_sanitize;
+  MethodHandle staticMethod_Unsafe_NeverZeroBranchfree_sanitize;
+  MethodHandle staticMethod_Unsafe_SimpleIncrement_sanitize;
+
+  public static MethodHandle handleForTargetMethod(ClassLoader classLoader)
+      throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException {
+    Class<?> targetClass = classLoader.loadClass(TARGET_CLASSNAME);
+    return MethodHandles.lookup().findStatic(targetClass, TARGET_METHOD, TARGET_TYPE);
+  }
+
+  public static MethodHandle instrumentWithStrategy(
+      EdgeCoverageStrategy strategy, Class<?> coverageMapClass)
+      throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException {
+    if (strategy == null) {
+      // Do not instrument the code by using the benchmark class' ClassLoader.
+      return handleForTargetMethod(CoverageInstrumentationBenchmark.class.getClassLoader());
+    }
+    // It's fine to reuse a single instrumentor here as we don't want to know which class received
+    // how many counters.
+    Instrumentor instrumentor = new EdgeCoverageInstrumentor(strategy, coverageMapClass, 0);
+    return handleForTargetMethod(new InstrumentingClassLoader(instrumentor, TARGET_PACKAGE));
+  }
+
+  @Setup
+  public void instrumentWithStrategies()
+      throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException {
+    uninstrumented_sanitize = instrumentWithStrategy(null, null);
+    local_DirectByteBuffer_NeverZero_sanitize = instrumentWithStrategy(
+        DirectByteBufferStrategy.INSTANCE, DirectByteBufferCoverageMap.class);
+    staticMethod_DirectByteBuffer_NeverZero_sanitize =
+        instrumentWithStrategy(new StaticMethodStrategy(), DirectByteBufferCoverageMap.class);
+    staticMethod_DirectByteBuffer2_NeverZero_sanitize =
+        instrumentWithStrategy(new StaticMethodStrategy(), DirectByteBuffer2CoverageMap.class);
+    staticMethod_Unsafe_NeverZero_sanitize =
+        instrumentWithStrategy(new StaticMethodStrategy(), UnsafeCoverageMap.class);
+    staticMethod_Unsafe_NeverZero2_sanitize =
+        instrumentWithStrategy(new StaticMethodStrategy(), Unsafe2CoverageMap.class);
+    staticMethod_Unsafe_SimpleIncrement_sanitize =
+        instrumentWithStrategy(new StaticMethodStrategy(), UnsafeSimpleIncrementCoverageMap.class);
+    staticMethod_Unsafe_NeverZeroBranchfree_sanitize =
+        instrumentWithStrategy(new StaticMethodStrategy(), UnsafeBranchfreeCoverageMap.class);
+  }
+
+  @Benchmark
+  public String uninstrumented() throws Throwable {
+    return (String) uninstrumented_sanitize.invokeExact(TARGET_ARG);
+  }
+
+  @Benchmark
+  public String local_DirectByteBuffer_NeverZero() throws Throwable {
+    return (String) local_DirectByteBuffer_NeverZero_sanitize.invokeExact(TARGET_ARG);
+  }
+
+  @Benchmark
+  public String staticMethod_DirectByteBuffer_NeverZero() throws Throwable {
+    return (String) staticMethod_DirectByteBuffer_NeverZero_sanitize.invokeExact(TARGET_ARG);
+  }
+
+  @Benchmark
+  public String staticMethod_DirectByteBuffer2_NeverZero() throws Throwable {
+    return (String) staticMethod_DirectByteBuffer2_NeverZero_sanitize.invokeExact(TARGET_ARG);
+  }
+
+  @Benchmark
+  public String staticMethod_Unsafe_NeverZero() throws Throwable {
+    return (String) staticMethod_Unsafe_NeverZero_sanitize.invokeExact(TARGET_ARG);
+  }
+
+  @Benchmark
+  public String staticMethod_Unsafe_NeverZero2() throws Throwable {
+    return (String) staticMethod_Unsafe_NeverZero2_sanitize.invokeExact(TARGET_ARG);
+  }
+
+  @Benchmark
+  public String staticMethod_Unsafe_SimpleIncrement() throws Throwable {
+    return (String) staticMethod_Unsafe_SimpleIncrement_sanitize.invokeExact(TARGET_ARG);
+  }
+
+  @Benchmark
+  public String staticMethod_Unsafe_NeverZeroBranchfree() throws Throwable {
+    return (String) staticMethod_Unsafe_NeverZeroBranchfree_sanitize.invokeExact(TARGET_ARG);
+  }
+}
+
+class InstrumentingClassLoader extends ClassLoader {
+  private final Instrumentor instrumentor;
+  private final String classNamePrefix;
+
+  InstrumentingClassLoader(Instrumentor instrumentor, String packageToInstrument) {
+    super(InstrumentingClassLoader.class.getClassLoader());
+    this.instrumentor = instrumentor;
+    this.classNamePrefix = packageToInstrument + ".";
+  }
+
+  @Override
+  public Class<?> loadClass(String name) throws ClassNotFoundException {
+    if (!name.startsWith(classNamePrefix)) {
+      return super.loadClass(name);
+    }
+    try (InputStream stream = super.getResourceAsStream(name.replace('.', '/') + ".class")) {
+      if (stream == null) {
+        throw new ClassNotFoundException(String.format("Failed to find class file for %s", name));
+      }
+      byte[] bytecode = readAllBytes(stream);
+      byte[] instrumentedBytecode = instrumentor.instrument(bytecode);
+      return defineClass(name, instrumentedBytecode, 0, instrumentedBytecode.length);
+    } catch (IOException e) {
+      throw new ClassNotFoundException(String.format("Failed to read class file for %s", name), e);
+    }
+  }
+
+  private static byte[] readAllBytes(InputStream in) throws IOException {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    byte[] buffer = new byte[64 * 104 * 1024];
+    int read;
+    while ((read = in.read(buffer)) != -1) {
+      out.write(buffer, 0, read);
+    }
+    return out.toByteArray();
+  }
+}