driver: Set a default -rss_limit_mb

This is necessary for a pure Java driver as we can no longer set -Xmx in
that situation. It is also much cleaner than hand-tuning -Xmx, but we
still keep the max heap size in the native driver for backwards
compatibility with existing crashing inputs.

1 files changed, 15 insertions, 0 deletions

diff --git a/driver/src/main/java/com/code_intelligence/jazzer/driver/Driver.java b/driver/src/main/java/com/code_intelligence/jazzer/driver/Driver.java
index 05e1a582..462b7023 100644
--- a/driver/src/main/java/com/code_intelligence/jazzer/driver/Driver.java
+++ b/driver/src/main/java/com/code_intelligence/jazzer/driver/Driver.java
@@ -86,6 +86,10 @@ public class Driver {
             });
     System.setProperty("jazzer.seed", seed);
 
+    if (args.stream().noneMatch(arg -> arg.startsWith("-rss_limit_mb="))) {
+      args.add(getDefaultRssLimitMbArg());
+    }
+
     // Do *not* modify system properties beyond this point - initializing Opt parses them as a side
     // effect.
 
@@ -95,4 +99,15 @@ public class Driver {
 
     return FuzzTargetRunner.startLibFuzzer(args);
   }
+
+  private static String getDefaultRssLimitMbArg() {
+    // Java OutOfMemoryErrors are strictly more informative than libFuzzer's out of memory crashes.
+    // We thus want to scale the default libFuzzer memory limit, which includes all memory used by
+    // the process including Jazzer's native and non-native memory footprint, such that:
+    // 1. we never reach it purely by allocating memory on the Java heap;
+    // 2. it is still reached if the fuzz target allocates excessively on the native heap.
+    // As a heuristic, we set the overall memory limit to 2 * the maximum size of the Java heap.
+    long maxHeapInBytes = Runtime.getRuntime().maxMemory();
+    return "-rss_limit_mb=" + (2 * maxHeapInBytes / (1024 * 1024));
+  }
 }