[mupdf] Improve custom allocator (#5505)

Previously the allocator handled small pointer values incorrectly
and prevented multiple executions (unexpectedly used when OSS-fuzz
uses LSAN to look for memory leaks).

1 files changed, 84 insertions, 65 deletions

diff --git a/projects/mupdf/pdf_fuzzer.cc b/projects/mupdf/pdf_fuzzer.cc
index fd8ad7faf..3e3f1fbb6 100644
--- a/projects/mupdf/pdf_fuzzer.cc
+++ b/projects/mupdf/pdf_fuzzer.cc
@@ -23,96 +23,115 @@
 
 #include <mupdf/fitz.h>
 
-#define ALIGNMENT 16
-#define MAX_ALLOCATION (1024 * 1024 * 1024)
+#define ALIGNMENT ((size_t) 16)
+#define KBYTE ((size_t) 1024)
+#define MBYTE (1024 * KBYTE)
+#define GBYTE (1024 * MBYTE)
+#define MAX_ALLOCATION (1 * GBYTE)
 
-static uint64_t total = 0;
+static size_t used;
 
-static void *
-fz_malloc_ossfuzz(void *opaque, size_t size)
+static void *fz_limit_reached_ossfuzz(size_t oldsize, size_t size)
 {
-	char *ptr = NULL;
+  if (oldsize == 0)
+    fprintf(stderr, "limit: %zu Mbyte used: %zu Mbyte allocation: %zu: limit reached\n", MAX_ALLOCATION / MBYTE, used / MBYTE, size);
+  else
+    fprintf(stderr, "limit: %zu Mbyte used: %zu Mbyte reallocation: %zu -> %zu: limit reached\n", MAX_ALLOCATION / MBYTE, used / MBYTE, oldsize, size);
+  fflush(0);
+  return NULL;
+}
 
-	if (size == 0)
-		return NULL;
-	if (size > SIZE_MAX - ALIGNMENT)
-		return NULL;
+static void *fz_malloc_ossfuzz(void *opaque, size_t size)
+{
+  char *ptr = NULL;
 
-	if (size > MAX_ALLOCATION - ALIGNMENT - total)
-		return NULL;
+  if (size == 0)
+    return NULL;
+  if (size > SIZE_MAX - ALIGNMENT)
+    return NULL;
+  if (size + ALIGNMENT > MAX_ALLOCATION - used)
+    return fz_limit_reached_ossfuzz(0, size + ALIGNMENT);
 
-	ptr = (char *) malloc(size + ALIGNMENT);
-	if (ptr == NULL)
-		return NULL;
+  ptr = (char *) malloc(size + ALIGNMENT);
+  if (ptr == NULL)
+    return NULL;
 
-	memcpy(ptr, &size, sizeof(size));
-	total += size + ALIGNMENT;
+  memcpy(ptr, &size, sizeof(size));
+  used += size + ALIGNMENT;
 
-	return ptr + ALIGNMENT;
+  return ptr + ALIGNMENT;
 }
 
-static void
-fz_free_ossfuzz(void *opaque, void *ptr)
+static void fz_free_ossfuzz(void *opaque, void *ptr)
 {
-	size_t size;
+  size_t size;
 
-	if (ptr == NULL)
-		return;
+  if (ptr == NULL)
+    return;
+  if (ptr < (void *) ALIGNMENT)
+    return;
 
-	ptr = ((char *) ptr) - ALIGNMENT;
+  ptr = (char *) ptr - ALIGNMENT;
+  memcpy(&size, ptr, sizeof(size));
 
-	memcpy(&size, ptr, sizeof(size));
-	total -= size - ALIGNMENT;
-	free(ptr);
+  used -= size + ALIGNMENT;
+  free(ptr);
 }
 
-static void *
-fz_realloc_ossfuzz(void *opaque, void *old, size_t size)
+static void *fz_realloc_ossfuzz(void *opaque, void *old, size_t size)
 {
-	size_t oldsize;
-	char *ptr;
-
-	if (old == NULL)
-		return fz_malloc_ossfuzz(opaque, size);
-	if (size == 0)
-	{
-		fz_free_ossfuzz(opaque, old);
-		return NULL;
-	}
-	if (size > SIZE_MAX - ALIGNMENT)
-		return NULL;
-
-	old = ((char *) old) - ALIGNMENT;
-	memcpy(&oldsize, old, sizeof(oldsize));
-
-	if (size > MAX_ALLOCATION - total + oldsize)
-		return NULL;
-
-	ptr = (char *) realloc(old, size + ALIGNMENT);
-	if (ptr == NULL)
-		return NULL;
-
-	total -= oldsize + ALIGNMENT;
-	memcpy(ptr, &size, sizeof(size));
-	total += size + ALIGNMENT;
-
-	return ptr + ALIGNMENT;
+  size_t oldsize;
+  char *ptr;
+
+  if (old == NULL)
+    return fz_malloc_ossfuzz(opaque, size);
+  if (old < (void *) ALIGNMENT)
+    return NULL;
+
+  if (size == 0) {
+    fz_free_ossfuzz(opaque, old);
+    return NULL;
+  }
+  if (size > SIZE_MAX - ALIGNMENT)
+    return NULL;
+
+  old = (char *) old - ALIGNMENT;
+  memcpy(&oldsize, old, sizeof(oldsize));
+
+  if (size + ALIGNMENT > MAX_ALLOCATION - used + oldsize + ALIGNMENT)
+    return fz_limit_reached_ossfuzz(oldsize + ALIGNMENT, size + ALIGNMENT);
+
+  ptr = (char *) realloc(old, size + ALIGNMENT);
+  if (ptr == NULL)
+    return NULL;
+
+  used -= oldsize + ALIGNMENT;
+  memcpy(ptr, &size, sizeof(size));
+  used += size + ALIGNMENT;
+
+  return ptr + ALIGNMENT;
 }
 
 static fz_alloc_context fz_alloc_ossfuzz =
 {
-	NULL,
-	fz_malloc_ossfuzz,
-	fz_realloc_ossfuzz,
-	fz_free_ossfuzz
+  NULL,
+  fz_malloc_ossfuzz,
+  fz_realloc_ossfuzz,
+  fz_free_ossfuzz
 };
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  fz_context *ctx = fz_new_context(&fz_alloc_ossfuzz, nullptr, FZ_STORE_DEFAULT);
+  fz_context *ctx;
+  fz_stream *stream;
+  fz_document *doc;
+  fz_pixmap *pix;
+
+  used = 0;
 
-  fz_stream *stream = NULL;
-  fz_document *doc = NULL;
-  fz_pixmap *pix = NULL;
+  ctx = fz_new_context(&fz_alloc_ossfuzz, nullptr, FZ_STORE_DEFAULT);
+  stream = NULL;
+  doc = NULL;
+  pix = NULL;
 
   fz_var(stream);
   fz_var(doc);