WIP get dlmalloc working with a mmap() style allocator out of the heap wrapper

11 files changed, 191 insertions, 67 deletions

diff --git a/include/kernel/vm.h b/include/kernel/vm.h
index 433e7649..ec34da3a 100644
--- a/include/kernel/vm.h
+++ b/include/kernel/vm.h
@@ -175,6 +175,8 @@ void *pmm_alloc_kpages(uint count, struct list_node *list);
     /* Helper routine for pmm_alloc_kpages. */
 static inline void *pmm_alloc_kpage(void) { return pmm_alloc_kpages(1, NULL); }
 
+size_t pmm_free_kpages(void *ptr, uint count);
+
 /* physical to virtual */
 void *paddr_to_kvaddr(paddr_t pa);
 
diff --git a/kernel/vm/pmm.c b/kernel/vm/pmm.c
index fea77c86..0b9afc17 100644
--- a/kernel/vm/pmm.c
+++ b/kernel/vm/pmm.c
@@ -262,6 +262,28 @@ void *pmm_alloc_kpages(uint count, struct list_node *list)
     return paddr_to_kvaddr(pa);
 }
 
+size_t pmm_free_kpages(void *_ptr, uint count)
+{
+    LTRACEF("ptr %p, count %u\n", _ptr, count);
+
+    uint8_t *ptr = (uint8_t *)_ptr;
+
+    struct list_node list;
+    list_initialize(&list);
+
+    while (count > 0) {
+        vm_page_t *p = address_to_page(kvaddr_to_paddr(ptr));
+        if (p) {
+            list_add_tail(&list, &p->node);
+        }
+
+        ptr += PAGE_SIZE;
+        count--;
+    }
+
+    return pmm_free(&list);
+}
+
 size_t pmm_alloc_contiguous(uint count, uint8_t alignment_log2, paddr_t *pa, struct list_node *list)
 {
     LTRACEF("count %u, align %u\n", count, alignment_log2);
diff --git a/lib/dlmalloc/dlmalloc.c b/lib/heap/dlmalloc/dlmalloc.c
index 8b8daae3..796d6353 100644
--- a/lib/dlmalloc/dlmalloc.c
+++ b/lib/heap/dlmalloc/dlmalloc.c
@@ -539,8 +539,31 @@ MAX_RELEASE_CHECK_RATE   default: 4095 unless not HAVE_MMAP
 #define LACKS_UNISTD_H
 #define LACKS_SYS_PARAM_H
 #define LACKS_SCHED_H
-#define HAVE_MMAP 0
-#define HAVE_MORECORE 1
+#define HAVE_MMAP 1
+#include <sys/types.h>
+#include <stdio.h>
+#include <lib/heap.h>
+#include <debug.h>
+
+static inline void *mmap(size_t len) {
+    void *ptr;
+    if (heap_grow_memory(&ptr, len) < 0)
+        return 0;
+
+    return ptr;
+}
+
+static inline int munmap(void *base, size_t len) {
+    heap_free_memory(base, len);
+    return 0;
+}
+
+#define MMAP(s) mmap(s)
+#define DIRECT_MMAP(s) mmap(s)
+#define MUNMAP(b, s) munmap(b, s)
+#define HAVE_MORECORE 0
+#define MORECORE dl_sbrk
+void *dl_sbrk(long incr);
 #define USE_LOCKS 2
 #include <debug.h>
 #define ABORT panic("dlmalloc abort\n")
diff --git a/lib/dlmalloc/include/lib/dlmalloc.h b/lib/heap/dlmalloc/include/lib/dlmalloc.h
index 8eb396f9..8eb396f9 100644
--- a/lib/dlmalloc/include/lib/dlmalloc.h
+++ b/lib/heap/dlmalloc/include/lib/dlmalloc.h
diff --git a/lib/dlmalloc/rules.mk b/lib/heap/dlmalloc/rules.mk
index 1c629a4a..1c629a4a 100644
--- a/lib/dlmalloc/rules.mk
+++ b/lib/heap/dlmalloc/rules.mk
diff --git a/lib/heap/heap_wrapper.c b/lib/heap/heap_wrapper.c
index 68df5f3f..0fc5d5ce 100644
--- a/lib/heap/heap_wrapper.c
+++ b/lib/heap/heap_wrapper.c
@@ -23,17 +23,82 @@
 #include <lib/heap.h>
 
 #include <trace.h>
+#include <debug.h>
+#include <assert.h>
 #include <string.h>
+#include <err.h>
 #include <list.h>
 #include <kernel/spinlock.h>
 #include <lib/console.h>
-#include <lib/miniheap.h>
 
 #define LOCAL_TRACE 0
 
 struct list_node delayed_free_list = LIST_INITIAL_VALUE(delayed_free_list);
 spin_lock_t delayed_free_lock = SPIN_LOCK_INITIAL_VALUE;
 
+#if WITH_LIB_HEAP_MINIHEAP
+#include <lib/miniheap.h>
+
+#define HEAP_ALLOC miniheap_alloc
+#define HEAP_FREE miniheap_free
+static inline void HEAP_INIT(void) { miniheap_init(NULL, 0); }
+#define HEAP_DUMP miniheap_dump
+
+#elif WITH_LIB_HEAP_DLMALLOC
+#include <lib/dlmalloc.h>
+
+static inline void *HEAP_ALLOC(size_t size, unsigned int alignment) {
+    if (alignment == 0)
+        return dlmalloc(size);
+    else
+        return dlmemalign(alignment, size);
+}
+
+static inline void HEAP_FREE(void *ptr) { dlfree(ptr); }
+static inline void HEAP_INIT(void) {}
+static inline void HEAP_DUMP(void) {}
+
+void *dl_sbrk(int incr) {
+    LTRACEF("incr %d\n", incr);
+
+    panic("what");
+}
+
+#else
+#error need to select valid heap implementation or provide wrapper
+#endif
+
+#if WITH_KERNEL_VM
+
+#include <kernel/vm.h>
+
+/* we will use kalloc routines to back our heap */
+#if !defined(HEAP_GROW_SIZE)
+#define HEAP_GROW_SIZE (64 * 1024) /* size the heap grows by when it runs out of memory */
+#endif
+
+STATIC_ASSERT(IS_PAGE_ALIGNED(HEAP_GROW_SIZE));
+
+#elif WITH_STATIC_HEAP
+
+#if !defined(HEAP_START) || !defined(HEAP_LEN)
+#error WITH_STATIC_HEAP set but no HEAP_START or HEAP_LEN defined
+#endif
+
+#else
+/* not a static vm, not using the kernel vm */
+extern int _end;
+extern int _end_of_ram;
+
+/* default to using up the rest of memory after the kernel ends */
+/* may be modified by other parts of the system */
+uintptr_t _heap_start = (uintptr_t)&_end;
+uintptr_t _heap_end = (uintptr_t)&_end_of_ram;
+
+#define HEAP_START ((uintptr_t)_heap_start)
+#define HEAP_LEN ((uintptr_t)_heap_end - HEAP_START)
+#endif
+
 static void heap_free_delayed_list(void)
 {
     struct list_node list;
@@ -64,19 +129,19 @@ void *heap_alloc(size_t size, unsigned int alignment)
         heap_free_delayed_list();
     }
 
-    return miniheap_alloc(size, alignment);
+    return HEAP_ALLOC(size, alignment);
 }
 
 void heap_free(void *ptr)
 {
     LTRACEF("ptr %p\n", ptr);
 
-    miniheap_free(ptr);
+    HEAP_FREE(ptr);
 }
 
 void heap_init(void)
 {
-    miniheap_init();
+    HEAP_INIT();
 }
 
 /* critical section time delayed free */
@@ -96,7 +161,7 @@ void heap_delayed_free(void *ptr)
 
 static void heap_dump(void)
 {
-    miniheap_dump();
+    HEAP_DUMP();
 
     printf("\tdelayed free list:\n");
     spin_lock_saved_state_t state;
@@ -108,6 +173,48 @@ static void heap_dump(void)
     spin_unlock_irqrestore(&delayed_free_lock, state);
 }
 
+/* called back from the heap implementation to allocate another block of memory */
+ssize_t heap_grow_memory(void **ptr, size_t size)
+{
+    LTRACEF("ptr %p, size 0x%zx\n", ptr, size);
+
+#if WITH_KERNEL_VM && !WITH_STATIC_HEAP
+    size = ROUNDUP(size, PAGE_SIZE);
+    LTRACEF("size now 0x%zx\n", size);
+
+    *ptr = pmm_alloc_kpages(size / PAGE_SIZE, NULL);
+    if (!*ptr) {
+        TRACEF("failed to grow kernel heap by 0x%zx bytes\n", size);
+        return ERR_NO_MEMORY;
+    }
+#else
+    static bool have_asked_for_memory = false;
+
+    if (have_asked_for_memory)
+        return ERR_NO_MEMORY;
+
+    *ptr = HEAP_START;
+    size = HEAP_LEN;
+    have_asked_for_memory = true;
+#endif
+
+    LTRACEF("returning %p, size 0x%zx\n", *ptr, size);
+
+    return size;
+}
+
+void heap_free_memory(void *ptr, size_t len)
+{
+    LTRACEF("ptr %p, len 0x%zx\n", ptr, len);
+
+#if WITH_KERNEL_VM && !WITH_STATIC_HEAP
+    DEBUG_ASSERT(IS_PAGE_ALIGNED((uintptr_t)ptr));
+    DEBUG_ASSERT(IS_PAGE_ALIGNED(len));
+
+    pmm_free_kpages(ptr, len / PAGE_SIZE);
+#endif
+}
+
 #if 0
 static void heap_test(void)
 {
diff --git a/lib/heap/include/lib/heap.h b/lib/heap/include/lib/heap.h
index 88799f4a..b26703f0 100644
--- a/lib/heap/include/lib/heap.h
+++ b/lib/heap/include/lib/heap.h
@@ -35,4 +35,7 @@ void heap_init(void);
 /* critical section time delayed free */
 void heap_delayed_free(void *);
 
+ssize_t heap_grow_memory(void **ptr, size_t len);
+void heap_free_memory(void *ptr, size_t len);
+
 __END_CDECLS;
diff --git a/lib/heap/miniheap/include/lib/miniheap.h b/lib/heap/miniheap/include/lib/miniheap.h
index ba62a5c6..66bc9e47 100644
--- a/lib/heap/miniheap/include/lib/miniheap.h
+++ b/lib/heap/miniheap/include/lib/miniheap.h
@@ -39,7 +39,7 @@ void miniheap_get_stats(struct miniheap_stats *ptr);
 void *miniheap_alloc(size_t, unsigned int alignment);
 void miniheap_free(void *);
 
-void miniheap_init(void);
+void miniheap_init(void *ptr, size_t len);
 void miniheap_dump(void);
 
 __END_CDECLS;
diff --git a/lib/heap/miniheap/miniheap.c b/lib/heap/miniheap/miniheap.c
index 9976dd9c..802bdb5a 100644
--- a/lib/heap/miniheap/miniheap.c
+++ b/lib/heap/miniheap/miniheap.c
@@ -33,6 +33,7 @@
 #include <kernel/thread.h>
 #include <kernel/mutex.h>
 #include <lib/miniheap.h>
+#include <lib/heap.h>
 
 #define LOCAL_TRACE 0
 
@@ -44,36 +45,6 @@
 
 #define HEAP_MAGIC 'HEAP'
 
-#if WITH_KERNEL_VM
-
-#include <kernel/vm.h>
-/* we will use kalloc routines to back our heap */
-#if !defined(HEAP_GROW_SIZE)
-#define HEAP_GROW_SIZE (4 * 1024 * 1024) /* size the heap grows by when it runs out of memory */
-#endif
-
-STATIC_ASSERT(IS_PAGE_ALIGNED(HEAP_GROW_SIZE));
-
-#elif WITH_STATIC_HEAP
-
-#if !defined(HEAP_START) || !defined(HEAP_LEN)
-#error WITH_STATIC_HEAP set but no HEAP_START or HEAP_LEN defined
-#endif
-
-#else
-/* not a static vm, not using the kernel vm */
-extern int _end;
-extern int _end_of_ram;
-
-/* default to using up the rest of memory after the kernel ends */
-/* may be modified by other parts of the system */
-uintptr_t _heap_start = (uintptr_t)&_end;
-uintptr_t _heap_end = (uintptr_t)&_end_of_ram;
-
-#define HEAP_START ((uintptr_t)_heap_start)
-#define HEAP_LEN ((uintptr_t)_heap_end - HEAP_START)
-#endif
-
 struct free_heap_chunk {
     struct list_node node;
     size_t len;
@@ -321,9 +292,7 @@ retry:
 #if WITH_KERNEL_VM
     /* try to grow the heap if we can */
     if (ptr == NULL && retry_count == 0) {
-        size_t growby = MAX(HEAP_GROW_SIZE, ROUNDUP(size, PAGE_SIZE));
-
-        ssize_t err = heap_grow(growby);
+        ssize_t err = heap_grow(size);
         if (err >= 0) {
             retry_count++;
             goto retry;
@@ -396,35 +365,30 @@ void miniheap_get_stats(struct miniheap_stats *ptr)
 
 static ssize_t heap_grow(size_t size)
 {
-#if WITH_KERNEL_VM
-    size = ROUNDUP(size, PAGE_SIZE);
-
-    void *ptr = pmm_alloc_kpages(size / PAGE_SIZE, NULL);
-    if (!ptr) {
+    void *ptr;
+    ssize_t allocated = heap_grow_memory(&ptr, size);
+    if (allocated <= 0) {
         TRACEF("failed to grow kernel heap by 0x%zx bytes\n", size);
         return ERR_NO_MEMORY;
     }
 
-    LTRACEF("growing heap by 0x%zx bytes, new ptr %p\n", size, ptr);
+    LTRACEF("growing heap by 0x%zx bytes, allocated 0x%zx, new ptr %p\n", size, allocated, ptr);
 
-    heap_insert_free_chunk(heap_create_free_chunk(ptr, size, true));
+    heap_insert_free_chunk(heap_create_free_chunk(ptr, allocated, true));
 
     /* change the heap start and end variables */
     if ((uintptr_t)ptr < (uintptr_t)theheap.base)
         theheap.base = ptr;
 
-    uintptr_t endptr = (uintptr_t)ptr + size;
+    uintptr_t endptr = (uintptr_t)ptr + allocated;
     if (endptr > (uintptr_t)theheap.base + theheap.len) {
         theheap.len = (uintptr_t)endptr - (uintptr_t)theheap.base;
     }
 
-    return size;
-#else
-    return ERR_NO_MEMORY;
-#endif
+    return allocated;
 }
 
-void miniheap_init(void)
+void miniheap_init(void *ptr, size_t len)
 {
     LTRACE_ENTRY;
 
@@ -435,22 +399,12 @@ void miniheap_init(void)
     list_initialize(&theheap.free_list);
 
     // set the heap range
-#if WITH_KERNEL_VM
-    theheap.base = pmm_alloc_kpages(HEAP_GROW_SIZE / PAGE_SIZE, NULL);
-    theheap.len = HEAP_GROW_SIZE;
-
-    if (theheap.base == 0) {
-        panic("HEAP: error allocating initial heap size\n");
-    }
-#else
-    theheap.base = (void *)HEAP_START;
-    theheap.len = HEAP_LEN;
-#endif
+    theheap.base = 0;
+    theheap.len = 0;
     theheap.remaining = 0; // will get set by heap_insert_free_chunk()
     theheap.low_watermark = theheap.len;
-    LTRACEF("base %p size %zd bytes\n", theheap.base, theheap.len);
 
     // create an initial free chunk
-    heap_insert_free_chunk(heap_create_free_chunk(theheap.base, theheap.len, false));
+    //heap_insert_free_chunk(heap_create_free_chunk(theheap.base, theheap.len, false));
 }
 
diff --git a/lib/heap/rules.mk b/lib/heap/rules.mk
index bc91b910..5095beb5 100644
--- a/lib/heap/rules.mk
+++ b/lib/heap/rules.mk
@@ -7,6 +7,17 @@ MODULE := $(LOCAL_DIR)
 MODULE_SRCS += \
 	$(LOCAL_DIR)/heap_wrapper.c
 
+# pick a heap implementation
+ifndef LK_HEAP_IMPLEMENTATION
+LK_HEAP_IMPLEMENTATION=miniheap
+endif
+ifeq ($(LK_HEAP_IMPLEMENTATION),miniheap)
 MODULE_DEPS := $(LOCAL_DIR)/miniheap
+endif
+ifeq ($(LK_HEAP_IMPLEMENTATION),dlmalloc)
+MODULE_DEPS := $(LOCAL_DIR)/dlmalloc
+endif
+
+GLOBAL_DEFINES += LK_HEAP_IMPLEMENTATION=$(LK_HEAP_IMPLEMENTATION)
 
 include make/module.mk
diff --git a/platform/qemu-virt/rules.mk b/platform/qemu-virt/rules.mk
index f0a6d890..9b85d4d8 100644
--- a/platform/qemu-virt/rules.mk
+++ b/platform/qemu-virt/rules.mk
@@ -13,6 +13,8 @@ ARM_CPU ?= cortex-a15
 endif
 WITH_SMP ?= 1
 
+LK_HEAP_IMPLEMENTATION ?= dlmalloc
+
 GLOBAL_INCLUDES += \
     $(LOCAL_DIR)/include