diff options
Diffstat (limited to 'smalloc.c')
| -rw-r--r-- | smalloc.c | 96 |
1 files changed, 31 insertions, 65 deletions
@@ -13,6 +13,7 @@ #include <limits.h> #include <fcntl.h> +#include "fio.h" #include "mutex.h" #include "arch/arch.h" #include "os/os.h" @@ -26,13 +27,17 @@ #define SMALLOC_BPL (SMALLOC_BPB * SMALLOC_BPI) #define INITIAL_SIZE 16*1024*1024 /* new pool size */ -#define MAX_POOLS 8 /* maximum number of pools to setup */ +#define INITIAL_POOLS 8 /* maximum number of pools to setup */ + +#define MAX_POOLS 16 #define SMALLOC_PRE_RED 0xdeadbeefU #define SMALLOC_POST_RED 0x5aa55aa5U unsigned int smalloc_pool_size = INITIAL_SIZE; +#ifdef SMALLOC_REDZONE static const int int_mask = sizeof(int) - 1; +#endif struct pool { struct fio_mutex *lock; /* protects this pool */ @@ -54,37 +59,6 @@ struct block_hdr { static struct pool mp[MAX_POOLS]; static unsigned int nr_pools; static unsigned int last_pool; -static struct fio_rwlock *lock; - -static inline void pool_lock(struct pool *pool) -{ - fio_mutex_down(pool->lock); -} - -static inline void pool_unlock(struct pool *pool) -{ - fio_mutex_up(pool->lock); -} - -static inline void global_read_lock(void) -{ - fio_rwlock_read(lock); -} - -static inline void global_read_unlock(void) -{ - fio_rwlock_unlock(lock); -} - -static inline void global_write_lock(void) -{ - fio_rwlock_write(lock); -} - -static inline void global_write_unlock(void) -{ - fio_rwlock_unlock(lock); -} static inline int ptr_valid(struct pool *pool, void *ptr) { @@ -178,12 +152,15 @@ static int find_next_zero(int word, int start) return ffz(word) + start; } -static int add_pool(struct pool *pool, unsigned int alloc_size) +static bool add_pool(struct pool *pool, unsigned int alloc_size) { int bitmap_blocks; int mmap_flags; void *ptr; + if (nr_pools == MAX_POOLS) + return false; + #ifdef SMALLOC_REDZONE alloc_size += sizeof(unsigned int); #endif @@ -211,32 +188,31 @@ static int add_pool(struct pool *pool, unsigned int alloc_size) if (ptr == MAP_FAILED) goto out_fail; - memset(ptr, 0, alloc_size); pool->map = ptr; pool->bitmap = (void *) ptr + (pool->nr_blocks * SMALLOC_BPL); + memset(pool->bitmap, 0, bitmap_blocks * sizeof(unsigned int)); pool->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED); if (!pool->lock) goto out_fail; nr_pools++; - return 0; + return true; out_fail: log_err("smalloc: failed adding pool\n"); if (pool->map) munmap(pool->map, pool->mmap_size); - return 1; + return false; } void sinit(void) { - int i, ret; - - lock = fio_rwlock_init(); + bool ret; + int i; - for (i = 0; i < MAX_POOLS; i++) { - ret = add_pool(&mp[i], INITIAL_SIZE); - if (ret) + for (i = 0; i < INITIAL_POOLS; i++) { + ret = add_pool(&mp[nr_pools], smalloc_pool_size); + if (!ret) break; } @@ -265,9 +241,6 @@ void scleanup(void) for (i = 0; i < nr_pools; i++) cleanup_pool(&mp[i]); - - if (lock) - fio_rwlock_remove(lock); } #ifdef SMALLOC_REDZONE @@ -276,7 +249,7 @@ static void *postred_ptr(struct block_hdr *hdr) uintptr_t ptr; ptr = (uintptr_t) hdr + hdr->size - sizeof(unsigned int); - ptr = (ptr + int_mask) & ~int_mask; + ptr = (uintptr_t) PTR_ALIGN(ptr, int_mask); return (void *) ptr; } @@ -336,12 +309,12 @@ static void sfree_pool(struct pool *pool, void *ptr) i = offset / SMALLOC_BPL; idx = (offset % SMALLOC_BPL) / SMALLOC_BPB; - pool_lock(pool); + fio_mutex_down(pool->lock); clear_blocks(pool, i, idx, size_to_blocks(hdr->size)); if (i < pool->next_non_full) pool->next_non_full = i; pool->free_blocks += size_to_blocks(hdr->size); - pool_unlock(pool); + fio_mutex_up(pool->lock); } void sfree(void *ptr) @@ -352,8 +325,6 @@ void sfree(void *ptr) if (!ptr) return; - global_read_lock(); - for (i = 0; i < nr_pools; i++) { if (ptr_valid(&mp[i], ptr)) { pool = &mp[i]; @@ -361,10 +332,12 @@ void sfree(void *ptr) } } - global_read_unlock(); + if (pool) { + sfree_pool(pool, ptr); + return; + } - assert(pool); - sfree_pool(pool, ptr); + log_err("smalloc: ptr %p not from smalloc pool\n", ptr); } static void *__smalloc_pool(struct pool *pool, size_t size) @@ -375,7 +348,7 @@ static void *__smalloc_pool(struct pool *pool, size_t size) unsigned int last_idx; void *ret = NULL; - pool_lock(pool); + fio_mutex_down(pool->lock); nr_blocks = size_to_blocks(size); if (nr_blocks > pool->free_blocks) @@ -418,7 +391,7 @@ static void *__smalloc_pool(struct pool *pool, size_t size) ret = pool->map + offset; } fail: - pool_unlock(pool); + fio_mutex_up(pool->lock); return ret; } @@ -457,7 +430,6 @@ void *smalloc(size_t size) if (size != (unsigned int) size) return NULL; - global_write_lock(); i = last_pool; end_pool = nr_pools; @@ -467,7 +439,6 @@ void *smalloc(size_t size) if (ptr) { last_pool = i; - global_write_unlock(); return ptr; } } @@ -480,19 +451,14 @@ void *smalloc(size_t size) break; } while (1); - global_write_unlock(); + log_err("smalloc: OOM. Consider using --alloc-size to increase the " + "shared memory available.\n"); return NULL; } void *scalloc(size_t nmemb, size_t size) { - void *ret; - - ret = smalloc(nmemb * size); - if (ret) - memset(ret, 0, nmemb * size); - - return ret; + return smalloc(nmemb * size); } char *smalloc_strdup(const char *str) |