From 354524779686d6d5518078fd681110157631c941 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 14 Jun 2016 14:28:47 -0700 Subject: Updates to generated files for 4.2.1 release. Bug: 28860984 Change-Id: If12daed270ec0a85cd151aaaa432d178c8389757 --- Android.bp | 6 +- android/test/run_jemalloc_tests.sh | 4 + include/jemalloc/internal/jemalloc_internal.h | 193 +++++++++++---------- include/jemalloc/internal/jemalloc_internal_defs.h | 9 + include/jemalloc/internal/private_namespace.h | 103 +++++++++-- include/jemalloc/internal/private_unnamespace.h | 103 +++++++++-- include/jemalloc/jemalloc.h | 18 +- include/jemalloc/jemalloc_macros.h | 18 +- src/android_je_iterate.c | 4 +- src/android_je_mallinfo.c | 32 ++-- test/include/test/jemalloc_test.h | 78 +++++---- 11 files changed, 372 insertions(+), 196 deletions(-) diff --git a/Android.bp b/Android.bp index 051b03c..f65c934 100644 --- a/Android.bp +++ b/Android.bp @@ -44,7 +44,6 @@ common_cflags = [ // usually decreases the amount of PSS used, but can increase // fragmentation. common_cflags += [ - "-DANDROID_ALWAYS_PURGE", "-DANDROID_MAX_ARENAS=2", "-DANDROID_TCACHE_NSLOTS_SMALL_MAX=8", "-DANDROID_TCACHE_NSLOTS_LARGE=16", @@ -118,6 +117,7 @@ lib_src_files = [ "src/ticker.c", "src/tsd.c", "src/util.c", + "src/witness.c", ] //----------------------------------------------------------------------- @@ -195,10 +195,13 @@ cc_library_static { // jemalloc unit tests //----------------------------------------------------------------------- unit_tests = [ + "test/unit/a0.c", + "test/unit/arena_reset.c", "test/unit/atomic.c", "test/unit/bitmap.c", "test/unit/ckh.c", "test/unit/decay.c", + "test/unit/fork.c", "test/unit/hash.c", "test/unit/junk.c", "test/unit/junk_alloc.c", @@ -229,6 +232,7 @@ unit_tests = [ "test/unit/ticker.c", "test/unit/tsd.c", "test/unit/util.c", + "test/unit/witness.c", "test/unit/zero.c", ] diff --git a/android/test/run_jemalloc_tests.sh b/android/test/run_jemalloc_tests.sh index e4a935d..ae479e7 100755 --- a/android/test/run_jemalloc_tests.sh +++ b/android/test/run_jemalloc_tests.sh @@ -17,10 +17,13 @@ UNIT_TEST_DIR="jemalloc_unittests" UNIT_TESTS=( \ + "a0" \ + "arena_reset" \ "atomic" \ "bitmap" \ "ckh" \ "decay" \ + "fork" \ "hash" \ "junk" \ "junk_alloc" \ @@ -51,6 +54,7 @@ UNIT_TESTS=( \ "ticker" \ "tsd" \ "util" \ + "witness" \ "zero" \ ) diff --git a/include/jemalloc/internal/jemalloc_internal.h b/include/jemalloc/internal/jemalloc_internal.h index 6f94356..a6902f4 100644 --- a/include/jemalloc/internal/jemalloc_internal.h +++ b/include/jemalloc/internal/jemalloc_internal.h @@ -161,6 +161,7 @@ static const bool config_cache_oblivious = #include #endif +#include "jemalloc/internal/ph.h" #define RB_COMPACT #include "jemalloc/internal/rb.h" #include "jemalloc/internal/qr.h" @@ -257,6 +258,9 @@ typedef unsigned szind_t; # ifdef __powerpc__ # define LG_QUANTUM 4 # endif +# ifdef __riscv__ +# define LG_QUANTUM 4 +# endif # ifdef __s390__ # define LG_QUANTUM 4 # endif @@ -367,6 +371,7 @@ typedef unsigned szind_t; #include "jemalloc/internal/smoothstep.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/witness.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" @@ -398,6 +403,7 @@ typedef unsigned szind_t; #include "jemalloc/internal/smoothstep.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/witness.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/bitmap.h" @@ -440,6 +446,9 @@ extern bool in_valgrind; /* Number of CPUs. */ extern unsigned ncpus; +/* Number of arenas used for automatic multiplexing of threads and arenas. */ +extern unsigned narenas_auto; + /* * Arenas that are used to service external requests. Not all elements of the * arenas array are necessarily used; arenas are created lazily as needed. @@ -463,14 +472,14 @@ void a0dalloc(void *ptr); void *bootstrap_malloc(size_t size); void *bootstrap_calloc(size_t num, size_t size); void bootstrap_free(void *ptr); -arena_t *arenas_extend(unsigned ind); unsigned narenas_total_get(void); -arena_t *arena_init(unsigned ind); +arena_t *arena_init(tsdn_t *tsdn, unsigned ind); arena_tdata_t *arena_tdata_get_hard(tsd_t *tsd, unsigned ind); -arena_t *arena_choose_hard(tsd_t *tsd); +arena_t *arena_choose_hard(tsd_t *tsd, bool internal); void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind); void thread_allocated_cleanup(tsd_t *tsd); void thread_deallocated_cleanup(tsd_t *tsd); +void iarena_cleanup(tsd_t *tsd); void arena_cleanup(tsd_t *tsd); void arenas_tdata_cleanup(tsd_t *tsd); void narenas_tdata_cleanup(tsd_t *tsd); @@ -490,6 +499,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/smoothstep.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/witness.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/bitmap.h" @@ -521,8 +531,9 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/smoothstep.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" -#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/tsd.h" +#include "jemalloc/internal/witness.h" +#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/base.h" @@ -542,10 +553,12 @@ size_t s2u_compute(size_t size); size_t s2u_lookup(size_t size); size_t s2u(size_t size); size_t sa2u(size_t size, size_t alignment); +arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal); arena_t *arena_choose(tsd_t *tsd, arena_t *arena); +arena_t *arena_ichoose(tsdn_t *tsdn, arena_t *arena); arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing); -arena_t *arena_get(unsigned ind, bool init_if_missing); +arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing); ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind); #endif @@ -780,19 +793,38 @@ sa2u(size_t size, size_t alignment) /* Choose an arena based on a per-thread value. */ JEMALLOC_INLINE arena_t * -arena_choose(tsd_t *tsd, arena_t *arena) +arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { arena_t *ret; if (arena != NULL) return (arena); - if (unlikely((ret = tsd_arena_get(tsd)) == NULL)) - ret = arena_choose_hard(tsd); + ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd); + if (unlikely(ret == NULL)) + ret = arena_choose_hard(tsd, internal); return (ret); } +JEMALLOC_INLINE arena_t * +arena_choose(tsd_t *tsd, arena_t *arena) +{ + + return (arena_choose_impl(tsd, arena, false)); +} + +JEMALLOC_INLINE arena_t * +arena_ichoose(tsdn_t *tsdn, arena_t *arena) +{ + + assert(!tsdn_null(tsdn) || arena != NULL); + + if (!tsdn_null(tsdn)) + return (arena_choose_impl(tsdn_tsd(tsdn), NULL, true)); + return (arena); +} + JEMALLOC_INLINE arena_tdata_t * arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) { @@ -819,7 +851,7 @@ arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) } JEMALLOC_INLINE arena_t * -arena_get(unsigned ind, bool init_if_missing) +arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) { arena_t *ret; @@ -829,7 +861,7 @@ arena_get(unsigned ind, bool init_if_missing) if (unlikely(ret == NULL)) { ret = atomic_read_p((void *)&arenas[ind]); if (init_if_missing && unlikely(ret == NULL)) - ret = arena_init(ind); + ret = arena_init(tsdn, ind); } return (ret); } @@ -863,30 +895,27 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) #ifndef JEMALLOC_ENABLE_INLINE arena_t *iaalloc(const void *ptr); -size_t isalloc(const void *ptr, bool demote); -void *iallocztm(tsd_t *tsd, size_t size, szind_t ind, bool zero, +size_t isalloc(tsdn_t *tsdn, const void *ptr, bool demote); +void *iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool is_metadata, arena_t *arena, bool slow_path); -void *imalloct(tsd_t *tsd, size_t size, szind_t ind, tcache_t *tcache, - arena_t *arena); -void *imalloc(tsd_t *tsd, size_t size, szind_t ind, bool slow_path); -void *icalloct(tsd_t *tsd, size_t size, szind_t ind, tcache_t *tcache, - arena_t *arena); -void *icalloc(tsd_t *tsd, size_t size, szind_t ind); -void *ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, +void *ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, + bool slow_path); +void *ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, tcache_t *tcache, bool is_metadata, arena_t *arena); -void *ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, +void *ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena); void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero); -size_t ivsalloc(const void *ptr, bool demote); +size_t ivsalloc(tsdn_t *tsdn, const void *ptr, bool demote); size_t u2rz(size_t usize); -size_t p2rz(const void *ptr); -void idalloctm(tsd_t *tsd, void *ptr, tcache_t *tcache, bool is_metadata, +size_t p2rz(tsdn_t *tsdn, const void *ptr); +void idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_metadata, bool slow_path); -void idalloct(tsd_t *tsd, void *ptr, tcache_t *tcache); void idalloc(tsd_t *tsd, void *ptr); void iqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path); -void isdalloct(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache); -void isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache); +void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, + bool slow_path); +void isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache, + bool slow_path); void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena); @@ -894,7 +923,7 @@ void *iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena); void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, bool zero); -bool ixalloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, +bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero); #endif @@ -910,102 +939,85 @@ iaalloc(const void *ptr) /* * Typical usage: + * tsdn_t *tsdn = [...] * void *ptr = [...] - * size_t sz = isalloc(ptr, config_prof); + * size_t sz = isalloc(tsdn, ptr, config_prof); */ JEMALLOC_ALWAYS_INLINE size_t -isalloc(const void *ptr, bool demote) +isalloc(tsdn_t *tsdn, const void *ptr, bool demote) { assert(ptr != NULL); /* Demotion only makes sense if config_prof is true. */ assert(config_prof || !demote); - return (arena_salloc(ptr, demote)); + return (arena_salloc(tsdn, ptr, demote)); } JEMALLOC_ALWAYS_INLINE void * -iallocztm(tsd_t *tsd, size_t size, szind_t ind, bool zero, tcache_t *tcache, +iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool is_metadata, arena_t *arena, bool slow_path) { void *ret; assert(size != 0); + assert(!is_metadata || tcache == NULL); + assert(!is_metadata || arena == NULL || arena->ind < narenas_auto); - ret = arena_malloc(tsd, arena, size, ind, zero, tcache, slow_path); + ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path); if (config_stats && is_metadata && likely(ret != NULL)) { - arena_metadata_allocated_add(iaalloc(ret), isalloc(ret, - config_prof)); + arena_metadata_allocated_add(iaalloc(ret), + isalloc(tsdn, ret, config_prof)); } return (ret); } JEMALLOC_ALWAYS_INLINE void * -imalloct(tsd_t *tsd, size_t size, szind_t ind, tcache_t *tcache, arena_t *arena) -{ - - return (iallocztm(tsd, size, ind, false, tcache, false, arena, true)); -} - -JEMALLOC_ALWAYS_INLINE void * -imalloc(tsd_t *tsd, size_t size, szind_t ind, bool slow_path) +ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) { - return (iallocztm(tsd, size, ind, false, tcache_get(tsd, true), false, - NULL, slow_path)); + return (iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd, true), + false, NULL, slow_path)); } JEMALLOC_ALWAYS_INLINE void * -icalloct(tsd_t *tsd, size_t size, szind_t ind, tcache_t *tcache, arena_t *arena) -{ - - return (iallocztm(tsd, size, ind, true, tcache, false, arena, true)); -} - -JEMALLOC_ALWAYS_INLINE void * -icalloc(tsd_t *tsd, size_t size, szind_t ind) -{ - - return (iallocztm(tsd, size, ind, true, tcache_get(tsd, true), false, - NULL, true)); -} - -JEMALLOC_ALWAYS_INLINE void * -ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, +ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, tcache_t *tcache, bool is_metadata, arena_t *arena) { void *ret; assert(usize != 0); assert(usize == sa2u(usize, alignment)); + assert(!is_metadata || tcache == NULL); + assert(!is_metadata || arena == NULL || arena->ind < narenas_auto); - ret = arena_palloc(tsd, arena, usize, alignment, zero, tcache); + ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache); assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); if (config_stats && is_metadata && likely(ret != NULL)) { - arena_metadata_allocated_add(iaalloc(ret), isalloc(ret, + arena_metadata_allocated_add(iaalloc(ret), isalloc(tsdn, ret, config_prof)); } return (ret); } JEMALLOC_ALWAYS_INLINE void * -ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, +ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) { - return (ipallocztm(tsd, usize, alignment, zero, tcache, false, arena)); + return (ipallocztm(tsdn, usize, alignment, zero, tcache, false, arena)); } JEMALLOC_ALWAYS_INLINE void * ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) { - return (ipallocztm(tsd, usize, alignment, zero, tcache_get(tsd, true), - false, NULL)); + return (ipallocztm(tsd_tsdn(tsd), usize, alignment, zero, + tcache_get(tsd, true), false, NULL)); } JEMALLOC_ALWAYS_INLINE size_t -ivsalloc(const void *ptr, bool demote) +ivsalloc(tsdn_t *tsdn, const void *ptr, bool demote) { extent_node_t *node; @@ -1017,7 +1029,7 @@ ivsalloc(const void *ptr, bool demote) assert(extent_node_addr_get(node) == ptr || extent_node_achunk_get(node)); - return (isalloc(ptr, demote)); + return (isalloc(tsdn, ptr, demote)); } JEMALLOC_INLINE size_t @@ -1035,39 +1047,34 @@ u2rz(size_t usize) } JEMALLOC_INLINE size_t -p2rz(const void *ptr) +p2rz(tsdn_t *tsdn, const void *ptr) { - size_t usize = isalloc(ptr, false); + size_t usize = isalloc(tsdn, ptr, false); return (u2rz(usize)); } JEMALLOC_ALWAYS_INLINE void -idalloctm(tsd_t *tsd, void *ptr, tcache_t *tcache, bool is_metadata, +idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_metadata, bool slow_path) { assert(ptr != NULL); + assert(!is_metadata || tcache == NULL); + assert(!is_metadata || iaalloc(ptr)->ind < narenas_auto); if (config_stats && is_metadata) { - arena_metadata_allocated_sub(iaalloc(ptr), isalloc(ptr, + arena_metadata_allocated_sub(iaalloc(ptr), isalloc(tsdn, ptr, config_prof)); } - arena_dalloc(tsd, ptr, tcache, slow_path); -} - -JEMALLOC_ALWAYS_INLINE void -idalloct(tsd_t *tsd, void *ptr, tcache_t *tcache) -{ - - idalloctm(tsd, ptr, tcache, false, true); + arena_dalloc(tsdn, ptr, tcache, slow_path); } JEMALLOC_ALWAYS_INLINE void idalloc(tsd_t *tsd, void *ptr) { - idalloctm(tsd, ptr, tcache_get(tsd, false), false, true); + idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd, false), false, true); } JEMALLOC_ALWAYS_INLINE void @@ -1077,24 +1084,25 @@ iqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) if (slow_path && config_fill && unlikely(opt_quarantine)) quarantine(tsd, ptr); else - idalloctm(tsd, ptr, tcache, false, slow_path); + idalloctm(tsd_tsdn(tsd), ptr, tcache, false, slow_path); } JEMALLOC_ALWAYS_INLINE void -isdalloct(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) +isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, + bool slow_path) { - arena_sdalloc(tsd, ptr, size, tcache); + arena_sdalloc(tsdn, ptr, size, tcache, slow_path); } JEMALLOC_ALWAYS_INLINE void -isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) +isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache, bool slow_path) { - if (config_fill && unlikely(opt_quarantine)) + if (slow_path && config_fill && unlikely(opt_quarantine)) quarantine(tsd, ptr); else - isdalloct(tsd, ptr, size, tcache); + isdalloct(tsd_tsdn(tsd), ptr, size, tcache, slow_path); } JEMALLOC_ALWAYS_INLINE void * @@ -1107,7 +1115,7 @@ iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, usize = sa2u(size + extra, alignment); if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) return (NULL); - p = ipalloct(tsd, usize, alignment, zero, tcache, arena); + p = ipalloct(tsd_tsdn(tsd), usize, alignment, zero, tcache, arena); if (p == NULL) { if (extra == 0) return (NULL); @@ -1115,7 +1123,8 @@ iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, usize = sa2u(size, alignment); if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) return (NULL); - p = ipalloct(tsd, usize, alignment, zero, tcache, arena); + p = ipalloct(tsd_tsdn(tsd), usize, alignment, zero, tcache, + arena); if (p == NULL) return (NULL); } @@ -1125,7 +1134,7 @@ iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(p, ptr, copysize); - isqalloc(tsd, ptr, oldsize, tcache); + isqalloc(tsd, ptr, oldsize, tcache, true); return (p); } @@ -1161,7 +1170,7 @@ iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, } JEMALLOC_ALWAYS_INLINE bool -ixalloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t extra, +ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero) { @@ -1174,7 +1183,7 @@ ixalloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t extra, return (true); } - return (arena_ralloc_no_move(tsd, ptr, oldsize, size, extra, zero)); + return (arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero)); } #endif diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h b/include/jemalloc/internal/jemalloc_internal_defs.h index c746c92..dc63ffd 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h +++ b/include/jemalloc/internal/jemalloc_internal_defs.h @@ -220,6 +220,15 @@ /* #undef JEMALLOC_ZONE */ /* #undef JEMALLOC_ZONE_VERSION */ +/* + * Methods for determining whether the OS overcommits. + * JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY: Linux's + * /proc/sys/vm.overcommit_memory file. + * JEMALLOC_SYSCTL_VM_OVERCOMMIT: FreeBSD's vm.overcommit sysctl. + */ +/* #undef JEMALLOC_SYSCTL_VM_OVERCOMMIT */ +#define JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY + /* * Methods for purging unused pages differ between operating systems. * diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h index e1cb05b..271cbf4 100644 --- a/include/jemalloc/internal/private_namespace.h +++ b/include/jemalloc/internal/private_namespace.h @@ -5,10 +5,12 @@ #define arena_basic_stats_merge JEMALLOC_N(arena_basic_stats_merge) #define arena_bin_index JEMALLOC_N(arena_bin_index) #define arena_bin_info JEMALLOC_N(arena_bin_info) -#define arena_bitselm_get JEMALLOC_N(arena_bitselm_get) +#define arena_bitselm_get_const JEMALLOC_N(arena_bitselm_get_const) +#define arena_bitselm_get_mutable JEMALLOC_N(arena_bitselm_get_mutable) #define arena_boot JEMALLOC_N(arena_boot) #define arena_choose JEMALLOC_N(arena_choose) #define arena_choose_hard JEMALLOC_N(arena_choose_hard) +#define arena_choose_impl JEMALLOC_N(arena_choose_impl) #define arena_chunk_alloc_huge JEMALLOC_N(arena_chunk_alloc_huge) #define arena_chunk_cache_maybe_insert JEMALLOC_N(arena_chunk_cache_maybe_insert) #define arena_chunk_cache_maybe_remove JEMALLOC_N(arena_chunk_cache_maybe_remove) @@ -34,6 +36,7 @@ #define arena_dss_prec_get JEMALLOC_N(arena_dss_prec_get) #define arena_dss_prec_set JEMALLOC_N(arena_dss_prec_set) #define arena_get JEMALLOC_N(arena_get) +#define arena_ichoose JEMALLOC_N(arena_ichoose) #define arena_init JEMALLOC_N(arena_init) #define arena_lg_dirty_mult_default_get JEMALLOC_N(arena_lg_dirty_mult_default_get) #define arena_lg_dirty_mult_default_set JEMALLOC_N(arena_lg_dirty_mult_default_set) @@ -60,7 +63,8 @@ #define arena_mapbits_unallocated_size_get JEMALLOC_N(arena_mapbits_unallocated_size_get) #define arena_mapbits_unallocated_size_set JEMALLOC_N(arena_mapbits_unallocated_size_set) #define arena_mapbits_unzeroed_get JEMALLOC_N(arena_mapbits_unzeroed_get) -#define arena_mapbitsp_get JEMALLOC_N(arena_mapbitsp_get) +#define arena_mapbitsp_get_const JEMALLOC_N(arena_mapbitsp_get_const) +#define arena_mapbitsp_get_mutable JEMALLOC_N(arena_mapbitsp_get_mutable) #define arena_mapbitsp_read JEMALLOC_N(arena_mapbitsp_read) #define arena_mapbitsp_write JEMALLOC_N(arena_mapbitsp_write) #define arena_maxrun JEMALLOC_N(arena_maxrun) @@ -69,7 +73,8 @@ #define arena_metadata_allocated_get JEMALLOC_N(arena_metadata_allocated_get) #define arena_metadata_allocated_sub JEMALLOC_N(arena_metadata_allocated_sub) #define arena_migrate JEMALLOC_N(arena_migrate) -#define arena_miscelm_get JEMALLOC_N(arena_miscelm_get) +#define arena_miscelm_get_const JEMALLOC_N(arena_miscelm_get_const) +#define arena_miscelm_get_mutable JEMALLOC_N(arena_miscelm_get_mutable) #define arena_miscelm_to_pageind JEMALLOC_N(arena_miscelm_to_pageind) #define arena_miscelm_to_rpages JEMALLOC_N(arena_miscelm_to_rpages) #define arena_new JEMALLOC_N(arena_new) @@ -81,7 +86,10 @@ #define arena_palloc JEMALLOC_N(arena_palloc) #define arena_postfork_child JEMALLOC_N(arena_postfork_child) #define arena_postfork_parent JEMALLOC_N(arena_postfork_parent) -#define arena_prefork JEMALLOC_N(arena_prefork) +#define arena_prefork0 JEMALLOC_N(arena_prefork0) +#define arena_prefork1 JEMALLOC_N(arena_prefork1) +#define arena_prefork2 JEMALLOC_N(arena_prefork2) +#define arena_prefork3 JEMALLOC_N(arena_prefork3) #define arena_prof_accum JEMALLOC_N(arena_prof_accum) #define arena_prof_accum_impl JEMALLOC_N(arena_prof_accum_impl) #define arena_prof_accum_locked JEMALLOC_N(arena_prof_accum_locked) @@ -97,6 +105,7 @@ #define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) #define arena_rd_to_miscelm JEMALLOC_N(arena_rd_to_miscelm) #define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) +#define arena_reset JEMALLOC_N(arena_reset) #define arena_run_regind JEMALLOC_N(arena_run_regind) #define arena_run_to_miscelm JEMALLOC_N(arena_run_to_miscelm) #define arena_salloc JEMALLOC_N(arena_salloc) @@ -123,6 +132,11 @@ #define atomic_sub_uint32 JEMALLOC_N(atomic_sub_uint32) #define atomic_sub_uint64 JEMALLOC_N(atomic_sub_uint64) #define atomic_sub_z JEMALLOC_N(atomic_sub_z) +#define atomic_write_p JEMALLOC_N(atomic_write_p) +#define atomic_write_u JEMALLOC_N(atomic_write_u) +#define atomic_write_uint32 JEMALLOC_N(atomic_write_uint32) +#define atomic_write_uint64 JEMALLOC_N(atomic_write_uint64) +#define atomic_write_z JEMALLOC_N(atomic_write_z) #define base_alloc JEMALLOC_N(base_alloc) #define base_boot JEMALLOC_N(base_boot) #define base_postfork_child JEMALLOC_N(base_postfork_child) @@ -148,7 +162,6 @@ #define chunk_alloc_mmap JEMALLOC_N(chunk_alloc_mmap) #define chunk_alloc_wrapper JEMALLOC_N(chunk_alloc_wrapper) #define chunk_boot JEMALLOC_N(chunk_boot) -#define chunk_dalloc_arena JEMALLOC_N(chunk_dalloc_arena) #define chunk_dalloc_cache JEMALLOC_N(chunk_dalloc_cache) #define chunk_dalloc_mmap JEMALLOC_N(chunk_dalloc_mmap) #define chunk_dalloc_wrapper JEMALLOC_N(chunk_dalloc_wrapper) @@ -168,7 +181,6 @@ #define chunk_postfork_child JEMALLOC_N(chunk_postfork_child) #define chunk_postfork_parent JEMALLOC_N(chunk_postfork_parent) #define chunk_prefork JEMALLOC_N(chunk_prefork) -#define chunk_purge_arena JEMALLOC_N(chunk_purge_arena) #define chunk_purge_wrapper JEMALLOC_N(chunk_purge_wrapper) #define chunk_register JEMALLOC_N(chunk_register) #define chunks_rtree JEMALLOC_N(chunks_rtree) @@ -200,6 +212,8 @@ #define extent_node_addr_set JEMALLOC_N(extent_node_addr_set) #define extent_node_arena_get JEMALLOC_N(extent_node_arena_get) #define extent_node_arena_set JEMALLOC_N(extent_node_arena_set) +#define extent_node_committed_get JEMALLOC_N(extent_node_committed_get) +#define extent_node_committed_set JEMALLOC_N(extent_node_committed_set) #define extent_node_dirty_insert JEMALLOC_N(extent_node_dirty_insert) #define extent_node_dirty_linkage_init JEMALLOC_N(extent_node_dirty_linkage_init) #define extent_node_dirty_remove JEMALLOC_N(extent_node_dirty_remove) @@ -210,6 +224,8 @@ #define extent_node_size_set JEMALLOC_N(extent_node_size_set) #define extent_node_zeroed_get JEMALLOC_N(extent_node_zeroed_get) #define extent_node_zeroed_set JEMALLOC_N(extent_node_zeroed_set) +#define extent_tree_ad_destroy JEMALLOC_N(extent_tree_ad_destroy) +#define extent_tree_ad_destroy_recurse JEMALLOC_N(extent_tree_ad_destroy_recurse) #define extent_tree_ad_empty JEMALLOC_N(extent_tree_ad_empty) #define extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first) #define extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert) @@ -227,6 +243,8 @@ #define extent_tree_ad_reverse_iter_recurse JEMALLOC_N(extent_tree_ad_reverse_iter_recurse) #define extent_tree_ad_reverse_iter_start JEMALLOC_N(extent_tree_ad_reverse_iter_start) #define extent_tree_ad_search JEMALLOC_N(extent_tree_ad_search) +#define extent_tree_szad_destroy JEMALLOC_N(extent_tree_szad_destroy) +#define extent_tree_szad_destroy_recurse JEMALLOC_N(extent_tree_szad_destroy_recurse) #define extent_tree_szad_empty JEMALLOC_N(extent_tree_szad_empty) #define extent_tree_szad_first JEMALLOC_N(extent_tree_szad_first) #define extent_tree_szad_insert JEMALLOC_N(extent_tree_szad_insert) @@ -273,14 +291,11 @@ #define huge_ralloc_no_move JEMALLOC_N(huge_ralloc_no_move) #define huge_salloc JEMALLOC_N(huge_salloc) #define iaalloc JEMALLOC_N(iaalloc) +#define ialloc JEMALLOC_N(ialloc) #define iallocztm JEMALLOC_N(iallocztm) -#define icalloc JEMALLOC_N(icalloc) -#define icalloct JEMALLOC_N(icalloct) +#define iarena_cleanup JEMALLOC_N(iarena_cleanup) #define idalloc JEMALLOC_N(idalloc) -#define idalloct JEMALLOC_N(idalloct) #define idalloctm JEMALLOC_N(idalloctm) -#define imalloc JEMALLOC_N(imalloc) -#define imalloct JEMALLOC_N(imalloct) #define in_valgrind JEMALLOC_N(in_valgrind) #define index2size JEMALLOC_N(index2size) #define index2size_compute JEMALLOC_N(index2size_compute) @@ -304,7 +319,11 @@ #define jemalloc_prefork JEMALLOC_N(jemalloc_prefork) #define large_maxclass JEMALLOC_N(large_maxclass) #define lg_floor JEMALLOC_N(lg_floor) +#define lg_prof_sample JEMALLOC_N(lg_prof_sample) #define malloc_cprintf JEMALLOC_N(malloc_cprintf) +#define malloc_mutex_assert_not_owner JEMALLOC_N(malloc_mutex_assert_not_owner) +#define malloc_mutex_assert_owner JEMALLOC_N(malloc_mutex_assert_owner) +#define malloc_mutex_boot JEMALLOC_N(malloc_mutex_boot) #define malloc_mutex_init JEMALLOC_N(malloc_mutex_init) #define malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock) #define malloc_mutex_postfork_child JEMALLOC_N(malloc_mutex_postfork_child) @@ -326,11 +345,13 @@ #define map_bias JEMALLOC_N(map_bias) #define map_misc_offset JEMALLOC_N(map_misc_offset) #define mb_write JEMALLOC_N(mb_write) -#define mutex_boot JEMALLOC_N(mutex_boot) +#define narenas_auto JEMALLOC_N(narenas_auto) #define narenas_tdata_cleanup JEMALLOC_N(narenas_tdata_cleanup) #define narenas_total_get JEMALLOC_N(narenas_total_get) #define ncpus JEMALLOC_N(ncpus) #define nhbins JEMALLOC_N(nhbins) +#define nhclasses JEMALLOC_N(nhclasses) +#define nlclasses JEMALLOC_N(nlclasses) #define nstime_add JEMALLOC_N(nstime_add) #define nstime_compare JEMALLOC_N(nstime_compare) #define nstime_copy JEMALLOC_N(nstime_copy) @@ -373,6 +394,7 @@ #define opt_xmalloc JEMALLOC_N(opt_xmalloc) #define opt_zero JEMALLOC_N(opt_zero) #define p2rz JEMALLOC_N(p2rz) +#define pages_boot JEMALLOC_N(pages_boot) #define pages_commit JEMALLOC_N(pages_commit) #define pages_decommit JEMALLOC_N(pages_decommit) #define pages_map JEMALLOC_N(pages_map) @@ -384,6 +406,7 @@ #define pow2_ceil_zu JEMALLOC_N(pow2_ceil_zu) #define prng_lg_range JEMALLOC_N(prng_lg_range) #define prng_range JEMALLOC_N(prng_range) +#define prof_active JEMALLOC_N(prof_active) #define prof_active_get JEMALLOC_N(prof_active_get) #define prof_active_get_unlocked JEMALLOC_N(prof_active_get_unlocked) #define prof_active_set JEMALLOC_N(prof_active_set) @@ -393,6 +416,7 @@ #define prof_boot0 JEMALLOC_N(prof_boot0) #define prof_boot1 JEMALLOC_N(prof_boot1) #define prof_boot2 JEMALLOC_N(prof_boot2) +#define prof_bt_count JEMALLOC_N(prof_bt_count) #define prof_dump_header JEMALLOC_N(prof_dump_header) #define prof_dump_open JEMALLOC_N(prof_dump_open) #define prof_free JEMALLOC_N(prof_free) @@ -410,7 +434,8 @@ #define prof_mdump JEMALLOC_N(prof_mdump) #define prof_postfork_child JEMALLOC_N(prof_postfork_child) #define prof_postfork_parent JEMALLOC_N(prof_postfork_parent) -#define prof_prefork JEMALLOC_N(prof_prefork) +#define prof_prefork0 JEMALLOC_N(prof_prefork0) +#define prof_prefork1 JEMALLOC_N(prof_prefork1) #define prof_realloc JEMALLOC_N(prof_realloc) #define prof_reset JEMALLOC_N(prof_reset) #define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update) @@ -419,6 +444,7 @@ #define prof_tctx_reset JEMALLOC_N(prof_tctx_reset) #define prof_tctx_set JEMALLOC_N(prof_tctx_set) #define prof_tdata_cleanup JEMALLOC_N(prof_tdata_cleanup) +#define prof_tdata_count JEMALLOC_N(prof_tdata_count) #define prof_tdata_get JEMALLOC_N(prof_tdata_get) #define prof_tdata_init JEMALLOC_N(prof_tdata_init) #define prof_tdata_reinit JEMALLOC_N(prof_tdata_reinit) @@ -470,8 +496,6 @@ #define tcache_alloc_large JEMALLOC_N(tcache_alloc_large) #define tcache_alloc_small JEMALLOC_N(tcache_alloc_small) #define tcache_alloc_small_hard JEMALLOC_N(tcache_alloc_small_hard) -#define tcache_arena_associate JEMALLOC_N(tcache_arena_associate) -#define tcache_arena_dissociate JEMALLOC_N(tcache_arena_dissociate) #define tcache_arena_reassociate JEMALLOC_N(tcache_arena_reassociate) #define tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large) #define tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small) @@ -506,38 +530,83 @@ #define ticker_ticks JEMALLOC_N(ticker_ticks) #define tsd_arena_get JEMALLOC_N(tsd_arena_get) #define tsd_arena_set JEMALLOC_N(tsd_arena_set) +#define tsd_arenap_get JEMALLOC_N(tsd_arenap_get) +#define tsd_arenas_tdata_bypass_get JEMALLOC_N(tsd_arenas_tdata_bypass_get) +#define tsd_arenas_tdata_bypass_set JEMALLOC_N(tsd_arenas_tdata_bypass_set) +#define tsd_arenas_tdata_bypassp_get JEMALLOC_N(tsd_arenas_tdata_bypassp_get) +#define tsd_arenas_tdata_get JEMALLOC_N(tsd_arenas_tdata_get) +#define tsd_arenas_tdata_set JEMALLOC_N(tsd_arenas_tdata_set) +#define tsd_arenas_tdatap_get JEMALLOC_N(tsd_arenas_tdatap_get) #define tsd_boot JEMALLOC_N(tsd_boot) #define tsd_boot0 JEMALLOC_N(tsd_boot0) #define tsd_boot1 JEMALLOC_N(tsd_boot1) #define tsd_booted JEMALLOC_N(tsd_booted) +#define tsd_booted_get JEMALLOC_N(tsd_booted_get) #define tsd_cleanup JEMALLOC_N(tsd_cleanup) #define tsd_cleanup_wrapper JEMALLOC_N(tsd_cleanup_wrapper) #define tsd_fetch JEMALLOC_N(tsd_fetch) #define tsd_get JEMALLOC_N(tsd_get) -#define tsd_wrapper_get JEMALLOC_N(tsd_wrapper_get) -#define tsd_wrapper_set JEMALLOC_N(tsd_wrapper_set) +#define tsd_iarena_get JEMALLOC_N(tsd_iarena_get) +#define tsd_iarena_set JEMALLOC_N(tsd_iarena_set) +#define tsd_iarenap_get JEMALLOC_N(tsd_iarenap_get) #define tsd_initialized JEMALLOC_N(tsd_initialized) #define tsd_init_check_recursion JEMALLOC_N(tsd_init_check_recursion) #define tsd_init_finish JEMALLOC_N(tsd_init_finish) #define tsd_init_head JEMALLOC_N(tsd_init_head) +#define tsd_narenas_tdata_get JEMALLOC_N(tsd_narenas_tdata_get) +#define tsd_narenas_tdata_set JEMALLOC_N(tsd_narenas_tdata_set) +#define tsd_narenas_tdatap_get JEMALLOC_N(tsd_narenas_tdatap_get) +#define tsd_wrapper_get JEMALLOC_N(tsd_wrapper_get) +#define tsd_wrapper_set JEMALLOC_N(tsd_wrapper_set) #define tsd_nominal JEMALLOC_N(tsd_nominal) #define tsd_prof_tdata_get JEMALLOC_N(tsd_prof_tdata_get) #define tsd_prof_tdata_set JEMALLOC_N(tsd_prof_tdata_set) +#define tsd_prof_tdatap_get JEMALLOC_N(tsd_prof_tdatap_get) #define tsd_quarantine_get JEMALLOC_N(tsd_quarantine_get) #define tsd_quarantine_set JEMALLOC_N(tsd_quarantine_set) +#define tsd_quarantinep_get JEMALLOC_N(tsd_quarantinep_get) #define tsd_set JEMALLOC_N(tsd_set) #define tsd_tcache_enabled_get JEMALLOC_N(tsd_tcache_enabled_get) #define tsd_tcache_enabled_set JEMALLOC_N(tsd_tcache_enabled_set) +#define tsd_tcache_enabledp_get JEMALLOC_N(tsd_tcache_enabledp_get) #define tsd_tcache_get JEMALLOC_N(tsd_tcache_get) #define tsd_tcache_set JEMALLOC_N(tsd_tcache_set) +#define tsd_tcachep_get JEMALLOC_N(tsd_tcachep_get) #define tsd_thread_allocated_get JEMALLOC_N(tsd_thread_allocated_get) #define tsd_thread_allocated_set JEMALLOC_N(tsd_thread_allocated_set) +#define tsd_thread_allocatedp_get JEMALLOC_N(tsd_thread_allocatedp_get) #define tsd_thread_deallocated_get JEMALLOC_N(tsd_thread_deallocated_get) #define tsd_thread_deallocated_set JEMALLOC_N(tsd_thread_deallocated_set) +#define tsd_thread_deallocatedp_get JEMALLOC_N(tsd_thread_deallocatedp_get) #define tsd_tls JEMALLOC_N(tsd_tls) #define tsd_tsd JEMALLOC_N(tsd_tsd) +#define tsd_tsdn JEMALLOC_N(tsd_tsdn) +#define tsd_witness_fork_get JEMALLOC_N(tsd_witness_fork_get) +#define tsd_witness_fork_set JEMALLOC_N(tsd_witness_fork_set) +#define tsd_witness_forkp_get JEMALLOC_N(tsd_witness_forkp_get) +#define tsd_witnesses_get JEMALLOC_N(tsd_witnesses_get) +#define tsd_witnesses_set JEMALLOC_N(tsd_witnesses_set) +#define tsd_witnessesp_get JEMALLOC_N(tsd_witnessesp_get) +#define tsdn_fetch JEMALLOC_N(tsdn_fetch) +#define tsdn_null JEMALLOC_N(tsdn_null) +#define tsdn_tsd JEMALLOC_N(tsdn_tsd) #define u2rz JEMALLOC_N(u2rz) #define valgrind_freelike_block JEMALLOC_N(valgrind_freelike_block) #define valgrind_make_mem_defined JEMALLOC_N(valgrind_make_mem_defined) #define valgrind_make_mem_noaccess JEMALLOC_N(valgrind_make_mem_noaccess) #define valgrind_make_mem_undefined JEMALLOC_N(valgrind_make_mem_undefined) +#define witness_assert_lockless JEMALLOC_N(witness_assert_lockless) +#define witness_assert_not_owner JEMALLOC_N(witness_assert_not_owner) +#define witness_assert_owner JEMALLOC_N(witness_assert_owner) +#define witness_fork_cleanup JEMALLOC_N(witness_fork_cleanup) +#define witness_init JEMALLOC_N(witness_init) +#define witness_lock JEMALLOC_N(witness_lock) +#define witness_lock_error JEMALLOC_N(witness_lock_error) +#define witness_lockless_error JEMALLOC_N(witness_lockless_error) +#define witness_not_owner_error JEMALLOC_N(witness_not_owner_error) +#define witness_owner_error JEMALLOC_N(witness_owner_error) +#define witness_postfork_child JEMALLOC_N(witness_postfork_child) +#define witness_postfork_parent JEMALLOC_N(witness_postfork_parent) +#define witness_prefork JEMALLOC_N(witness_prefork) +#define witness_unlock JEMALLOC_N(witness_unlock) +#define witnesses_cleanup JEMALLOC_N(witnesses_cleanup) diff --git a/include/jemalloc/internal/private_unnamespace.h b/include/jemalloc/internal/private_unnamespace.h index e620fc7..b5a79bd 100644 --- a/include/jemalloc/internal/private_unnamespace.h +++ b/include/jemalloc/internal/private_unnamespace.h @@ -5,10 +5,12 @@ #undef arena_basic_stats_merge #undef arena_bin_index #undef arena_bin_info -#undef arena_bitselm_get +#undef arena_bitselm_get_const +#undef arena_bitselm_get_mutable #undef arena_boot #undef arena_choose #undef arena_choose_hard +#undef arena_choose_impl #undef arena_chunk_alloc_huge #undef arena_chunk_cache_maybe_insert #undef arena_chunk_cache_maybe_remove @@ -34,6 +36,7 @@ #undef arena_dss_prec_get #undef arena_dss_prec_set #undef arena_get +#undef arena_ichoose #undef arena_init #undef arena_lg_dirty_mult_default_get #undef arena_lg_dirty_mult_default_set @@ -60,7 +63,8 @@ #undef arena_mapbits_unallocated_size_get #undef arena_mapbits_unallocated_size_set #undef arena_mapbits_unzeroed_get -#undef arena_mapbitsp_get +#undef arena_mapbitsp_get_const +#undef arena_mapbitsp_get_mutable #undef arena_mapbitsp_read #undef arena_mapbitsp_write #undef arena_maxrun @@ -69,7 +73,8 @@ #undef arena_metadata_allocated_get #undef arena_metadata_allocated_sub #undef arena_migrate -#undef arena_miscelm_get +#undef arena_miscelm_get_const +#undef arena_miscelm_get_mutable #undef arena_miscelm_to_pageind #undef arena_miscelm_to_rpages #undef arena_new @@ -81,7 +86,10 @@ #undef arena_palloc #undef arena_postfork_child #undef arena_postfork_parent -#undef arena_prefork +#undef arena_prefork0 +#undef arena_prefork1 +#undef arena_prefork2 +#undef arena_prefork3 #undef arena_prof_accum #undef arena_prof_accum_impl #undef arena_prof_accum_locked @@ -97,6 +105,7 @@ #undef arena_ralloc_no_move #undef arena_rd_to_miscelm #undef arena_redzone_corruption +#undef arena_reset #undef arena_run_regind #undef arena_run_to_miscelm #undef arena_salloc @@ -123,6 +132,11 @@ #undef atomic_sub_uint32 #undef atomic_sub_uint64 #undef atomic_sub_z +#undef atomic_write_p +#undef atomic_write_u +#undef atomic_write_uint32 +#undef atomic_write_uint64 +#undef atomic_write_z #undef base_alloc #undef base_boot #undef base_postfork_child @@ -148,7 +162,6 @@ #undef chunk_alloc_mmap #undef chunk_alloc_wrapper #undef chunk_boot -#undef chunk_dalloc_arena #undef chunk_dalloc_cache #undef chunk_dalloc_mmap #undef chunk_dalloc_wrapper @@ -168,7 +181,6 @@ #undef chunk_postfork_child #undef chunk_postfork_parent #undef chunk_prefork -#undef chunk_purge_arena #undef chunk_purge_wrapper #undef chunk_register #undef chunks_rtree @@ -200,6 +212,8 @@ #undef extent_node_addr_set #undef extent_node_arena_get #undef extent_node_arena_set +#undef extent_node_committed_get +#undef extent_node_committed_set #undef extent_node_dirty_insert #undef extent_node_dirty_linkage_init #undef extent_node_dirty_remove @@ -210,6 +224,8 @@ #undef extent_node_size_set #undef extent_node_zeroed_get #undef extent_node_zeroed_set +#undef extent_tree_ad_destroy +#undef extent_tree_ad_destroy_recurse #undef extent_tree_ad_empty #undef extent_tree_ad_first #undef extent_tree_ad_insert @@ -227,6 +243,8 @@ #undef extent_tree_ad_reverse_iter_recurse #undef extent_tree_ad_reverse_iter_start #undef extent_tree_ad_search +#undef extent_tree_szad_destroy +#undef extent_tree_szad_destroy_recurse #undef extent_tree_szad_empty #undef extent_tree_szad_first #undef extent_tree_szad_insert @@ -273,14 +291,11 @@ #undef huge_ralloc_no_move #undef huge_salloc #undef iaalloc +#undef ialloc #undef iallocztm -#undef icalloc -#undef icalloct +#undef iarena_cleanup #undef idalloc -#undef idalloct #undef idalloctm -#undef imalloc -#undef imalloct #undef in_valgrind #undef index2size #undef index2size_compute @@ -304,7 +319,11 @@ #undef jemalloc_prefork #undef large_maxclass #undef lg_floor +#undef lg_prof_sample #undef malloc_cprintf +#undef malloc_mutex_assert_not_owner +#undef malloc_mutex_assert_owner +#undef malloc_mutex_boot #undef malloc_mutex_init #undef malloc_mutex_lock #undef malloc_mutex_postfork_child @@ -326,11 +345,13 @@ #undef map_bias #undef map_misc_offset #undef mb_write -#undef mutex_boot +#undef narenas_auto #undef narenas_tdata_cleanup #undef narenas_total_get #undef ncpus #undef nhbins +#undef nhclasses +#undef nlclasses #undef nstime_add #undef nstime_compare #undef nstime_copy @@ -373,6 +394,7 @@ #undef opt_xmalloc #undef opt_zero #undef p2rz +#undef pages_boot #undef pages_commit #undef pages_decommit #undef pages_map @@ -384,6 +406,7 @@ #undef pow2_ceil_zu #undef prng_lg_range #undef prng_range +#undef prof_active #undef prof_active_get #undef prof_active_get_unlocked #undef prof_active_set @@ -393,6 +416,7 @@ #undef prof_boot0 #undef prof_boot1 #undef prof_boot2 +#undef prof_bt_count #undef prof_dump_header #undef prof_dump_open #undef prof_free @@ -410,7 +434,8 @@ #undef prof_mdump #undef prof_postfork_child #undef prof_postfork_parent -#undef prof_prefork +#undef prof_prefork0 +#undef prof_prefork1 #undef prof_realloc #undef prof_reset #undef prof_sample_accum_update @@ -419,6 +444,7 @@ #undef prof_tctx_reset #undef prof_tctx_set #undef prof_tdata_cleanup +#undef prof_tdata_count #undef prof_tdata_get #undef prof_tdata_init #undef prof_tdata_reinit @@ -470,8 +496,6 @@ #undef tcache_alloc_large #undef tcache_alloc_small #undef tcache_alloc_small_hard -#undef tcache_arena_associate -#undef tcache_arena_dissociate #undef tcache_arena_reassociate #undef tcache_bin_flush_large #undef tcache_bin_flush_small @@ -506,38 +530,83 @@ #undef ticker_ticks #undef tsd_arena_get #undef tsd_arena_set +#undef tsd_arenap_get +#undef tsd_arenas_tdata_bypass_get +#undef tsd_arenas_tdata_bypass_set +#undef tsd_arenas_tdata_bypassp_get +#undef tsd_arenas_tdata_get +#undef tsd_arenas_tdata_set +#undef tsd_arenas_tdatap_get #undef tsd_boot #undef tsd_boot0 #undef tsd_boot1 #undef tsd_booted +#undef tsd_booted_get #undef tsd_cleanup #undef tsd_cleanup_wrapper #undef tsd_fetch #undef tsd_get -#undef tsd_wrapper_get -#undef tsd_wrapper_set +#undef tsd_iarena_get +#undef tsd_iarena_set +#undef tsd_iarenap_get #undef tsd_initialized #undef tsd_init_check_recursion #undef tsd_init_finish #undef tsd_init_head +#undef tsd_narenas_tdata_get +#undef tsd_narenas_tdata_set +#undef tsd_narenas_tdatap_get +#undef tsd_wrapper_get +#undef tsd_wrapper_set #undef tsd_nominal #undef tsd_prof_tdata_get #undef tsd_prof_tdata_set +#undef tsd_prof_tdatap_get #undef tsd_quarantine_get #undef tsd_quarantine_set +#undef tsd_quarantinep_get #undef tsd_set #undef tsd_tcache_enabled_get #undef tsd_tcache_enabled_set +#undef tsd_tcache_enabledp_get #undef tsd_tcache_get #undef tsd_tcache_set +#undef tsd_tcachep_get #undef tsd_thread_allocated_get #undef tsd_thread_allocated_set +#undef tsd_thread_allocatedp_get #undef tsd_thread_deallocated_get #undef tsd_thread_deallocated_set +#undef tsd_thread_deallocatedp_get #undef tsd_tls #undef tsd_tsd +#undef tsd_tsdn +#undef tsd_witness_fork_get +#undef tsd_witness_fork_set +#undef tsd_witness_forkp_get +#undef tsd_witnesses_get +#undef tsd_witnesses_set +#undef tsd_witnessesp_get +#undef tsdn_fetch +#undef tsdn_null +#undef tsdn_tsd #undef u2rz #undef valgrind_freelike_block #undef valgrind_make_mem_defined #undef valgrind_make_mem_noaccess #undef valgrind_make_mem_undefined +#undef witness_assert_lockless +#undef witness_assert_not_owner +#undef witness_assert_owner +#undef witness_fork_cleanup +#undef witness_init +#undef witness_lock +#undef witness_lock_error +#undef witness_lockless_error +#undef witness_not_owner_error +#undef witness_owner_error +#undef witness_postfork_child +#undef witness_postfork_parent +#undef witness_prefork +#undef witness_unlock +#undef witnesses_cleanup diff --git a/include/jemalloc/jemalloc.h b/include/jemalloc/jemalloc.h index bfc95ec..5ed13cb 100644 --- a/include/jemalloc/jemalloc.h +++ b/include/jemalloc/jemalloc.h @@ -94,20 +94,20 @@ extern "C" { #include #include -#define JEMALLOC_VERSION "4.1.0-4-g33184bf69813087bf1885b0993685f9d03320c69" +#define JEMALLOC_VERSION "4.2.1-0-g3de035335255d553bdb344c32ffdb603816195d8" #define JEMALLOC_VERSION_MAJOR 4 -#define JEMALLOC_VERSION_MINOR 1 -#define JEMALLOC_VERSION_BUGFIX 0 -#define JEMALLOC_VERSION_NREV 4 -#define JEMALLOC_VERSION_GID "33184bf69813087bf1885b0993685f9d03320c69" +#define JEMALLOC_VERSION_MINOR 2 +#define JEMALLOC_VERSION_BUGFIX 1 +#define JEMALLOC_VERSION_NREV 0 +#define JEMALLOC_VERSION_GID "3de035335255d553bdb344c32ffdb603816195d8" # define MALLOCX_LG_ALIGN(la) ((int)(la)) # if LG_SIZEOF_PTR == 2 -# define MALLOCX_ALIGN(a) ((int)(ffs(a)-1)) +# define MALLOCX_ALIGN(a) ((int)(ffs((int)(a))-1)) # else # define MALLOCX_ALIGN(a) \ - ((int)(((a) < (size_t)INT_MAX) ? ffs((int)(a))-1 : \ - ffs((int)((a)>>32))+31)) + ((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 : \ + ffs((int)(((size_t)(a))>>32))+31)) # endif # define MALLOCX_ZERO ((int)0x40) /* @@ -119,7 +119,7 @@ extern "C" { /* * Bias arena index bits so that 0 encodes "use an automatically chosen arena". */ -# define MALLOCX_ARENA(a) ((int)(((a)+1) << 20)) +# define MALLOCX_ARENA(a) ((((int)(a))+1) << 20) #if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) # define JEMALLOC_CXX_THROW throw() diff --git a/include/jemalloc/jemalloc_macros.h b/include/jemalloc/jemalloc_macros.h index d4a3794..04d143f 100644 --- a/include/jemalloc/jemalloc_macros.h +++ b/include/jemalloc/jemalloc_macros.h @@ -4,20 +4,20 @@ #include #include -#define JEMALLOC_VERSION "4.1.0-4-g33184bf69813087bf1885b0993685f9d03320c69" +#define JEMALLOC_VERSION "4.2.1-0-g3de035335255d553bdb344c32ffdb603816195d8" #define JEMALLOC_VERSION_MAJOR 4 -#define JEMALLOC_VERSION_MINOR 1 -#define JEMALLOC_VERSION_BUGFIX 0 -#define JEMALLOC_VERSION_NREV 4 -#define JEMALLOC_VERSION_GID "33184bf69813087bf1885b0993685f9d03320c69" +#define JEMALLOC_VERSION_MINOR 2 +#define JEMALLOC_VERSION_BUGFIX 1 +#define JEMALLOC_VERSION_NREV 0 +#define JEMALLOC_VERSION_GID "3de035335255d553bdb344c32ffdb603816195d8" # define MALLOCX_LG_ALIGN(la) ((int)(la)) # if LG_SIZEOF_PTR == 2 -# define MALLOCX_ALIGN(a) ((int)(ffs(a)-1)) +# define MALLOCX_ALIGN(a) ((int)(ffs((int)(a))-1)) # else # define MALLOCX_ALIGN(a) \ - ((int)(((a) < (size_t)INT_MAX) ? ffs((int)(a))-1 : \ - ffs((int)((a)>>32))+31)) + ((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 : \ + ffs((int)(((size_t)(a))>>32))+31)) # endif # define MALLOCX_ZERO ((int)0x40) /* @@ -29,7 +29,7 @@ /* * Bias arena index bits so that 0 encodes "use an automatically chosen arena". */ -# define MALLOCX_ARENA(a) ((int)(((a)+1) << 20)) +# define MALLOCX_ARENA(a) ((((int)(a))+1) << 20) #if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) # define JEMALLOC_CXX_THROW throw() diff --git a/src/android_je_iterate.c b/src/android_je_iterate.c index 3fbbdc9..7aa4099 100644 --- a/src/android_je_iterate.c +++ b/src/android_je_iterate.c @@ -101,7 +101,7 @@ static void je_iterate_chunk(arena_chunk_t *chunk, void *rpages; size = arena_mapbits_large_size_get(chunk, pageind); - rpages = arena_miscelm_to_rpages(arena_miscelm_get(chunk, pageind)); + rpages = arena_miscelm_to_rpages(arena_miscelm_get_mutable(chunk, pageind)); callback((uintptr_t)rpages, size, arg); } else { /* Run of small allocations */ @@ -110,7 +110,7 @@ static void je_iterate_chunk(arena_chunk_t *chunk, assert(arena_mapbits_small_runind_get(chunk, pageind) == pageind); binind = arena_mapbits_binind_get(chunk, pageind); - run = &arena_miscelm_get(chunk, pageind)->run; + run = &arena_miscelm_get_mutable(chunk, pageind)->run; assert(run->binind == binind); size = arena_bin_info[binind].run_size; diff --git a/src/android_je_mallinfo.c b/src/android_je_mallinfo.c index 1637edd..802a3eb 100644 --- a/src/android_je_mallinfo.c +++ b/src/android_je_mallinfo.c @@ -21,25 +21,25 @@ struct mallinfo je_mallinfo() { struct mallinfo mi; memset(&mi, 0, sizeof(mi)); - malloc_mutex_lock(&arenas_lock); + malloc_mutex_lock(TSDN_NULL, &arenas_lock); for (unsigned i = 0; i < narenas_auto; i++) { if (arenas[i] != NULL) { - malloc_mutex_lock(&arenas[i]->lock); + malloc_mutex_lock(TSDN_NULL, &arenas[i]->lock); mi.hblkhd += arenas[i]->stats.mapped; mi.uordblks += arenas[i]->stats.allocated_large; mi.uordblks += arenas[i]->stats.allocated_huge; - malloc_mutex_unlock(&arenas[i]->lock); + malloc_mutex_unlock(TSDN_NULL, &arenas[i]->lock); for (unsigned j = 0; j < NBINS; j++) { arena_bin_t* bin = &arenas[i]->bins[j]; - malloc_mutex_lock(&bin->lock); + malloc_mutex_lock(TSDN_NULL, &bin->lock); mi.uordblks += arena_bin_info[j].reg_size * bin->stats.curregs; - malloc_mutex_unlock(&bin->lock); + malloc_mutex_unlock(TSDN_NULL, &bin->lock); } } } - malloc_mutex_unlock(&arenas_lock); + malloc_mutex_unlock(TSDN_NULL, &arenas_lock); mi.fordblks = mi.hblkhd - mi.uordblks; mi.usmblks = mi.hblkhd; return mi; @@ -57,25 +57,25 @@ struct mallinfo __mallinfo_arena_info(size_t aidx) { struct mallinfo mi; memset(&mi, 0, sizeof(mi)); - malloc_mutex_lock(&arenas_lock); + malloc_mutex_lock(TSDN_NULL, &arenas_lock); if (aidx < narenas_auto) { if (arenas[aidx] != NULL) { - malloc_mutex_lock(&arenas[aidx]->lock); + malloc_mutex_lock(TSDN_NULL, &arenas[aidx]->lock); mi.hblkhd = arenas[aidx]->stats.mapped; mi.ordblks = arenas[aidx]->stats.allocated_large; mi.uordblks = arenas[aidx]->stats.allocated_huge; - malloc_mutex_unlock(&arenas[aidx]->lock); + malloc_mutex_unlock(TSDN_NULL, &arenas[aidx]->lock); for (unsigned j = 0; j < NBINS; j++) { arena_bin_t* bin = &arenas[aidx]->bins[j]; - malloc_mutex_lock(&bin->lock); + malloc_mutex_lock(TSDN_NULL, &bin->lock); mi.fsmblks += arena_bin_info[j].reg_size * bin->stats.curregs; - malloc_mutex_unlock(&bin->lock); + malloc_mutex_unlock(TSDN_NULL, &bin->lock); } } } - malloc_mutex_unlock(&arenas_lock); + malloc_mutex_unlock(TSDN_NULL, &arenas_lock); return mi; } @@ -83,18 +83,18 @@ struct mallinfo __mallinfo_bin_info(size_t aidx, size_t bidx) { struct mallinfo mi; memset(&mi, 0, sizeof(mi)); - malloc_mutex_lock(&arenas_lock); + malloc_mutex_lock(TSDN_NULL, &arenas_lock); if (aidx < narenas_auto && bidx < NBINS) { if (arenas[aidx] != NULL) { arena_bin_t* bin = &arenas[aidx]->bins[bidx]; - malloc_mutex_lock(&bin->lock); + malloc_mutex_lock(TSDN_NULL, &bin->lock); mi.ordblks = arena_bin_info[bidx].reg_size * bin->stats.curregs; mi.uordblks = bin->stats.nmalloc; mi.fordblks = bin->stats.ndalloc; - malloc_mutex_unlock(&bin->lock); + malloc_mutex_unlock(TSDN_NULL, &bin->lock); } } - malloc_mutex_unlock(&arenas_lock); + malloc_mutex_unlock(TSDN_NULL, &arenas_lock); return mi; } diff --git a/test/include/test/jemalloc_test.h b/test/include/test/jemalloc_test.h index 8348ac5..ec3a599 100644 --- a/test/include/test/jemalloc_test.h +++ b/test/include/test/jemalloc_test.h @@ -19,39 +19,6 @@ # include #endif -/******************************************************************************/ -/* - * Define always-enabled assertion macros, so that test assertions execute even - * if assertions are disabled in the library code. These definitions must - * exist prior to including "jemalloc/internal/util.h". - */ -#define assert(e) do { \ - if (!(e)) { \ - malloc_printf( \ - ": %s:%d: Failed assertion: \"%s\"\n", \ - __FILE__, __LINE__, #e); \ - abort(); \ - } \ -} while (0) - -#define not_reached() do { \ - malloc_printf( \ - ": %s:%d: Unreachable code reached\n", \ - __FILE__, __LINE__); \ - abort(); \ -} while (0) - -#define not_implemented() do { \ - malloc_printf(": %s:%d: Not implemented\n", \ - __FILE__, __LINE__); \ - abort(); \ -} while (0) - -#define assert_not_implemented(e) do { \ - if (!(e)) \ - not_implemented(); \ -} while (0) - #include "test/jemalloc_test_defs.h" #ifdef JEMALLOC_OSSPIN @@ -86,6 +53,14 @@ # include "jemalloc/internal/jemalloc_internal_defs.h" # include "jemalloc/internal/jemalloc_internal_macros.h" +static const bool config_debug = +#ifdef JEMALLOC_DEBUG + true +#else + false +#endif + ; + # define JEMALLOC_N(n) je_##n # include "jemalloc/internal/private_namespace.h" @@ -149,3 +124,40 @@ #include "test/thd.h" #define MEXP 19937 #include "test/SFMT.h" + +/******************************************************************************/ +/* + * Define always-enabled assertion macros, so that test assertions execute even + * if assertions are disabled in the library code. + */ +#undef assert +#undef not_reached +#undef not_implemented +#undef assert_not_implemented + +#define assert(e) do { \ + if (!(e)) { \ + malloc_printf( \ + ": %s:%d: Failed assertion: \"%s\"\n", \ + __FILE__, __LINE__, #e); \ + abort(); \ + } \ +} while (0) + +#define not_reached() do { \ + malloc_printf( \ + ": %s:%d: Unreachable code reached\n", \ + __FILE__, __LINE__); \ + abort(); \ +} while (0) + +#define not_implemented() do { \ + malloc_printf(": %s:%d: Not implemented\n", \ + __FILE__, __LINE__); \ + abort(); \ +} while (0) + +#define assert_not_implemented(e) do { \ + if (!(e)) \ + not_implemented(); \ +} while (0) -- cgit v1.2.3