diff options
author | Christopher Ferris <cferris@google.com> | 2016-12-08 11:56:18 -0800 |
---|---|---|
committer | Christopher Ferris <cferris@google.com> | 2016-12-12 11:27:08 -0800 |
commit | bf9b0180f3e75b710d1f419fff343885ee1b3880 (patch) | |
tree | d6d90045696788a36c0166add55a8399cb6accc4 /include | |
parent | b5d0373650f3274c28b4355533e70ae3e745f039 (diff) | |
parent | f1f76357313e7dcad7262f17a48ff0a2e005fcdc (diff) | |
download | jemalloc-bf9b0180f3e75b710d1f419fff343885ee1b3880.tar.gz |
Update to jemalloc 4.4.0.
Merge remote-tracking branch 'aosp/upstream-new' into upgrade
Includes regenerating the necessary files.
Bug: 33321361
Test: Built for x86_64/arm64. Built the no tcache and tcache enabled
Test: configs (nexus 9/nexus 6p). Compared the before after running
Test: all dumps through the memory replay and verified no unexpected
Test: increases. Ran bionic unit tests in both configs, 32 bit and 64 bit
Test: variants. Ran the jemalloc unit tests in both configs.
Change-Id: I2e8f3305cd1717c7efced69718fff90797f21068
Diffstat (limited to 'include')
-rw-r--r-- | include/jemalloc/internal/arena.h | 23 | ||||
-rw-r--r-- | include/jemalloc/internal/chunk.h | 11 | ||||
-rw-r--r-- | include/jemalloc/internal/extent.h | 41 | ||||
-rw-r--r-- | include/jemalloc/internal/jemalloc_internal.h | 4 | ||||
-rw-r--r-- | include/jemalloc/internal/jemalloc_internal.h.in | 4 | ||||
-rw-r--r-- | include/jemalloc/internal/jemalloc_internal_defs.h | 35 | ||||
-rw-r--r-- | include/jemalloc/internal/jemalloc_internal_defs.h.in | 34 | ||||
-rw-r--r-- | include/jemalloc/internal/pages.h | 2 | ||||
-rw-r--r-- | include/jemalloc/internal/private_namespace.h | 43 | ||||
-rw-r--r-- | include/jemalloc/internal/private_symbols.txt | 43 | ||||
-rw-r--r-- | include/jemalloc/internal/private_unnamespace.h | 43 | ||||
-rw-r--r-- | include/jemalloc/internal/stats.h | 8 | ||||
-rw-r--r-- | include/jemalloc/internal/util.h | 8 | ||||
-rw-r--r-- | include/jemalloc/internal/valgrind.h | 26 | ||||
-rw-r--r-- | include/jemalloc/jemalloc.h | 12 | ||||
-rw-r--r-- | include/jemalloc/jemalloc_macros.h | 8 |
16 files changed, 216 insertions, 129 deletions
diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 9e06f42..1ab91ea 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -197,6 +197,14 @@ struct arena_chunk_s { extent_node_t node; /* + * True if memory could be backed by transparent huge pages. This is + * only directly relevant to Linux, since it is the only supported + * platform on which jemalloc interacts with explicit transparent huge + * page controls. + */ + bool hugepage; + + /* * Map of pages within chunk that keeps track of free/large/small. The * first map_bias entries are omitted, since the chunk header does not * need to be tracked in the map. This omission saves a header page @@ -380,10 +388,12 @@ struct arena_s { dss_prec_t dss_prec; - /* Extant arena chunks. */ ql_head(extent_node_t) achunks; + /* Extent serial number generator state. */ + size_t extent_sn_next; + /* * In order to avoid rapid chunk allocation/deallocation when an arena * oscillates right on the cusp of needing a new chunk, cache the most @@ -459,9 +469,9 @@ struct arena_s { * orderings are needed, which is why there are two trees with the same * contents. */ - extent_tree_t chunks_szad_cached; + extent_tree_t chunks_szsnad_cached; extent_tree_t chunks_ad_cached; - extent_tree_t chunks_szad_retained; + extent_tree_t chunks_szsnad_retained; extent_tree_t chunks_ad_retained; malloc_mutex_t chunks_mtx; @@ -528,13 +538,13 @@ void arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, extent_node_t *arena_node_alloc(tsdn_t *tsdn, arena_t *arena); void arena_node_dalloc(tsdn_t *tsdn, arena_t *arena, extent_node_t *node); void *arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, - size_t alignment, bool *zero); + size_t alignment, size_t *sn, bool *zero); void arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, - size_t usize); + size_t usize, size_t sn); void arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t oldsize, size_t usize); void arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, - void *chunk, size_t oldsize, size_t usize); + void *chunk, size_t oldsize, size_t usize, size_t sn); bool arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t oldsize, size_t usize, bool *zero); ssize_t arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena); @@ -607,6 +617,7 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, unsigned arena_nthreads_get(arena_t *arena, bool internal); void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal); +size_t arena_extent_sn_next(arena_t *arena); arena_t *arena_new(tsdn_t *tsdn, unsigned ind); void arena_boot(void); void arena_prefork0(tsdn_t *tsdn, arena_t *arena); diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index eee0172..9b47f1c 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -62,15 +62,16 @@ void chunk_deregister(const void *chunk, const extent_node_t *node); void *chunk_alloc_base(size_t size); void *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, - bool *zero, bool *commit, bool dalloc_node); + size_t *sn, bool *zero, bool *commit, bool dalloc_node); void *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, - bool *zero, bool *commit); + size_t *sn, bool *zero, bool *commit); void chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool committed); -void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool zeroed, + chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t sn, bool committed); +void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, + chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t sn, + bool zeroed, bool committed); bool chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t offset, size_t length); diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 49d76a5..168ffe6 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -19,6 +19,20 @@ struct extent_node_s { size_t en_size; /* + * Serial number (potentially non-unique). + * + * In principle serial numbers can wrap around on 32-bit systems if + * JEMALLOC_MUNMAP is defined, but as long as comparison functions fall + * back on address comparison for equal serial numbers, stable (if + * imperfect) ordering is maintained. + * + * Serial numbers may not be unique even in the absence of wrap-around, + * e.g. when splitting an extent and assigning the same serial number to + * both resulting adjacent extents. + */ + size_t en_sn; + + /* * The zeroed flag is used by chunk recycling code to track whether * memory is zero-filled. */ @@ -45,8 +59,8 @@ struct extent_node_s { qr(extent_node_t) cc_link; union { - /* Linkage for the size/address-ordered tree. */ - rb_node(extent_node_t) szad_link; + /* Linkage for the size/sn/address-ordered tree. */ + rb_node(extent_node_t) szsnad_link; /* Linkage for arena's achunks, huge, and node_cache lists. */ ql_elm(extent_node_t) ql_link; @@ -61,7 +75,7 @@ typedef rb_tree(extent_node_t) extent_tree_t; /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -rb_proto(, extent_tree_szad_, extent_tree_t, extent_node_t) +rb_proto(, extent_tree_szsnad_, extent_tree_t, extent_node_t) rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t) @@ -73,6 +87,7 @@ rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t) arena_t *extent_node_arena_get(const extent_node_t *node); void *extent_node_addr_get(const extent_node_t *node); size_t extent_node_size_get(const extent_node_t *node); +size_t extent_node_sn_get(const extent_node_t *node); bool extent_node_zeroed_get(const extent_node_t *node); bool extent_node_committed_get(const extent_node_t *node); bool extent_node_achunk_get(const extent_node_t *node); @@ -80,12 +95,13 @@ prof_tctx_t *extent_node_prof_tctx_get(const extent_node_t *node); void extent_node_arena_set(extent_node_t *node, arena_t *arena); void extent_node_addr_set(extent_node_t *node, void *addr); void extent_node_size_set(extent_node_t *node, size_t size); +void extent_node_sn_set(extent_node_t *node, size_t sn); void extent_node_zeroed_set(extent_node_t *node, bool zeroed); void extent_node_committed_set(extent_node_t *node, bool committed); void extent_node_achunk_set(extent_node_t *node, bool achunk); void extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx); void extent_node_init(extent_node_t *node, arena_t *arena, void *addr, - size_t size, bool zeroed, bool committed); + size_t size, size_t sn, bool zeroed, bool committed); void extent_node_dirty_linkage_init(extent_node_t *node); void extent_node_dirty_insert(extent_node_t *node, arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty); @@ -114,6 +130,13 @@ extent_node_size_get(const extent_node_t *node) return (node->en_size); } +JEMALLOC_INLINE size_t +extent_node_sn_get(const extent_node_t *node) +{ + + return (node->en_sn); +} + JEMALLOC_INLINE bool extent_node_zeroed_get(const extent_node_t *node) { @@ -165,6 +188,13 @@ extent_node_size_set(extent_node_t *node, size_t size) } JEMALLOC_INLINE void +extent_node_sn_set(extent_node_t *node, size_t sn) +{ + + node->en_sn = sn; +} + +JEMALLOC_INLINE void extent_node_zeroed_set(extent_node_t *node, bool zeroed) { @@ -194,12 +224,13 @@ extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx) JEMALLOC_INLINE void extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size, - bool zeroed, bool committed) + size_t sn, bool zeroed, bool committed) { extent_node_arena_set(node, arena); extent_node_addr_set(node, addr); extent_node_size_set(node, size); + extent_node_sn_set(node, sn); extent_node_zeroed_set(node, zeroed); extent_node_committed_set(node, committed); extent_node_achunk_set(node, false); diff --git a/include/jemalloc/internal/jemalloc_internal.h b/include/jemalloc/internal/jemalloc_internal.h index 13e6407..36e7345 100644 --- a/include/jemalloc/internal/jemalloc_internal.h +++ b/include/jemalloc/internal/jemalloc_internal.h @@ -337,7 +337,7 @@ typedef unsigned szind_t; /* Return the nearest aligned address at or below a. */ #define ALIGNMENT_ADDR2BASE(a, alignment) \ - ((void *)((uintptr_t)(a) & (-(alignment)))) + ((void *)((uintptr_t)(a) & ((~(alignment)) + 1))) /* Return the offset between a and the nearest aligned address at or below a. */ #define ALIGNMENT_ADDR2OFFSET(a, alignment) \ @@ -345,7 +345,7 @@ typedef unsigned szind_t; /* Return the smallest alignment multiple that is >= s. */ #define ALIGNMENT_CEILING(s, alignment) \ - (((s) + (alignment - 1)) & (-(alignment))) + (((s) + (alignment - 1)) & ((~(alignment)) + 1)) /* Declare a variable-length array. */ #if __STDC_VERSION__ < 199901L diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index fdc8fef..e7ace7d 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -337,7 +337,7 @@ typedef unsigned szind_t; /* Return the nearest aligned address at or below a. */ #define ALIGNMENT_ADDR2BASE(a, alignment) \ - ((void *)((uintptr_t)(a) & (-(alignment)))) + ((void *)((uintptr_t)(a) & ((~(alignment)) + 1))) /* Return the offset between a and the nearest aligned address at or below a. */ #define ALIGNMENT_ADDR2OFFSET(a, alignment) \ @@ -345,7 +345,7 @@ typedef unsigned szind_t; /* Return the smallest alignment multiple that is >= s. */ #define ALIGNMENT_CEILING(s, alignment) \ - (((s) + (alignment - 1)) & (-(alignment))) + (((s) + (alignment - 1)) & ((~(alignment)) + 1)) /* Declare a variable-length array. */ #if __STDC_VERSION__ < 199901L diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h b/include/jemalloc/internal/jemalloc_internal_defs.h index 2c1d8be..1899bf3 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h +++ b/include/jemalloc/internal/jemalloc_internal_defs.h @@ -61,11 +61,6 @@ #define JEMALLOC_HAVE_BUILTIN_CLZ /* - * Defined if madvise(2) is available. - */ -#define JEMALLOC_HAVE_MADVISE - -/* * Defined if os_unfair_lock_*() functions are available, as provided by Darwin. */ /* #undef JEMALLOC_OS_UNFAIR_LOCK */ @@ -76,9 +71,9 @@ */ /* #undef JEMALLOC_OSSPIN */ -/* Defined if syscall(2) is available. */ +/* Defined if syscall(2) is usable. */ /* Syscalls are available in Android, but avoid them for security reasons. */ -/* #undef JEMALLOC_HAVE_SYSCALL */ +/* #undef JEMALLOC_USE_SYSCALL */ /* * Defined if secure_getenv(3) is available. @@ -90,6 +85,9 @@ */ /* #undef JEMALLOC_HAVE_ISSETUGID */ +/* Defined if pthread_atfork(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_ATFORK + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ @@ -259,18 +257,27 @@ /* #undef JEMALLOC_SYSCTL_VM_OVERCOMMIT */ #define JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY +/* Defined if madvise(2) is available. */ +#define JEMALLOC_HAVE_MADVISE + /* * Methods for purging unused pages differ between operating systems. * - * madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages, - * such that new pages will be demand-zeroed if - * the address region is later touched. - * madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being - * unused, such that they will be discarded rather - * than swapped out. + * madvise(..., MADV_FREE) : This marks pages as being unused, such that they + * will be discarded rather than swapped out. + * madvise(..., MADV_DONTNEED) : This immediately discards pages, such that + * new pages will be demand-zeroed if the + * address region is later touched. */ -#define JEMALLOC_PURGE_MADVISE_DONTNEED /* #undef JEMALLOC_PURGE_MADVISE_FREE */ +#define JEMALLOC_PURGE_MADVISE_DONTNEED + +/* + * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE + * arguments to madvise(2). + */ +/* ANDROID: Do not enable huge pages because it can increase PSS. */ +/* #undef JEMALLOC_THP */ /* Define if operating system has alloca.h header. */ #define JEMALLOC_HAS_ALLOCA_H 1 diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 9b3dca5..def4ba5 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -56,11 +56,6 @@ #undef JEMALLOC_HAVE_BUILTIN_CLZ /* - * Defined if madvise(2) is available. - */ -#undef JEMALLOC_HAVE_MADVISE - -/* * Defined if os_unfair_lock_*() functions are available, as provided by Darwin. */ #undef JEMALLOC_OS_UNFAIR_LOCK @@ -71,8 +66,8 @@ */ #undef JEMALLOC_OSSPIN -/* Defined if syscall(2) is available. */ -#undef JEMALLOC_HAVE_SYSCALL +/* Defined if syscall(2) is usable. */ +#undef JEMALLOC_USE_SYSCALL /* * Defined if secure_getenv(3) is available. @@ -84,6 +79,9 @@ */ #undef JEMALLOC_HAVE_ISSETUGID +/* Defined if pthread_atfork(3) is available. */ +#undef JEMALLOC_HAVE_PTHREAD_ATFORK + /* * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. */ @@ -252,18 +250,26 @@ #undef JEMALLOC_SYSCTL_VM_OVERCOMMIT #undef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY +/* Defined if madvise(2) is available. */ +#undef JEMALLOC_HAVE_MADVISE + /* * Methods for purging unused pages differ between operating systems. * - * madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages, - * such that new pages will be demand-zeroed if - * the address region is later touched. - * madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being - * unused, such that they will be discarded rather - * than swapped out. + * madvise(..., MADV_FREE) : This marks pages as being unused, such that they + * will be discarded rather than swapped out. + * madvise(..., MADV_DONTNEED) : This immediately discards pages, such that + * new pages will be demand-zeroed if the + * address region is later touched. */ -#undef JEMALLOC_PURGE_MADVISE_DONTNEED #undef JEMALLOC_PURGE_MADVISE_FREE +#undef JEMALLOC_PURGE_MADVISE_DONTNEED + +/* + * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE + * arguments to madvise(2). + */ +#undef JEMALLOC_THP /* Define if operating system has alloca.h header. */ #undef JEMALLOC_HAS_ALLOCA_H diff --git a/include/jemalloc/internal/pages.h b/include/jemalloc/internal/pages.h index e21effd..4ae9f15 100644 --- a/include/jemalloc/internal/pages.h +++ b/include/jemalloc/internal/pages.h @@ -16,6 +16,8 @@ void *pages_trim(void *addr, size_t alloc_size, size_t leadsize, bool pages_commit(void *addr, size_t size); bool pages_decommit(void *addr, size_t size); bool pages_purge(void *addr, size_t size); +bool pages_huge(void *addr, size_t size); +bool pages_nohuge(void *addr, size_t size); void pages_boot(void); #endif /* JEMALLOC_H_EXTERNS */ diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h index fec0c20..6bc0e2a 100644 --- a/include/jemalloc/internal/private_namespace.h +++ b/include/jemalloc/internal/private_namespace.h @@ -36,6 +36,7 @@ #define arena_decay_time_set JEMALLOC_N(arena_decay_time_set) #define arena_dss_prec_get JEMALLOC_N(arena_dss_prec_get) #define arena_dss_prec_set JEMALLOC_N(arena_dss_prec_set) +#define arena_extent_sn_next JEMALLOC_N(arena_extent_sn_next) #define arena_get JEMALLOC_N(arena_get) #define arena_ichoose JEMALLOC_N(arena_ichoose) #define arena_init JEMALLOC_N(arena_init) @@ -218,6 +219,8 @@ #define extent_node_prof_tctx_set JEMALLOC_N(extent_node_prof_tctx_set) #define extent_node_size_get JEMALLOC_N(extent_node_size_get) #define extent_node_size_set JEMALLOC_N(extent_node_size_set) +#define extent_node_sn_get JEMALLOC_N(extent_node_sn_get) +#define extent_node_sn_set JEMALLOC_N(extent_node_sn_set) #define extent_node_zeroed_get JEMALLOC_N(extent_node_zeroed_get) #define extent_node_zeroed_set JEMALLOC_N(extent_node_zeroed_set) #define extent_tree_ad_destroy JEMALLOC_N(extent_tree_ad_destroy) @@ -239,25 +242,25 @@ #define extent_tree_ad_reverse_iter_recurse JEMALLOC_N(extent_tree_ad_reverse_iter_recurse) #define extent_tree_ad_reverse_iter_start JEMALLOC_N(extent_tree_ad_reverse_iter_start) #define extent_tree_ad_search JEMALLOC_N(extent_tree_ad_search) -#define extent_tree_szad_destroy JEMALLOC_N(extent_tree_szad_destroy) -#define extent_tree_szad_destroy_recurse JEMALLOC_N(extent_tree_szad_destroy_recurse) -#define extent_tree_szad_empty JEMALLOC_N(extent_tree_szad_empty) -#define extent_tree_szad_first JEMALLOC_N(extent_tree_szad_first) -#define extent_tree_szad_insert JEMALLOC_N(extent_tree_szad_insert) -#define extent_tree_szad_iter JEMALLOC_N(extent_tree_szad_iter) -#define extent_tree_szad_iter_recurse JEMALLOC_N(extent_tree_szad_iter_recurse) -#define extent_tree_szad_iter_start JEMALLOC_N(extent_tree_szad_iter_start) -#define extent_tree_szad_last JEMALLOC_N(extent_tree_szad_last) -#define extent_tree_szad_new JEMALLOC_N(extent_tree_szad_new) -#define extent_tree_szad_next JEMALLOC_N(extent_tree_szad_next) -#define extent_tree_szad_nsearch JEMALLOC_N(extent_tree_szad_nsearch) -#define extent_tree_szad_prev JEMALLOC_N(extent_tree_szad_prev) -#define extent_tree_szad_psearch JEMALLOC_N(extent_tree_szad_psearch) -#define extent_tree_szad_remove JEMALLOC_N(extent_tree_szad_remove) -#define extent_tree_szad_reverse_iter JEMALLOC_N(extent_tree_szad_reverse_iter) -#define extent_tree_szad_reverse_iter_recurse JEMALLOC_N(extent_tree_szad_reverse_iter_recurse) -#define extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start) -#define extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search) +#define extent_tree_szsnad_destroy JEMALLOC_N(extent_tree_szsnad_destroy) +#define extent_tree_szsnad_destroy_recurse JEMALLOC_N(extent_tree_szsnad_destroy_recurse) +#define extent_tree_szsnad_empty JEMALLOC_N(extent_tree_szsnad_empty) +#define extent_tree_szsnad_first JEMALLOC_N(extent_tree_szsnad_first) +#define extent_tree_szsnad_insert JEMALLOC_N(extent_tree_szsnad_insert) +#define extent_tree_szsnad_iter JEMALLOC_N(extent_tree_szsnad_iter) +#define extent_tree_szsnad_iter_recurse JEMALLOC_N(extent_tree_szsnad_iter_recurse) +#define extent_tree_szsnad_iter_start JEMALLOC_N(extent_tree_szsnad_iter_start) +#define extent_tree_szsnad_last JEMALLOC_N(extent_tree_szsnad_last) +#define extent_tree_szsnad_new JEMALLOC_N(extent_tree_szsnad_new) +#define extent_tree_szsnad_next JEMALLOC_N(extent_tree_szsnad_next) +#define extent_tree_szsnad_nsearch JEMALLOC_N(extent_tree_szsnad_nsearch) +#define extent_tree_szsnad_prev JEMALLOC_N(extent_tree_szsnad_prev) +#define extent_tree_szsnad_psearch JEMALLOC_N(extent_tree_szsnad_psearch) +#define extent_tree_szsnad_remove JEMALLOC_N(extent_tree_szsnad_remove) +#define extent_tree_szsnad_reverse_iter JEMALLOC_N(extent_tree_szsnad_reverse_iter) +#define extent_tree_szsnad_reverse_iter_recurse JEMALLOC_N(extent_tree_szsnad_reverse_iter_recurse) +#define extent_tree_szsnad_reverse_iter_start JEMALLOC_N(extent_tree_szsnad_reverse_iter_start) +#define extent_tree_szsnad_search JEMALLOC_N(extent_tree_szsnad_search) #define ffs_llu JEMALLOC_N(ffs_llu) #define ffs_lu JEMALLOC_N(ffs_lu) #define ffs_u JEMALLOC_N(ffs_u) @@ -394,7 +397,9 @@ #define pages_boot JEMALLOC_N(pages_boot) #define pages_commit JEMALLOC_N(pages_commit) #define pages_decommit JEMALLOC_N(pages_decommit) +#define pages_huge JEMALLOC_N(pages_huge) #define pages_map JEMALLOC_N(pages_map) +#define pages_nohuge JEMALLOC_N(pages_nohuge) #define pages_purge JEMALLOC_N(pages_purge) #define pages_trim JEMALLOC_N(pages_trim) #define pages_unmap JEMALLOC_N(pages_unmap) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 87c8c9b..c1c6c40 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -36,6 +36,7 @@ arena_decay_time_get arena_decay_time_set arena_dss_prec_get arena_dss_prec_set +arena_extent_sn_next arena_get arena_ichoose arena_init @@ -218,6 +219,8 @@ extent_node_prof_tctx_get extent_node_prof_tctx_set extent_node_size_get extent_node_size_set +extent_node_sn_get +extent_node_sn_set extent_node_zeroed_get extent_node_zeroed_set extent_tree_ad_destroy @@ -239,25 +242,25 @@ extent_tree_ad_reverse_iter extent_tree_ad_reverse_iter_recurse extent_tree_ad_reverse_iter_start extent_tree_ad_search -extent_tree_szad_destroy -extent_tree_szad_destroy_recurse -extent_tree_szad_empty -extent_tree_szad_first -extent_tree_szad_insert -extent_tree_szad_iter -extent_tree_szad_iter_recurse -extent_tree_szad_iter_start -extent_tree_szad_last -extent_tree_szad_new -extent_tree_szad_next -extent_tree_szad_nsearch -extent_tree_szad_prev -extent_tree_szad_psearch -extent_tree_szad_remove -extent_tree_szad_reverse_iter -extent_tree_szad_reverse_iter_recurse -extent_tree_szad_reverse_iter_start -extent_tree_szad_search +extent_tree_szsnad_destroy +extent_tree_szsnad_destroy_recurse +extent_tree_szsnad_empty +extent_tree_szsnad_first +extent_tree_szsnad_insert +extent_tree_szsnad_iter +extent_tree_szsnad_iter_recurse +extent_tree_szsnad_iter_start +extent_tree_szsnad_last +extent_tree_szsnad_new +extent_tree_szsnad_next +extent_tree_szsnad_nsearch +extent_tree_szsnad_prev +extent_tree_szsnad_psearch +extent_tree_szsnad_remove +extent_tree_szsnad_reverse_iter +extent_tree_szsnad_reverse_iter_recurse +extent_tree_szsnad_reverse_iter_start +extent_tree_szsnad_search ffs_llu ffs_lu ffs_u @@ -394,7 +397,9 @@ p2rz pages_boot pages_commit pages_decommit +pages_huge pages_map +pages_nohuge pages_purge pages_trim pages_unmap diff --git a/include/jemalloc/internal/private_unnamespace.h b/include/jemalloc/internal/private_unnamespace.h index dfb9bc2..44530f7 100644 --- a/include/jemalloc/internal/private_unnamespace.h +++ b/include/jemalloc/internal/private_unnamespace.h @@ -36,6 +36,7 @@ #undef arena_decay_time_set #undef arena_dss_prec_get #undef arena_dss_prec_set +#undef arena_extent_sn_next #undef arena_get #undef arena_ichoose #undef arena_init @@ -218,6 +219,8 @@ #undef extent_node_prof_tctx_set #undef extent_node_size_get #undef extent_node_size_set +#undef extent_node_sn_get +#undef extent_node_sn_set #undef extent_node_zeroed_get #undef extent_node_zeroed_set #undef extent_tree_ad_destroy @@ -239,25 +242,25 @@ #undef extent_tree_ad_reverse_iter_recurse #undef extent_tree_ad_reverse_iter_start #undef extent_tree_ad_search -#undef extent_tree_szad_destroy -#undef extent_tree_szad_destroy_recurse -#undef extent_tree_szad_empty -#undef extent_tree_szad_first -#undef extent_tree_szad_insert -#undef extent_tree_szad_iter -#undef extent_tree_szad_iter_recurse -#undef extent_tree_szad_iter_start -#undef extent_tree_szad_last -#undef extent_tree_szad_new -#undef extent_tree_szad_next -#undef extent_tree_szad_nsearch -#undef extent_tree_szad_prev -#undef extent_tree_szad_psearch -#undef extent_tree_szad_remove -#undef extent_tree_szad_reverse_iter -#undef extent_tree_szad_reverse_iter_recurse -#undef extent_tree_szad_reverse_iter_start -#undef extent_tree_szad_search +#undef extent_tree_szsnad_destroy +#undef extent_tree_szsnad_destroy_recurse +#undef extent_tree_szsnad_empty +#undef extent_tree_szsnad_first +#undef extent_tree_szsnad_insert +#undef extent_tree_szsnad_iter +#undef extent_tree_szsnad_iter_recurse +#undef extent_tree_szsnad_iter_start +#undef extent_tree_szsnad_last +#undef extent_tree_szsnad_new +#undef extent_tree_szsnad_next +#undef extent_tree_szsnad_nsearch +#undef extent_tree_szsnad_prev +#undef extent_tree_szsnad_psearch +#undef extent_tree_szsnad_remove +#undef extent_tree_szsnad_reverse_iter +#undef extent_tree_szsnad_reverse_iter_recurse +#undef extent_tree_szsnad_reverse_iter_start +#undef extent_tree_szsnad_search #undef ffs_llu #undef ffs_lu #undef ffs_u @@ -394,7 +397,9 @@ #undef pages_boot #undef pages_commit #undef pages_decommit +#undef pages_huge #undef pages_map +#undef pages_nohuge #undef pages_purge #undef pages_trim #undef pages_unmap diff --git a/include/jemalloc/internal/stats.h b/include/jemalloc/internal/stats.h index b621817..04e7dae 100644 --- a/include/jemalloc/internal/stats.h +++ b/include/jemalloc/internal/stats.h @@ -175,25 +175,21 @@ stats_cactive_get(void) JEMALLOC_INLINE void stats_cactive_add(size_t size) { - UNUSED size_t cactive; assert(size > 0); assert((size & chunksize_mask) == 0); - cactive = atomic_add_z(&stats_cactive, size); - assert(cactive - size < cactive); + atomic_add_z(&stats_cactive, size); } JEMALLOC_INLINE void stats_cactive_sub(size_t size) { - UNUSED size_t cactive; assert(size > 0); assert((size & chunksize_mask) == 0); - cactive = atomic_sub_z(&stats_cactive, size); - assert(cactive + size > cactive); + atomic_sub_z(&stats_cactive, size); } #endif diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index aee00d6..4b56d65 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -41,8 +41,12 @@ #define MALLOC_PRINTF_BUFSIZE 4096 /* Junk fill patterns. */ -#define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) -#define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) +#ifndef JEMALLOC_ALLOC_JUNK +# define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) +#endif +#ifndef JEMALLOC_FREE_JUNK +# define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) +#endif /* * Wrap a cpp argument that contains commas such that it isn't broken up into diff --git a/include/jemalloc/internal/valgrind.h b/include/jemalloc/internal/valgrind.h index 1a86808..877a142 100644 --- a/include/jemalloc/internal/valgrind.h +++ b/include/jemalloc/internal/valgrind.h @@ -36,13 +36,25 @@ zero); \ } \ } while (0) -#define JEMALLOC_VALGRIND_REALLOC(maybe_moved, tsdn, ptr, usize, \ - ptr_maybe_null, old_ptr, old_usize, old_rzsize, old_ptr_maybe_null, \ - zero) do { \ +#define JEMALLOC_VALGRIND_REALLOC_MOVED_no(ptr, old_ptr) \ + (false) +#define JEMALLOC_VALGRIND_REALLOC_MOVED_maybe(ptr, old_ptr) \ + ((ptr) != (old_ptr)) +#define JEMALLOC_VALGRIND_REALLOC_PTR_NULL_no(ptr) \ + (false) +#define JEMALLOC_VALGRIND_REALLOC_PTR_NULL_maybe(ptr) \ + (ptr == NULL) +#define JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_no(old_ptr) \ + (false) +#define JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_maybe(old_ptr) \ + (old_ptr == NULL) +#define JEMALLOC_VALGRIND_REALLOC(moved, tsdn, ptr, usize, ptr_null, \ + old_ptr, old_usize, old_rzsize, old_ptr_null, zero) do { \ if (unlikely(in_valgrind)) { \ size_t rzsize = p2rz(tsdn, ptr); \ \ - if (!maybe_moved || ptr == old_ptr) { \ + if (!JEMALLOC_VALGRIND_REALLOC_MOVED_##moved(ptr, \ + old_ptr)) { \ VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \ usize, rzsize); \ if (zero && old_usize < usize) { \ @@ -51,11 +63,13 @@ old_usize), usize - old_usize); \ } \ } else { \ - if (!old_ptr_maybe_null || old_ptr != NULL) { \ + if (!JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_## \ + old_ptr_null(old_ptr)) { \ valgrind_freelike_block(old_ptr, \ old_rzsize); \ } \ - if (!ptr_maybe_null || ptr != NULL) { \ + if (!JEMALLOC_VALGRIND_REALLOC_PTR_NULL_## \ + ptr_null(ptr)) { \ size_t copy_size = (old_usize < usize) \ ? old_usize : usize; \ size_t tail_size = usize - copy_size; \ diff --git a/include/jemalloc/jemalloc.h b/include/jemalloc/jemalloc.h index 7706510..bbd3ca5 100644 --- a/include/jemalloc/jemalloc.h +++ b/include/jemalloc/jemalloc.h @@ -24,7 +24,7 @@ extern "C" { */ #define JEMALLOC_OVERRIDE_MEMALIGN #ifndef __LP64__ -#define JEMALLOC_OVERRIDE_VALLOC +#define JEMALLOC_OVERRIDE_VALLOC #endif /* @@ -41,7 +41,7 @@ extern "C" { * with C++. The only justification for this is to match the prototypes that * glibc defines. */ -/* #undef JEMALLOC_USE_CXX_THROW */ +/* #undef JEMALLOC_USE_CXX_THROW */ #ifdef _MSC_VER # ifdef _WIN64 @@ -94,12 +94,12 @@ extern "C" { #include <limits.h> #include <strings.h> -#define JEMALLOC_VERSION "4.3.1-0-g0110fa8451af905affd77c3bea0d545fee2251b2" +#define JEMALLOC_VERSION "4.4.0-0-gf1f76357313e7dcad7262f17a48ff0a2e005fcdc" #define JEMALLOC_VERSION_MAJOR 4 -#define JEMALLOC_VERSION_MINOR 3 -#define JEMALLOC_VERSION_BUGFIX 1 +#define JEMALLOC_VERSION_MINOR 4 +#define JEMALLOC_VERSION_BUGFIX 0 #define JEMALLOC_VERSION_NREV 0 -#define JEMALLOC_VERSION_GID "0110fa8451af905affd77c3bea0d545fee2251b2" +#define JEMALLOC_VERSION_GID "f1f76357313e7dcad7262f17a48ff0a2e005fcdc" # define MALLOCX_LG_ALIGN(la) ((int)(la)) # if LG_SIZEOF_PTR == 2 diff --git a/include/jemalloc/jemalloc_macros.h b/include/jemalloc/jemalloc_macros.h index d7b2f96..de9f164 100644 --- a/include/jemalloc/jemalloc_macros.h +++ b/include/jemalloc/jemalloc_macros.h @@ -4,12 +4,12 @@ #include <limits.h> #include <strings.h> -#define JEMALLOC_VERSION "4.3.1-0-g0110fa8451af905affd77c3bea0d545fee2251b2" +#define JEMALLOC_VERSION "4.4.0-0-gf1f76357313e7dcad7262f17a48ff0a2e005fcdc" #define JEMALLOC_VERSION_MAJOR 4 -#define JEMALLOC_VERSION_MINOR 3 -#define JEMALLOC_VERSION_BUGFIX 1 +#define JEMALLOC_VERSION_MINOR 4 +#define JEMALLOC_VERSION_BUGFIX 0 #define JEMALLOC_VERSION_NREV 0 -#define JEMALLOC_VERSION_GID "0110fa8451af905affd77c3bea0d545fee2251b2" +#define JEMALLOC_VERSION_GID "f1f76357313e7dcad7262f17a48ff0a2e005fcdc" # define MALLOCX_LG_ALIGN(la) ((int)(la)) # if LG_SIZEOF_PTR == 2 |