diff options
| author | Christopher Ferris <cferris@google.com> | 2016-11-08 14:47:48 -0800 |
|---|---|---|
| committer | Christopher Ferris <cferris@google.com> | 2016-11-10 12:20:42 -0800 |
| commit | fb1f094f163a3bf15d8958ba845e83559c0e6dfe (patch) | |
| tree | b0605297b74ae918b10f34a37916e79e2ba63d13 /include | |
| parent | 00fa4482d89786a43fb6f76d6e86996e1587e16f (diff) | |
| parent | 0110fa8451af905affd77c3bea0d545fee2251b2 (diff) | |
| download | jemalloc-fb1f094f163a3bf15d8958ba845e83559c0e6dfe.tar.gz | |
Merge remote-tracking branch 'aosp/upstream-new' into fix
Included in this change are all of the updated generated files.
Bug: 32673024
Test: Built the angler build (normal config) and the volantis build
Test: (svelte config). Ran memory_replay 32 bit and 64 bit on both
Test: platforms before and after and verified results are similar.
Test: Ran bionic unit tests and jemalloc unit tests.
Test: Verified that two jemalloc unit test failures are due to
Test: Android extension that puts all large chunks on arena 0.
Change-Id: I12428bdbe15f51383489c9a1d72d687499fff01b
Diffstat (limited to 'include')
27 files changed, 2047 insertions, 1479 deletions
diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 4062660..9e06f42 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -48,6 +48,7 @@ typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; typedef struct arena_chunk_s arena_chunk_t; typedef struct arena_bin_info_s arena_bin_info_t; +typedef struct arena_decay_s arena_decay_t; typedef struct arena_bin_s arena_bin_t; typedef struct arena_s arena_t; typedef struct arena_tdata_s arena_tdata_t; @@ -263,6 +264,49 @@ struct arena_bin_info_s { uint32_t reg0_offset; }; +struct arena_decay_s { + /* + * Approximate time in seconds from the creation of a set of unused + * dirty pages until an equivalent set of unused dirty pages is purged + * and/or reused. + */ + ssize_t time; + /* time / SMOOTHSTEP_NSTEPS. */ + nstime_t interval; + /* + * Time at which the current decay interval logically started. We do + * not actually advance to a new epoch until sometime after it starts + * because of scheduling and computation delays, and it is even possible + * to completely skip epochs. In all cases, during epoch advancement we + * merge all relevant activity into the most recently recorded epoch. + */ + nstime_t epoch; + /* Deadline randomness generator. */ + uint64_t jitter_state; + /* + * Deadline for current epoch. This is the sum of interval and per + * epoch jitter which is a uniform random variable in [0..interval). + * Epochs always advance by precise multiples of interval, but we + * randomize the deadline to reduce the likelihood of arenas purging in + * lockstep. + */ + nstime_t deadline; + /* + * Number of dirty pages at beginning of current epoch. During epoch + * advancement we use the delta between arena->decay.ndirty and + * arena->ndirty to determine how many dirty pages, if any, were + * generated. + */ + size_t ndirty; + /* + * Trailing log of how many unused dirty pages were generated during + * each of the past SMOOTHSTEP_NSTEPS decay epochs, where the last + * element is the most recent epoch. Corresponding epoch times are + * relative to epoch. + */ + size_t backlog[SMOOTHSTEP_NSTEPS]; +}; + struct arena_bin_s { /* * All operations on runcur, runs, and stats require that lock be @@ -332,7 +376,7 @@ struct arena_s { * PRNG state for cache index randomization of large allocation base * pointers. */ - uint64_t offset_state; + size_t offset_state; dss_prec_t dss_prec; @@ -400,52 +444,8 @@ struct arena_s { arena_runs_dirty_link_t runs_dirty; extent_node_t chunks_cache; - /* - * Approximate time in seconds from the creation of a set of unused - * dirty pages until an equivalent set of unused dirty pages is purged - * and/or reused. - */ - ssize_t decay_time; - /* decay_time / SMOOTHSTEP_NSTEPS. */ - nstime_t decay_interval; - /* - * Time at which the current decay interval logically started. We do - * not actually advance to a new epoch until sometime after it starts - * because of scheduling and computation delays, and it is even possible - * to completely skip epochs. In all cases, during epoch advancement we - * merge all relevant activity into the most recently recorded epoch. - */ - nstime_t decay_epoch; - /* decay_deadline randomness generator. */ - uint64_t decay_jitter_state; - /* - * Deadline for current epoch. This is the sum of decay_interval and - * per epoch jitter which is a uniform random variable in - * [0..decay_interval). Epochs always advance by precise multiples of - * decay_interval, but we randomize the deadline to reduce the - * likelihood of arenas purging in lockstep. - */ - nstime_t decay_deadline; - /* - * Number of dirty pages at beginning of current epoch. During epoch - * advancement we use the delta between decay_ndirty and ndirty to - * determine how many dirty pages, if any, were generated, and record - * the result in decay_backlog. - */ - size_t decay_ndirty; - /* - * Memoized result of arena_decay_backlog_npages_limit() corresponding - * to the current contents of decay_backlog, i.e. the limit on how many - * pages are allowed to exist for the decay epochs. - */ - size_t decay_backlog_npages_limit; - /* - * Trailing log of how many unused dirty pages were generated during - * each of the past SMOOTHSTEP_NSTEPS decay epochs, where the last - * element is the most recent epoch. Corresponding epoch times are - * relative to decay_epoch. - */ - size_t decay_backlog[SMOOTHSTEP_NSTEPS]; + /* Decay-based purging state. */ + arena_decay_t decay; /* Extant huge allocations. */ ql_head(extent_node_t) huge; @@ -476,10 +476,12 @@ struct arena_s { arena_bin_t bins[NBINS]; /* - * Quantized address-ordered heaps of this arena's available runs. The - * heaps are used for first-best-fit run allocation. + * Size-segregated address-ordered heaps of this arena's available runs, + * used for first-best-fit run allocation. Runs are quantized, i.e. + * they reside in the last heap which corresponds to a size class less + * than or equal to the run size. */ - arena_run_heap_t runs_avail[1]; /* Dynamically sized. */ + arena_run_heap_t runs_avail[NPSIZES]; }; /* Used in conjunction with tsd for fast arena-related context lookup. */ @@ -511,7 +513,6 @@ extern size_t map_bias; /* Number of arena chunk header pages. */ extern size_t map_misc_offset; extern size_t arena_maxrun; /* Max run size for arenas. */ extern size_t large_maxclass; /* Max large size class. */ -extern size_t run_quantize_max; /* Max run_quantize_*() input. */ extern unsigned nlclasses; /* Number of large size classes. */ extern unsigned nhclasses; /* Number of huge size classes. */ @@ -607,7 +608,7 @@ unsigned arena_nthreads_get(arena_t *arena, bool internal); void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal); arena_t *arena_new(tsdn_t *tsdn, unsigned ind); -bool arena_boot(void); +void arena_boot(void); void arena_prefork0(tsdn_t *tsdn, arena_t *arena); void arena_prefork1(tsdn_t *tsdn, arena_t *arena); void arena_prefork2(tsdn_t *tsdn, arena_t *arena); diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index d6ecdab..eee0172 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -62,7 +62,7 @@ void chunk_deregister(const void *chunk, const extent_node_t *node); void *chunk_alloc_base(size_t size); void *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, - bool *zero, bool dalloc_node); + bool *zero, bool *commit, bool dalloc_node); void *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit); @@ -75,9 +75,6 @@ bool chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t offset, size_t length); bool chunk_boot(void); -void chunk_prefork(tsdn_t *tsdn); -void chunk_postfork_parent(tsdn_t *tsdn); -void chunk_postfork_child(tsdn_t *tsdn); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/chunk_dss.h b/include/jemalloc/internal/chunk_dss.h index 724fa57..da8511b 100644 --- a/include/jemalloc/internal/chunk_dss.h +++ b/include/jemalloc/internal/chunk_dss.h @@ -21,15 +21,13 @@ extern const char *dss_prec_names[]; /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -dss_prec_t chunk_dss_prec_get(tsdn_t *tsdn); -bool chunk_dss_prec_set(tsdn_t *tsdn, dss_prec_t dss_prec); +dss_prec_t chunk_dss_prec_get(void); +bool chunk_dss_prec_set(dss_prec_t dss_prec); void *chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit); -bool chunk_in_dss(tsdn_t *tsdn, void *chunk); -bool chunk_dss_boot(void); -void chunk_dss_prefork(tsdn_t *tsdn); -void chunk_dss_postfork_parent(tsdn_t *tsdn); -void chunk_dss_postfork_child(tsdn_t *tsdn); +bool chunk_in_dss(void *chunk); +bool chunk_dss_mergeable(void *chunk_a, void *chunk_b); +void chunk_dss_boot(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/ckh.h b/include/jemalloc/internal/ckh.h index 46e151c..f75ad90 100644 --- a/include/jemalloc/internal/ckh.h +++ b/include/jemalloc/internal/ckh.h @@ -64,13 +64,13 @@ struct ckh_s { /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -bool ckh_new(tsdn_t *tsdn, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, +bool ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp); -void ckh_delete(tsdn_t *tsdn, ckh_t *ckh); +void ckh_delete(tsd_t *tsd, ckh_t *ckh); size_t ckh_count(ckh_t *ckh); bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data); -bool ckh_insert(tsdn_t *tsdn, ckh_t *ckh, const void *key, const void *data); -bool ckh_remove(tsdn_t *tsdn, ckh_t *ckh, const void *searchkey, void **key, +bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data); +bool ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, void **data); bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data); void ckh_string_hash(const void *key, size_t r_hash[2]); diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index b5fa9e6..22184d9 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -17,7 +17,7 @@ bool huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, void *huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t usize, size_t alignment, bool zero, tcache_t *tcache); #ifdef JEMALLOC_JET -typedef void (huge_dalloc_junk_t)(tsdn_t *, void *, size_t); +typedef void (huge_dalloc_junk_t)(void *, size_t); extern huge_dalloc_junk_t *huge_dalloc_junk; #endif void huge_dalloc(tsdn_t *tsdn, void *ptr); diff --git a/include/jemalloc/internal/jemalloc_internal.h b/include/jemalloc/internal/jemalloc_internal.h index a6902f4..13e6407 100644 --- a/include/jemalloc/internal/jemalloc_internal.h +++ b/include/jemalloc/internal/jemalloc_internal.h @@ -162,7 +162,9 @@ static const bool config_cache_oblivious = #endif #include "jemalloc/internal/ph.h" +#ifndef __PGI #define RB_COMPACT +#endif #include "jemalloc/internal/rb.h" #include "jemalloc/internal/qr.h" #include "jemalloc/internal/ql.h" @@ -185,6 +187,9 @@ static const bool config_cache_oblivious = #include "jemalloc/internal/jemalloc_internal_macros.h" +/* Page size index type. */ +typedef unsigned pszind_t; + /* Size class index type. */ typedef unsigned szind_t; @@ -234,7 +239,7 @@ typedef unsigned szind_t; # ifdef __alpha__ # define LG_QUANTUM 4 # endif -# if (defined(__sparc64__) || defined(__sparcv9)) +# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) # define LG_QUANTUM 4 # endif # if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) @@ -364,6 +369,7 @@ typedef unsigned szind_t; #include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/spin.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/ckh.h" @@ -396,6 +402,7 @@ typedef unsigned szind_t; #include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/spin.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/ckh.h" @@ -456,10 +463,15 @@ extern unsigned narenas_auto; extern arena_t **arenas; /* + * pind2sz_tab encodes the same information as could be computed by + * pind2sz_compute(). + */ +extern size_t const pind2sz_tab[NPSIZES]; +/* * index2size_tab encodes the same information as could be computed (at * unacceptable cost in some code paths) by index2size_compute(). */ -extern size_t const index2size_tab[NSIZES+1]; +extern size_t const index2size_tab[NSIZES]; /* * size2index_tab is a compact lookup table that rounds request sizes up to * size classes. In order to reduce cache footprint, the table is compressed, @@ -467,6 +479,7 @@ extern size_t const index2size_tab[NSIZES+1]; */ extern uint8_t const size2index_tab[]; +arena_t *a0get(void); void *a0malloc(size_t size); void a0dalloc(void *ptr); void *bootstrap_malloc(size_t size); @@ -492,6 +505,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/spin.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/ckh.h" @@ -524,6 +538,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/spin.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/ckh.h" @@ -543,6 +558,11 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/huge.h" #ifndef JEMALLOC_ENABLE_INLINE +pszind_t psz2ind(size_t psz); +size_t pind2sz_compute(pszind_t pind); +size_t pind2sz_lookup(pszind_t pind); +size_t pind2sz(pszind_t pind); +size_t psz2u(size_t psz); szind_t size2index_compute(size_t size); szind_t size2index_lookup(size_t size); szind_t size2index(size_t size); @@ -555,7 +575,7 @@ size_t s2u(size_t size); size_t sa2u(size_t size, size_t alignment); arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal); arena_t *arena_choose(tsd_t *tsd, arena_t *arena); -arena_t *arena_ichoose(tsdn_t *tsdn, arena_t *arena); +arena_t *arena_ichoose(tsd_t *tsd, arena_t *arena); arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing); arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing); @@ -563,10 +583,90 @@ ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) +JEMALLOC_INLINE pszind_t +psz2ind(size_t psz) +{ + + if (unlikely(psz > HUGE_MAXCLASS)) + return (NPSIZES); + { + pszind_t x = lg_floor((psz<<1)-1); + pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x - + (LG_SIZE_CLASS_GROUP + LG_PAGE); + pszind_t grp = shift << LG_SIZE_CLASS_GROUP; + + pszind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? + LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; + + size_t delta_inverse_mask = ZI(-1) << lg_delta; + pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & + ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + + pszind_t ind = grp + mod; + return (ind); + } +} + +JEMALLOC_INLINE size_t +pind2sz_compute(pszind_t pind) +{ + + { + size_t grp = pind >> LG_SIZE_CLASS_GROUP; + size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + + size_t grp_size_mask = ~((!!grp)-1); + size_t grp_size = ((ZU(1) << (LG_PAGE + + (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + + size_t shift = (grp == 0) ? 1 : grp; + size_t lg_delta = shift + (LG_PAGE-1); + size_t mod_size = (mod+1) << lg_delta; + + size_t sz = grp_size + mod_size; + return (sz); + } +} + +JEMALLOC_INLINE size_t +pind2sz_lookup(pszind_t pind) +{ + size_t ret = (size_t)pind2sz_tab[pind]; + assert(ret == pind2sz_compute(pind)); + return (ret); +} + +JEMALLOC_INLINE size_t +pind2sz(pszind_t pind) +{ + + assert(pind < NPSIZES); + return (pind2sz_lookup(pind)); +} + +JEMALLOC_INLINE size_t +psz2u(size_t psz) +{ + + if (unlikely(psz > HUGE_MAXCLASS)) + return (0); + { + size_t x = lg_floor((psz<<1)-1); + size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? + LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; + size_t delta = ZU(1) << lg_delta; + size_t delta_mask = delta - 1; + size_t usize = (psz + delta_mask) & ~delta_mask; + return (usize); + } +} + JEMALLOC_INLINE szind_t size2index_compute(size_t size) { + if (unlikely(size > HUGE_MAXCLASS)) + return (NSIZES); #if (NTBINS != 0) if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { szind_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; @@ -575,9 +675,7 @@ size2index_compute(size_t size) } #endif { - szind_t x = unlikely(ZI(size) < 0) ? ((size<<1) ? - (ZU(1)<<(LG_SIZEOF_PTR+3)) : ((ZU(1)<<(LG_SIZEOF_PTR+3))-1)) - : lg_floor((size<<1)-1); + szind_t x = lg_floor((size<<1)-1); szind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); szind_t grp = shift << LG_SIZE_CLASS_GROUP; @@ -663,6 +761,8 @@ JEMALLOC_ALWAYS_INLINE size_t s2u_compute(size_t size) { + if (unlikely(size > HUGE_MAXCLASS)) + return (0); #if (NTBINS > 0) if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; @@ -672,9 +772,7 @@ s2u_compute(size_t size) } #endif { - size_t x = unlikely(ZI(size) < 0) ? ((size<<1) ? - (ZU(1)<<(LG_SIZEOF_PTR+3)) : ((ZU(1)<<(LG_SIZEOF_PTR+3))-1)) - : lg_floor((size<<1)-1); + size_t x = lg_floor((size<<1)-1); size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; size_t delta = ZU(1) << lg_delta; @@ -815,14 +913,10 @@ arena_choose(tsd_t *tsd, arena_t *arena) } JEMALLOC_INLINE arena_t * -arena_ichoose(tsdn_t *tsdn, arena_t *arena) +arena_ichoose(tsd_t *tsd, arena_t *arena) { - assert(!tsdn_null(tsdn) || arena != NULL); - - if (!tsdn_null(tsdn)) - return (arena_choose_impl(tsdn_tsd(tsdn), NULL, true)); - return (arena); + return (arena_choose_impl(tsd, arena, true)); } JEMALLOC_INLINE arena_tdata_t * diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 8f82edd..fdc8fef 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -162,7 +162,9 @@ static const bool config_cache_oblivious = #endif #include "jemalloc/internal/ph.h" +#ifndef __PGI #define RB_COMPACT +#endif #include "jemalloc/internal/rb.h" #include "jemalloc/internal/qr.h" #include "jemalloc/internal/ql.h" @@ -185,6 +187,9 @@ static const bool config_cache_oblivious = #include "jemalloc/internal/jemalloc_internal_macros.h" +/* Page size index type. */ +typedef unsigned pszind_t; + /* Size class index type. */ typedef unsigned szind_t; @@ -234,7 +239,7 @@ typedef unsigned szind_t; # ifdef __alpha__ # define LG_QUANTUM 4 # endif -# if (defined(__sparc64__) || defined(__sparcv9)) +# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) # define LG_QUANTUM 4 # endif # if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) @@ -364,6 +369,7 @@ typedef unsigned szind_t; #include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/spin.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/ckh.h" @@ -396,6 +402,7 @@ typedef unsigned szind_t; #include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/spin.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/ckh.h" @@ -456,10 +463,15 @@ extern unsigned narenas_auto; extern arena_t **arenas; /* + * pind2sz_tab encodes the same information as could be computed by + * pind2sz_compute(). + */ +extern size_t const pind2sz_tab[NPSIZES]; +/* * index2size_tab encodes the same information as could be computed (at * unacceptable cost in some code paths) by index2size_compute(). */ -extern size_t const index2size_tab[NSIZES+1]; +extern size_t const index2size_tab[NSIZES]; /* * size2index_tab is a compact lookup table that rounds request sizes up to * size classes. In order to reduce cache footprint, the table is compressed, @@ -467,6 +479,7 @@ extern size_t const index2size_tab[NSIZES+1]; */ extern uint8_t const size2index_tab[]; +arena_t *a0get(void); void *a0malloc(size_t size); void a0dalloc(void *ptr); void *bootstrap_malloc(size_t size); @@ -492,6 +505,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/spin.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/ckh.h" @@ -524,6 +538,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/spin.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/ckh.h" @@ -543,6 +558,11 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/huge.h" #ifndef JEMALLOC_ENABLE_INLINE +pszind_t psz2ind(size_t psz); +size_t pind2sz_compute(pszind_t pind); +size_t pind2sz_lookup(pszind_t pind); +size_t pind2sz(pszind_t pind); +size_t psz2u(size_t psz); szind_t size2index_compute(size_t size); szind_t size2index_lookup(size_t size); szind_t size2index(size_t size); @@ -555,7 +575,7 @@ size_t s2u(size_t size); size_t sa2u(size_t size, size_t alignment); arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal); arena_t *arena_choose(tsd_t *tsd, arena_t *arena); -arena_t *arena_ichoose(tsdn_t *tsdn, arena_t *arena); +arena_t *arena_ichoose(tsd_t *tsd, arena_t *arena); arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing); arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing); @@ -563,10 +583,90 @@ ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) +JEMALLOC_INLINE pszind_t +psz2ind(size_t psz) +{ + + if (unlikely(psz > HUGE_MAXCLASS)) + return (NPSIZES); + { + pszind_t x = lg_floor((psz<<1)-1); + pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x - + (LG_SIZE_CLASS_GROUP + LG_PAGE); + pszind_t grp = shift << LG_SIZE_CLASS_GROUP; + + pszind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? + LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; + + size_t delta_inverse_mask = ZI(-1) << lg_delta; + pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & + ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + + pszind_t ind = grp + mod; + return (ind); + } +} + +JEMALLOC_INLINE size_t +pind2sz_compute(pszind_t pind) +{ + + { + size_t grp = pind >> LG_SIZE_CLASS_GROUP; + size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + + size_t grp_size_mask = ~((!!grp)-1); + size_t grp_size = ((ZU(1) << (LG_PAGE + + (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + + size_t shift = (grp == 0) ? 1 : grp; + size_t lg_delta = shift + (LG_PAGE-1); + size_t mod_size = (mod+1) << lg_delta; + + size_t sz = grp_size + mod_size; + return (sz); + } +} + +JEMALLOC_INLINE size_t +pind2sz_lookup(pszind_t pind) +{ + size_t ret = (size_t)pind2sz_tab[pind]; + assert(ret == pind2sz_compute(pind)); + return (ret); +} + +JEMALLOC_INLINE size_t +pind2sz(pszind_t pind) +{ + + assert(pind < NPSIZES); + return (pind2sz_lookup(pind)); +} + +JEMALLOC_INLINE size_t +psz2u(size_t psz) +{ + + if (unlikely(psz > HUGE_MAXCLASS)) + return (0); + { + size_t x = lg_floor((psz<<1)-1); + size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? + LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; + size_t delta = ZU(1) << lg_delta; + size_t delta_mask = delta - 1; + size_t usize = (psz + delta_mask) & ~delta_mask; + return (usize); + } +} + JEMALLOC_INLINE szind_t size2index_compute(size_t size) { + if (unlikely(size > HUGE_MAXCLASS)) + return (NSIZES); #if (NTBINS != 0) if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { szind_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; @@ -575,9 +675,7 @@ size2index_compute(size_t size) } #endif { - szind_t x = unlikely(ZI(size) < 0) ? ((size<<1) ? - (ZU(1)<<(LG_SIZEOF_PTR+3)) : ((ZU(1)<<(LG_SIZEOF_PTR+3))-1)) - : lg_floor((size<<1)-1); + szind_t x = lg_floor((size<<1)-1); szind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); szind_t grp = shift << LG_SIZE_CLASS_GROUP; @@ -663,6 +761,8 @@ JEMALLOC_ALWAYS_INLINE size_t s2u_compute(size_t size) { + if (unlikely(size > HUGE_MAXCLASS)) + return (0); #if (NTBINS > 0) if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; @@ -672,9 +772,7 @@ s2u_compute(size_t size) } #endif { - size_t x = unlikely(ZI(size) < 0) ? ((size<<1) ? - (ZU(1)<<(LG_SIZEOF_PTR+3)) : ((ZU(1)<<(LG_SIZEOF_PTR+3))-1)) - : lg_floor((size<<1)-1); + size_t x = lg_floor((size<<1)-1); size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; size_t delta = ZU(1) << lg_delta; @@ -815,14 +913,10 @@ arena_choose(tsd_t *tsd, arena_t *arena) } JEMALLOC_INLINE arena_t * -arena_ichoose(tsdn_t *tsdn, arena_t *arena) +arena_ichoose(tsd_t *tsd, arena_t *arena) { - assert(!tsdn_null(tsdn) || arena != NULL); - - if (!tsdn_null(tsdn)) - return (arena_choose_impl(tsdn_tsd(tsdn), NULL, true)); - return (arena); + return (arena_choose_impl(tsd, arena, true)); } JEMALLOC_INLINE arena_tdata_t * diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h index 2b8ca5d..c907d91 100644 --- a/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/include/jemalloc/internal/jemalloc_internal_decls.h @@ -17,8 +17,18 @@ # include <sys/uio.h> # endif # include <pthread.h> +# ifdef JEMALLOC_OS_UNFAIR_LOCK +# include <os/lock.h> +# endif +# ifdef JEMALLOC_GLIBC_MALLOC_HOOK +# include <sched.h> +# endif # include <errno.h> # include <sys/time.h> +# include <time.h> +# ifdef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME +# include <mach/mach_time.h> +# endif #endif #include <sys/types.h> diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h b/include/jemalloc/internal/jemalloc_internal_defs.h index dc63ffd..2d00aaa 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h +++ b/include/jemalloc/internal/jemalloc_internal_defs.h @@ -28,7 +28,7 @@ #endif /* Defined if C11 atomics are available. */ -#define JEMALLOC_C11ATOMICS +#define JEMALLOC_C11ATOMICS 1 /* Defined if the equivalent of FreeBSD's atomic(9) functions are available. */ /* #undef JEMALLOC_ATOMIC9 */ @@ -66,11 +66,19 @@ #define JEMALLOC_HAVE_MADVISE /* + * Defined if os_unfair_lock_*() functions are available, as provided by Darwin. + */ +/* #undef JEMALLOC_OS_UNFAIR_LOCK */ + +/* * Defined if OSSpin*() functions are available, as provided by Darwin, and * documented in the spinlock(3) manual page. */ /* #undef JEMALLOC_OSSPIN */ +/* Defined if syscall(2) is available. */ +#define JEMALLOC_HAVE_SYSCALL + /* * Defined if secure_getenv(3) is available. */ @@ -82,6 +90,21 @@ /* #undef JEMALLOC_HAVE_ISSETUGID */ /* + * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 + +/* + * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 + +/* + * Defined if mach_absolute_time() is available. + */ +/* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ + +/* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc * bootstrapping will cause recursion into the pthreads library. Therefore, if @@ -195,6 +218,12 @@ /* #undef JEMALLOC_TLS */ /* + * Used to mark unreachable code to quiet "end of non-void" compiler warnings. + * Don't use this directly; instead use unreachable() from util.h + */ +#define JEMALLOC_INTERNAL_UNREACHABLE __builtin_unreachable + +/* * ffs*() functions to use for bitmapping. Don't use these directly; instead, * use ffs_*() from util.h. */ diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 7de0cf7..9b3dca5 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -61,11 +61,19 @@ #undef JEMALLOC_HAVE_MADVISE /* + * Defined if os_unfair_lock_*() functions are available, as provided by Darwin. + */ +#undef JEMALLOC_OS_UNFAIR_LOCK + +/* * Defined if OSSpin*() functions are available, as provided by Darwin, and * documented in the spinlock(3) manual page. */ #undef JEMALLOC_OSSPIN +/* Defined if syscall(2) is available. */ +#undef JEMALLOC_HAVE_SYSCALL + /* * Defined if secure_getenv(3) is available. */ @@ -77,6 +85,21 @@ #undef JEMALLOC_HAVE_ISSETUGID /* + * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. + */ +#undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE + +/* + * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. + */ +#undef JEMALLOC_HAVE_CLOCK_MONOTONIC + +/* + * Defined if mach_absolute_time() is available. + */ +#undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME + +/* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc * bootstrapping will cause recursion into the pthreads library. Therefore, if @@ -189,6 +212,12 @@ #undef JEMALLOC_TLS /* + * Used to mark unreachable code to quiet "end of non-void" compiler warnings. + * Don't use this directly; instead use unreachable() from util.h + */ +#undef JEMALLOC_INTERNAL_UNREACHABLE + +/* * ffs*() functions to use for bitmapping. Don't use these directly; instead, * use ffs_*() from util.h. */ diff --git a/include/jemalloc/internal/mb.h b/include/jemalloc/internal/mb.h index 437c86f..5384728 100644 --- a/include/jemalloc/internal/mb.h +++ b/include/jemalloc/internal/mb.h @@ -105,8 +105,8 @@ mb_write(void) malloc_mutex_t mtx; malloc_mutex_init(&mtx, "mb", WITNESS_RANK_OMIT); - malloc_mutex_lock(NULL, &mtx); - malloc_mutex_unlock(NULL, &mtx); + malloc_mutex_lock(TSDN_NULL, &mtx); + malloc_mutex_unlock(TSDN_NULL, &mtx); } #endif #endif diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h index 5221799..b442d2d 100644 --- a/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h @@ -5,6 +5,9 @@ typedef struct malloc_mutex_s malloc_mutex_t; #ifdef _WIN32 # define MALLOC_MUTEX_INITIALIZER +#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) +# define MALLOC_MUTEX_INITIALIZER \ + {OS_UNFAIR_LOCK_INIT, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)} #elif (defined(JEMALLOC_OSSPIN)) # define MALLOC_MUTEX_INITIALIZER {0, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)} #elif (defined(JEMALLOC_MUTEX_INIT_CB)) @@ -35,6 +38,8 @@ struct malloc_mutex_s { # else CRITICAL_SECTION lock; # endif +#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) + os_unfair_lock lock; #elif (defined(JEMALLOC_OSSPIN)) OSSpinLock lock; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) @@ -88,6 +93,8 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) # else EnterCriticalSection(&mutex->lock); # endif +#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) + os_unfair_lock_lock(&mutex->lock); #elif (defined(JEMALLOC_OSSPIN)) OSSpinLockLock(&mutex->lock); #else @@ -109,6 +116,8 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) # else LeaveCriticalSection(&mutex->lock); # endif +#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) + os_unfair_lock_unlock(&mutex->lock); #elif (defined(JEMALLOC_OSSPIN)) OSSpinLockUnlock(&mutex->lock); #else diff --git a/include/jemalloc/internal/nstime.h b/include/jemalloc/internal/nstime.h index dc293b7..93b27dc 100644 --- a/include/jemalloc/internal/nstime.h +++ b/include/jemalloc/internal/nstime.h @@ -1,9 +1,6 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES -#define JEMALLOC_CLOCK_GETTIME defined(_POSIX_MONOTONIC_CLOCK) \ - && _POSIX_MONOTONIC_CLOCK >= 0 - typedef struct nstime_s nstime_t; /* Maximum supported number of seconds (~584 years). */ @@ -34,9 +31,12 @@ void nstime_imultiply(nstime_t *time, uint64_t multiplier); void nstime_idivide(nstime_t *time, uint64_t divisor); uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor); #ifdef JEMALLOC_JET +typedef bool (nstime_monotonic_t)(void); +extern nstime_monotonic_t *nstime_monotonic; typedef bool (nstime_update_t)(nstime_t *); extern nstime_update_t *nstime_update; #else +bool nstime_monotonic(void); bool nstime_update(nstime_t *time); #endif diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h index 271cbf4..fec0c20 100644 --- a/include/jemalloc/internal/private_namespace.h +++ b/include/jemalloc/internal/private_namespace.h @@ -1,4 +1,5 @@ #define a0dalloc JEMALLOC_N(a0dalloc) +#define a0get JEMALLOC_N(a0get) #define a0malloc JEMALLOC_N(a0malloc) #define arena_aalloc JEMALLOC_N(arena_aalloc) #define arena_alloc_junk_small JEMALLOC_N(arena_alloc_junk_small) @@ -167,20 +168,15 @@ #define chunk_dalloc_wrapper JEMALLOC_N(chunk_dalloc_wrapper) #define chunk_deregister JEMALLOC_N(chunk_deregister) #define chunk_dss_boot JEMALLOC_N(chunk_dss_boot) -#define chunk_dss_postfork_child JEMALLOC_N(chunk_dss_postfork_child) -#define chunk_dss_postfork_parent JEMALLOC_N(chunk_dss_postfork_parent) +#define chunk_dss_mergeable JEMALLOC_N(chunk_dss_mergeable) #define chunk_dss_prec_get JEMALLOC_N(chunk_dss_prec_get) #define chunk_dss_prec_set JEMALLOC_N(chunk_dss_prec_set) -#define chunk_dss_prefork JEMALLOC_N(chunk_dss_prefork) #define chunk_hooks_default JEMALLOC_N(chunk_hooks_default) #define chunk_hooks_get JEMALLOC_N(chunk_hooks_get) #define chunk_hooks_set JEMALLOC_N(chunk_hooks_set) #define chunk_in_dss JEMALLOC_N(chunk_in_dss) #define chunk_lookup JEMALLOC_N(chunk_lookup) #define chunk_npages JEMALLOC_N(chunk_npages) -#define chunk_postfork_child JEMALLOC_N(chunk_postfork_child) -#define chunk_postfork_parent JEMALLOC_N(chunk_postfork_parent) -#define chunk_prefork JEMALLOC_N(chunk_prefork) #define chunk_purge_wrapper JEMALLOC_N(chunk_purge_wrapper) #define chunk_register JEMALLOC_N(chunk_register) #define chunks_rtree JEMALLOC_N(chunks_rtree) @@ -360,6 +356,7 @@ #define nstime_imultiply JEMALLOC_N(nstime_imultiply) #define nstime_init JEMALLOC_N(nstime_init) #define nstime_init2 JEMALLOC_N(nstime_init2) +#define nstime_monotonic JEMALLOC_N(nstime_monotonic) #define nstime_ns JEMALLOC_N(nstime_ns) #define nstime_nsec JEMALLOC_N(nstime_nsec) #define nstime_sec JEMALLOC_N(nstime_sec) @@ -401,11 +398,22 @@ #define pages_purge JEMALLOC_N(pages_purge) #define pages_trim JEMALLOC_N(pages_trim) #define pages_unmap JEMALLOC_N(pages_unmap) +#define pind2sz JEMALLOC_N(pind2sz) +#define pind2sz_compute JEMALLOC_N(pind2sz_compute) +#define pind2sz_lookup JEMALLOC_N(pind2sz_lookup) +#define pind2sz_tab JEMALLOC_N(pind2sz_tab) #define pow2_ceil_u32 JEMALLOC_N(pow2_ceil_u32) #define pow2_ceil_u64 JEMALLOC_N(pow2_ceil_u64) #define pow2_ceil_zu JEMALLOC_N(pow2_ceil_zu) -#define prng_lg_range JEMALLOC_N(prng_lg_range) -#define prng_range JEMALLOC_N(prng_range) +#define prng_lg_range_u32 JEMALLOC_N(prng_lg_range_u32) +#define prng_lg_range_u64 JEMALLOC_N(prng_lg_range_u64) +#define prng_lg_range_zu JEMALLOC_N(prng_lg_range_zu) +#define prng_range_u32 JEMALLOC_N(prng_range_u32) +#define prng_range_u64 JEMALLOC_N(prng_range_u64) +#define prng_range_zu JEMALLOC_N(prng_range_zu) +#define prng_state_next_u32 JEMALLOC_N(prng_state_next_u32) +#define prng_state_next_u64 JEMALLOC_N(prng_state_next_u64) +#define prng_state_next_zu JEMALLOC_N(prng_state_next_zu) #define prof_active JEMALLOC_N(prof_active) #define prof_active_get JEMALLOC_N(prof_active_get) #define prof_active_get_unlocked JEMALLOC_N(prof_active_get_unlocked) @@ -454,12 +462,13 @@ #define prof_thread_active_set JEMALLOC_N(prof_thread_active_set) #define prof_thread_name_get JEMALLOC_N(prof_thread_name_get) #define prof_thread_name_set JEMALLOC_N(prof_thread_name_set) +#define psz2ind JEMALLOC_N(psz2ind) +#define psz2u JEMALLOC_N(psz2u) #define purge_mode_names JEMALLOC_N(purge_mode_names) #define quarantine JEMALLOC_N(quarantine) #define quarantine_alloc_hook JEMALLOC_N(quarantine_alloc_hook) #define quarantine_alloc_hook_work JEMALLOC_N(quarantine_alloc_hook_work) #define quarantine_cleanup JEMALLOC_N(quarantine_cleanup) -#define register_zone JEMALLOC_N(register_zone) #define rtree_child_read JEMALLOC_N(rtree_child_read) #define rtree_child_read_hard JEMALLOC_N(rtree_child_read_hard) #define rtree_child_tryread JEMALLOC_N(rtree_child_tryread) @@ -477,7 +486,6 @@ #define rtree_val_write JEMALLOC_N(rtree_val_write) #define run_quantize_ceil JEMALLOC_N(run_quantize_ceil) #define run_quantize_floor JEMALLOC_N(run_quantize_floor) -#define run_quantize_max JEMALLOC_N(run_quantize_max) #define s2u JEMALLOC_N(s2u) #define s2u_compute JEMALLOC_N(s2u_compute) #define s2u_lookup JEMALLOC_N(s2u_lookup) @@ -487,6 +495,8 @@ #define size2index_compute JEMALLOC_N(size2index_compute) #define size2index_lookup JEMALLOC_N(size2index_lookup) #define size2index_tab JEMALLOC_N(size2index_tab) +#define spin_adaptive JEMALLOC_N(spin_adaptive) +#define spin_init JEMALLOC_N(spin_init) #define stats_cactive JEMALLOC_N(stats_cactive) #define stats_cactive_add JEMALLOC_N(stats_cactive_add) #define stats_cactive_get JEMALLOC_N(stats_cactive_get) @@ -545,7 +555,9 @@ #define tsd_cleanup JEMALLOC_N(tsd_cleanup) #define tsd_cleanup_wrapper JEMALLOC_N(tsd_cleanup_wrapper) #define tsd_fetch JEMALLOC_N(tsd_fetch) +#define tsd_fetch_impl JEMALLOC_N(tsd_fetch_impl) #define tsd_get JEMALLOC_N(tsd_get) +#define tsd_get_allocates JEMALLOC_N(tsd_get_allocates) #define tsd_iarena_get JEMALLOC_N(tsd_iarena_get) #define tsd_iarena_set JEMALLOC_N(tsd_iarena_set) #define tsd_iarenap_get JEMALLOC_N(tsd_iarenap_get) @@ -604,9 +616,11 @@ #define witness_lock_error JEMALLOC_N(witness_lock_error) #define witness_lockless_error JEMALLOC_N(witness_lockless_error) #define witness_not_owner_error JEMALLOC_N(witness_not_owner_error) +#define witness_owner JEMALLOC_N(witness_owner) #define witness_owner_error JEMALLOC_N(witness_owner_error) #define witness_postfork_child JEMALLOC_N(witness_postfork_child) #define witness_postfork_parent JEMALLOC_N(witness_postfork_parent) #define witness_prefork JEMALLOC_N(witness_prefork) #define witness_unlock JEMALLOC_N(witness_unlock) #define witnesses_cleanup JEMALLOC_N(witnesses_cleanup) +#define zone_register JEMALLOC_N(zone_register) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index f2b6a55..87c8c9b 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -1,4 +1,5 @@ a0dalloc +a0get a0malloc arena_aalloc arena_alloc_junk_small @@ -167,20 +168,15 @@ chunk_dalloc_mmap chunk_dalloc_wrapper chunk_deregister chunk_dss_boot -chunk_dss_postfork_child -chunk_dss_postfork_parent +chunk_dss_mergeable chunk_dss_prec_get chunk_dss_prec_set -chunk_dss_prefork chunk_hooks_default chunk_hooks_get chunk_hooks_set chunk_in_dss chunk_lookup chunk_npages -chunk_postfork_child -chunk_postfork_parent -chunk_prefork chunk_purge_wrapper chunk_register chunks_rtree @@ -360,6 +356,7 @@ nstime_idivide nstime_imultiply nstime_init nstime_init2 +nstime_monotonic nstime_ns nstime_nsec nstime_sec @@ -401,11 +398,22 @@ pages_map pages_purge pages_trim pages_unmap +pind2sz +pind2sz_compute +pind2sz_lookup +pind2sz_tab pow2_ceil_u32 pow2_ceil_u64 pow2_ceil_zu -prng_lg_range -prng_range +prng_lg_range_u32 +prng_lg_range_u64 +prng_lg_range_zu +prng_range_u32 +prng_range_u64 +prng_range_zu +prng_state_next_u32 +prng_state_next_u64 +prng_state_next_zu prof_active prof_active_get prof_active_get_unlocked @@ -454,12 +462,13 @@ prof_thread_active_init_set prof_thread_active_set prof_thread_name_get prof_thread_name_set +psz2ind +psz2u purge_mode_names quarantine quarantine_alloc_hook quarantine_alloc_hook_work quarantine_cleanup -register_zone rtree_child_read rtree_child_read_hard rtree_child_tryread @@ -477,7 +486,6 @@ rtree_val_read rtree_val_write run_quantize_ceil run_quantize_floor -run_quantize_max s2u s2u_compute s2u_lookup @@ -487,6 +495,8 @@ size2index size2index_compute size2index_lookup size2index_tab +spin_adaptive +spin_init stats_cactive stats_cactive_add stats_cactive_get @@ -545,7 +555,9 @@ tsd_booted_get tsd_cleanup tsd_cleanup_wrapper tsd_fetch +tsd_fetch_impl tsd_get +tsd_get_allocates tsd_iarena_get tsd_iarena_set tsd_iarenap_get @@ -604,9 +616,11 @@ witness_lock witness_lock_error witness_lockless_error witness_not_owner_error +witness_owner witness_owner_error witness_postfork_child witness_postfork_parent witness_prefork witness_unlock witnesses_cleanup +zone_register diff --git a/include/jemalloc/internal/private_unnamespace.h b/include/jemalloc/internal/private_unnamespace.h index b5a79bd..dfb9bc2 100644 --- a/include/jemalloc/internal/private_unnamespace.h +++ b/include/jemalloc/internal/private_unnamespace.h @@ -1,4 +1,5 @@ #undef a0dalloc +#undef a0get #undef a0malloc #undef arena_aalloc #undef arena_alloc_junk_small @@ -167,20 +168,15 @@ #undef chunk_dalloc_wrapper #undef chunk_deregister #undef chunk_dss_boot -#undef chunk_dss_postfork_child -#undef chunk_dss_postfork_parent +#undef chunk_dss_mergeable #undef chunk_dss_prec_get #undef chunk_dss_prec_set -#undef chunk_dss_prefork #undef chunk_hooks_default #undef chunk_hooks_get #undef chunk_hooks_set #undef chunk_in_dss #undef chunk_lookup #undef chunk_npages -#undef chunk_postfork_child -#undef chunk_postfork_parent -#undef chunk_prefork #undef chunk_purge_wrapper #undef chunk_register #undef chunks_rtree @@ -360,6 +356,7 @@ #undef nstime_imultiply #undef nstime_init #undef nstime_init2 +#undef nstime_monotonic #undef nstime_ns #undef nstime_nsec #undef nstime_sec @@ -401,11 +398,22 @@ #undef pages_purge #undef pages_trim #undef pages_unmap +#undef pind2sz +#undef pind2sz_compute +#undef pind2sz_lookup +#undef pind2sz_tab #undef pow2_ceil_u32 #undef pow2_ceil_u64 #undef pow2_ceil_zu -#undef prng_lg_range -#undef prng_range +#undef prng_lg_range_u32 +#undef prng_lg_range_u64 +#undef prng_lg_range_zu +#undef prng_range_u32 +#undef prng_range_u64 +#undef prng_range_zu +#undef prng_state_next_u32 +#undef prng_state_next_u64 +#undef prng_state_next_zu #undef prof_active #undef prof_active_get #undef prof_active_get_unlocked @@ -454,12 +462,13 @@ #undef prof_thread_active_set #undef prof_thread_name_get #undef prof_thread_name_set +#undef psz2ind +#undef psz2u #undef purge_mode_names #undef quarantine #undef quarantine_alloc_hook #undef quarantine_alloc_hook_work #undef quarantine_cleanup -#undef register_zone #undef rtree_child_read #undef rtree_child_read_hard #undef rtree_child_tryread @@ -477,7 +486,6 @@ #undef rtree_val_write #undef run_quantize_ceil #undef run_quantize_floor -#undef run_quantize_max #undef s2u #undef s2u_compute #undef s2u_lookup @@ -487,6 +495,8 @@ #undef size2index_compute #undef size2index_lookup #undef size2index_tab +#undef spin_adaptive +#undef spin_init #undef stats_cactive #undef stats_cactive_add #undef stats_cactive_get @@ -545,7 +555,9 @@ #undef tsd_cleanup #undef tsd_cleanup_wrapper #undef tsd_fetch +#undef tsd_fetch_impl #undef tsd_get +#undef tsd_get_allocates #undef tsd_iarena_get #undef tsd_iarena_set #undef tsd_iarenap_get @@ -604,9 +616,11 @@ #undef witness_lock_error #undef witness_lockless_error #undef witness_not_owner_error +#undef witness_owner #undef witness_owner_error #undef witness_postfork_child #undef witness_postfork_parent #undef witness_prefork #undef witness_unlock #undef witnesses_cleanup +#undef zone_register diff --git a/include/jemalloc/internal/prng.h b/include/jemalloc/internal/prng.h index 5830f8b..c2bda19 100644 --- a/include/jemalloc/internal/prng.h +++ b/include/jemalloc/internal/prng.h @@ -19,8 +19,12 @@ * the next has a cycle of 4, etc. For this reason, we prefer to use the upper * bits. */ -#define PRNG_A UINT64_C(6364136223846793005) -#define PRNG_C UINT64_C(1442695040888963407) + +#define PRNG_A_32 UINT32_C(1103515241) +#define PRNG_C_32 UINT32_C(12347) + +#define PRNG_A_64 UINT64_C(6364136223846793005) +#define PRNG_C_64 UINT64_C(1442695040888963407) #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ @@ -35,28 +39,133 @@ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -uint64_t prng_lg_range(uint64_t *state, unsigned lg_range); -uint64_t prng_range(uint64_t *state, uint64_t range); +uint32_t prng_state_next_u32(uint32_t state); +uint64_t prng_state_next_u64(uint64_t state); +size_t prng_state_next_zu(size_t state); + +uint32_t prng_lg_range_u32(uint32_t *state, unsigned lg_range, + bool atomic); +uint64_t prng_lg_range_u64(uint64_t *state, unsigned lg_range); +size_t prng_lg_range_zu(size_t *state, unsigned lg_range, bool atomic); + +uint32_t prng_range_u32(uint32_t *state, uint32_t range, bool atomic); +uint64_t prng_range_u64(uint64_t *state, uint64_t range); +size_t prng_range_zu(size_t *state, size_t range, bool atomic); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PRNG_C_)) +JEMALLOC_ALWAYS_INLINE uint32_t +prng_state_next_u32(uint32_t state) +{ + + return ((state * PRNG_A_32) + PRNG_C_32); +} + JEMALLOC_ALWAYS_INLINE uint64_t -prng_lg_range(uint64_t *state, unsigned lg_range) +prng_state_next_u64(uint64_t state) { - uint64_t ret; + + return ((state * PRNG_A_64) + PRNG_C_64); +} + +JEMALLOC_ALWAYS_INLINE size_t +prng_state_next_zu(size_t state) +{ + +#if LG_SIZEOF_PTR == 2 + return ((state * PRNG_A_32) + PRNG_C_32); +#elif LG_SIZEOF_PTR == 3 + return ((state * PRNG_A_64) + PRNG_C_64); +#else +#error Unsupported pointer size +#endif +} + +JEMALLOC_ALWAYS_INLINE uint32_t +prng_lg_range_u32(uint32_t *state, unsigned lg_range, bool atomic) +{ + uint32_t ret, state1; + + assert(lg_range > 0); + assert(lg_range <= 32); + + if (atomic) { + uint32_t state0; + + do { + state0 = atomic_read_uint32(state); + state1 = prng_state_next_u32(state0); + } while (atomic_cas_uint32(state, state0, state1)); + } else { + state1 = prng_state_next_u32(*state); + *state = state1; + } + ret = state1 >> (32 - lg_range); + + return (ret); +} + +/* 64-bit atomic operations cannot be supported on all relevant platforms. */ +JEMALLOC_ALWAYS_INLINE uint64_t +prng_lg_range_u64(uint64_t *state, unsigned lg_range) +{ + uint64_t ret, state1; assert(lg_range > 0); assert(lg_range <= 64); - ret = (*state * PRNG_A) + PRNG_C; - *state = ret; - ret >>= (64 - lg_range); + state1 = prng_state_next_u64(*state); + *state = state1; + ret = state1 >> (64 - lg_range); + + return (ret); +} + +JEMALLOC_ALWAYS_INLINE size_t +prng_lg_range_zu(size_t *state, unsigned lg_range, bool atomic) +{ + size_t ret, state1; + + assert(lg_range > 0); + assert(lg_range <= ZU(1) << (3 + LG_SIZEOF_PTR)); + + if (atomic) { + size_t state0; + + do { + state0 = atomic_read_z(state); + state1 = prng_state_next_zu(state0); + } while (atomic_cas_z(state, state0, state1)); + } else { + state1 = prng_state_next_zu(*state); + *state = state1; + } + ret = state1 >> ((ZU(1) << (3 + LG_SIZEOF_PTR)) - lg_range); + + return (ret); +} + +JEMALLOC_ALWAYS_INLINE uint32_t +prng_range_u32(uint32_t *state, uint32_t range, bool atomic) +{ + uint32_t ret; + unsigned lg_range; + + assert(range > 1); + + /* Compute the ceiling of lg(range). */ + lg_range = ffs_u32(pow2_ceil_u32(range)) - 1; + + /* Generate a result in [0..range) via repeated trial. */ + do { + ret = prng_lg_range_u32(state, lg_range, atomic); + } while (ret >= range); return (ret); } JEMALLOC_ALWAYS_INLINE uint64_t -prng_range(uint64_t *state, uint64_t range) +prng_range_u64(uint64_t *state, uint64_t range) { uint64_t ret; unsigned lg_range; @@ -68,7 +177,26 @@ prng_range(uint64_t *state, uint64_t range) /* Generate a result in [0..range) via repeated trial. */ do { - ret = prng_lg_range(state, lg_range); + ret = prng_lg_range_u64(state, lg_range); + } while (ret >= range); + + return (ret); +} + +JEMALLOC_ALWAYS_INLINE size_t +prng_range_zu(size_t *state, size_t range, bool atomic) +{ + size_t ret; + unsigned lg_range; + + assert(range > 1); + + /* Compute the ceiling of lg(range). */ + lg_range = ffs_u64(pow2_ceil_u64(range)) - 1; + + /* Generate a result in [0..range) via repeated trial. */ + do { + ret = prng_lg_range_zu(state, lg_range, atomic); } while (ret >= range); return (ret); diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 21dff5f..8293b71 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -299,9 +299,9 @@ extern prof_dump_header_t *prof_dump_header; void prof_idump(tsdn_t *tsdn); bool prof_mdump(tsd_t *tsd, const char *filename); void prof_gdump(tsdn_t *tsdn); -prof_tdata_t *prof_tdata_init(tsdn_t *tsdn); +prof_tdata_t *prof_tdata_init(tsd_t *tsd); prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata); -void prof_reset(tsdn_t *tsdn, size_t lg_sample); +void prof_reset(tsd_t *tsd, size_t lg_sample); void prof_tdata_cleanup(tsd_t *tsd); bool prof_active_get(tsdn_t *tsdn); bool prof_active_set(tsdn_t *tsdn, bool active); @@ -315,7 +315,7 @@ bool prof_gdump_get(tsdn_t *tsdn); bool prof_gdump_set(tsdn_t *tsdn, bool active); void prof_boot0(void); void prof_boot1(void); -bool prof_boot2(tsdn_t *tsdn); +bool prof_boot2(tsd_t *tsd); void prof_prefork0(tsdn_t *tsdn); void prof_prefork1(tsdn_t *tsdn); void prof_postfork_parent(tsdn_t *tsdn); @@ -384,7 +384,7 @@ prof_tdata_get(tsd_t *tsd, bool create) if (create) { if (unlikely(tdata == NULL)) { if (tsd_nominal(tsd)) { - tdata = prof_tdata_init(tsd_tsdn(tsd)); + tdata = prof_tdata_init(tsd); tsd_prof_tdata_set(tsd, tdata); } } else if (unlikely(tdata->expired)) { diff --git a/include/jemalloc/internal/size_classes.h b/include/jemalloc/internal/size_classes.h index f33390d..e4edc4b 100644 --- a/include/jemalloc/internal/size_classes.h +++ b/include/jemalloc/internal/size_classes.h @@ -7,13 +7,13 @@ * be defined prior to inclusion, and it in turn defines: * * LG_SIZE_CLASS_GROUP: Lg of size class count for each size doubling. - * SIZE_CLASSES: Complete table of - * SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) - * tuples. + * SIZE_CLASSES: Complete table of SC(index, lg_grp, lg_delta, ndelta, psz, + * bin, lg_delta_lookup) tuples. * index: Size class index. * lg_grp: Lg group base size (no deltas added). * lg_delta: Lg delta to previous size class. * ndelta: Delta multiplier. size == 1<<lg_grp + ndelta<<lg_delta + * psz: 'yes' if a multiple of the page size, 'no' otherwise. * bin: 'yes' if a small bin size class, 'no' otherwise. * lg_delta_lookup: Same as lg_delta if a lookup table size class, 'no' * otherwise. @@ -21,6 +21,7 @@ * NLBINS: Number of bins supported by the lookup table. * NBINS: Number of small size class bins. * NSIZES: Number of size classes. + * NPSIZES: Number of size classes that are a multiple of (1U << LG_PAGE). * LG_TINY_MAXCLASS: Lg of maximum tiny size class. * LOOKUP_MAXCLASS: Maximum size class included in lookup table. * SMALL_MAXCLASS: Maximum small size class. @@ -32,146 +33,147 @@ #if (LG_SIZEOF_PTR == 2 && LG_TINY_MIN == 3 && LG_QUANTUM == 3 && LG_PAGE == 12) #define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup */ \ - SC( 0, 3, 3, 0, yes, 3) \ - SC( 1, 3, 3, 1, yes, 3) \ - SC( 2, 3, 3, 2, yes, 3) \ - SC( 3, 3, 3, 3, yes, 3) \ - \ - SC( 4, 5, 3, 1, yes, 3) \ - SC( 5, 5, 3, 2, yes, 3) \ - SC( 6, 5, 3, 3, yes, 3) \ - SC( 7, 5, 3, 4, yes, 3) \ - \ - SC( 8, 6, 4, 1, yes, 4) \ - SC( 9, 6, 4, 2, yes, 4) \ - SC( 10, 6, 4, 3, yes, 4) \ - SC( 11, 6, 4, 4, yes, 4) \ - \ - SC( 12, 7, 5, 1, yes, 5) \ - SC( 13, 7, 5, 2, yes, 5) \ - SC( 14, 7, 5, 3, yes, 5) \ - SC( 15, 7, 5, 4, yes, 5) \ - \ - SC( 16, 8, 6, 1, yes, 6) \ - SC( 17, 8, 6, 2, yes, 6) \ - SC( 18, 8, 6, 3, yes, 6) \ - SC( 19, 8, 6, 4, yes, 6) \ - \ - SC( 20, 9, 7, 1, yes, 7) \ - SC( 21, 9, 7, 2, yes, 7) \ - SC( 22, 9, 7, 3, yes, 7) \ - SC( 23, 9, 7, 4, yes, 7) \ - \ - SC( 24, 10, 8, 1, yes, 8) \ - SC( 25, 10, 8, 2, yes, 8) \ - SC( 26, 10, 8, 3, yes, 8) \ - SC( 27, 10, 8, 4, yes, 8) \ - \ - SC( 28, 11, 9, 1, yes, 9) \ - SC( 29, 11, 9, 2, yes, 9) \ - SC( 30, 11, 9, 3, yes, 9) \ - SC( 31, 11, 9, 4, yes, 9) \ - \ - SC( 32, 12, 10, 1, yes, no) \ - SC( 33, 12, 10, 2, yes, no) \ - SC( 34, 12, 10, 3, yes, no) \ - SC( 35, 12, 10, 4, yes, no) \ - \ - SC( 36, 13, 11, 1, yes, no) \ - SC( 37, 13, 11, 2, yes, no) \ - SC( 38, 13, 11, 3, yes, no) \ - SC( 39, 13, 11, 4, no, no) \ - \ - SC( 40, 14, 12, 1, no, no) \ - SC( 41, 14, 12, 2, no, no) \ - SC( 42, 14, 12, 3, no, no) \ - SC( 43, 14, 12, 4, no, no) \ - \ - SC( 44, 15, 13, 1, no, no) \ - SC( 45, 15, 13, 2, no, no) \ - SC( 46, 15, 13, 3, no, no) \ - SC( 47, 15, 13, 4, no, no) \ - \ - SC( 48, 16, 14, 1, no, no) \ - SC( 49, 16, 14, 2, no, no) \ - SC( 50, 16, 14, 3, no, no) \ - SC( 51, 16, 14, 4, no, no) \ - \ - SC( 52, 17, 15, 1, no, no) \ - SC( 53, 17, 15, 2, no, no) \ - SC( 54, 17, 15, 3, no, no) \ - SC( 55, 17, 15, 4, no, no) \ - \ - SC( 56, 18, 16, 1, no, no) \ - SC( 57, 18, 16, 2, no, no) \ - SC( 58, 18, 16, 3, no, no) \ - SC( 59, 18, 16, 4, no, no) \ - \ - SC( 60, 19, 17, 1, no, no) \ - SC( 61, 19, 17, 2, no, no) \ - SC( 62, 19, 17, 3, no, no) \ - SC( 63, 19, 17, 4, no, no) \ - \ - SC( 64, 20, 18, 1, no, no) \ - SC( 65, 20, 18, 2, no, no) \ - SC( 66, 20, 18, 3, no, no) \ - SC( 67, 20, 18, 4, no, no) \ - \ - SC( 68, 21, 19, 1, no, no) \ - SC( 69, 21, 19, 2, no, no) \ - SC( 70, 21, 19, 3, no, no) \ - SC( 71, 21, 19, 4, no, no) \ - \ - SC( 72, 22, 20, 1, no, no) \ - SC( 73, 22, 20, 2, no, no) \ - SC( 74, 22, 20, 3, no, no) \ - SC( 75, 22, 20, 4, no, no) \ - \ - SC( 76, 23, 21, 1, no, no) \ - SC( 77, 23, 21, 2, no, no) \ - SC( 78, 23, 21, 3, no, no) \ - SC( 79, 23, 21, 4, no, no) \ - \ - SC( 80, 24, 22, 1, no, no) \ - SC( 81, 24, 22, 2, no, no) \ - SC( 82, 24, 22, 3, no, no) \ - SC( 83, 24, 22, 4, no, no) \ - \ - SC( 84, 25, 23, 1, no, no) \ - SC( 85, 25, 23, 2, no, no) \ - SC( 86, 25, 23, 3, no, no) \ - SC( 87, 25, 23, 4, no, no) \ - \ - SC( 88, 26, 24, 1, no, no) \ - SC( 89, 26, 24, 2, no, no) \ - SC( 90, 26, 24, 3, no, no) \ - SC( 91, 26, 24, 4, no, no) \ - \ - SC( 92, 27, 25, 1, no, no) \ - SC( 93, 27, 25, 2, no, no) \ - SC( 94, 27, 25, 3, no, no) \ - SC( 95, 27, 25, 4, no, no) \ - \ - SC( 96, 28, 26, 1, no, no) \ - SC( 97, 28, 26, 2, no, no) \ - SC( 98, 28, 26, 3, no, no) \ - SC( 99, 28, 26, 4, no, no) \ - \ - SC(100, 29, 27, 1, no, no) \ - SC(101, 29, 27, 2, no, no) \ - SC(102, 29, 27, 3, no, no) \ - SC(103, 29, 27, 4, no, no) \ - \ - SC(104, 30, 28, 1, no, no) \ - SC(105, 30, 28, 2, no, no) \ - SC(106, 30, 28, 3, no, no) \ + /* index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup */ \ + SC( 0, 3, 3, 0, no, yes, 3) \ + SC( 1, 3, 3, 1, no, yes, 3) \ + SC( 2, 3, 3, 2, no, yes, 3) \ + SC( 3, 3, 3, 3, no, yes, 3) \ + \ + SC( 4, 5, 3, 1, no, yes, 3) \ + SC( 5, 5, 3, 2, no, yes, 3) \ + SC( 6, 5, 3, 3, no, yes, 3) \ + SC( 7, 5, 3, 4, no, yes, 3) \ + \ + SC( 8, 6, 4, 1, no, yes, 4) \ + SC( 9, 6, 4, 2, no, yes, 4) \ + SC( 10, 6, 4, 3, no, yes, 4) \ + SC( 11, 6, 4, 4, no, yes, 4) \ + \ + SC( 12, 7, 5, 1, no, yes, 5) \ + SC( 13, 7, 5, 2, no, yes, 5) \ + SC( 14, 7, 5, 3, no, yes, 5) \ + SC( 15, 7, 5, 4, no, yes, 5) \ + \ + SC( 16, 8, 6, 1, no, yes, 6) \ + SC( 17, 8, 6, 2, no, yes, 6) \ + SC( 18, 8, 6, 3, no, yes, 6) \ + SC( 19, 8, 6, 4, no, yes, 6) \ + \ + SC( 20, 9, 7, 1, no, yes, 7) \ + SC( 21, 9, 7, 2, no, yes, 7) \ + SC( 22, 9, 7, 3, no, yes, 7) \ + SC( 23, 9, 7, 4, no, yes, 7) \ + \ + SC( 24, 10, 8, 1, no, yes, 8) \ + SC( 25, 10, 8, 2, no, yes, 8) \ + SC( 26, 10, 8, 3, no, yes, 8) \ + SC( 27, 10, 8, 4, no, yes, 8) \ + \ + SC( 28, 11, 9, 1, no, yes, 9) \ + SC( 29, 11, 9, 2, no, yes, 9) \ + SC( 30, 11, 9, 3, no, yes, 9) \ + SC( 31, 11, 9, 4, yes, yes, 9) \ + \ + SC( 32, 12, 10, 1, no, yes, no) \ + SC( 33, 12, 10, 2, no, yes, no) \ + SC( 34, 12, 10, 3, no, yes, no) \ + SC( 35, 12, 10, 4, yes, yes, no) \ + \ + SC( 36, 13, 11, 1, no, yes, no) \ + SC( 37, 13, 11, 2, yes, yes, no) \ + SC( 38, 13, 11, 3, no, yes, no) \ + SC( 39, 13, 11, 4, yes, no, no) \ + \ + SC( 40, 14, 12, 1, yes, no, no) \ + SC( 41, 14, 12, 2, yes, no, no) \ + SC( 42, 14, 12, 3, yes, no, no) \ + SC( 43, 14, 12, 4, yes, no, no) \ + \ + SC( 44, 15, 13, 1, yes, no, no) \ + SC( 45, 15, 13, 2, yes, no, no) \ + SC( 46, 15, 13, 3, yes, no, no) \ + SC( 47, 15, 13, 4, yes, no, no) \ + \ + SC( 48, 16, 14, 1, yes, no, no) \ + SC( 49, 16, 14, 2, yes, no, no) \ + SC( 50, 16, 14, 3, yes, no, no) \ + SC( 51, 16, 14, 4, yes, no, no) \ + \ + SC( 52, 17, 15, 1, yes, no, no) \ + SC( 53, 17, 15, 2, yes, no, no) \ + SC( 54, 17, 15, 3, yes, no, no) \ + SC( 55, 17, 15, 4, yes, no, no) \ + \ + SC( 56, 18, 16, 1, yes, no, no) \ + SC( 57, 18, 16, 2, yes, no, no) \ + SC( 58, 18, 16, 3, yes, no, no) \ + SC( 59, 18, 16, 4, yes, no, no) \ + \ + SC( 60, 19, 17, 1, yes, no, no) \ + SC( 61, 19, 17, 2, yes, no, no) \ + SC( 62, 19, 17, 3, yes, no, no) \ + SC( 63, 19, 17, 4, yes, no, no) \ + \ + SC( 64, 20, 18, 1, yes, no, no) \ + SC( 65, 20, 18, 2, yes, no, no) \ + SC( 66, 20, 18, 3, yes, no, no) \ + SC( 67, 20, 18, 4, yes, no, no) \ + \ + SC( 68, 21, 19, 1, yes, no, no) \ + SC( 69, 21, 19, 2, yes, no, no) \ + SC( 70, 21, 19, 3, yes, no, no) \ + SC( 71, 21, 19, 4, yes, no, no) \ + \ + SC( 72, 22, 20, 1, yes, no, no) \ + SC( 73, 22, 20, 2, yes, no, no) \ + SC( 74, 22, 20, 3, yes, no, no) \ + SC( 75, 22, 20, 4, yes, no, no) \ + \ + SC( 76, 23, 21, 1, yes, no, no) \ + SC( 77, 23, 21, 2, yes, no, no) \ + SC( 78, 23, 21, 3, yes, no, no) \ + SC( 79, 23, 21, 4, yes, no, no) \ + \ + SC( 80, 24, 22, 1, yes, no, no) \ + SC( 81, 24, 22, 2, yes, no, no) \ + SC( 82, 24, 22, 3, yes, no, no) \ + SC( 83, 24, 22, 4, yes, no, no) \ + \ + SC( 84, 25, 23, 1, yes, no, no) \ + SC( 85, 25, 23, 2, yes, no, no) \ + SC( 86, 25, 23, 3, yes, no, no) \ + SC( 87, 25, 23, 4, yes, no, no) \ + \ + SC( 88, 26, 24, 1, yes, no, no) \ + SC( 89, 26, 24, 2, yes, no, no) \ + SC( 90, 26, 24, 3, yes, no, no) \ + SC( 91, 26, 24, 4, yes, no, no) \ + \ + SC( 92, 27, 25, 1, yes, no, no) \ + SC( 93, 27, 25, 2, yes, no, no) \ + SC( 94, 27, 25, 3, yes, no, no) \ + SC( 95, 27, 25, 4, yes, no, no) \ + \ + SC( 96, 28, 26, 1, yes, no, no) \ + SC( 97, 28, 26, 2, yes, no, no) \ + SC( 98, 28, 26, 3, yes, no, no) \ + SC( 99, 28, 26, 4, yes, no, no) \ + \ + SC(100, 29, 27, 1, yes, no, no) \ + SC(101, 29, 27, 2, yes, no, no) \ + SC(102, 29, 27, 3, yes, no, no) \ + SC(103, 29, 27, 4, yes, no, no) \ + \ + SC(104, 30, 28, 1, yes, no, no) \ + SC(105, 30, 28, 2, yes, no, no) \ + SC(106, 30, 28, 3, yes, no, no) \ #define SIZE_CLASSES_DEFINED #define NTBINS 0 #define NLBINS 32 #define NBINS 39 #define NSIZES 107 +#define NPSIZES 71 #define LG_TINY_MAXCLASS "NA" #define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) #define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) @@ -181,143 +183,144 @@ #if (LG_SIZEOF_PTR == 2 && LG_TINY_MIN == 3 && LG_QUANTUM == 4 && LG_PAGE == 12) #define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup */ \ - SC( 0, 3, 3, 0, yes, 3) \ - \ - SC( 1, 3, 3, 1, yes, 3) \ - SC( 2, 4, 4, 1, yes, 4) \ - SC( 3, 4, 4, 2, yes, 4) \ - SC( 4, 4, 4, 3, yes, 4) \ - \ - SC( 5, 6, 4, 1, yes, 4) \ - SC( 6, 6, 4, 2, yes, 4) \ - SC( 7, 6, 4, 3, yes, 4) \ - SC( 8, 6, 4, 4, yes, 4) \ - \ - SC( 9, 7, 5, 1, yes, 5) \ - SC( 10, 7, 5, 2, yes, 5) \ - SC( 11, 7, 5, 3, yes, 5) \ - SC( 12, 7, 5, 4, yes, 5) \ - \ - SC( 13, 8, 6, 1, yes, 6) \ - SC( 14, 8, 6, 2, yes, 6) \ - SC( 15, 8, 6, 3, yes, 6) \ - SC( 16, 8, 6, 4, yes, 6) \ - \ - SC( 17, 9, 7, 1, yes, 7) \ - SC( 18, 9, 7, 2, yes, 7) \ - SC( 19, 9, 7, 3, yes, 7) \ - SC( 20, 9, 7, 4, yes, 7) \ - \ - SC( 21, 10, 8, 1, yes, 8) \ - SC( 22, 10, 8, 2, yes, 8) \ - SC( 23, 10, 8, 3, yes, 8) \ - SC( 24, 10, 8, 4, yes, 8) \ - \ - SC( 25, 11, 9, 1, yes, 9) \ - SC( 26, 11, 9, 2, yes, 9) \ - SC( 27, 11, 9, 3, yes, 9) \ - SC( 28, 11, 9, 4, yes, 9) \ - \ - SC( 29, 12, 10, 1, yes, no) \ - SC( 30, 12, 10, 2, yes, no) \ - SC( 31, 12, 10, 3, yes, no) \ - SC( 32, 12, 10, 4, yes, no) \ - \ - SC( 33, 13, 11, 1, yes, no) \ - SC( 34, 13, 11, 2, yes, no) \ - SC( 35, 13, 11, 3, yes, no) \ - SC( 36, 13, 11, 4, no, no) \ - \ - SC( 37, 14, 12, 1, no, no) \ - SC( 38, 14, 12, 2, no, no) \ - SC( 39, 14, 12, 3, no, no) \ - SC( 40, 14, 12, 4, no, no) \ - \ - SC( 41, 15, 13, 1, no, no) \ - SC( 42, 15, 13, 2, no, no) \ - SC( 43, 15, 13, 3, no, no) \ - SC( 44, 15, 13, 4, no, no) \ - \ - SC( 45, 16, 14, 1, no, no) \ - SC( 46, 16, 14, 2, no, no) \ - SC( 47, 16, 14, 3, no, no) \ - SC( 48, 16, 14, 4, no, no) \ - \ - SC( 49, 17, 15, 1, no, no) \ - SC( 50, 17, 15, 2, no, no) \ - SC( 51, 17, 15, 3, no, no) \ - SC( 52, 17, 15, 4, no, no) \ - \ - SC( 53, 18, 16, 1, no, no) \ - SC( 54, 18, 16, 2, no, no) \ - SC( 55, 18, 16, 3, no, no) \ - SC( 56, 18, 16, 4, no, no) \ - \ - SC( 57, 19, 17, 1, no, no) \ - SC( 58, 19, 17, 2, no, no) \ - SC( 59, 19, 17, 3, no, no) \ - SC( 60, 19, 17, 4, no, no) \ - \ - SC( 61, 20, 18, 1, no, no) \ - SC( 62, 20, 18, 2, no, no) \ - SC( 63, 20, 18, 3, no, no) \ - SC( 64, 20, 18, 4, no, no) \ - \ - SC( 65, 21, 19, 1, no, no) \ - SC( 66, 21, 19, 2, no, no) \ - SC( 67, 21, 19, 3, no, no) \ - SC( 68, 21, 19, 4, no, no) \ - \ - SC( 69, 22, 20, 1, no, no) \ - SC( 70, 22, 20, 2, no, no) \ - SC( 71, 22, 20, 3, no, no) \ - SC( 72, 22, 20, 4, no, no) \ - \ - SC( 73, 23, 21, 1, no, no) \ - SC( 74, 23, 21, 2, no, no) \ - SC( 75, 23, 21, 3, no, no) \ - SC( 76, 23, 21, 4, no, no) \ - \ - SC( 77, 24, 22, 1, no, no) \ - SC( 78, 24, 22, 2, no, no) \ - SC( 79, 24, 22, 3, no, no) \ - SC( 80, 24, 22, 4, no, no) \ - \ - SC( 81, 25, 23, 1, no, no) \ - SC( 82, 25, 23, 2, no, no) \ - SC( 83, 25, 23, 3, no, no) \ - SC( 84, 25, 23, 4, no, no) \ - \ - SC( 85, 26, 24, 1, no, no) \ - SC( 86, 26, 24, 2, no, no) \ - SC( 87, 26, 24, 3, no, no) \ - SC( 88, 26, 24, 4, no, no) \ - \ - SC( 89, 27, 25, 1, no, no) \ - SC( 90, 27, 25, 2, no, no) \ - SC( 91, 27, 25, 3, no, no) \ - SC( 92, 27, 25, 4, no, no) \ - \ - SC( 93, 28, 26, 1, no, no) \ - SC( 94, 28, 26, 2, no, no) \ - SC( 95, 28, 26, 3, no, no) \ - SC( 96, 28, 26, 4, no, no) \ - \ - SC( 97, 29, 27, 1, no, no) \ - SC( 98, 29, 27, 2, no, no) \ - SC( 99, 29, 27, 3, no, no) \ - SC(100, 29, 27, 4, no, no) \ - \ - SC(101, 30, 28, 1, no, no) \ - SC(102, 30, 28, 2, no, no) \ - SC(103, 30, 28, 3, no, no) \ + /* index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup */ \ + SC( 0, 3, 3, 0, no, yes, 3) \ + \ + SC( 1, 3, 3, 1, no, yes, 3) \ + SC( 2, 4, 4, 1, no, yes, 4) \ + SC( 3, 4, 4, 2, no, yes, 4) \ + SC( 4, 4, 4, 3, no, yes, 4) \ + \ + SC( 5, 6, 4, 1, no, yes, 4) \ + SC( 6, 6, 4, 2, no, yes, 4) \ + SC( 7, 6, 4, 3, no, yes, 4) \ + SC( 8, 6, 4, 4, no, yes, 4) \ + \ + SC( 9, 7, 5, 1, no, yes, 5) \ + SC( 10, 7, 5, 2, no, yes, 5) \ + SC( 11, 7, 5, 3, no, yes, 5) \ + SC( 12, 7, 5, 4, no, yes, 5) \ + \ + SC( 13, 8, 6, 1, no, yes, 6) \ + SC( 14, 8, 6, 2, no, yes, 6) \ + SC( 15, 8, 6, 3, no, yes, 6) \ + SC( 16, 8, 6, 4, no, yes, 6) \ + \ + SC( 17, 9, 7, 1, no, yes, 7) \ + SC( 18, 9, 7, 2, no, yes, 7) \ + SC( 19, 9, 7, 3, no, yes, 7) \ + SC( 20, 9, 7, 4, no, yes, 7) \ + \ + SC( 21, 10, 8, 1, no, yes, 8) \ + SC( 22, 10, 8, 2, no, yes, 8) \ + SC( 23, 10, 8, 3, no, yes, 8) \ + SC( 24, 10, 8, 4, no, yes, 8) \ + \ + SC( 25, 11, 9, 1, no, yes, 9) \ + SC( 26, 11, 9, 2, no, yes, 9) \ + SC( 27, 11, 9, 3, no, yes, 9) \ + SC( 28, 11, 9, 4, yes, yes, 9) \ + \ + SC( 29, 12, 10, 1, no, yes, no) \ + SC( 30, 12, 10, 2, no, yes, no) \ + SC( 31, 12, 10, 3, no, yes, no) \ + SC( 32, 12, 10, 4, yes, yes, no) \ + \ + SC( 33, 13, 11, 1, no, yes, no) \ + SC( 34, 13, 11, 2, yes, yes, no) \ + SC( 35, 13, 11, 3, no, yes, no) \ + SC( 36, 13, 11, 4, yes, no, no) \ + \ + SC( 37, 14, 12, 1, yes, no, no) \ + SC( 38, 14, 12, 2, yes, no, no) \ + SC( 39, 14, 12, 3, yes, no, no) \ + SC( 40, 14, 12, 4, yes, no, no) \ + \ + SC( 41, 15, 13, 1, yes, no, no) \ + SC( 42, 15, 13, 2, yes, no, no) \ + SC( 43, 15, 13, 3, yes, no, no) \ + SC( 44, 15, 13, 4, yes, no, no) \ + \ + SC( 45, 16, 14, 1, yes, no, no) \ + SC( 46, 16, 14, 2, yes, no, no) \ + SC( 47, 16, 14, 3, yes, no, no) \ + SC( 48, 16, 14, 4, yes, no, no) \ + \ + SC( 49, 17, 15, 1, yes, no, no) \ + SC( 50, 17, 15, 2, yes, no, no) \ + SC( 51, 17, 15, 3, yes, no, no) \ + SC( 52, 17, 15, 4, yes, no, no) \ + \ + SC( 53, 18, 16, 1, yes, no, no) \ + SC( 54, 18, 16, 2, yes, no, no) \ + SC( 55, 18, 16, 3, yes, no, no) \ + SC( 56, 18, 16, 4, yes, no, no) \ + \ + SC( 57, 19, 17, 1, yes, no, no) \ + SC( 58, 19, 17, 2, yes, no, no) \ + SC( 59, 19, 17, 3, yes, no, no) \ + SC( 60, 19, 17, 4, yes, no, no) \ + \ + SC( 61, 20, 18, 1, yes, no, no) \ + SC( 62, 20, 18, 2, yes, no, no) \ + SC( 63, 20, 18, 3, yes, no, no) \ + SC( 64, 20, 18, 4, yes, no, no) \ + \ + SC( 65, 21, 19, 1, yes, no, no) \ + SC( 66, 21, 19, 2, yes, no, no) \ + SC( 67, 21, 19, 3, yes, no, no) \ + SC( 68, 21, 19, 4, yes, no, no) \ + \ + SC( 69, 22, 20, 1, yes, no, no) \ + SC( 70, 22, 20, 2, yes, no, no) \ + SC( 71, 22, 20, 3, yes, no, no) \ + SC( 72, 22, 20, 4, yes, no, no) \ + \ + SC( 73, 23, 21, 1, yes, no, no) \ + SC( 74, 23, 21, 2, yes, no, no) \ + SC( 75, 23, 21, 3, yes, no, no) \ + SC( 76, 23, 21, 4, yes, no, no) \ + \ + SC( 77, 24, 22, 1, yes, no, no) \ + SC( 78, 24, 22, 2, yes, no, no) \ + SC( 79, 24, 22, 3, yes, no, no) \ + SC( 80, 24, 22, 4, yes, no, no) \ + \ + SC( 81, 25, 23, 1, yes, no, no) \ + SC( 82, 25, 23, 2, yes, no, no) \ + SC( 83, 25, 23, 3, yes, no, no) \ + SC( 84, 25, 23, 4, yes, no, no) \ + \ + SC( 85, 26, 24, 1, yes, no, no) \ + SC( 86, 26, 24, 2, yes, no, no) \ + SC( 87, 26, 24, 3, yes, no, no) \ + SC( 88, 26, 24, 4, yes, no, no) \ + \ + SC( 89, 27, 25, 1, yes, no, no) \ + SC( 90, 27, 25, 2, yes, no, no) \ + SC( 91, 27, 25, 3, yes, no, no) \ + SC( 92, 27, 25, 4, yes, no, no) \ + \ + SC( 93, 28, 26, 1, yes, no, no) \ + SC( 94, 28, 26, 2, yes, no, no) \ + SC( 95, 28, 26, 3, yes, no, no) \ + SC( 96, 28, 26, 4, yes, no, no) \ + \ + SC( 97, 29, 27, 1, yes, no, no) \ + SC( 98, 29, 27, 2, yes, no, no) \ + SC( 99, 29, 27, 3, yes, no, no) \ + SC(100, 29, 27, 4, yes, no, no) \ + \ + SC(101, 30, 28, 1, yes, no, no) \ + SC(102, 30, 28, 2, yes, no, no) \ + SC(103, 30, 28, 3, yes, no, no) \ #define SIZE_CLASSES_DEFINED #define NTBINS 1 #define NLBINS 29 #define NBINS 36 #define NSIZES 104 +#define NPSIZES 71 #define LG_TINY_MAXCLASS 3 #define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) #define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) @@ -327,141 +330,142 @@ #if (LG_SIZEOF_PTR == 2 && LG_TINY_MIN == 4 && LG_QUANTUM == 4 && LG_PAGE == 12) #define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup */ \ - SC( 0, 4, 4, 0, yes, 4) \ - SC( 1, 4, 4, 1, yes, 4) \ - SC( 2, 4, 4, 2, yes, 4) \ - SC( 3, 4, 4, 3, yes, 4) \ - \ - SC( 4, 6, 4, 1, yes, 4) \ - SC( 5, 6, 4, 2, yes, 4) \ - SC( 6, 6, 4, 3, yes, 4) \ - SC( 7, 6, 4, 4, yes, 4) \ - \ - SC( 8, 7, 5, 1, yes, 5) \ - SC( 9, 7, 5, 2, yes, 5) \ - SC( 10, 7, 5, 3, yes, 5) \ - SC( 11, 7, 5, 4, yes, 5) \ - \ - SC( 12, 8, 6, 1, yes, 6) \ - SC( 13, 8, 6, 2, yes, 6) \ - SC( 14, 8, 6, 3, yes, 6) \ - SC( 15, 8, 6, 4, yes, 6) \ - \ - SC( 16, 9, 7, 1, yes, 7) \ - SC( 17, 9, 7, 2, yes, 7) \ - SC( 18, 9, 7, 3, yes, 7) \ - SC( 19, 9, 7, 4, yes, 7) \ - \ - SC( 20, 10, 8, 1, yes, 8) \ - SC( 21, 10, 8, 2, yes, 8) \ - SC( 22, 10, 8, 3, yes, 8) \ - SC( 23, 10, 8, 4, yes, 8) \ - \ - SC( 24, 11, 9, 1, yes, 9) \ - SC( 25, 11, 9, 2, yes, 9) \ - SC( 26, 11, 9, 3, yes, 9) \ - SC( 27, 11, 9, 4, yes, 9) \ - \ - SC( 28, 12, 10, 1, yes, no) \ - SC( 29, 12, 10, 2, yes, no) \ - SC( 30, 12, 10, 3, yes, no) \ - SC( 31, 12, 10, 4, yes, no) \ - \ - SC( 32, 13, 11, 1, yes, no) \ - SC( 33, 13, 11, 2, yes, no) \ - SC( 34, 13, 11, 3, yes, no) \ - SC( 35, 13, 11, 4, no, no) \ - \ - SC( 36, 14, 12, 1, no, no) \ - SC( 37, 14, 12, 2, no, no) \ - SC( 38, 14, 12, 3, no, no) \ - SC( 39, 14, 12, 4, no, no) \ - \ - SC( 40, 15, 13, 1, no, no) \ - SC( 41, 15, 13, 2, no, no) \ - SC( 42, 15, 13, 3, no, no) \ - SC( 43, 15, 13, 4, no, no) \ - \ - SC( 44, 16, 14, 1, no, no) \ - SC( 45, 16, 14, 2, no, no) \ - SC( 46, 16, 14, 3, no, no) \ - SC( 47, 16, 14, 4, no, no) \ - \ - SC( 48, 17, 15, 1, no, no) \ - SC( 49, 17, 15, 2, no, no) \ - SC( 50, 17, 15, 3, no, no) \ - SC( 51, 17, 15, 4, no, no) \ - \ - SC( 52, 18, 16, 1, no, no) \ - SC( 53, 18, 16, 2, no, no) \ - SC( 54, 18, 16, 3, no, no) \ - SC( 55, 18, 16, 4, no, no) \ - \ - SC( 56, 19, 17, 1, no, no) \ - SC( 57, 19, 17, 2, no, no) \ - SC( 58, 19, 17, 3, no, no) \ - SC( 59, 19, 17, 4, no, no) \ - \ - SC( 60, 20, 18, 1, no, no) \ - SC( 61, 20, 18, 2, no, no) \ - SC( 62, 20, 18, 3, no, no) \ - SC( 63, 20, 18, 4, no, no) \ - \ - SC( 64, 21, 19, 1, no, no) \ - SC( 65, 21, 19, 2, no, no) \ - SC( 66, 21, 19, 3, no, no) \ - SC( 67, 21, 19, 4, no, no) \ - \ - SC( 68, 22, 20, 1, no, no) \ - SC( 69, 22, 20, 2, no, no) \ - SC( 70, 22, 20, 3, no, no) \ - SC( 71, 22, 20, 4, no, no) \ - \ - SC( 72, 23, 21, 1, no, no) \ - SC( 73, 23, 21, 2, no, no) \ - SC( 74, 23, 21, 3, no, no) \ - SC( 75, 23, 21, 4, no, no) \ - \ - SC( 76, 24, 22, 1, no, no) \ - SC( 77, 24, 22, 2, no, no) \ - SC( 78, 24, 22, 3, no, no) \ - SC( 79, 24, 22, 4, no, no) \ - \ - SC( 80, 25, 23, 1, no, no) \ - SC( 81, 25, 23, 2, no, no) \ - SC( 82, 25, 23, 3, no, no) \ - SC( 83, 25, 23, 4, no, no) \ - \ - SC( 84, 26, 24, 1, no, no) \ - SC( 85, 26, 24, 2, no, no) \ - SC( 86, 26, 24, 3, no, no) \ - SC( 87, 26, 24, 4, no, no) \ - \ - SC( 88, 27, 25, 1, no, no) \ - SC( 89, 27, 25, 2, no, no) \ - SC( 90, 27, 25, 3, no, no) \ - SC( 91, 27, 25, 4, no, no) \ - \ - SC( 92, 28, 26, 1, no, no) \ - SC( 93, 28, 26, 2, no, no) \ - SC( 94, 28, 26, 3, no, no) \ - SC( 95, 28, 26, 4, no, no) \ - \ - SC( 96, 29, 27, 1, no, no) \ - SC( 97, 29, 27, 2, no, no) \ - SC( 98, 29, 27, 3, no, no) \ - SC( 99, 29, 27, 4, no, no) \ - \ - SC(100, 30, 28, 1, no, no) \ - SC(101, 30, 28, 2, no, no) \ - SC(102, 30, 28, 3, no, no) \ + /* index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup */ \ + SC( 0, 4, 4, 0, no, yes, 4) \ + SC( 1, 4, 4, 1, no, yes, 4) \ + SC( 2, 4, 4, 2, no, yes, 4) \ + SC( 3, 4, 4, 3, no, yes, 4) \ + \ + SC( 4, 6, 4, 1, no, yes, 4) \ + SC( 5, 6, 4, 2, no, yes, 4) \ + SC( 6, 6, 4, 3, no, yes, 4) \ + SC( 7, 6, 4, 4, no, yes, 4) \ + \ + SC( 8, 7, 5, 1, no, yes, 5) \ + SC( 9, 7, 5, 2, no, yes, 5) \ + SC( 10, 7, 5, 3, no, yes, 5) \ + SC( 11, 7, 5, 4, no, yes, 5) \ + \ + SC( 12, 8, 6, 1, no, yes, 6) \ + SC( 13, 8, 6, 2, no, yes, 6) \ + SC( 14, 8, 6, 3, no, yes, 6) \ + SC( 15, 8, 6, 4, no, yes, 6) \ + \ + SC( 16, 9, 7, 1, no, yes, 7) \ + SC( 17, 9, 7, 2, no, yes, 7) \ + SC( 18, 9, 7, 3, no, yes, 7) \ + SC( 19, 9, 7, 4, no, yes, 7) \ + \ + SC( 20, 10, 8, 1, no, yes, 8) \ + SC( 21, 10, 8, 2, no, yes, 8) \ + SC( 22, 10, 8, 3, no, yes, 8) \ + SC( 23, 10, 8, 4, no, yes, 8) \ + \ + SC( 24, 11, 9, 1, no, yes, 9) \ + SC( 25, 11, 9, 2, no, yes, 9) \ + SC( 26, 11, 9, 3, no, yes, 9) \ + SC( 27, 11, 9, 4, yes, yes, 9) \ + \ + SC( 28, 12, 10, 1, no, yes, no) \ + SC( 29, 12, 10, 2, no, yes, no) \ + SC( 30, 12, 10, 3, no, yes, no) \ + SC( 31, 12, 10, 4, yes, yes, no) \ + \ + SC( 32, 13, 11, 1, no, yes, no) \ + SC( 33, 13, 11, 2, yes, yes, no) \ + SC( 34, 13, 11, 3, no, yes, no) \ + SC( 35, 13, 11, 4, yes, no, no) \ + \ + SC( 36, 14, 12, 1, yes, no, no) \ + SC( 37, 14, 12, 2, yes, no, no) \ + SC( 38, 14, 12, 3, yes, no, no) \ + SC( 39, 14, 12, 4, yes, no, no) \ + \ + SC( 40, 15, 13, 1, yes, no, no) \ + SC( 41, 15, 13, 2, yes, no, no) \ + SC( 42, 15, 13, 3, yes, no, no) \ + SC( 43, 15, 13, 4, yes, no, no) \ + \ + SC( 44, 16, 14, 1, yes, no, no) \ + SC( 45, 16, 14, 2, yes, no, no) \ + SC( 46, 16, 14, 3, yes, no, no) \ + SC( 47, 16, 14, 4, yes, no, no) \ + \ + SC( 48, 17, 15, 1, yes, no, no) \ + SC( 49, 17, 15, 2, yes, no, no) \ + SC( 50, 17, 15, 3, yes, no, no) \ + SC( 51, 17, 15, 4, yes, no, no) \ + \ + SC( 52, 18, 16, 1, yes, no, no) \ + SC( 53, 18, 16, 2, yes, no, no) \ + SC( 54, 18, 16, 3, yes, no, no) \ + SC( 55, 18, 16, 4, yes, no, no) \ + \ + SC( 56, 19, 17, 1, yes, no, no) \ + SC( 57, 19, 17, 2, yes, no, no) \ + SC( 58, 19, 17, 3, yes, no, no) \ + SC( 59, 19, 17, 4, yes, no, no) \ + \ + SC( 60, 20, 18, 1, yes, no, no) \ + SC( 61, 20, 18, 2, yes, no, no) \ + SC( 62, 20, 18, 3, yes, no, no) \ + SC( 63, 20, 18, 4, yes, no, no) \ + \ + SC( 64, 21, 19, 1, yes, no, no) \ + SC( 65, 21, 19, 2, yes, no, no) \ + SC( 66, 21, 19, 3, yes, no, no) \ + SC( 67, 21, 19, 4, yes, no, no) \ + \ + SC( 68, 22, 20, 1, yes, no, no) \ + SC( 69, 22, 20, 2, yes, no, no) \ + SC( 70, 22, 20, 3, yes, no, no) \ + SC( 71, 22, 20, 4, yes, no, no) \ + \ + SC( 72, 23, 21, 1, yes, no, no) \ + SC( 73, 23, 21, 2, yes, no, no) \ + SC( 74, 23, 21, 3, yes, no, no) \ + SC( 75, 23, 21, 4, yes, no, no) \ + \ + SC( 76, 24, 22, 1, yes, no, no) \ + SC( 77, 24, 22, 2, yes, no, no) \ + SC( 78, 24, 22, 3, yes, no, no) \ + SC( 79, 24, 22, 4, yes, no, no) \ + \ + SC( 80, 25, 23, 1, yes, no, no) \ + SC( 81, 25, 23, 2, yes, no, no) \ + SC( 82, 25, 23, 3, yes, no, no) \ + SC( 83, 25, 23, 4, yes, no, no) \ + \ + SC( 84, 26, 24, 1, yes, no, no) \ + SC( 85, 26, 24, 2, yes, no, no) \ + SC( 86, 26, 24, 3, yes, no, no) \ + SC( 87, 26, 24, 4, yes, no, no) \ + \ + SC( 88, 27, 25, 1, yes, no, no) \ + SC( 89, 27, 25, 2, yes, no, no) \ + SC( 90, 27, 25, 3, yes, no, no) \ + SC( 91, 27, 25, 4, yes, no, no) \ + \ + SC( 92, 28, 26, 1, yes, no, no) \ + SC( 93, 28, 26, 2, yes, no, no) \ + SC( 94, 28, 26, 3, yes, no, no) \ + SC( 95, 28, 26, 4, yes, no, no) \ + \ + SC( 96, 29, 27, 1, yes, no, no) \ + SC( 97, 29, 27, 2, yes, no, no) \ + SC( 98, 29, 27, 3, yes, no, no) \ + SC( 99, 29, 27, 4, yes, no, no) \ + \ + SC(100, 30, 28, 1, yes, no, no) \ + SC(101, 30, 28, 2, yes, no, no) \ + SC(102, 30, 28, 3, yes, no, no) \ #define SIZE_CLASSES_DEFINED #define NTBINS 0 #define NLBINS 28 #define NBINS 35 #define NSIZES 103 +#define NPSIZES 71 #define LG_TINY_MAXCLASS "NA" #define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) #define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) @@ -471,306 +475,307 @@ #if (LG_SIZEOF_PTR == 3 && LG_TINY_MIN == 3 && LG_QUANTUM == 3 && LG_PAGE == 12) #define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup */ \ - SC( 0, 3, 3, 0, yes, 3) \ - SC( 1, 3, 3, 1, yes, 3) \ - SC( 2, 3, 3, 2, yes, 3) \ - SC( 3, 3, 3, 3, yes, 3) \ - \ - SC( 4, 5, 3, 1, yes, 3) \ - SC( 5, 5, 3, 2, yes, 3) \ - SC( 6, 5, 3, 3, yes, 3) \ - SC( 7, 5, 3, 4, yes, 3) \ - \ - SC( 8, 6, 4, 1, yes, 4) \ - SC( 9, 6, 4, 2, yes, 4) \ - SC( 10, 6, 4, 3, yes, 4) \ - SC( 11, 6, 4, 4, yes, 4) \ - \ - SC( 12, 7, 5, 1, yes, 5) \ - SC( 13, 7, 5, 2, yes, 5) \ - SC( 14, 7, 5, 3, yes, 5) \ - SC( 15, 7, 5, 4, yes, 5) \ - \ - SC( 16, 8, 6, 1, yes, 6) \ - SC( 17, 8, 6, 2, yes, 6) \ - SC( 18, 8, 6, 3, yes, 6) \ - SC( 19, 8, 6, 4, yes, 6) \ - \ - SC( 20, 9, 7, 1, yes, 7) \ - SC( 21, 9, 7, 2, yes, 7) \ - SC( 22, 9, 7, 3, yes, 7) \ - SC( 23, 9, 7, 4, yes, 7) \ - \ - SC( 24, 10, 8, 1, yes, 8) \ - SC( 25, 10, 8, 2, yes, 8) \ - SC( 26, 10, 8, 3, yes, 8) \ - SC( 27, 10, 8, 4, yes, 8) \ - \ - SC( 28, 11, 9, 1, yes, 9) \ - SC( 29, 11, 9, 2, yes, 9) \ - SC( 30, 11, 9, 3, yes, 9) \ - SC( 31, 11, 9, 4, yes, 9) \ - \ - SC( 32, 12, 10, 1, yes, no) \ - SC( 33, 12, 10, 2, yes, no) \ - SC( 34, 12, 10, 3, yes, no) \ - SC( 35, 12, 10, 4, yes, no) \ - \ - SC( 36, 13, 11, 1, yes, no) \ - SC( 37, 13, 11, 2, yes, no) \ - SC( 38, 13, 11, 3, yes, no) \ - SC( 39, 13, 11, 4, no, no) \ - \ - SC( 40, 14, 12, 1, no, no) \ - SC( 41, 14, 12, 2, no, no) \ - SC( 42, 14, 12, 3, no, no) \ - SC( 43, 14, 12, 4, no, no) \ - \ - SC( 44, 15, 13, 1, no, no) \ - SC( 45, 15, 13, 2, no, no) \ - SC( 46, 15, 13, 3, no, no) \ - SC( 47, 15, 13, 4, no, no) \ - \ - SC( 48, 16, 14, 1, no, no) \ - SC( 49, 16, 14, 2, no, no) \ - SC( 50, 16, 14, 3, no, no) \ - SC( 51, 16, 14, 4, no, no) \ - \ - SC( 52, 17, 15, 1, no, no) \ - SC( 53, 17, 15, 2, no, no) \ - SC( 54, 17, 15, 3, no, no) \ - SC( 55, 17, 15, 4, no, no) \ - \ - SC( 56, 18, 16, 1, no, no) \ - SC( 57, 18, 16, 2, no, no) \ - SC( 58, 18, 16, 3, no, no) \ - SC( 59, 18, 16, 4, no, no) \ - \ - SC( 60, 19, 17, 1, no, no) \ - SC( 61, 19, 17, 2, no, no) \ - SC( 62, 19, 17, 3, no, no) \ - SC( 63, 19, 17, 4, no, no) \ - \ - SC( 64, 20, 18, 1, no, no) \ - SC( 65, 20, 18, 2, no, no) \ - SC( 66, 20, 18, 3, no, no) \ - SC( 67, 20, 18, 4, no, no) \ - \ - SC( 68, 21, 19, 1, no, no) \ - SC( 69, 21, 19, 2, no, no) \ - SC( 70, 21, 19, 3, no, no) \ - SC( 71, 21, 19, 4, no, no) \ - \ - SC( 72, 22, 20, 1, no, no) \ - SC( 73, 22, 20, 2, no, no) \ - SC( 74, 22, 20, 3, no, no) \ - SC( 75, 22, 20, 4, no, no) \ - \ - SC( 76, 23, 21, 1, no, no) \ - SC( 77, 23, 21, 2, no, no) \ - SC( 78, 23, 21, 3, no, no) \ - SC( 79, 23, 21, 4, no, no) \ - \ - SC( 80, 24, 22, 1, no, no) \ - SC( 81, 24, 22, 2, no, no) \ - SC( 82, 24, 22, 3, no, no) \ - SC( 83, 24, 22, 4, no, no) \ - \ - SC( 84, 25, 23, 1, no, no) \ - SC( 85, 25, 23, 2, no, no) \ - SC( 86, 25, 23, 3, no, no) \ - SC( 87, 25, 23, 4, no, no) \ - \ - SC( 88, 26, 24, 1, no, no) \ - SC( 89, 26, 24, 2, no, no) \ - SC( 90, 26, 24, 3, no, no) \ - SC( 91, 26, 24, 4, no, no) \ - \ - SC( 92, 27, 25, 1, no, no) \ - SC( 93, 27, 25, 2, no, no) \ - SC( 94, 27, 25, 3, no, no) \ - SC( 95, 27, 25, 4, no, no) \ - \ - SC( 96, 28, 26, 1, no, no) \ - SC( 97, 28, 26, 2, no, no) \ - SC( 98, 28, 26, 3, no, no) \ - SC( 99, 28, 26, 4, no, no) \ - \ - SC(100, 29, 27, 1, no, no) \ - SC(101, 29, 27, 2, no, no) \ - SC(102, 29, 27, 3, no, no) \ - SC(103, 29, 27, 4, no, no) \ - \ - SC(104, 30, 28, 1, no, no) \ - SC(105, 30, 28, 2, no, no) \ - SC(106, 30, 28, 3, no, no) \ - SC(107, 30, 28, 4, no, no) \ - \ - SC(108, 31, 29, 1, no, no) \ - SC(109, 31, 29, 2, no, no) \ - SC(110, 31, 29, 3, no, no) \ - SC(111, 31, 29, 4, no, no) \ - \ - SC(112, 32, 30, 1, no, no) \ - SC(113, 32, 30, 2, no, no) \ - SC(114, 32, 30, 3, no, no) \ - SC(115, 32, 30, 4, no, no) \ - \ - SC(116, 33, 31, 1, no, no) \ - SC(117, 33, 31, 2, no, no) \ - SC(118, 33, 31, 3, no, no) \ - SC(119, 33, 31, 4, no, no) \ - \ - SC(120, 34, 32, 1, no, no) \ - SC(121, 34, 32, 2, no, no) \ - SC(122, 34, 32, 3, no, no) \ - SC(123, 34, 32, 4, no, no) \ - \ - SC(124, 35, 33, 1, no, no) \ - SC(125, 35, 33, 2, no, no) \ - SC(126, 35, 33, 3, no, no) \ - SC(127, 35, 33, 4, no, no) \ - \ - SC(128, 36, 34, 1, no, no) \ - SC(129, 36, 34, 2, no, no) \ - SC(130, 36, 34, 3, no, no) \ - SC(131, 36, 34, 4, no, no) \ - \ - SC(132, 37, 35, 1, no, no) \ - SC(133, 37, 35, 2, no, no) \ - SC(134, 37, 35, 3, no, no) \ - SC(135, 37, 35, 4, no, no) \ - \ - SC(136, 38, 36, 1, no, no) \ - SC(137, 38, 36, 2, no, no) \ - SC(138, 38, 36, 3, no, no) \ - SC(139, 38, 36, 4, no, no) \ - \ - SC(140, 39, 37, 1, no, no) \ - SC(141, 39, 37, 2, no, no) \ - SC(142, 39, 37, 3, no, no) \ - SC(143, 39, 37, 4, no, no) \ - \ - SC(144, 40, 38, 1, no, no) \ - SC(145, 40, 38, 2, no, no) \ - SC(146, 40, 38, 3, no, no) \ - SC(147, 40, 38, 4, no, no) \ - \ - SC(148, 41, 39, 1, no, no) \ - SC(149, 41, 39, 2, no, no) \ - SC(150, 41, 39, 3, no, no) \ - SC(151, 41, 39, 4, no, no) \ - \ - SC(152, 42, 40, 1, no, no) \ - SC(153, 42, 40, 2, no, no) \ - SC(154, 42, 40, 3, no, no) \ - SC(155, 42, 40, 4, no, no) \ - \ - SC(156, 43, 41, 1, no, no) \ - SC(157, 43, 41, 2, no, no) \ - SC(158, 43, 41, 3, no, no) \ - SC(159, 43, 41, 4, no, no) \ - \ - SC(160, 44, 42, 1, no, no) \ - SC(161, 44, 42, 2, no, no) \ - SC(162, 44, 42, 3, no, no) \ - SC(163, 44, 42, 4, no, no) \ - \ - SC(164, 45, 43, 1, no, no) \ - SC(165, 45, 43, 2, no, no) \ - SC(166, 45, 43, 3, no, no) \ - SC(167, 45, 43, 4, no, no) \ - \ - SC(168, 46, 44, 1, no, no) \ - SC(169, 46, 44, 2, no, no) \ - SC(170, 46, 44, 3, no, no) \ - SC(171, 46, 44, 4, no, no) \ - \ - SC(172, 47, 45, 1, no, no) \ - SC(173, 47, 45, 2, no, no) \ - SC(174, 47, 45, 3, no, no) \ - SC(175, 47, 45, 4, no, no) \ - \ - SC(176, 48, 46, 1, no, no) \ - SC(177, 48, 46, 2, no, no) \ - SC(178, 48, 46, 3, no, no) \ - SC(179, 48, 46, 4, no, no) \ - \ - SC(180, 49, 47, 1, no, no) \ - SC(181, 49, 47, 2, no, no) \ - SC(182, 49, 47, 3, no, no) \ - SC(183, 49, 47, 4, no, no) \ - \ - SC(184, 50, 48, 1, no, no) \ - SC(185, 50, 48, 2, no, no) \ - SC(186, 50, 48, 3, no, no) \ - SC(187, 50, 48, 4, no, no) \ - \ - SC(188, 51, 49, 1, no, no) \ - SC(189, 51, 49, 2, no, no) \ - SC(190, 51, 49, 3, no, no) \ - SC(191, 51, 49, 4, no, no) \ - \ - SC(192, 52, 50, 1, no, no) \ - SC(193, 52, 50, 2, no, no) \ - SC(194, 52, 50, 3, no, no) \ - SC(195, 52, 50, 4, no, no) \ - \ - SC(196, 53, 51, 1, no, no) \ - SC(197, 53, 51, 2, no, no) \ - SC(198, 53, 51, 3, no, no) \ - SC(199, 53, 51, 4, no, no) \ - \ - SC(200, 54, 52, 1, no, no) \ - SC(201, 54, 52, 2, no, no) \ - SC(202, 54, 52, 3, no, no) \ - SC(203, 54, 52, 4, no, no) \ - \ - SC(204, 55, 53, 1, no, no) \ - SC(205, 55, 53, 2, no, no) \ - SC(206, 55, 53, 3, no, no) \ - SC(207, 55, 53, 4, no, no) \ - \ - SC(208, 56, 54, 1, no, no) \ - SC(209, 56, 54, 2, no, no) \ - SC(210, 56, 54, 3, no, no) \ - SC(211, 56, 54, 4, no, no) \ - \ - SC(212, 57, 55, 1, no, no) \ - SC(213, 57, 55, 2, no, no) \ - SC(214, 57, 55, 3, no, no) \ - SC(215, 57, 55, 4, no, no) \ - \ - SC(216, 58, 56, 1, no, no) \ - SC(217, 58, 56, 2, no, no) \ - SC(218, 58, 56, 3, no, no) \ - SC(219, 58, 56, 4, no, no) \ - \ - SC(220, 59, 57, 1, no, no) \ - SC(221, 59, 57, 2, no, no) \ - SC(222, 59, 57, 3, no, no) \ - SC(223, 59, 57, 4, no, no) \ - \ - SC(224, 60, 58, 1, no, no) \ - SC(225, 60, 58, 2, no, no) \ - SC(226, 60, 58, 3, no, no) \ - SC(227, 60, 58, 4, no, no) \ - \ - SC(228, 61, 59, 1, no, no) \ - SC(229, 61, 59, 2, no, no) \ - SC(230, 61, 59, 3, no, no) \ - SC(231, 61, 59, 4, no, no) \ - \ - SC(232, 62, 60, 1, no, no) \ - SC(233, 62, 60, 2, no, no) \ - SC(234, 62, 60, 3, no, no) \ + /* index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup */ \ + SC( 0, 3, 3, 0, no, yes, 3) \ + SC( 1, 3, 3, 1, no, yes, 3) \ + SC( 2, 3, 3, 2, no, yes, 3) \ + SC( 3, 3, 3, 3, no, yes, 3) \ + \ + SC( 4, 5, 3, 1, no, yes, 3) \ + SC( 5, 5, 3, 2, no, yes, 3) \ + SC( 6, 5, 3, 3, no, yes, 3) \ + SC( 7, 5, 3, 4, no, yes, 3) \ + \ + SC( 8, 6, 4, 1, no, yes, 4) \ + SC( 9, 6, 4, 2, no, yes, 4) \ + SC( 10, 6, 4, 3, no, yes, 4) \ + SC( 11, 6, 4, 4, no, yes, 4) \ + \ + SC( 12, 7, 5, 1, no, yes, 5) \ + SC( 13, 7, 5, 2, no, yes, 5) \ + SC( 14, 7, 5, 3, no, yes, 5) \ + SC( 15, 7, 5, 4, no, yes, 5) \ + \ + SC( 16, 8, 6, 1, no, yes, 6) \ + SC( 17, 8, 6, 2, no, yes, 6) \ + SC( 18, 8, 6, 3, no, yes, 6) \ + SC( 19, 8, 6, 4, no, yes, 6) \ + \ + SC( 20, 9, 7, 1, no, yes, 7) \ + SC( 21, 9, 7, 2, no, yes, 7) \ + SC( 22, 9, 7, 3, no, yes, 7) \ + SC( 23, 9, 7, 4, no, yes, 7) \ + \ + SC( 24, 10, 8, 1, no, yes, 8) \ + SC( 25, 10, 8, 2, no, yes, 8) \ + SC( 26, 10, 8, 3, no, yes, 8) \ + SC( 27, 10, 8, 4, no, yes, 8) \ + \ + SC( 28, 11, 9, 1, no, yes, 9) \ + SC( 29, 11, 9, 2, no, yes, 9) \ + SC( 30, 11, 9, 3, no, yes, 9) \ + SC( 31, 11, 9, 4, yes, yes, 9) \ + \ + SC( 32, 12, 10, 1, no, yes, no) \ + SC( 33, 12, 10, 2, no, yes, no) \ + SC( 34, 12, 10, 3, no, yes, no) \ + SC( 35, 12, 10, 4, yes, yes, no) \ + \ + SC( 36, 13, 11, 1, no, yes, no) \ + SC( 37, 13, 11, 2, yes, yes, no) \ + SC( 38, 13, 11, 3, no, yes, no) \ + SC( 39, 13, 11, 4, yes, no, no) \ + \ + SC( 40, 14, 12, 1, yes, no, no) \ + SC( 41, 14, 12, 2, yes, no, no) \ + SC( 42, 14, 12, 3, yes, no, no) \ + SC( 43, 14, 12, 4, yes, no, no) \ + \ + SC( 44, 15, 13, 1, yes, no, no) \ + SC( 45, 15, 13, 2, yes, no, no) \ + SC( 46, 15, 13, 3, yes, no, no) \ + SC( 47, 15, 13, 4, yes, no, no) \ + \ + SC( 48, 16, 14, 1, yes, no, no) \ + SC( 49, 16, 14, 2, yes, no, no) \ + SC( 50, 16, 14, 3, yes, no, no) \ + SC( 51, 16, 14, 4, yes, no, no) \ + \ + SC( 52, 17, 15, 1, yes, no, no) \ + SC( 53, 17, 15, 2, yes, no, no) \ + SC( 54, 17, 15, 3, yes, no, no) \ + SC( 55, 17, 15, 4, yes, no, no) \ + \ + SC( 56, 18, 16, 1, yes, no, no) \ + SC( 57, 18, 16, 2, yes, no, no) \ + SC( 58, 18, 16, 3, yes, no, no) \ + SC( 59, 18, 16, 4, yes, no, no) \ + \ + SC( 60, 19, 17, 1, yes, no, no) \ + SC( 61, 19, 17, 2, yes, no, no) \ + SC( 62, 19, 17, 3, yes, no, no) \ + SC( 63, 19, 17, 4, yes, no, no) \ + \ + SC( 64, 20, 18, 1, yes, no, no) \ + SC( 65, 20, 18, 2, yes, no, no) \ + SC( 66, 20, 18, 3, yes, no, no) \ + SC( 67, 20, 18, 4, yes, no, no) \ + \ + SC( 68, 21, 19, 1, yes, no, no) \ + SC( 69, 21, 19, 2, yes, no, no) \ + SC( 70, 21, 19, 3, yes, no, no) \ + SC( 71, 21, 19, 4, yes, no, no) \ + \ + SC( 72, 22, 20, 1, yes, no, no) \ + SC( 73, 22, 20, 2, yes, no, no) \ + SC( 74, 22, 20, 3, yes, no, no) \ + SC( 75, 22, 20, 4, yes, no, no) \ + \ + SC( 76, 23, 21, 1, yes, no, no) \ + SC( 77, 23, 21, 2, yes, no, no) \ + SC( 78, 23, 21, 3, yes, no, no) \ + SC( 79, 23, 21, 4, yes, no, no) \ + \ + SC( 80, 24, 22, 1, yes, no, no) \ + SC( 81, 24, 22, 2, yes, no, no) \ + SC( 82, 24, 22, 3, yes, no, no) \ + SC( 83, 24, 22, 4, yes, no, no) \ + \ + SC( 84, 25, 23, 1, yes, no, no) \ + SC( 85, 25, 23, 2, yes, no, no) \ + SC( 86, 25, 23, 3, yes, no, no) \ + SC( 87, 25, 23, 4, yes, no, no) \ + \ + SC( 88, 26, 24, 1, yes, no, no) \ + SC( 89, 26, 24, 2, yes, no, no) \ + SC( 90, 26, 24, 3, yes, no, no) \ + SC( 91, 26, 24, 4, yes, no, no) \ + \ + SC( 92, 27, 25, 1, yes, no, no) \ + SC( 93, 27, 25, 2, yes, no, no) \ + SC( 94, 27, 25, 3, yes, no, no) \ + SC( 95, 27, 25, 4, yes, no, no) \ + \ + SC( 96, 28, 26, 1, yes, no, no) \ + SC( 97, 28, 26, 2, yes, no, no) \ + SC( 98, 28, 26, 3, yes, no, no) \ + SC( 99, 28, 26, 4, yes, no, no) \ + \ + SC(100, 29, 27, 1, yes, no, no) \ + SC(101, 29, 27, 2, yes, no, no) \ + SC(102, 29, 27, 3, yes, no, no) \ + SC(103, 29, 27, 4, yes, no, no) \ + \ + SC(104, 30, 28, 1, yes, no, no) \ + SC(105, 30, 28, 2, yes, no, no) \ + SC(106, 30, 28, 3, yes, no, no) \ + SC(107, 30, 28, 4, yes, no, no) \ + \ + SC(108, 31, 29, 1, yes, no, no) \ + SC(109, 31, 29, 2, yes, no, no) \ + SC(110, 31, 29, 3, yes, no, no) \ + SC(111, 31, 29, 4, yes, no, no) \ + \ + SC(112, 32, 30, 1, yes, no, no) \ + SC(113, 32, 30, 2, yes, no, no) \ + SC(114, 32, 30, 3, yes, no, no) \ + SC(115, 32, 30, 4, yes, no, no) \ + \ + SC(116, 33, 31, 1, yes, no, no) \ + SC(117, 33, 31, 2, yes, no, no) \ + SC(118, 33, 31, 3, yes, no, no) \ + SC(119, 33, 31, 4, yes, no, no) \ + \ + SC(120, 34, 32, 1, yes, no, no) \ + SC(121, 34, 32, 2, yes, no, no) \ + SC(122, 34, 32, 3, yes, no, no) \ + SC(123, 34, 32, 4, yes, no, no) \ + \ + SC(124, 35, 33, 1, yes, no, no) \ + SC(125, 35, 33, 2, yes, no, no) \ + SC(126, 35, 33, 3, yes, no, no) \ + SC(127, 35, 33, 4, yes, no, no) \ + \ + SC(128, 36, 34, 1, yes, no, no) \ + SC(129, 36, 34, 2, yes, no, no) \ + SC(130, 36, 34, 3, yes, no, no) \ + SC(131, 36, 34, 4, yes, no, no) \ + \ + SC(132, 37, 35, 1, yes, no, no) \ + SC(133, 37, 35, 2, yes, no, no) \ + SC(134, 37, 35, 3, yes, no, no) \ + SC(135, 37, 35, 4, yes, no, no) \ + \ + SC(136, 38, 36, 1, yes, no, no) \ + SC(137, 38, 36, 2, yes, no, no) \ + SC(138, 38, 36, 3, yes, no, no) \ + SC(139, 38, 36, 4, yes, no, no) \ + \ + SC(140, 39, 37, 1, yes, no, no) \ + SC(141, 39, 37, 2, yes, no, no) \ + SC(142, 39, 37, 3, yes, no, no) \ + SC(143, 39, 37, 4, yes, no, no) \ + \ + SC(144, 40, 38, 1, yes, no, no) \ + SC(145, 40, 38, 2, yes, no, no) \ + SC(146, 40, 38, 3, yes, no, no) \ + SC(147, 40, 38, 4, yes, no, no) \ + \ + SC(148, 41, 39, 1, yes, no, no) \ + SC(149, 41, 39, 2, yes, no, no) \ + SC(150, 41, 39, 3, yes, no, no) \ + SC(151, 41, 39, 4, yes, no, no) \ + \ + SC(152, 42, 40, 1, yes, no, no) \ + SC(153, 42, 40, 2, yes, no, no) \ + SC(154, 42, 40, 3, yes, no, no) \ + SC(155, 42, 40, 4, yes, no, no) \ + \ + SC(156, 43, 41, 1, yes, no, no) \ + SC(157, 43, 41, 2, yes, no, no) \ + SC(158, 43, 41, 3, yes, no, no) \ + SC(159, 43, 41, 4, yes, no, no) \ + \ + SC(160, 44, 42, 1, yes, no, no) \ + SC(161, 44, 42, 2, yes, no, no) \ + SC(162, 44, 42, 3, yes, no, no) \ + SC(163, 44, 42, 4, yes, no, no) \ + \ + SC(164, 45, 43, 1, yes, no, no) \ + SC(165, 45, 43, 2, yes, no, no) \ + SC(166, 45, 43, 3, yes, no, no) \ + SC(167, 45, 43, 4, yes, no, no) \ + \ + SC(168, 46, 44, 1, yes, no, no) \ + SC(169, 46, 44, 2, yes, no, no) \ + SC(170, 46, 44, 3, yes, no, no) \ + SC(171, 46, 44, 4, yes, no, no) \ + \ + SC(172, 47, 45, 1, yes, no, no) \ + SC(173, 47, 45, 2, yes, no, no) \ + SC(174, 47, 45, 3, yes, no, no) \ + SC(175, 47, 45, 4, yes, no, no) \ + \ + SC(176, 48, 46, 1, yes, no, no) \ + SC(177, 48, 46, 2, yes, no, no) \ + SC(178, 48, 46, 3, yes, no, no) \ + SC(179, 48, 46, 4, yes, no, no) \ + \ + SC(180, 49, 47, 1, yes, no, no) \ + SC(181, 49, 47, 2, yes, no, no) \ + SC(182, 49, 47, 3, yes, no, no) \ + SC(183, 49, 47, 4, yes, no, no) \ + \ + SC(184, 50, 48, 1, yes, no, no) \ + SC(185, 50, 48, 2, yes, no, no) \ + SC(186, 50, 48, 3, yes, no, no) \ + SC(187, 50, 48, 4, yes, no, no) \ + \ + SC(188, 51, 49, 1, yes, no, no) \ + SC(189, 51, 49, 2, yes, no, no) \ + SC(190, 51, 49, 3, yes, no, no) \ + SC(191, 51, 49, 4, yes, no, no) \ + \ + SC(192, 52, 50, 1, yes, no, no) \ + SC(193, 52, 50, 2, yes, no, no) \ + SC(194, 52, 50, 3, yes, no, no) \ + SC(195, 52, 50, 4, yes, no, no) \ + \ + SC(196, 53, 51, 1, yes, no, no) \ + SC(197, 53, 51, 2, yes, no, no) \ + SC(198, 53, 51, 3, yes, no, no) \ + SC(199, 53, 51, 4, yes, no, no) \ + \ + SC(200, 54, 52, 1, yes, no, no) \ + SC(201, 54, 52, 2, yes, no, no) \ + SC(202, 54, 52, 3, yes, no, no) \ + SC(203, 54, 52, 4, yes, no, no) \ + \ + SC(204, 55, 53, 1, yes, no, no) \ + SC(205, 55, 53, 2, yes, no, no) \ + SC(206, 55, 53, 3, yes, no, no) \ + SC(207, 55, 53, 4, yes, no, no) \ + \ + SC(208, 56, 54, 1, yes, no, no) \ + SC(209, 56, 54, 2, yes, no, no) \ + SC(210, 56, 54, 3, yes, no, no) \ + SC(211, 56, 54, 4, yes, no, no) \ + \ + SC(212, 57, 55, 1, yes, no, no) \ + SC(213, 57, 55, 2, yes, no, no) \ + SC(214, 57, 55, 3, yes, no, no) \ + SC(215, 57, 55, 4, yes, no, no) \ + \ + SC(216, 58, 56, 1, yes, no, no) \ + SC(217, 58, 56, 2, yes, no, no) \ + SC(218, 58, 56, 3, yes, no, no) \ + SC(219, 58, 56, 4, yes, no, no) \ + \ + SC(220, 59, 57, 1, yes, no, no) \ + SC(221, 59, 57, 2, yes, no, no) \ + SC(222, 59, 57, 3, yes, no, no) \ + SC(223, 59, 57, 4, yes, no, no) \ + \ + SC(224, 60, 58, 1, yes, no, no) \ + SC(225, 60, 58, 2, yes, no, no) \ + SC(226, 60, 58, 3, yes, no, no) \ + SC(227, 60, 58, 4, yes, no, no) \ + \ + SC(228, 61, 59, 1, yes, no, no) \ + SC(229, 61, 59, 2, yes, no, no) \ + SC(230, 61, 59, 3, yes, no, no) \ + SC(231, 61, 59, 4, yes, no, no) \ + \ + SC(232, 62, 60, 1, yes, no, no) \ + SC(233, 62, 60, 2, yes, no, no) \ + SC(234, 62, 60, 3, yes, no, no) \ #define SIZE_CLASSES_DEFINED #define NTBINS 0 #define NLBINS 32 #define NBINS 39 #define NSIZES 235 +#define NPSIZES 199 #define LG_TINY_MAXCLASS "NA" #define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) #define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) @@ -780,303 +785,304 @@ #if (LG_SIZEOF_PTR == 3 && LG_TINY_MIN == 3 && LG_QUANTUM == 4 && LG_PAGE == 12) #define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup */ \ - SC( 0, 3, 3, 0, yes, 3) \ - \ - SC( 1, 3, 3, 1, yes, 3) \ - SC( 2, 4, 4, 1, yes, 4) \ - SC( 3, 4, 4, 2, yes, 4) \ - SC( 4, 4, 4, 3, yes, 4) \ - \ - SC( 5, 6, 4, 1, yes, 4) \ - SC( 6, 6, 4, 2, yes, 4) \ - SC( 7, 6, 4, 3, yes, 4) \ - SC( 8, 6, 4, 4, yes, 4) \ - \ - SC( 9, 7, 5, 1, yes, 5) \ - SC( 10, 7, 5, 2, yes, 5) \ - SC( 11, 7, 5, 3, yes, 5) \ - SC( 12, 7, 5, 4, yes, 5) \ - \ - SC( 13, 8, 6, 1, yes, 6) \ - SC( 14, 8, 6, 2, yes, 6) \ - SC( 15, 8, 6, 3, yes, 6) \ - SC( 16, 8, 6, 4, yes, 6) \ - \ - SC( 17, 9, 7, 1, yes, 7) \ - SC( 18, 9, 7, 2, yes, 7) \ - SC( 19, 9, 7, 3, yes, 7) \ - SC( 20, 9, 7, 4, yes, 7) \ - \ - SC( 21, 10, 8, 1, yes, 8) \ - SC( 22, 10, 8, 2, yes, 8) \ - SC( 23, 10, 8, 3, yes, 8) \ - SC( 24, 10, 8, 4, yes, 8) \ - \ - SC( 25, 11, 9, 1, yes, 9) \ - SC( 26, 11, 9, 2, yes, 9) \ - SC( 27, 11, 9, 3, yes, 9) \ - SC( 28, 11, 9, 4, yes, 9) \ - \ - SC( 29, 12, 10, 1, yes, no) \ - SC( 30, 12, 10, 2, yes, no) \ - SC( 31, 12, 10, 3, yes, no) \ - SC( 32, 12, 10, 4, yes, no) \ - \ - SC( 33, 13, 11, 1, yes, no) \ - SC( 34, 13, 11, 2, yes, no) \ - SC( 35, 13, 11, 3, yes, no) \ - SC( 36, 13, 11, 4, no, no) \ - \ - SC( 37, 14, 12, 1, no, no) \ - SC( 38, 14, 12, 2, no, no) \ - SC( 39, 14, 12, 3, no, no) \ - SC( 40, 14, 12, 4, no, no) \ - \ - SC( 41, 15, 13, 1, no, no) \ - SC( 42, 15, 13, 2, no, no) \ - SC( 43, 15, 13, 3, no, no) \ - SC( 44, 15, 13, 4, no, no) \ - \ - SC( 45, 16, 14, 1, no, no) \ - SC( 46, 16, 14, 2, no, no) \ - SC( 47, 16, 14, 3, no, no) \ - SC( 48, 16, 14, 4, no, no) \ - \ - SC( 49, 17, 15, 1, no, no) \ - SC( 50, 17, 15, 2, no, no) \ - SC( 51, 17, 15, 3, no, no) \ - SC( 52, 17, 15, 4, no, no) \ - \ - SC( 53, 18, 16, 1, no, no) \ - SC( 54, 18, 16, 2, no, no) \ - SC( 55, 18, 16, 3, no, no) \ - SC( 56, 18, 16, 4, no, no) \ - \ - SC( 57, 19, 17, 1, no, no) \ - SC( 58, 19, 17, 2, no, no) \ - SC( 59, 19, 17, 3, no, no) \ - SC( 60, 19, 17, 4, no, no) \ - \ - SC( 61, 20, 18, 1, no, no) \ - SC( 62, 20, 18, 2, no, no) \ - SC( 63, 20, 18, 3, no, no) \ - SC( 64, 20, 18, 4, no, no) \ - \ - SC( 65, 21, 19, 1, no, no) \ - SC( 66, 21, 19, 2, no, no) \ - SC( 67, 21, 19, 3, no, no) \ - SC( 68, 21, 19, 4, no, no) \ - \ - SC( 69, 22, 20, 1, no, no) \ - SC( 70, 22, 20, 2, no, no) \ - SC( 71, 22, 20, 3, no, no) \ - SC( 72, 22, 20, 4, no, no) \ - \ - SC( 73, 23, 21, 1, no, no) \ - SC( 74, 23, 21, 2, no, no) \ - SC( 75, 23, 21, 3, no, no) \ - SC( 76, 23, 21, 4, no, no) \ - \ - SC( 77, 24, 22, 1, no, no) \ - SC( 78, 24, 22, 2, no, no) \ - SC( 79, 24, 22, 3, no, no) \ - SC( 80, 24, 22, 4, no, no) \ - \ - SC( 81, 25, 23, 1, no, no) \ - SC( 82, 25, 23, 2, no, no) \ - SC( 83, 25, 23, 3, no, no) \ - SC( 84, 25, 23, 4, no, no) \ - \ - SC( 85, 26, 24, 1, no, no) \ - SC( 86, 26, 24, 2, no, no) \ - SC( 87, 26, 24, 3, no, no) \ - SC( 88, 26, 24, 4, no, no) \ - \ - SC( 89, 27, 25, 1, no, no) \ - SC( 90, 27, 25, 2, no, no) \ - SC( 91, 27, 25, 3, no, no) \ - SC( 92, 27, 25, 4, no, no) \ - \ - SC( 93, 28, 26, 1, no, no) \ - SC( 94, 28, 26, 2, no, no) \ - SC( 95, 28, 26, 3, no, no) \ - SC( 96, 28, 26, 4, no, no) \ - \ - SC( 97, 29, 27, 1, no, no) \ - SC( 98, 29, 27, 2, no, no) \ - SC( 99, 29, 27, 3, no, no) \ - SC(100, 29, 27, 4, no, no) \ - \ - SC(101, 30, 28, 1, no, no) \ - SC(102, 30, 28, 2, no, no) \ - SC(103, 30, 28, 3, no, no) \ - SC(104, 30, 28, 4, no, no) \ - \ - SC(105, 31, 29, 1, no, no) \ - SC(106, 31, 29, 2, no, no) \ - SC(107, 31, 29, 3, no, no) \ - SC(108, 31, 29, 4, no, no) \ - \ - SC(109, 32, 30, 1, no, no) \ - SC(110, 32, 30, 2, no, no) \ - SC(111, 32, 30, 3, no, no) \ - SC(112, 32, 30, 4, no, no) \ - \ - SC(113, 33, 31, 1, no, no) \ - SC(114, 33, 31, 2, no, no) \ - SC(115, 33, 31, 3, no, no) \ - SC(116, 33, 31, 4, no, no) \ - \ - SC(117, 34, 32, 1, no, no) \ - SC(118, 34, 32, 2, no, no) \ - SC(119, 34, 32, 3, no, no) \ - SC(120, 34, 32, 4, no, no) \ - \ - SC(121, 35, 33, 1, no, no) \ - SC(122, 35, 33, 2, no, no) \ - SC(123, 35, 33, 3, no, no) \ - SC(124, 35, 33, 4, no, no) \ - \ - SC(125, 36, 34, 1, no, no) \ - SC(126, 36, 34, 2, no, no) \ - SC(127, 36, 34, 3, no, no) \ - SC(128, 36, 34, 4, no, no) \ - \ - SC(129, 37, 35, 1, no, no) \ - SC(130, 37, 35, 2, no, no) \ - SC(131, 37, 35, 3, no, no) \ - SC(132, 37, 35, 4, no, no) \ - \ - SC(133, 38, 36, 1, no, no) \ - SC(134, 38, 36, 2, no, no) \ - SC(135, 38, 36, 3, no, no) \ - SC(136, 38, 36, 4, no, no) \ - \ - SC(137, 39, 37, 1, no, no) \ - SC(138, 39, 37, 2, no, no) \ - SC(139, 39, 37, 3, no, no) \ - SC(140, 39, 37, 4, no, no) \ - \ - SC(141, 40, 38, 1, no, no) \ - SC(142, 40, 38, 2, no, no) \ - SC(143, 40, 38, 3, no, no) \ - SC(144, 40, 38, 4, no, no) \ - \ - SC(145, 41, 39, 1, no, no) \ - SC(146, 41, 39, 2, no, no) \ - SC(147, 41, 39, 3, no, no) \ - SC(148, 41, 39, 4, no, no) \ - \ - SC(149, 42, 40, 1, no, no) \ - SC(150, 42, 40, 2, no, no) \ - SC(151, 42, 40, 3, no, no) \ - SC(152, 42, 40, 4, no, no) \ - \ - SC(153, 43, 41, 1, no, no) \ - SC(154, 43, 41, 2, no, no) \ - SC(155, 43, 41, 3, no, no) \ - SC(156, 43, 41, 4, no, no) \ - \ - SC(157, 44, 42, 1, no, no) \ - SC(158, 44, 42, 2, no, no) \ - SC(159, 44, 42, 3, no, no) \ - SC(160, 44, 42, 4, no, no) \ - \ - SC(161, 45, 43, 1, no, no) \ - SC(162, 45, 43, 2, no, no) \ - SC(163, 45, 43, 3, no, no) \ - SC(164, 45, 43, 4, no, no) \ - \ - SC(165, 46, 44, 1, no, no) \ - SC(166, 46, 44, 2, no, no) \ - SC(167, 46, 44, 3, no, no) \ - SC(168, 46, 44, 4, no, no) \ - \ - SC(169, 47, 45, 1, no, no) \ - SC(170, 47, 45, 2, no, no) \ - SC(171, 47, 45, 3, no, no) \ - SC(172, 47, 45, 4, no, no) \ - \ - SC(173, 48, 46, 1, no, no) \ - SC(174, 48, 46, 2, no, no) \ - SC(175, 48, 46, 3, no, no) \ - SC(176, 48, 46, 4, no, no) \ - \ - SC(177, 49, 47, 1, no, no) \ - SC(178, 49, 47, 2, no, no) \ - SC(179, 49, 47, 3, no, no) \ - SC(180, 49, 47, 4, no, no) \ - \ - SC(181, 50, 48, 1, no, no) \ - SC(182, 50, 48, 2, no, no) \ - SC(183, 50, 48, 3, no, no) \ - SC(184, 50, 48, 4, no, no) \ - \ - SC(185, 51, 49, 1, no, no) \ - SC(186, 51, 49, 2, no, no) \ - SC(187, 51, 49, 3, no, no) \ - SC(188, 51, 49, 4, no, no) \ - \ - SC(189, 52, 50, 1, no, no) \ - SC(190, 52, 50, 2, no, no) \ - SC(191, 52, 50, 3, no, no) \ - SC(192, 52, 50, 4, no, no) \ - \ - SC(193, 53, 51, 1, no, no) \ - SC(194, 53, 51, 2, no, no) \ - SC(195, 53, 51, 3, no, no) \ - SC(196, 53, 51, 4, no, no) \ - \ - SC(197, 54, 52, 1, no, no) \ - SC(198, 54, 52, 2, no, no) \ - SC(199, 54, 52, 3, no, no) \ - SC(200, 54, 52, 4, no, no) \ - \ - SC(201, 55, 53, 1, no, no) \ - SC(202, 55, 53, 2, no, no) \ - SC(203, 55, 53, 3, no, no) \ - SC(204, 55, 53, 4, no, no) \ - \ - SC(205, 56, 54, 1, no, no) \ - SC(206, 56, 54, 2, no, no) \ - SC(207, 56, 54, 3, no, no) \ - SC(208, 56, 54, 4, no, no) \ - \ - SC(209, 57, 55, 1, no, no) \ - SC(210, 57, 55, 2, no, no) \ - SC(211, 57, 55, 3, no, no) \ - SC(212, 57, 55, 4, no, no) \ - \ - SC(213, 58, 56, 1, no, no) \ - SC(214, 58, 56, 2, no, no) \ - SC(215, 58, 56, 3, no, no) \ - SC(216, 58, 56, 4, no, no) \ - \ - SC(217, 59, 57, 1, no, no) \ - SC(218, 59, 57, 2, no, no) \ - SC(219, 59, 57, 3, no, no) \ - SC(220, 59, 57, 4, no, no) \ - \ - SC(221, 60, 58, 1, no, no) \ - SC(222, 60, 58, 2, no, no) \ - SC(223, 60, 58, 3, no, no) \ - SC(224, 60, 58, 4, no, no) \ - \ - SC(225, 61, 59, 1, no, no) \ - SC(226, 61, 59, 2, no, no) \ - SC(227, 61, 59, 3, no, no) \ - SC(228, 61, 59, 4, no, no) \ - \ - SC(229, 62, 60, 1, no, no) \ - SC(230, 62, 60, 2, no, no) \ - SC(231, 62, 60, 3, no, no) \ + /* index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup */ \ + SC( 0, 3, 3, 0, no, yes, 3) \ + \ + SC( 1, 3, 3, 1, no, yes, 3) \ + SC( 2, 4, 4, 1, no, yes, 4) \ + SC( 3, 4, 4, 2, no, yes, 4) \ + SC( 4, 4, 4, 3, no, yes, 4) \ + \ + SC( 5, 6, 4, 1, no, yes, 4) \ + SC( 6, 6, 4, 2, no, yes, 4) \ + SC( 7, 6, 4, 3, no, yes, 4) \ + SC( 8, 6, 4, 4, no, yes, 4) \ + \ + SC( 9, 7, 5, 1, no, yes, 5) \ + SC( 10, 7, 5, 2, no, yes, 5) \ + SC( 11, 7, 5, 3, no, yes, 5) \ + SC( 12, 7, 5, 4, no, yes, 5) \ + \ + SC( 13, 8, 6, 1, no, yes, 6) \ + SC( 14, 8, 6, 2, no, yes, 6) \ + SC( 15, 8, 6, 3, no, yes, 6) \ + SC( 16, 8, 6, 4, no, yes, 6) \ + \ + SC( 17, 9, 7, 1, no, yes, 7) \ + SC( 18, 9, 7, 2, no, yes, 7) \ + SC( 19, 9, 7, 3, no, yes, 7) \ + SC( 20, 9, 7, 4, no, yes, 7) \ + \ + SC( 21, 10, 8, 1, no, yes, 8) \ + SC( 22, 10, 8, 2, no, yes, 8) \ + SC( 23, 10, 8, 3, no, yes, 8) \ + SC( 24, 10, 8, 4, no, yes, 8) \ + \ + SC( 25, 11, 9, 1, no, yes, 9) \ + SC( 26, 11, 9, 2, no, yes, 9) \ + SC( 27, 11, 9, 3, no, yes, 9) \ + SC( 28, 11, 9, 4, yes, yes, 9) \ + \ + SC( 29, 12, 10, 1, no, yes, no) \ + SC( 30, 12, 10, 2, no, yes, no) \ + SC( 31, 12, 10, 3, no, yes, no) \ + SC( 32, 12, 10, 4, yes, yes, no) \ + \ + SC( 33, 13, 11, 1, no, yes, no) \ + SC( 34, 13, 11, 2, yes, yes, no) \ + SC( 35, 13, 11, 3, no, yes, no) \ + SC( 36, 13, 11, 4, yes, no, no) \ + \ + SC( 37, 14, 12, 1, yes, no, no) \ + SC( 38, 14, 12, 2, yes, no, no) \ + SC( 39, 14, 12, 3, yes, no, no) \ + SC( 40, 14, 12, 4, yes, no, no) \ + \ + SC( 41, 15, 13, 1, yes, no, no) \ + SC( 42, 15, 13, 2, yes, no, no) \ + SC( 43, 15, 13, 3, yes, no, no) \ + SC( 44, 15, 13, 4, yes, no, no) \ + \ + SC( 45, 16, 14, 1, yes, no, no) \ + SC( 46, 16, 14, 2, yes, no, no) \ + SC( 47, 16, 14, 3, yes, no, no) \ + SC( 48, 16, 14, 4, yes, no, no) \ + \ + SC( 49, 17, 15, 1, yes, no, no) \ + SC( 50, 17, 15, 2, yes, no, no) \ + SC( 51, 17, 15, 3, yes, no, no) \ + SC( 52, 17, 15, 4, yes, no, no) \ + \ + SC( 53, 18, 16, 1, yes, no, no) \ + SC( 54, 18, 16, 2, yes, no, no) \ + SC( 55, 18, 16, 3, yes, no, no) \ + SC( 56, 18, 16, 4, yes, no, no) \ + \ + SC( 57, 19, 17, 1, yes, no, no) \ + SC( 58, 19, 17, 2, yes, no, no) \ + SC( 59, 19, 17, 3, yes, no, no) \ + SC( 60, 19, 17, 4, yes, no, no) \ + \ + SC( 61, 20, 18, 1, yes, no, no) \ + SC( 62, 20, 18, 2, yes, no, no) \ + SC( 63, 20, 18, 3, yes, no, no) \ + SC( 64, 20, 18, 4, yes, no, no) \ + \ + SC( 65, 21, 19, 1, yes, no, no) \ + SC( 66, 21, 19, 2, yes, no, no) \ + SC( 67, 21, 19, 3, yes, no, no) \ + SC( 68, 21, 19, 4, yes, no, no) \ + \ + SC( 69, 22, 20, 1, yes, no, no) \ + SC( 70, 22, 20, 2, yes, no, no) \ + SC( 71, 22, 20, 3, yes, no, no) \ + SC( 72, 22, 20, 4, yes, no, no) \ + \ + SC( 73, 23, 21, 1, yes, no, no) \ + SC( 74, 23, 21, 2, yes, no, no) \ + SC( 75, 23, 21, 3, yes, no, no) \ + SC( 76, 23, 21, 4, yes, no, no) \ + \ + SC( 77, 24, 22, 1, yes, no, no) \ + SC( 78, 24, 22, 2, yes, no, no) \ + SC( 79, 24, 22, 3, yes, no, no) \ + SC( 80, 24, 22, 4, yes, no, no) \ + \ + SC( 81, 25, 23, 1, yes, no, no) \ + SC( 82, 25, 23, 2, yes, no, no) \ + SC( 83, 25, 23, 3, yes, no, no) \ + SC( 84, 25, 23, 4, yes, no, no) \ + \ + SC( 85, 26, 24, 1, yes, no, no) \ + SC( 86, 26, 24, 2, yes, no, no) \ + SC( 87, 26, 24, 3, yes, no, no) \ + SC( 88, 26, 24, 4, yes, no, no) \ + \ + SC( 89, 27, 25, 1, yes, no, no) \ + SC( 90, 27, 25, 2, yes, no, no) \ + SC( 91, 27, 25, 3, yes, no, no) \ + SC( 92, 27, 25, 4, yes, no, no) \ + \ + SC( 93, 28, 26, 1, yes, no, no) \ + SC( 94, 28, 26, 2, yes, no, no) \ + SC( 95, 28, 26, 3, yes, no, no) \ + SC( 96, 28, 26, 4, yes, no, no) \ + \ + SC( 97, 29, 27, 1, yes, no, no) \ + SC( 98, 29, 27, 2, yes, no, no) \ + SC( 99, 29, 27, 3, yes, no, no) \ + SC(100, 29, 27, 4, yes, no, no) \ + \ + SC(101, 30, 28, 1, yes, no, no) \ + SC(102, 30, 28, 2, yes, no, no) \ + SC(103, 30, 28, 3, yes, no, no) \ + SC(104, 30, 28, 4, yes, no, no) \ + \ + SC(105, 31, 29, 1, yes, no, no) \ + SC(106, 31, 29, 2, yes, no, no) \ + SC(107, 31, 29, 3, yes, no, no) \ + SC(108, 31, 29, 4, yes, no, no) \ + \ + SC(109, 32, 30, 1, yes, no, no) \ + SC(110, 32, 30, 2, yes, no, no) \ + SC(111, 32, 30, 3, yes, no, no) \ + SC(112, 32, 30, 4, yes, no, no) \ + \ + SC(113, 33, 31, 1, yes, no, no) \ + SC(114, 33, 31, 2, yes, no, no) \ + SC(115, 33, 31, 3, yes, no, no) \ + SC(116, 33, 31, 4, yes, no, no) \ + \ + SC(117, 34, 32, 1, yes, no, no) \ + SC(118, 34, 32, 2, yes, no, no) \ + SC(119, 34, 32, 3, yes, no, no) \ + SC(120, 34, 32, 4, yes, no, no) \ + \ + SC(121, 35, 33, 1, yes, no, no) \ + SC(122, 35, 33, 2, yes, no, no) \ + SC(123, 35, 33, 3, yes, no, no) \ + SC(124, 35, 33, 4, yes, no, no) \ + \ + SC(125, 36, 34, 1, yes, no, no) \ + SC(126, 36, 34, 2, yes, no, no) \ + SC(127, 36, 34, 3, yes, no, no) \ + SC(128, 36, 34, 4, yes, no, no) \ + \ + SC(129, 37, 35, 1, yes, no, no) \ + SC(130, 37, 35, 2, yes, no, no) \ + SC(131, 37, 35, 3, yes, no, no) \ + SC(132, 37, 35, 4, yes, no, no) \ + \ + SC(133, 38, 36, 1, yes, no, no) \ + SC(134, 38, 36, 2, yes, no, no) \ + SC(135, 38, 36, 3, yes, no, no) \ + SC(136, 38, 36, 4, yes, no, no) \ + \ + SC(137, 39, 37, 1, yes, no, no) \ + SC(138, 39, 37, 2, yes, no, no) \ + SC(139, 39, 37, 3, yes, no, no) \ + SC(140, 39, 37, 4, yes, no, no) \ + \ + SC(141, 40, 38, 1, yes, no, no) \ + SC(142, 40, 38, 2, yes, no, no) \ + SC(143, 40, 38, 3, yes, no, no) \ + SC(144, 40, 38, 4, yes, no, no) \ + \ + SC(145, 41, 39, 1, yes, no, no) \ + SC(146, 41, 39, 2, yes, no, no) \ + SC(147, 41, 39, 3, yes, no, no) \ + SC(148, 41, 39, 4, yes, no, no) \ + \ + SC(149, 42, 40, 1, yes, no, no) \ + SC(150, 42, 40, 2, yes, no, no) \ + SC(151, 42, 40, 3, yes, no, no) \ + SC(152, 42, 40, 4, yes, no, no) \ + \ + SC(153, 43, 41, 1, yes, no, no) \ + SC(154, 43, 41, 2, yes, no, no) \ + SC(155, 43, 41, 3, yes, no, no) \ + SC(156, 43, 41, 4, yes, no, no) \ + \ + SC(157, 44, 42, 1, yes, no, no) \ + SC(158, 44, 42, 2, yes, no, no) \ + SC(159, 44, 42, 3, yes, no, no) \ + SC(160, 44, 42, 4, yes, no, no) \ + \ + SC(161, 45, 43, 1, yes, no, no) \ + SC(162, 45, 43, 2, yes, no, no) \ + SC(163, 45, 43, 3, yes, no, no) \ + SC(164, 45, 43, 4, yes, no, no) \ + \ + SC(165, 46, 44, 1, yes, no, no) \ + SC(166, 46, 44, 2, yes, no, no) \ + SC(167, 46, 44, 3, yes, no, no) \ + SC(168, 46, 44, 4, yes, no, no) \ + \ + SC(169, 47, 45, 1, yes, no, no) \ + SC(170, 47, 45, 2, yes, no, no) \ + SC(171, 47, 45, 3, yes, no, no) \ + SC(172, 47, 45, 4, yes, no, no) \ + \ + SC(173, 48, 46, 1, yes, no, no) \ + SC(174, 48, 46, 2, yes, no, no) \ + SC(175, 48, 46, 3, yes, no, no) \ + SC(176, 48, 46, 4, yes, no, no) \ + \ + SC(177, 49, 47, 1, yes, no, no) \ + SC(178, 49, 47, 2, yes, no, no) \ + SC(179, 49, 47, 3, yes, no, no) \ + SC(180, 49, 47, 4, yes, no, no) \ + \ + SC(181, 50, 48, 1, yes, no, no) \ + SC(182, 50, 48, 2, yes, no, no) \ + SC(183, 50, 48, 3, yes, no, no) \ + SC(184, 50, 48, 4, yes, no, no) \ + \ + SC(185, 51, 49, 1, yes, no, no) \ + SC(186, 51, 49, 2, yes, no, no) \ + SC(187, 51, 49, 3, yes, no, no) \ + SC(188, 51, 49, 4, yes, no, no) \ + \ + SC(189, 52, 50, 1, yes, no, no) \ + SC(190, 52, 50, 2, yes, no, no) \ + SC(191, 52, 50, 3, yes, no, no) \ + SC(192, 52, 50, 4, yes, no, no) \ + \ + SC(193, 53, 51, 1, yes, no, no) \ + SC(194, 53, 51, 2, yes, no, no) \ + SC(195, 53, 51, 3, yes, no, no) \ + SC(196, 53, 51, 4, yes, no, no) \ + \ + SC(197, 54, 52, 1, yes, no, no) \ + SC(198, 54, 52, 2, yes, no, no) \ + SC(199, 54, 52, 3, yes, no, no) \ + SC(200, 54, 52, 4, yes, no, no) \ + \ + SC(201, 55, 53, 1, yes, no, no) \ + SC(202, 55, 53, 2, yes, no, no) \ + SC(203, 55, 53, 3, yes, no, no) \ + SC(204, 55, 53, 4, yes, no, no) \ + \ + SC(205, 56, 54, 1, yes, no, no) \ + SC(206, 56, 54, 2, yes, no, no) \ + SC(207, 56, 54, 3, yes, no, no) \ + SC(208, 56, 54, 4, yes, no, no) \ + \ + SC(209, 57, 55, 1, yes, no, no) \ + SC(210, 57, 55, 2, yes, no, no) \ + SC(211, 57, 55, 3, yes, no, no) \ + SC(212, 57, 55, 4, yes, no, no) \ + \ + SC(213, 58, 56, 1, yes, no, no) \ + SC(214, 58, 56, 2, yes, no, no) \ + SC(215, 58, 56, 3, yes, no, no) \ + SC(216, 58, 56, 4, yes, no, no) \ + \ + SC(217, 59, 57, 1, yes, no, no) \ + SC(218, 59, 57, 2, yes, no, no) \ + SC(219, 59, 57, 3, yes, no, no) \ + SC(220, 59, 57, 4, yes, no, no) \ + \ + SC(221, 60, 58, 1, yes, no, no) \ + SC(222, 60, 58, 2, yes, no, no) \ + SC(223, 60, 58, 3, yes, no, no) \ + SC(224, 60, 58, 4, yes, no, no) \ + \ + SC(225, 61, 59, 1, yes, no, no) \ + SC(226, 61, 59, 2, yes, no, no) \ + SC(227, 61, 59, 3, yes, no, no) \ + SC(228, 61, 59, 4, yes, no, no) \ + \ + SC(229, 62, 60, 1, yes, no, no) \ + SC(230, 62, 60, 2, yes, no, no) \ + SC(231, 62, 60, 3, yes, no, no) \ #define SIZE_CLASSES_DEFINED #define NTBINS 1 #define NLBINS 29 #define NBINS 36 #define NSIZES 232 +#define NPSIZES 199 #define LG_TINY_MAXCLASS 3 #define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) #define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) @@ -1086,301 +1092,302 @@ #if (LG_SIZEOF_PTR == 3 && LG_TINY_MIN == 4 && LG_QUANTUM == 4 && LG_PAGE == 12) #define SIZE_CLASSES \ - /* index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup */ \ - SC( 0, 4, 4, 0, yes, 4) \ - SC( 1, 4, 4, 1, yes, 4) \ - SC( 2, 4, 4, 2, yes, 4) \ - SC( 3, 4, 4, 3, yes, 4) \ - \ - SC( 4, 6, 4, 1, yes, 4) \ - SC( 5, 6, 4, 2, yes, 4) \ - SC( 6, 6, 4, 3, yes, 4) \ - SC( 7, 6, 4, 4, yes, 4) \ - \ - SC( 8, 7, 5, 1, yes, 5) \ - SC( 9, 7, 5, 2, yes, 5) \ - SC( 10, 7, 5, 3, yes, 5) \ - SC( 11, 7, 5, 4, yes, 5) \ - \ - SC( 12, 8, 6, 1, yes, 6) \ - SC( 13, 8, 6, 2, yes, 6) \ - SC( 14, 8, 6, 3, yes, 6) \ - SC( 15, 8, 6, 4, yes, 6) \ - \ - SC( 16, 9, 7, 1, yes, 7) \ - SC( 17, 9, 7, 2, yes, 7) \ - SC( 18, 9, 7, 3, yes, 7) \ - SC( 19, 9, 7, 4, yes, 7) \ - \ - SC( 20, 10, 8, 1, yes, 8) \ - SC( 21, 10, 8, 2, yes, 8) \ - SC( 22, 10, 8, 3, yes, 8) \ - SC( 23, 10, 8, 4, yes, 8) \ - \ - SC( 24, 11, 9, 1, yes, 9) \ - SC( 25, 11, 9, 2, yes, 9) \ - SC( 26, 11, 9, 3, yes, 9) \ - SC( 27, 11, 9, 4, yes, 9) \ - \ - SC( 28, 12, 10, 1, yes, no) \ - SC( 29, 12, 10, 2, yes, no) \ - SC( 30, 12, 10, 3, yes, no) \ - SC( 31, 12, 10, 4, yes, no) \ - \ - SC( 32, 13, 11, 1, yes, no) \ - SC( 33, 13, 11, 2, yes, no) \ - SC( 34, 13, 11, 3, yes, no) \ - SC( 35, 13, 11, 4, no, no) \ - \ - SC( 36, 14, 12, 1, no, no) \ - SC( 37, 14, 12, 2, no, no) \ - SC( 38, 14, 12, 3, no, no) \ - SC( 39, 14, 12, 4, no, no) \ - \ - SC( 40, 15, 13, 1, no, no) \ - SC( 41, 15, 13, 2, no, no) \ - SC( 42, 15, 13, 3, no, no) \ - SC( 43, 15, 13, 4, no, no) \ - \ - SC( 44, 16, 14, 1, no, no) \ - SC( 45, 16, 14, 2, no, no) \ - SC( 46, 16, 14, 3, no, no) \ - SC( 47, 16, 14, 4, no, no) \ - \ - SC( 48, 17, 15, 1, no, no) \ - SC( 49, 17, 15, 2, no, no) \ - SC( 50, 17, 15, 3, no, no) \ - SC( 51, 17, 15, 4, no, no) \ - \ - SC( 52, 18, 16, 1, no, no) \ - SC( 53, 18, 16, 2, no, no) \ - SC( 54, 18, 16, 3, no, no) \ - SC( 55, 18, 16, 4, no, no) \ - \ - SC( 56, 19, 17, 1, no, no) \ - SC( 57, 19, 17, 2, no, no) \ - SC( 58, 19, 17, 3, no, no) \ - SC( 59, 19, 17, 4, no, no) \ - \ - SC( 60, 20, 18, 1, no, no) \ - SC( 61, 20, 18, 2, no, no) \ - SC( 62, 20, 18, 3, no, no) \ - SC( 63, 20, 18, 4, no, no) \ - \ - SC( 64, 21, 19, 1, no, no) \ - SC( 65, 21, 19, 2, no, no) \ - SC( 66, 21, 19, 3, no, no) \ - SC( 67, 21, 19, 4, no, no) \ - \ - SC( 68, 22, 20, 1, no, no) \ - SC( 69, 22, 20, 2, no, no) \ - SC( 70, 22, 20, 3, no, no) \ - SC( 71, 22, 20, 4, no, no) \ - \ - SC( 72, 23, 21, 1, no, no) \ - SC( 73, 23, 21, 2, no, no) \ - SC( 74, 23, 21, 3, no, no) \ - SC( 75, 23, 21, 4, no, no) \ - \ - SC( 76, 24, 22, 1, no, no) \ - SC( 77, 24, 22, 2, no, no) \ - SC( 78, 24, 22, 3, no, no) \ - SC( 79, 24, 22, 4, no, no) \ - \ - SC( 80, 25, 23, 1, no, no) \ - SC( 81, 25, 23, 2, no, no) \ - SC( 82, 25, 23, 3, no, no) \ - SC( 83, 25, 23, 4, no, no) \ - \ - SC( 84, 26, 24, 1, no, no) \ - SC( 85, 26, 24, 2, no, no) \ - SC( 86, 26, 24, 3, no, no) \ - SC( 87, 26, 24, 4, no, no) \ - \ - SC( 88, 27, 25, 1, no, no) \ - SC( 89, 27, 25, 2, no, no) \ - SC( 90, 27, 25, 3, no, no) \ - SC( 91, 27, 25, 4, no, no) \ - \ - SC( 92, 28, 26, 1, no, no) \ - SC( 93, 28, 26, 2, no, no) \ - SC( 94, 28, 26, 3, no, no) \ - SC( 95, 28, 26, 4, no, no) \ - \ - SC( 96, 29, 27, 1, no, no) \ - SC( 97, 29, 27, 2, no, no) \ - SC( 98, 29, 27, 3, no, no) \ - SC( 99, 29, 27, 4, no, no) \ - \ - SC(100, 30, 28, 1, no, no) \ - SC(101, 30, 28, 2, no, no) \ - SC(102, 30, 28, 3, no, no) \ - SC(103, 30, 28, 4, no, no) \ - \ - SC(104, 31, 29, 1, no, no) \ - SC(105, 31, 29, 2, no, no) \ - SC(106, 31, 29, 3, no, no) \ - SC(107, 31, 29, 4, no, no) \ - \ - SC(108, 32, 30, 1, no, no) \ - SC(109, 32, 30, 2, no, no) \ - SC(110, 32, 30, 3, no, no) \ - SC(111, 32, 30, 4, no, no) \ - \ - SC(112, 33, 31, 1, no, no) \ - SC(113, 33, 31, 2, no, no) \ - SC(114, 33, 31, 3, no, no) \ - SC(115, 33, 31, 4, no, no) \ - \ - SC(116, 34, 32, 1, no, no) \ - SC(117, 34, 32, 2, no, no) \ - SC(118, 34, 32, 3, no, no) \ - SC(119, 34, 32, 4, no, no) \ - \ - SC(120, 35, 33, 1, no, no) \ - SC(121, 35, 33, 2, no, no) \ - SC(122, 35, 33, 3, no, no) \ - SC(123, 35, 33, 4, no, no) \ - \ - SC(124, 36, 34, 1, no, no) \ - SC(125, 36, 34, 2, no, no) \ - SC(126, 36, 34, 3, no, no) \ - SC(127, 36, 34, 4, no, no) \ - \ - SC(128, 37, 35, 1, no, no) \ - SC(129, 37, 35, 2, no, no) \ - SC(130, 37, 35, 3, no, no) \ - SC(131, 37, 35, 4, no, no) \ - \ - SC(132, 38, 36, 1, no, no) \ - SC(133, 38, 36, 2, no, no) \ - SC(134, 38, 36, 3, no, no) \ - SC(135, 38, 36, 4, no, no) \ - \ - SC(136, 39, 37, 1, no, no) \ - SC(137, 39, 37, 2, no, no) \ - SC(138, 39, 37, 3, no, no) \ - SC(139, 39, 37, 4, no, no) \ - \ - SC(140, 40, 38, 1, no, no) \ - SC(141, 40, 38, 2, no, no) \ - SC(142, 40, 38, 3, no, no) \ - SC(143, 40, 38, 4, no, no) \ - \ - SC(144, 41, 39, 1, no, no) \ - SC(145, 41, 39, 2, no, no) \ - SC(146, 41, 39, 3, no, no) \ - SC(147, 41, 39, 4, no, no) \ - \ - SC(148, 42, 40, 1, no, no) \ - SC(149, 42, 40, 2, no, no) \ - SC(150, 42, 40, 3, no, no) \ - SC(151, 42, 40, 4, no, no) \ - \ - SC(152, 43, 41, 1, no, no) \ - SC(153, 43, 41, 2, no, no) \ - SC(154, 43, 41, 3, no, no) \ - SC(155, 43, 41, 4, no, no) \ - \ - SC(156, 44, 42, 1, no, no) \ - SC(157, 44, 42, 2, no, no) \ - SC(158, 44, 42, 3, no, no) \ - SC(159, 44, 42, 4, no, no) \ - \ - SC(160, 45, 43, 1, no, no) \ - SC(161, 45, 43, 2, no, no) \ - SC(162, 45, 43, 3, no, no) \ - SC(163, 45, 43, 4, no, no) \ - \ - SC(164, 46, 44, 1, no, no) \ - SC(165, 46, 44, 2, no, no) \ - SC(166, 46, 44, 3, no, no) \ - SC(167, 46, 44, 4, no, no) \ - \ - SC(168, 47, 45, 1, no, no) \ - SC(169, 47, 45, 2, no, no) \ - SC(170, 47, 45, 3, no, no) \ - SC(171, 47, 45, 4, no, no) \ - \ - SC(172, 48, 46, 1, no, no) \ - SC(173, 48, 46, 2, no, no) \ - SC(174, 48, 46, 3, no, no) \ - SC(175, 48, 46, 4, no, no) \ - \ - SC(176, 49, 47, 1, no, no) \ - SC(177, 49, 47, 2, no, no) \ - SC(178, 49, 47, 3, no, no) \ - SC(179, 49, 47, 4, no, no) \ - \ - SC(180, 50, 48, 1, no, no) \ - SC(181, 50, 48, 2, no, no) \ - SC(182, 50, 48, 3, no, no) \ - SC(183, 50, 48, 4, no, no) \ - \ - SC(184, 51, 49, 1, no, no) \ - SC(185, 51, 49, 2, no, no) \ - SC(186, 51, 49, 3, no, no) \ - SC(187, 51, 49, 4, no, no) \ - \ - SC(188, 52, 50, 1, no, no) \ - SC(189, 52, 50, 2, no, no) \ - SC(190, 52, 50, 3, no, no) \ - SC(191, 52, 50, 4, no, no) \ - \ - SC(192, 53, 51, 1, no, no) \ - SC(193, 53, 51, 2, no, no) \ - SC(194, 53, 51, 3, no, no) \ - SC(195, 53, 51, 4, no, no) \ - \ - SC(196, 54, 52, 1, no, no) \ - SC(197, 54, 52, 2, no, no) \ - SC(198, 54, 52, 3, no, no) \ - SC(199, 54, 52, 4, no, no) \ - \ - SC(200, 55, 53, 1, no, no) \ - SC(201, 55, 53, 2, no, no) \ - SC(202, 55, 53, 3, no, no) \ - SC(203, 55, 53, 4, no, no) \ - \ - SC(204, 56, 54, 1, no, no) \ - SC(205, 56, 54, 2, no, no) \ - SC(206, 56, 54, 3, no, no) \ - SC(207, 56, 54, 4, no, no) \ - \ - SC(208, 57, 55, 1, no, no) \ - SC(209, 57, 55, 2, no, no) \ - SC(210, 57, 55, 3, no, no) \ - SC(211, 57, 55, 4, no, no) \ - \ - SC(212, 58, 56, 1, no, no) \ - SC(213, 58, 56, 2, no, no) \ - SC(214, 58, 56, 3, no, no) \ - SC(215, 58, 56, 4, no, no) \ - \ - SC(216, 59, 57, 1, no, no) \ - SC(217, 59, 57, 2, no, no) \ - SC(218, 59, 57, 3, no, no) \ - SC(219, 59, 57, 4, no, no) \ - \ - SC(220, 60, 58, 1, no, no) \ - SC(221, 60, 58, 2, no, no) \ - SC(222, 60, 58, 3, no, no) \ - SC(223, 60, 58, 4, no, no) \ - \ - SC(224, 61, 59, 1, no, no) \ - SC(225, 61, 59, 2, no, no) \ - SC(226, 61, 59, 3, no, no) \ - SC(227, 61, 59, 4, no, no) \ - \ - SC(228, 62, 60, 1, no, no) \ - SC(229, 62, 60, 2, no, no) \ - SC(230, 62, 60, 3, no, no) \ + /* index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup */ \ + SC( 0, 4, 4, 0, no, yes, 4) \ + SC( 1, 4, 4, 1, no, yes, 4) \ + SC( 2, 4, 4, 2, no, yes, 4) \ + SC( 3, 4, 4, 3, no, yes, 4) \ + \ + SC( 4, 6, 4, 1, no, yes, 4) \ + SC( 5, 6, 4, 2, no, yes, 4) \ + SC( 6, 6, 4, 3, no, yes, 4) \ + SC( 7, 6, 4, 4, no, yes, 4) \ + \ + SC( 8, 7, 5, 1, no, yes, 5) \ + SC( 9, 7, 5, 2, no, yes, 5) \ + SC( 10, 7, 5, 3, no, yes, 5) \ + SC( 11, 7, 5, 4, no, yes, 5) \ + \ + SC( 12, 8, 6, 1, no, yes, 6) \ + SC( 13, 8, 6, 2, no, yes, 6) \ + SC( 14, 8, 6, 3, no, yes, 6) \ + SC( 15, 8, 6, 4, no, yes, 6) \ + \ + SC( 16, 9, 7, 1, no, yes, 7) \ + SC( 17, 9, 7, 2, no, yes, 7) \ + SC( 18, 9, 7, 3, no, yes, 7) \ + SC( 19, 9, 7, 4, no, yes, 7) \ + \ + SC( 20, 10, 8, 1, no, yes, 8) \ + SC( 21, 10, 8, 2, no, yes, 8) \ + SC( 22, 10, 8, 3, no, yes, 8) \ + SC( 23, 10, 8, 4, no, yes, 8) \ + \ + SC( 24, 11, 9, 1, no, yes, 9) \ + SC( 25, 11, 9, 2, no, yes, 9) \ + SC( 26, 11, 9, 3, no, yes, 9) \ + SC( 27, 11, 9, 4, yes, yes, 9) \ + \ + SC( 28, 12, 10, 1, no, yes, no) \ + SC( 29, 12, 10, 2, no, yes, no) \ + SC( 30, 12, 10, 3, no, yes, no) \ + SC( 31, 12, 10, 4, yes, yes, no) \ + \ + SC( 32, 13, 11, 1, no, yes, no) \ + SC( 33, 13, 11, 2, yes, yes, no) \ + SC( 34, 13, 11, 3, no, yes, no) \ + SC( 35, 13, 11, 4, yes, no, no) \ + \ + SC( 36, 14, 12, 1, yes, no, no) \ + SC( 37, 14, 12, 2, yes, no, no) \ + SC( 38, 14, 12, 3, yes, no, no) \ + SC( 39, 14, 12, 4, yes, no, no) \ + \ + SC( 40, 15, 13, 1, yes, no, no) \ + SC( 41, 15, 13, 2, yes, no, no) \ + SC( 42, 15, 13, 3, yes, no, no) \ + SC( 43, 15, 13, 4, yes, no, no) \ + \ + SC( 44, 16, 14, 1, yes, no, no) \ + SC( 45, 16, 14, 2, yes, no, no) \ + SC( 46, 16, 14, 3, yes, no, no) \ + SC( 47, 16, 14, 4, yes, no, no) \ + \ + SC( 48, 17, 15, 1, yes, no, no) \ + SC( 49, 17, 15, 2, yes, no, no) \ + SC( 50, 17, 15, 3, yes, no, no) \ + SC( 51, 17, 15, 4, yes, no, no) \ + \ + SC( 52, 18, 16, 1, yes, no, no) \ + SC( 53, 18, 16, 2, yes, no, no) \ + SC( 54, 18, 16, 3, yes, no, no) \ + SC( 55, 18, 16, 4, yes, no, no) \ + \ + SC( 56, 19, 17, 1, yes, no, no) \ + SC( 57, 19, 17, 2, yes, no, no) \ + SC( 58, 19, 17, 3, yes, no, no) \ + SC( 59, 19, 17, 4, yes, no, no) \ + \ + SC( 60, 20, 18, 1, yes, no, no) \ + SC( 61, 20, 18, 2, yes, no, no) \ + SC( 62, 20, 18, 3, yes, no, no) \ + SC( 63, 20, 18, 4, yes, no, no) \ + \ + SC( 64, 21, 19, 1, yes, no, no) \ + SC( 65, 21, 19, 2, yes, no, no) \ + SC( 66, 21, 19, 3, yes, no, no) \ + SC( 67, 21, 19, 4, yes, no, no) \ + \ + SC( 68, 22, 20, 1, yes, no, no) \ + SC( 69, 22, 20, 2, yes, no, no) \ + SC( 70, 22, 20, 3, yes, no, no) \ + SC( 71, 22, 20, 4, yes, no, no) \ + \ + SC( 72, 23, 21, 1, yes, no, no) \ + SC( 73, 23, 21, 2, yes, no, no) \ + SC( 74, 23, 21, 3, yes, no, no) \ + SC( 75, 23, 21, 4, yes, no, no) \ + \ + SC( 76, 24, 22, 1, yes, no, no) \ + SC( 77, 24, 22, 2, yes, no, no) \ + SC( 78, 24, 22, 3, yes, no, no) \ + SC( 79, 24, 22, 4, yes, no, no) \ + \ + SC( 80, 25, 23, 1, yes, no, no) \ + SC( 81, 25, 23, 2, yes, no, no) \ + SC( 82, 25, 23, 3, yes, no, no) \ + SC( 83, 25, 23, 4, yes, no, no) \ + \ + SC( 84, 26, 24, 1, yes, no, no) \ + SC( 85, 26, 24, 2, yes, no, no) \ + SC( 86, 26, 24, 3, yes, no, no) \ + SC( 87, 26, 24, 4, yes, no, no) \ + \ + SC( 88, 27, 25, 1, yes, no, no) \ + SC( 89, 27, 25, 2, yes, no, no) \ + SC( 90, 27, 25, 3, yes, no, no) \ + SC( 91, 27, 25, 4, yes, no, no) \ + \ + SC( 92, 28, 26, 1, yes, no, no) \ + SC( 93, 28, 26, 2, yes, no, no) \ + SC( 94, 28, 26, 3, yes, no, no) \ + SC( 95, 28, 26, 4, yes, no, no) \ + \ + SC( 96, 29, 27, 1, yes, no, no) \ + SC( 97, 29, 27, 2, yes, no, no) \ + SC( 98, 29, 27, 3, yes, no, no) \ + SC( 99, 29, 27, 4, yes, no, no) \ + \ + SC(100, 30, 28, 1, yes, no, no) \ + SC(101, 30, 28, 2, yes, no, no) \ + SC(102, 30, 28, 3, yes, no, no) \ + SC(103, 30, 28, 4, yes, no, no) \ + \ + SC(104, 31, 29, 1, yes, no, no) \ + SC(105, 31, 29, 2, yes, no, no) \ + SC(106, 31, 29, 3, yes, no, no) \ + SC(107, 31, 29, 4, yes, no, no) \ + \ + SC(108, 32, 30, 1, yes, no, no) \ + SC(109, 32, 30, 2, yes, no, no) \ + SC(110, 32, 30, 3, yes, no, no) \ + SC(111, 32, 30, 4, yes, no, no) \ + \ + SC(112, 33, 31, 1, yes, no, no) \ + SC(113, 33, 31, 2, yes, no, no) \ + SC(114, 33, 31, 3, yes, no, no) \ + SC(115, 33, 31, 4, yes, no, no) \ + \ + SC(116, 34, 32, 1, yes, no, no) \ + SC(117, 34, 32, 2, yes, no, no) \ + SC(118, 34, 32, 3, yes, no, no) \ + SC(119, 34, 32, 4, yes, no, no) \ + \ + SC(120, 35, 33, 1, yes, no, no) \ + SC(121, 35, 33, 2, yes, no, no) \ + SC(122, 35, 33, 3, yes, no, no) \ + SC(123, 35, 33, 4, yes, no, no) \ + \ + SC(124, 36, 34, 1, yes, no, no) \ + SC(125, 36, 34, 2, yes, no, no) \ + SC(126, 36, 34, 3, yes, no, no) \ + SC(127, 36, 34, 4, yes, no, no) \ + \ + SC(128, 37, 35, 1, yes, no, no) \ + SC(129, 37, 35, 2, yes, no, no) \ + SC(130, 37, 35, 3, yes, no, no) \ + SC(131, 37, 35, 4, yes, no, no) \ + \ + SC(132, 38, 36, 1, yes, no, no) \ + SC(133, 38, 36, 2, yes, no, no) \ + SC(134, 38, 36, 3, yes, no, no) \ + SC(135, 38, 36, 4, yes, no, no) \ + \ + SC(136, 39, 37, 1, yes, no, no) \ + SC(137, 39, 37, 2, yes, no, no) \ + SC(138, 39, 37, 3, yes, no, no) \ + SC(139, 39, 37, 4, yes, no, no) \ + \ + SC(140, 40, 38, 1, yes, no, no) \ + SC(141, 40, 38, 2, yes, no, no) \ + SC(142, 40, 38, 3, yes, no, no) \ + SC(143, 40, 38, 4, yes, no, no) \ + \ + SC(144, 41, 39, 1, yes, no, no) \ + SC(145, 41, 39, 2, yes, no, no) \ + SC(146, 41, 39, 3, yes, no, no) \ + SC(147, 41, 39, 4, yes, no, no) \ + \ + SC(148, 42, 40, 1, yes, no, no) \ + SC(149, 42, 40, 2, yes, no, no) \ + SC(150, 42, 40, 3, yes, no, no) \ + SC(151, 42, 40, 4, yes, no, no) \ + \ + SC(152, 43, 41, 1, yes, no, no) \ + SC(153, 43, 41, 2, yes, no, no) \ + SC(154, 43, 41, 3, yes, no, no) \ + SC(155, 43, 41, 4, yes, no, no) \ + \ + SC(156, 44, 42, 1, yes, no, no) \ + SC(157, 44, 42, 2, yes, no, no) \ + SC(158, 44, 42, 3, yes, no, no) \ + SC(159, 44, 42, 4, yes, no, no) \ + \ + SC(160, 45, 43, 1, yes, no, no) \ + SC(161, 45, 43, 2, yes, no, no) \ + SC(162, 45, 43, 3, yes, no, no) \ + SC(163, 45, 43, 4, yes, no, no) \ + \ + SC(164, 46, 44, 1, yes, no, no) \ + SC(165, 46, 44, 2, yes, no, no) \ + SC(166, 46, 44, 3, yes, no, no) \ + SC(167, 46, 44, 4, yes, no, no) \ + \ + SC(168, 47, 45, 1, yes, no, no) \ + SC(169, 47, 45, 2, yes, no, no) \ + SC(170, 47, 45, 3, yes, no, no) \ + SC(171, 47, 45, 4, yes, no, no) \ + \ + SC(172, 48, 46, 1, yes, no, no) \ + SC(173, 48, 46, 2, yes, no, no) \ + SC(174, 48, 46, 3, yes, no, no) \ + SC(175, 48, 46, 4, yes, no, no) \ + \ + SC(176, 49, 47, 1, yes, no, no) \ + SC(177, 49, 47, 2, yes, no, no) \ + SC(178, 49, 47, 3, yes, no, no) \ + SC(179, 49, 47, 4, yes, no, no) \ + \ + SC(180, 50, 48, 1, yes, no, no) \ + SC(181, 50, 48, 2, yes, no, no) \ + SC(182, 50, 48, 3, yes, no, no) \ + SC(183, 50, 48, 4, yes, no, no) \ + \ + SC(184, 51, 49, 1, yes, no, no) \ + SC(185, 51, 49, 2, yes, no, no) \ + SC(186, 51, 49, 3, yes, no, no) \ + SC(187, 51, 49, 4, yes, no, no) \ + \ + SC(188, 52, 50, 1, yes, no, no) \ + SC(189, 52, 50, 2, yes, no, no) \ + SC(190, 52, 50, 3, yes, no, no) \ + SC(191, 52, 50, 4, yes, no, no) \ + \ + SC(192, 53, 51, 1, yes, no, no) \ + SC(193, 53, 51, 2, yes, no, no) \ + SC(194, 53, 51, 3, yes, no, no) \ + SC(195, 53, 51, 4, yes, no, no) \ + \ + SC(196, 54, 52, 1, yes, no, no) \ + SC(197, 54, 52, 2, yes, no, no) \ + SC(198, 54, 52, 3, yes, no, no) \ + SC(199, 54, 52, 4, yes, no, no) \ + \ + SC(200, 55, 53, 1, yes, no, no) \ + SC(201, 55, 53, 2, yes, no, no) \ + SC(202, 55, 53, 3, yes, no, no) \ + SC(203, 55, 53, 4, yes, no, no) \ + \ + SC(204, 56, 54, 1, yes, no, no) \ + SC(205, 56, 54, 2, yes, no, no) \ + SC(206, 56, 54, 3, yes, no, no) \ + SC(207, 56, 54, 4, yes, no, no) \ + \ + SC(208, 57, 55, 1, yes, no, no) \ + SC(209, 57, 55, 2, yes, no, no) \ + SC(210, 57, 55, 3, yes, no, no) \ + SC(211, 57, 55, 4, yes, no, no) \ + \ + SC(212, 58, 56, 1, yes, no, no) \ + SC(213, 58, 56, 2, yes, no, no) \ + SC(214, 58, 56, 3, yes, no, no) \ + SC(215, 58, 56, 4, yes, no, no) \ + \ + SC(216, 59, 57, 1, yes, no, no) \ + SC(217, 59, 57, 2, yes, no, no) \ + SC(218, 59, 57, 3, yes, no, no) \ + SC(219, 59, 57, 4, yes, no, no) \ + \ + SC(220, 60, 58, 1, yes, no, no) \ + SC(221, 60, 58, 2, yes, no, no) \ + SC(222, 60, 58, 3, yes, no, no) \ + SC(223, 60, 58, 4, yes, no, no) \ + \ + SC(224, 61, 59, 1, yes, no, no) \ + SC(225, 61, 59, 2, yes, no, no) \ + SC(226, 61, 59, 3, yes, no, no) \ + SC(227, 61, 59, 4, yes, no, no) \ + \ + SC(228, 62, 60, 1, yes, no, no) \ + SC(229, 62, 60, 2, yes, no, no) \ + SC(230, 62, 60, 3, yes, no, no) \ #define SIZE_CLASSES_DEFINED #define NTBINS 0 #define NLBINS 28 #define NBINS 35 #define NSIZES 231 +#define NPSIZES 199 #define LG_TINY_MAXCLASS "NA" #define LOOKUP_MAXCLASS ((((size_t)1) << 11) + (((size_t)4) << 9)) #define SMALL_MAXCLASS ((((size_t)1) << 13) + (((size_t)3) << 11)) diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh index 2b0ca29..f6fbce4 100755 --- a/include/jemalloc/internal/size_classes.sh +++ b/include/jemalloc/internal/size_classes.sh @@ -48,6 +48,21 @@ size_class() { lg_p=$5 lg_kmax=$6 + if [ ${lg_delta} -ge ${lg_p} ] ; then + psz="yes" + else + pow2 ${lg_p}; p=${pow2_result} + pow2 ${lg_grp}; grp=${pow2_result} + pow2 ${lg_delta}; delta=${pow2_result} + sz=$((${grp} + ${delta} * ${ndelta})) + npgs=$((${sz} / ${p})) + if [ ${sz} -eq $((${npgs} * ${p})) ] ; then + psz="yes" + else + psz="no" + fi + fi + lg ${ndelta}; lg_ndelta=${lg_result}; pow2 ${lg_ndelta} if [ ${pow2_result} -lt ${ndelta} ] ; then rem="yes" @@ -74,14 +89,15 @@ size_class() { else lg_delta_lookup="no" fi - printf ' SC(%3d, %6d, %8d, %6d, %3s, %2s) \\\n' ${index} ${lg_grp} ${lg_delta} ${ndelta} ${bin} ${lg_delta_lookup} + printf ' SC(%3d, %6d, %8d, %6d, %3s, %3s, %2s) \\\n' ${index} ${lg_grp} ${lg_delta} ${ndelta} ${psz} ${bin} ${lg_delta_lookup} # Defined upon return: - # - lg_delta_lookup (${lg_delta} or "no") + # - psz ("yes" or "no") # - bin ("yes" or "no") + # - lg_delta_lookup (${lg_delta} or "no") } sep_line() { - echo " \\" + echo " \\" } size_classes() { @@ -95,12 +111,13 @@ size_classes() { pow2 ${lg_g}; g=${pow2_result} echo "#define SIZE_CLASSES \\" - echo " /* index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup */ \\" + echo " /* index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup */ \\" ntbins=0 nlbins=0 lg_tiny_maxclass='"NA"' nbins=0 + npsizes=0 # Tiny size classes. ndelta=0 @@ -112,6 +129,9 @@ size_classes() { if [ ${lg_delta_lookup} != "no" ] ; then nlbins=$((${index} + 1)) fi + if [ ${psz} = "yes" ] ; then + npsizes=$((${npsizes} + 1)) + fi if [ ${bin} != "no" ] ; then nbins=$((${index} + 1)) fi @@ -133,11 +153,17 @@ size_classes() { index=$((${index} + 1)) lg_grp=$((${lg_grp} + 1)) lg_delta=$((${lg_delta} + 1)) + if [ ${psz} = "yes" ] ; then + npsizes=$((${npsizes} + 1)) + fi fi while [ ${ndelta} -lt ${g} ] ; do size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} index=$((${index} + 1)) ndelta=$((${ndelta} + 1)) + if [ ${psz} = "yes" ] ; then + npsizes=$((${npsizes} + 1)) + fi done # All remaining groups. @@ -157,6 +183,9 @@ size_classes() { # Final written value is correct: lookup_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" fi + if [ ${psz} = "yes" ] ; then + npsizes=$((${npsizes} + 1)) + fi if [ ${bin} != "no" ] ; then nbins=$((${index} + 1)) # Final written value is correct: @@ -183,6 +212,7 @@ size_classes() { # - nlbins # - nbins # - nsizes + # - npsizes # - lg_tiny_maxclass # - lookup_maxclass # - small_maxclass @@ -200,13 +230,13 @@ cat <<EOF * be defined prior to inclusion, and it in turn defines: * * LG_SIZE_CLASS_GROUP: Lg of size class count for each size doubling. - * SIZE_CLASSES: Complete table of - * SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) - * tuples. + * SIZE_CLASSES: Complete table of SC(index, lg_grp, lg_delta, ndelta, psz, + * bin, lg_delta_lookup) tuples. * index: Size class index. * lg_grp: Lg group base size (no deltas added). * lg_delta: Lg delta to previous size class. * ndelta: Delta multiplier. size == 1<<lg_grp + ndelta<<lg_delta + * psz: 'yes' if a multiple of the page size, 'no' otherwise. * bin: 'yes' if a small bin size class, 'no' otherwise. * lg_delta_lookup: Same as lg_delta if a lookup table size class, 'no' * otherwise. @@ -214,6 +244,7 @@ cat <<EOF * NLBINS: Number of bins supported by the lookup table. * NBINS: Number of small size class bins. * NSIZES: Number of size classes. + * NPSIZES: Number of size classes that are a multiple of (1U << LG_PAGE). * LG_TINY_MAXCLASS: Lg of maximum tiny size class. * LOOKUP_MAXCLASS: Maximum size class included in lookup table. * SMALL_MAXCLASS: Maximum small size class. @@ -238,6 +269,7 @@ for lg_z in ${lg_zarr} ; do echo "#define NLBINS ${nlbins}" echo "#define NBINS ${nbins}" echo "#define NSIZES ${nsizes}" + echo "#define NPSIZES ${npsizes}" echo "#define LG_TINY_MAXCLASS ${lg_tiny_maxclass}" echo "#define LOOKUP_MAXCLASS ${lookup_maxclass}" echo "#define SMALL_MAXCLASS ${small_maxclass}" diff --git a/include/jemalloc/internal/spin.h b/include/jemalloc/internal/spin.h new file mode 100644 index 0000000..9ef5ceb --- /dev/null +++ b/include/jemalloc/internal/spin.h @@ -0,0 +1,51 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +typedef struct spin_s spin_t; + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +struct spin_s { + unsigned iteration; +}; + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +void spin_init(spin_t *spin); +void spin_adaptive(spin_t *spin); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_SPIN_C_)) +JEMALLOC_INLINE void +spin_init(spin_t *spin) +{ + + spin->iteration = 0; +} + +JEMALLOC_INLINE void +spin_adaptive(spin_t *spin) +{ + volatile uint64_t i; + + for (i = 0; i < (KQU(1) << spin->iteration); i++) + CPU_SPINWAIT; + + if (spin->iteration < 63) + spin->iteration++; +} + +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ + diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 165dd1b..4d0a8ff 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -157,7 +157,7 @@ tcache_t *tcache_create(tsdn_t *tsdn, arena_t *arena); void tcache_cleanup(tsd_t *tsd); void tcache_enabled_cleanup(tsd_t *tsd); void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena); -bool tcaches_create(tsdn_t *tsdn, unsigned *r_ind); +bool tcaches_create(tsd_t *tsd, unsigned *r_ind); void tcaches_flush(tsd_t *tsd, unsigned ind); void tcaches_destroy(tsd_t *tsd, unsigned ind); bool tcache_boot(tsdn_t *tsdn); diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index bf11341..9055aca 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -48,7 +48,7 @@ typedef enum { * * bool example_tsd_boot(void) {...} * bool example_tsd_booted_get(void) {...} - * example_t *example_tsd_get() {...} + * example_t *example_tsd_get(bool init) {...} * void example_tsd_set(example_t *val) {...} * * Note that all of the functions deal in terms of (a_type *) rather than @@ -105,7 +105,7 @@ a_name##tsd_boot(void); \ a_attr bool \ a_name##tsd_booted_get(void); \ a_attr a_type * \ -a_name##tsd_get(void); \ +a_name##tsd_get(bool init); \ a_attr void \ a_name##tsd_set(a_type *val); @@ -213,9 +213,15 @@ a_name##tsd_booted_get(void) \ \ return (a_name##tsd_booted); \ } \ +a_attr bool \ +a_name##tsd_get_allocates(void) \ +{ \ + \ + return (false); \ +} \ /* Get/set. */ \ a_attr a_type * \ -a_name##tsd_get(void) \ +a_name##tsd_get(bool init) \ { \ \ assert(a_name##tsd_booted); \ @@ -264,9 +270,15 @@ a_name##tsd_booted_get(void) \ \ return (a_name##tsd_booted); \ } \ +a_attr bool \ +a_name##tsd_get_allocates(void) \ +{ \ + \ + return (false); \ +} \ /* Get/set. */ \ a_attr a_type * \ -a_name##tsd_get(void) \ +a_name##tsd_get(bool init) \ { \ \ assert(a_name##tsd_booted); \ @@ -325,14 +337,14 @@ a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ } \ } \ a_attr a_name##tsd_wrapper_t * \ -a_name##tsd_wrapper_get(void) \ +a_name##tsd_wrapper_get(bool init) \ { \ DWORD error = GetLastError(); \ a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ TlsGetValue(a_name##tsd_tsd); \ SetLastError(error); \ \ - if (unlikely(wrapper == NULL)) { \ + if (init && unlikely(wrapper == NULL)) { \ wrapper = (a_name##tsd_wrapper_t *) \ malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ if (wrapper == NULL) { \ @@ -392,14 +404,22 @@ a_name##tsd_booted_get(void) \ \ return (a_name##tsd_booted); \ } \ +a_attr bool \ +a_name##tsd_get_allocates(void) \ +{ \ + \ + return (true); \ +} \ /* Get/set. */ \ a_attr a_type * \ -a_name##tsd_get(void) \ +a_name##tsd_get(bool init) \ { \ a_name##tsd_wrapper_t *wrapper; \ \ assert(a_name##tsd_booted); \ - wrapper = a_name##tsd_wrapper_get(); \ + wrapper = a_name##tsd_wrapper_get(init); \ + if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \ + return (NULL); \ return (&wrapper->val); \ } \ a_attr void \ @@ -408,7 +428,7 @@ a_name##tsd_set(a_type *val) \ a_name##tsd_wrapper_t *wrapper; \ \ assert(a_name##tsd_booted); \ - wrapper = a_name##tsd_wrapper_get(); \ + wrapper = a_name##tsd_wrapper_get(true); \ wrapper->val = *(val); \ if (a_cleanup != malloc_tsd_no_cleanup) \ wrapper->initialized = true; \ @@ -452,12 +472,12 @@ a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ } \ } \ a_attr a_name##tsd_wrapper_t * \ -a_name##tsd_wrapper_get(void) \ +a_name##tsd_wrapper_get(bool init) \ { \ a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ pthread_getspecific(a_name##tsd_tsd); \ \ - if (unlikely(wrapper == NULL)) { \ + if (init && unlikely(wrapper == NULL)) { \ tsd_init_block_t block; \ wrapper = tsd_init_check_recursion( \ &a_name##tsd_init_head, &block); \ @@ -520,14 +540,22 @@ a_name##tsd_booted_get(void) \ \ return (a_name##tsd_booted); \ } \ +a_attr bool \ +a_name##tsd_get_allocates(void) \ +{ \ + \ + return (true); \ +} \ /* Get/set. */ \ a_attr a_type * \ -a_name##tsd_get(void) \ +a_name##tsd_get(bool init) \ { \ a_name##tsd_wrapper_t *wrapper; \ \ assert(a_name##tsd_booted); \ - wrapper = a_name##tsd_wrapper_get(); \ + wrapper = a_name##tsd_wrapper_get(init); \ + if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \ + return (NULL); \ return (&wrapper->val); \ } \ a_attr void \ @@ -536,7 +564,7 @@ a_name##tsd_set(a_type *val) \ a_name##tsd_wrapper_t *wrapper; \ \ assert(a_name##tsd_booted); \ - wrapper = a_name##tsd_wrapper_get(); \ + wrapper = a_name##tsd_wrapper_get(true); \ wrapper->val = *(val); \ if (a_cleanup != malloc_tsd_no_cleanup) \ wrapper->initialized = true; \ @@ -639,6 +667,7 @@ void tsd_cleanup(void *arg); #ifndef JEMALLOC_ENABLE_INLINE malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t) +tsd_t *tsd_fetch_impl(bool init); tsd_t *tsd_fetch(void); tsdn_t *tsd_tsdn(tsd_t *tsd); bool tsd_nominal(tsd_t *tsd); @@ -658,9 +687,13 @@ malloc_tsd_externs(, tsd_t) malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup) JEMALLOC_ALWAYS_INLINE tsd_t * -tsd_fetch(void) +tsd_fetch_impl(bool init) { - tsd_t *tsd = tsd_get(); + tsd_t *tsd = tsd_get(init); + + if (!init && tsd_get_allocates() && tsd == NULL) + return (NULL); + assert(tsd != NULL); if (unlikely(tsd->state != tsd_state_nominal)) { if (tsd->state == tsd_state_uninitialized) { @@ -677,6 +710,13 @@ tsd_fetch(void) return (tsd); } +JEMALLOC_ALWAYS_INLINE tsd_t * +tsd_fetch(void) +{ + + return (tsd_fetch_impl(true)); +} + JEMALLOC_ALWAYS_INLINE tsdn_t * tsd_tsdn(tsd_t *tsd) { @@ -723,7 +763,7 @@ tsdn_fetch(void) if (!tsd_booted_get()) return (NULL); - return (tsd_tsdn(tsd_fetch())); + return (tsd_tsdn(tsd_fetch_impl(false))); } JEMALLOC_ALWAYS_INLINE bool diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index a0c2203..aee00d6 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -61,30 +61,20 @@ # define JEMALLOC_CC_SILENCE_INIT(v) #endif -#define JEMALLOC_GNUC_PREREQ(major, minor) \ - (!defined(__clang__) && \ - (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))) -#ifndef __has_builtin -# define __has_builtin(builtin) (0) -#endif -#define JEMALLOC_CLANG_HAS_BUILTIN(builtin) \ - (defined(__clang__) && __has_builtin(builtin)) - #ifdef __GNUC__ # define likely(x) __builtin_expect(!!(x), 1) # define unlikely(x) __builtin_expect(!!(x), 0) -# if JEMALLOC_GNUC_PREREQ(4, 6) || \ - JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable) -# define unreachable() __builtin_unreachable() -# else -# define unreachable() abort() -# endif #else # define likely(x) !!(x) # define unlikely(x) !!(x) -# define unreachable() abort() #endif +#if !defined(JEMALLOC_INTERNAL_UNREACHABLE) +# error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure +#endif + +#define unreachable() JEMALLOC_INTERNAL_UNREACHABLE() + #include "jemalloc/internal/assert.h" /* Use to assert a particular configuration, e.g., cassert(config_debug). */ diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h index d78dca2..cdf15d7 100644 --- a/include/jemalloc/internal/witness.h +++ b/include/jemalloc/internal/witness.h @@ -108,6 +108,7 @@ void witness_postfork_child(tsd_t *tsd); #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE +bool witness_owner(tsd_t *tsd, const witness_t *witness); void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness); void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness); void witness_assert_lockless(tsdn_t *tsdn); @@ -116,12 +117,25 @@ void witness_unlock(tsdn_t *tsdn, witness_t *witness); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_)) +JEMALLOC_INLINE bool +witness_owner(tsd_t *tsd, const witness_t *witness) +{ + witness_list_t *witnesses; + witness_t *w; + + witnesses = tsd_witnessesp_get(tsd); + ql_foreach(w, witnesses, link) { + if (w == witness) + return (true); + } + + return (false); +} + JEMALLOC_INLINE void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) { tsd_t *tsd; - witness_list_t *witnesses; - witness_t *w; if (!config_debug) return; @@ -132,11 +146,8 @@ witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) if (witness->rank == WITNESS_RANK_OMIT) return; - witnesses = tsd_witnessesp_get(tsd); - ql_foreach(w, witnesses, link) { - if (w == witness) - return; - } + if (witness_owner(tsd, witness)) + return; witness_owner_error(witness); } @@ -238,10 +249,16 @@ witness_unlock(tsdn_t *tsdn, witness_t *witness) if (witness->rank == WITNESS_RANK_OMIT) return; - witness_assert_owner(tsdn, witness); - - witnesses = tsd_witnessesp_get(tsd); - ql_remove(witnesses, witness, link); + /* + * Check whether owner before removal, rather than relying on + * witness_assert_owner() to abort, so that unit tests can test this + * function's failure mode without causing undefined behavior. + */ + if (witness_owner(tsd, witness)) { + witnesses = tsd_witnessesp_get(tsd); + ql_remove(witnesses, witness, link); + } else + witness_assert_owner(tsdn, witness); } #endif diff --git a/include/jemalloc/jemalloc.h b/include/jemalloc/jemalloc.h index c983dc8..7706510 100644 --- a/include/jemalloc/jemalloc.h +++ b/include/jemalloc/jemalloc.h @@ -94,12 +94,12 @@ extern "C" { #include <limits.h> #include <strings.h> -#define JEMALLOC_VERSION "4.2.1-0-g3de035335255d553bdb344c32ffdb603816195d8" +#define JEMALLOC_VERSION "4.3.1-0-g0110fa8451af905affd77c3bea0d545fee2251b2" #define JEMALLOC_VERSION_MAJOR 4 -#define JEMALLOC_VERSION_MINOR 2 +#define JEMALLOC_VERSION_MINOR 3 #define JEMALLOC_VERSION_BUGFIX 1 #define JEMALLOC_VERSION_NREV 0 -#define JEMALLOC_VERSION_GID "3de035335255d553bdb344c32ffdb603816195d8" +#define JEMALLOC_VERSION_GID "0110fa8451af905affd77c3bea0d545fee2251b2" # define MALLOCX_LG_ALIGN(la) ((int)(la)) # if LG_SIZEOF_PTR == 2 @@ -127,7 +127,7 @@ extern "C" { # define JEMALLOC_CXX_THROW #endif -#if defined(_MSC_VER) +#if _MSC_VER # define JEMALLOC_ATTR(s) # define JEMALLOC_ALIGNED(s) __declspec(align(s)) # define JEMALLOC_ALLOC_SIZE(s) diff --git a/include/jemalloc/jemalloc_macros.h b/include/jemalloc/jemalloc_macros.h index 04d143f..d7b2f96 100644 --- a/include/jemalloc/jemalloc_macros.h +++ b/include/jemalloc/jemalloc_macros.h @@ -4,12 +4,12 @@ #include <limits.h> #include <strings.h> -#define JEMALLOC_VERSION "4.2.1-0-g3de035335255d553bdb344c32ffdb603816195d8" +#define JEMALLOC_VERSION "4.3.1-0-g0110fa8451af905affd77c3bea0d545fee2251b2" #define JEMALLOC_VERSION_MAJOR 4 -#define JEMALLOC_VERSION_MINOR 2 +#define JEMALLOC_VERSION_MINOR 3 #define JEMALLOC_VERSION_BUGFIX 1 #define JEMALLOC_VERSION_NREV 0 -#define JEMALLOC_VERSION_GID "3de035335255d553bdb344c32ffdb603816195d8" +#define JEMALLOC_VERSION_GID "0110fa8451af905affd77c3bea0d545fee2251b2" # define MALLOCX_LG_ALIGN(la) ((int)(la)) # if LG_SIZEOF_PTR == 2 |