Updates to generated files for 4.2.1 release.android-wear-n-preview-2android-wear-7.1.1_r1android-n-preview-5android-n-iot-preview-2nougat-mr1-wear-releasen-iot-preview-2

Bug: 28860984
Change-Id: If12daed270ec0a85cd151aaaa432d178c8389757

11 files changed, 372 insertions, 196 deletions

diff --git a/Android.bp b/Android.bp
index 051b03c..f65c934 100644
--- a/Android.bp
+++ b/Android.bp
@@ -44,7 +44,6 @@ common_cflags = [
 //     usually decreases the amount of PSS used, but can increase
 //     fragmentation.
 common_cflags += [
-    "-DANDROID_ALWAYS_PURGE",
     "-DANDROID_MAX_ARENAS=2",
     "-DANDROID_TCACHE_NSLOTS_SMALL_MAX=8",
     "-DANDROID_TCACHE_NSLOTS_LARGE=16",
@@ -118,6 +117,7 @@ lib_src_files = [
     "src/ticker.c",
     "src/tsd.c",
     "src/util.c",
+    "src/witness.c",
 ]
 
 //-----------------------------------------------------------------------
@@ -195,10 +195,13 @@ cc_library_static {
 // jemalloc unit tests
 //-----------------------------------------------------------------------
 unit_tests = [
+    "test/unit/a0.c",
+    "test/unit/arena_reset.c",
     "test/unit/atomic.c",
     "test/unit/bitmap.c",
     "test/unit/ckh.c",
     "test/unit/decay.c",
+    "test/unit/fork.c",
     "test/unit/hash.c",
     "test/unit/junk.c",
     "test/unit/junk_alloc.c",
@@ -229,6 +232,7 @@ unit_tests = [
     "test/unit/ticker.c",
     "test/unit/tsd.c",
     "test/unit/util.c",
+    "test/unit/witness.c",
     "test/unit/zero.c",
 ]
 
diff --git a/android/test/run_jemalloc_tests.sh b/android/test/run_jemalloc_tests.sh
index e4a935d..ae479e7 100755
--- a/android/test/run_jemalloc_tests.sh
+++ b/android/test/run_jemalloc_tests.sh
@@ -17,10 +17,13 @@
 UNIT_TEST_DIR="jemalloc_unittests"
 
 UNIT_TESTS=( \
+  "a0" \
+  "arena_reset" \
   "atomic" \
   "bitmap" \
   "ckh" \
   "decay" \
+  "fork" \
   "hash" \
   "junk" \
   "junk_alloc" \
@@ -51,6 +54,7 @@ UNIT_TESTS=( \
   "ticker" \
   "tsd" \
   "util" \
+  "witness" \
   "zero" \
 )
 
diff --git a/include/jemalloc/internal/jemalloc_internal.h b/include/jemalloc/internal/jemalloc_internal.h
index 6f94356..a6902f4 100644
--- a/include/jemalloc/internal/jemalloc_internal.h
+++ b/include/jemalloc/internal/jemalloc_internal.h
@@ -161,6 +161,7 @@ static const bool config_cache_oblivious =
 #include <malloc/malloc.h>
 #endif
 
+#include "jemalloc/internal/ph.h"
 #define	RB_COMPACT
 #include "jemalloc/internal/rb.h"
 #include "jemalloc/internal/qr.h"
@@ -257,6 +258,9 @@ typedef unsigned szind_t;
 #  ifdef __powerpc__
 #    define LG_QUANTUM		4
 #  endif
+#  ifdef __riscv__
+#    define LG_QUANTUM		4
+#  endif
 #  ifdef __s390__
 #    define LG_QUANTUM		4
 #  endif
@@ -367,6 +371,7 @@ typedef unsigned szind_t;
 #include "jemalloc/internal/smoothstep.h"
 #include "jemalloc/internal/stats.h"
 #include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/witness.h"
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/tsd.h"
 #include "jemalloc/internal/mb.h"
@@ -398,6 +403,7 @@ typedef unsigned szind_t;
 #include "jemalloc/internal/smoothstep.h"
 #include "jemalloc/internal/stats.h"
 #include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/witness.h"
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/mb.h"
 #include "jemalloc/internal/bitmap.h"
@@ -440,6 +446,9 @@ extern bool	in_valgrind;
 /* Number of CPUs. */
 extern unsigned	ncpus;
 
+/* Number of arenas used for automatic multiplexing of threads and arenas. */
+extern unsigned	narenas_auto;
+
 /*
  * Arenas that are used to service external requests.  Not all elements of the
  * arenas array are necessarily used; arenas are created lazily as needed.
@@ -463,14 +472,14 @@ void	a0dalloc(void *ptr);
 void	*bootstrap_malloc(size_t size);
 void	*bootstrap_calloc(size_t num, size_t size);
 void	bootstrap_free(void *ptr);
-arena_t	*arenas_extend(unsigned ind);
 unsigned	narenas_total_get(void);
-arena_t	*arena_init(unsigned ind);
+arena_t	*arena_init(tsdn_t *tsdn, unsigned ind);
 arena_tdata_t	*arena_tdata_get_hard(tsd_t *tsd, unsigned ind);
-arena_t	*arena_choose_hard(tsd_t *tsd);
+arena_t	*arena_choose_hard(tsd_t *tsd, bool internal);
 void	arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind);
 void	thread_allocated_cleanup(tsd_t *tsd);
 void	thread_deallocated_cleanup(tsd_t *tsd);
+void	iarena_cleanup(tsd_t *tsd);
 void	arena_cleanup(tsd_t *tsd);
 void	arenas_tdata_cleanup(tsd_t *tsd);
 void	narenas_tdata_cleanup(tsd_t *tsd);
@@ -490,6 +499,7 @@ void	jemalloc_postfork_child(void);
 #include "jemalloc/internal/smoothstep.h"
 #include "jemalloc/internal/stats.h"
 #include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/witness.h"
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/mb.h"
 #include "jemalloc/internal/bitmap.h"
@@ -521,8 +531,9 @@ void	jemalloc_postfork_child(void);
 #include "jemalloc/internal/smoothstep.h"
 #include "jemalloc/internal/stats.h"
 #include "jemalloc/internal/ctl.h"
-#include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/tsd.h"
+#include "jemalloc/internal/witness.h"
+#include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/mb.h"
 #include "jemalloc/internal/extent.h"
 #include "jemalloc/internal/base.h"
@@ -542,10 +553,12 @@ size_t	s2u_compute(size_t size);
 size_t	s2u_lookup(size_t size);
 size_t	s2u(size_t size);
 size_t	sa2u(size_t size, size_t alignment);
+arena_t	*arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal);
 arena_t	*arena_choose(tsd_t *tsd, arena_t *arena);
+arena_t	*arena_ichoose(tsdn_t *tsdn, arena_t *arena);
 arena_tdata_t	*arena_tdata_get(tsd_t *tsd, unsigned ind,
     bool refresh_if_missing);
-arena_t	*arena_get(unsigned ind, bool init_if_missing);
+arena_t	*arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing);
 ticker_t	*decay_ticker_get(tsd_t *tsd, unsigned ind);
 #endif
 
@@ -780,19 +793,38 @@ sa2u(size_t size, size_t alignment)
 
 /* Choose an arena based on a per-thread value. */
 JEMALLOC_INLINE arena_t *
-arena_choose(tsd_t *tsd, arena_t *arena)
+arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal)
 {
 	arena_t *ret;
 
 	if (arena != NULL)
 		return (arena);
 
-	if (unlikely((ret = tsd_arena_get(tsd)) == NULL))
-		ret = arena_choose_hard(tsd);
+	ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd);
+	if (unlikely(ret == NULL))
+		ret = arena_choose_hard(tsd, internal);
 
 	return (ret);
 }
 
+JEMALLOC_INLINE arena_t *
+arena_choose(tsd_t *tsd, arena_t *arena)
+{
+
+	return (arena_choose_impl(tsd, arena, false));
+}
+
+JEMALLOC_INLINE arena_t *
+arena_ichoose(tsdn_t *tsdn, arena_t *arena)
+{
+
+	assert(!tsdn_null(tsdn) || arena != NULL);
+
+	if (!tsdn_null(tsdn))
+		return (arena_choose_impl(tsdn_tsd(tsdn), NULL, true));
+	return (arena);
+}
+
 JEMALLOC_INLINE arena_tdata_t *
 arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing)
 {
@@ -819,7 +851,7 @@ arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing)
 }
 
 JEMALLOC_INLINE arena_t *
-arena_get(unsigned ind, bool init_if_missing)
+arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing)
 {
 	arena_t *ret;
 
@@ -829,7 +861,7 @@ arena_get(unsigned ind, bool init_if_missing)
 	if (unlikely(ret == NULL)) {
 		ret = atomic_read_p((void *)&arenas[ind]);
 		if (init_if_missing && unlikely(ret == NULL))
-			ret = arena_init(ind);
+			ret = arena_init(tsdn, ind);
 	}
 	return (ret);
 }
@@ -863,30 +895,27 @@ decay_ticker_get(tsd_t *tsd, unsigned ind)
 
 #ifndef JEMALLOC_ENABLE_INLINE
 arena_t	*iaalloc(const void *ptr);
-size_t	isalloc(const void *ptr, bool demote);
-void	*iallocztm(tsd_t *tsd, size_t size, szind_t ind, bool zero,
+size_t	isalloc(tsdn_t *tsdn, const void *ptr, bool demote);
+void	*iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
     tcache_t *tcache, bool is_metadata, arena_t *arena, bool slow_path);
-void	*imalloct(tsd_t *tsd, size_t size, szind_t ind, tcache_t *tcache,
-    arena_t *arena);
-void	*imalloc(tsd_t *tsd, size_t size, szind_t ind, bool slow_path);
-void	*icalloct(tsd_t *tsd, size_t size, szind_t ind, tcache_t *tcache,
-    arena_t *arena);
-void	*icalloc(tsd_t *tsd, size_t size, szind_t ind);
-void	*ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
+void	*ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero,
+    bool slow_path);
+void	*ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
     tcache_t *tcache, bool is_metadata, arena_t *arena);
-void	*ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
+void	*ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
     tcache_t *tcache, arena_t *arena);
 void	*ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
-size_t	ivsalloc(const void *ptr, bool demote);
+size_t	ivsalloc(tsdn_t *tsdn, const void *ptr, bool demote);
 size_t	u2rz(size_t usize);
-size_t	p2rz(const void *ptr);
-void	idalloctm(tsd_t *tsd, void *ptr, tcache_t *tcache, bool is_metadata,
+size_t	p2rz(tsdn_t *tsdn, const void *ptr);
+void	idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_metadata,
     bool slow_path);
-void	idalloct(tsd_t *tsd, void *ptr, tcache_t *tcache);
 void	idalloc(tsd_t *tsd, void *ptr);
 void	iqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path);
-void	isdalloct(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache);
-void	isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache);
+void	isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
+    bool slow_path);
+void	isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache,
+    bool slow_path);
 void	*iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
     size_t extra, size_t alignment, bool zero, tcache_t *tcache,
     arena_t *arena);
@@ -894,7 +923,7 @@ void	*iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
     size_t alignment, bool zero, tcache_t *tcache, arena_t *arena);
 void	*iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
     size_t alignment, bool zero);
-bool	ixalloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
+bool	ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
     size_t extra, size_t alignment, bool zero);
 #endif
 
@@ -910,102 +939,85 @@ iaalloc(const void *ptr)
 
 /*
  * Typical usage:
+ *   tsdn_t *tsdn = [...]
  *   void *ptr = [...]
- *   size_t sz = isalloc(ptr, config_prof);
+ *   size_t sz = isalloc(tsdn, ptr, config_prof);
  */
 JEMALLOC_ALWAYS_INLINE size_t
-isalloc(const void *ptr, bool demote)
+isalloc(tsdn_t *tsdn, const void *ptr, bool demote)
 {
 
 	assert(ptr != NULL);
 	/* Demotion only makes sense if config_prof is true. */
 	assert(config_prof || !demote);
 
-	return (arena_salloc(ptr, demote));
+	return (arena_salloc(tsdn, ptr, demote));
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-iallocztm(tsd_t *tsd, size_t size, szind_t ind, bool zero, tcache_t *tcache,
+iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache,
     bool is_metadata, arena_t *arena, bool slow_path)
 {
 	void *ret;
 
 	assert(size != 0);
+	assert(!is_metadata || tcache == NULL);
+	assert(!is_metadata || arena == NULL || arena->ind < narenas_auto);
 
-	ret = arena_malloc(tsd, arena, size, ind, zero, tcache, slow_path);
+	ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path);
 	if (config_stats && is_metadata && likely(ret != NULL)) {
-		arena_metadata_allocated_add(iaalloc(ret), isalloc(ret,
-		    config_prof));
+		arena_metadata_allocated_add(iaalloc(ret),
+		    isalloc(tsdn, ret, config_prof));
 	}
 	return (ret);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-imalloct(tsd_t *tsd, size_t size, szind_t ind, tcache_t *tcache, arena_t *arena)
-{
-
-	return (iallocztm(tsd, size, ind, false, tcache, false, arena, true));
-}
-
-JEMALLOC_ALWAYS_INLINE void *
-imalloc(tsd_t *tsd, size_t size, szind_t ind, bool slow_path)
+ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path)
 {
 
-	return (iallocztm(tsd, size, ind, false, tcache_get(tsd, true), false,
-	    NULL, slow_path));
+	return (iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd, true),
+	    false, NULL, slow_path));
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-icalloct(tsd_t *tsd, size_t size, szind_t ind, tcache_t *tcache, arena_t *arena)
-{
-
-	return (iallocztm(tsd, size, ind, true, tcache, false, arena, true));
-}
-
-JEMALLOC_ALWAYS_INLINE void *
-icalloc(tsd_t *tsd, size_t size, szind_t ind)
-{
-
-	return (iallocztm(tsd, size, ind, true, tcache_get(tsd, true), false,
-	    NULL, true));
-}
-
-JEMALLOC_ALWAYS_INLINE void *
-ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
+ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
     tcache_t *tcache, bool is_metadata, arena_t *arena)
 {
 	void *ret;
 
 	assert(usize != 0);
 	assert(usize == sa2u(usize, alignment));
+	assert(!is_metadata || tcache == NULL);
+	assert(!is_metadata || arena == NULL || arena->ind < narenas_auto);
 
-	ret = arena_palloc(tsd, arena, usize, alignment, zero, tcache);
+	ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache);
 	assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
 	if (config_stats && is_metadata && likely(ret != NULL)) {
-		arena_metadata_allocated_add(iaalloc(ret), isalloc(ret,
+		arena_metadata_allocated_add(iaalloc(ret), isalloc(tsdn, ret,
 		    config_prof));
 	}
 	return (ret);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
+ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
     tcache_t *tcache, arena_t *arena)
 {
 
-	return (ipallocztm(tsd, usize, alignment, zero, tcache, false, arena));
+	return (ipallocztm(tsdn, usize, alignment, zero, tcache, false, arena));
 }
 
 JEMALLOC_ALWAYS_INLINE void *
 ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero)
 {
 
-	return (ipallocztm(tsd, usize, alignment, zero, tcache_get(tsd, true),
-	    false, NULL));
+	return (ipallocztm(tsd_tsdn(tsd), usize, alignment, zero,
+	    tcache_get(tsd, true), false, NULL));
 }
 
 JEMALLOC_ALWAYS_INLINE size_t
-ivsalloc(const void *ptr, bool demote)
+ivsalloc(tsdn_t *tsdn, const void *ptr, bool demote)
 {
 	extent_node_t *node;
 
@@ -1017,7 +1029,7 @@ ivsalloc(const void *ptr, bool demote)
 	assert(extent_node_addr_get(node) == ptr ||
 	    extent_node_achunk_get(node));
 
-	return (isalloc(ptr, demote));
+	return (isalloc(tsdn, ptr, demote));
 }
 
 JEMALLOC_INLINE size_t
@@ -1035,39 +1047,34 @@ u2rz(size_t usize)
 }
 
 JEMALLOC_INLINE size_t
-p2rz(const void *ptr)
+p2rz(tsdn_t *tsdn, const void *ptr)
 {
-	size_t usize = isalloc(ptr, false);
+	size_t usize = isalloc(tsdn, ptr, false);
 
 	return (u2rz(usize));
 }
 
 JEMALLOC_ALWAYS_INLINE void
-idalloctm(tsd_t *tsd, void *ptr, tcache_t *tcache, bool is_metadata,
+idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_metadata,
     bool slow_path)
 {
 
 	assert(ptr != NULL);
+	assert(!is_metadata || tcache == NULL);
+	assert(!is_metadata || iaalloc(ptr)->ind < narenas_auto);
 	if (config_stats && is_metadata) {
-		arena_metadata_allocated_sub(iaalloc(ptr), isalloc(ptr,
+		arena_metadata_allocated_sub(iaalloc(ptr), isalloc(tsdn, ptr,
 		    config_prof));
 	}
 
-	arena_dalloc(tsd, ptr, tcache, slow_path);
-}
-
-JEMALLOC_ALWAYS_INLINE void
-idalloct(tsd_t *tsd, void *ptr, tcache_t *tcache)
-{
-
-	idalloctm(tsd, ptr, tcache, false, true);
+	arena_dalloc(tsdn, ptr, tcache, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void
 idalloc(tsd_t *tsd, void *ptr)
 {
 
-	idalloctm(tsd, ptr, tcache_get(tsd, false), false, true);
+	idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd, false), false, true);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -1077,24 +1084,25 @@ iqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
 	if (slow_path && config_fill && unlikely(opt_quarantine))
 		quarantine(tsd, ptr);
 	else
-		idalloctm(tsd, ptr, tcache, false, slow_path);
+		idalloctm(tsd_tsdn(tsd), ptr, tcache, false, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void
-isdalloct(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache)
+isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
+    bool slow_path)
 {
 
-	arena_sdalloc(tsd, ptr, size, tcache);
+	arena_sdalloc(tsdn, ptr, size, tcache, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void
-isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache)
+isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache, bool slow_path)
 {
 
-	if (config_fill && unlikely(opt_quarantine))
+	if (slow_path && config_fill && unlikely(opt_quarantine))
 		quarantine(tsd, ptr);
 	else
-		isdalloct(tsd, ptr, size, tcache);
+		isdalloct(tsd_tsdn(tsd), ptr, size, tcache, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
@@ -1107,7 +1115,7 @@ iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
 	usize = sa2u(size + extra, alignment);
 	if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
 		return (NULL);
-	p = ipalloct(tsd, usize, alignment, zero, tcache, arena);
+	p = ipalloct(tsd_tsdn(tsd), usize, alignment, zero, tcache, arena);
 	if (p == NULL) {
 		if (extra == 0)
 			return (NULL);
@@ -1115,7 +1123,8 @@ iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
 		usize = sa2u(size, alignment);
 		if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
 			return (NULL);
-		p = ipalloct(tsd, usize, alignment, zero, tcache, arena);
+		p = ipalloct(tsd_tsdn(tsd), usize, alignment, zero, tcache,
+		    arena);
 		if (p == NULL)
 			return (NULL);
 	}
@@ -1125,7 +1134,7 @@ iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
 	 */
 	copysize = (size < oldsize) ? size : oldsize;
 	memcpy(p, ptr, copysize);
-	isqalloc(tsd, ptr, oldsize, tcache);
+	isqalloc(tsd, ptr, oldsize, tcache, true);
 	return (p);
 }
 
@@ -1161,7 +1170,7 @@ iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
 }
 
 JEMALLOC_ALWAYS_INLINE bool
-ixalloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t extra,
+ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
     size_t alignment, bool zero)
 {
 
@@ -1174,7 +1183,7 @@ ixalloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t extra,
 		return (true);
 	}
 
-	return (arena_ralloc_no_move(tsd, ptr, oldsize, size, extra, zero));
+	return (arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero));
 }
 #endif
 
diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h b/include/jemalloc/internal/jemalloc_internal_defs.h
index c746c92..dc63ffd 100644
--- a/include/jemalloc/internal/jemalloc_internal_defs.h
+++ b/include/jemalloc/internal/jemalloc_internal_defs.h
@@ -221,6 +221,15 @@
 /* #undef JEMALLOC_ZONE_VERSION */
 
 /*
+ * Methods for determining whether the OS overcommits.
+ * JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY: Linux's
+ *                                         /proc/sys/vm.overcommit_memory file.
+ * JEMALLOC_SYSCTL_VM_OVERCOMMIT: FreeBSD's vm.overcommit sysctl.
+ */
+/* #undef JEMALLOC_SYSCTL_VM_OVERCOMMIT */
+#define JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY 
+
+/*
  * Methods for purging unused pages differ between operating systems.
  *
  *   madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages,
diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h
index e1cb05b..271cbf4 100644
--- a/include/jemalloc/internal/private_namespace.h
+++ b/include/jemalloc/internal/private_namespace.h
@@ -5,10 +5,12 @@
 #define	arena_basic_stats_merge JEMALLOC_N(arena_basic_stats_merge)
 #define	arena_bin_index JEMALLOC_N(arena_bin_index)
 #define	arena_bin_info JEMALLOC_N(arena_bin_info)
-#define	arena_bitselm_get JEMALLOC_N(arena_bitselm_get)
+#define	arena_bitselm_get_const JEMALLOC_N(arena_bitselm_get_const)
+#define	arena_bitselm_get_mutable JEMALLOC_N(arena_bitselm_get_mutable)
 #define	arena_boot JEMALLOC_N(arena_boot)
 #define	arena_choose JEMALLOC_N(arena_choose)
 #define	arena_choose_hard JEMALLOC_N(arena_choose_hard)
+#define	arena_choose_impl JEMALLOC_N(arena_choose_impl)
 #define	arena_chunk_alloc_huge JEMALLOC_N(arena_chunk_alloc_huge)
 #define	arena_chunk_cache_maybe_insert JEMALLOC_N(arena_chunk_cache_maybe_insert)
 #define	arena_chunk_cache_maybe_remove JEMALLOC_N(arena_chunk_cache_maybe_remove)
@@ -34,6 +36,7 @@
 #define	arena_dss_prec_get JEMALLOC_N(arena_dss_prec_get)
 #define	arena_dss_prec_set JEMALLOC_N(arena_dss_prec_set)
 #define	arena_get JEMALLOC_N(arena_get)
+#define	arena_ichoose JEMALLOC_N(arena_ichoose)
 #define	arena_init JEMALLOC_N(arena_init)
 #define	arena_lg_dirty_mult_default_get JEMALLOC_N(arena_lg_dirty_mult_default_get)
 #define	arena_lg_dirty_mult_default_set JEMALLOC_N(arena_lg_dirty_mult_default_set)
@@ -60,7 +63,8 @@
 #define	arena_mapbits_unallocated_size_get JEMALLOC_N(arena_mapbits_unallocated_size_get)
 #define	arena_mapbits_unallocated_size_set JEMALLOC_N(arena_mapbits_unallocated_size_set)
 #define	arena_mapbits_unzeroed_get JEMALLOC_N(arena_mapbits_unzeroed_get)
-#define	arena_mapbitsp_get JEMALLOC_N(arena_mapbitsp_get)
+#define	arena_mapbitsp_get_const JEMALLOC_N(arena_mapbitsp_get_const)
+#define	arena_mapbitsp_get_mutable JEMALLOC_N(arena_mapbitsp_get_mutable)
 #define	arena_mapbitsp_read JEMALLOC_N(arena_mapbitsp_read)
 #define	arena_mapbitsp_write JEMALLOC_N(arena_mapbitsp_write)
 #define	arena_maxrun JEMALLOC_N(arena_maxrun)
@@ -69,7 +73,8 @@
 #define	arena_metadata_allocated_get JEMALLOC_N(arena_metadata_allocated_get)
 #define	arena_metadata_allocated_sub JEMALLOC_N(arena_metadata_allocated_sub)
 #define	arena_migrate JEMALLOC_N(arena_migrate)
-#define	arena_miscelm_get JEMALLOC_N(arena_miscelm_get)
+#define	arena_miscelm_get_const JEMALLOC_N(arena_miscelm_get_const)
+#define	arena_miscelm_get_mutable JEMALLOC_N(arena_miscelm_get_mutable)
 #define	arena_miscelm_to_pageind JEMALLOC_N(arena_miscelm_to_pageind)
 #define	arena_miscelm_to_rpages JEMALLOC_N(arena_miscelm_to_rpages)
 #define	arena_new JEMALLOC_N(arena_new)
@@ -81,7 +86,10 @@
 #define	arena_palloc JEMALLOC_N(arena_palloc)
 #define	arena_postfork_child JEMALLOC_N(arena_postfork_child)
 #define	arena_postfork_parent JEMALLOC_N(arena_postfork_parent)
-#define	arena_prefork JEMALLOC_N(arena_prefork)
+#define	arena_prefork0 JEMALLOC_N(arena_prefork0)
+#define	arena_prefork1 JEMALLOC_N(arena_prefork1)
+#define	arena_prefork2 JEMALLOC_N(arena_prefork2)
+#define	arena_prefork3 JEMALLOC_N(arena_prefork3)
 #define	arena_prof_accum JEMALLOC_N(arena_prof_accum)
 #define	arena_prof_accum_impl JEMALLOC_N(arena_prof_accum_impl)
 #define	arena_prof_accum_locked JEMALLOC_N(arena_prof_accum_locked)
@@ -97,6 +105,7 @@
 #define	arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move)
 #define	arena_rd_to_miscelm JEMALLOC_N(arena_rd_to_miscelm)
 #define	arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption)
+#define	arena_reset JEMALLOC_N(arena_reset)
 #define	arena_run_regind JEMALLOC_N(arena_run_regind)
 #define	arena_run_to_miscelm JEMALLOC_N(arena_run_to_miscelm)
 #define	arena_salloc JEMALLOC_N(arena_salloc)
@@ -123,6 +132,11 @@
 #define	atomic_sub_uint32 JEMALLOC_N(atomic_sub_uint32)
 #define	atomic_sub_uint64 JEMALLOC_N(atomic_sub_uint64)
 #define	atomic_sub_z JEMALLOC_N(atomic_sub_z)
+#define	atomic_write_p JEMALLOC_N(atomic_write_p)
+#define	atomic_write_u JEMALLOC_N(atomic_write_u)
+#define	atomic_write_uint32 JEMALLOC_N(atomic_write_uint32)
+#define	atomic_write_uint64 JEMALLOC_N(atomic_write_uint64)
+#define	atomic_write_z JEMALLOC_N(atomic_write_z)
 #define	base_alloc JEMALLOC_N(base_alloc)
 #define	base_boot JEMALLOC_N(base_boot)
 #define	base_postfork_child JEMALLOC_N(base_postfork_child)
@@ -148,7 +162,6 @@
 #define	chunk_alloc_mmap JEMALLOC_N(chunk_alloc_mmap)
 #define	chunk_alloc_wrapper JEMALLOC_N(chunk_alloc_wrapper)
 #define	chunk_boot JEMALLOC_N(chunk_boot)
-#define	chunk_dalloc_arena JEMALLOC_N(chunk_dalloc_arena)
 #define	chunk_dalloc_cache JEMALLOC_N(chunk_dalloc_cache)
 #define	chunk_dalloc_mmap JEMALLOC_N(chunk_dalloc_mmap)
 #define	chunk_dalloc_wrapper JEMALLOC_N(chunk_dalloc_wrapper)
@@ -168,7 +181,6 @@
 #define	chunk_postfork_child JEMALLOC_N(chunk_postfork_child)
 #define	chunk_postfork_parent JEMALLOC_N(chunk_postfork_parent)
 #define	chunk_prefork JEMALLOC_N(chunk_prefork)
-#define	chunk_purge_arena JEMALLOC_N(chunk_purge_arena)
 #define	chunk_purge_wrapper JEMALLOC_N(chunk_purge_wrapper)
 #define	chunk_register JEMALLOC_N(chunk_register)
 #define	chunks_rtree JEMALLOC_N(chunks_rtree)
@@ -200,6 +212,8 @@
 #define	extent_node_addr_set JEMALLOC_N(extent_node_addr_set)
 #define	extent_node_arena_get JEMALLOC_N(extent_node_arena_get)
 #define	extent_node_arena_set JEMALLOC_N(extent_node_arena_set)
+#define	extent_node_committed_get JEMALLOC_N(extent_node_committed_get)
+#define	extent_node_committed_set JEMALLOC_N(extent_node_committed_set)
 #define	extent_node_dirty_insert JEMALLOC_N(extent_node_dirty_insert)
 #define	extent_node_dirty_linkage_init JEMALLOC_N(extent_node_dirty_linkage_init)
 #define	extent_node_dirty_remove JEMALLOC_N(extent_node_dirty_remove)
@@ -210,6 +224,8 @@
 #define	extent_node_size_set JEMALLOC_N(extent_node_size_set)
 #define	extent_node_zeroed_get JEMALLOC_N(extent_node_zeroed_get)
 #define	extent_node_zeroed_set JEMALLOC_N(extent_node_zeroed_set)
+#define	extent_tree_ad_destroy JEMALLOC_N(extent_tree_ad_destroy)
+#define	extent_tree_ad_destroy_recurse JEMALLOC_N(extent_tree_ad_destroy_recurse)
 #define	extent_tree_ad_empty JEMALLOC_N(extent_tree_ad_empty)
 #define	extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first)
 #define	extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert)
@@ -227,6 +243,8 @@
 #define	extent_tree_ad_reverse_iter_recurse JEMALLOC_N(extent_tree_ad_reverse_iter_recurse)
 #define	extent_tree_ad_reverse_iter_start JEMALLOC_N(extent_tree_ad_reverse_iter_start)
 #define	extent_tree_ad_search JEMALLOC_N(extent_tree_ad_search)
+#define	extent_tree_szad_destroy JEMALLOC_N(extent_tree_szad_destroy)
+#define	extent_tree_szad_destroy_recurse JEMALLOC_N(extent_tree_szad_destroy_recurse)
 #define	extent_tree_szad_empty JEMALLOC_N(extent_tree_szad_empty)
 #define	extent_tree_szad_first JEMALLOC_N(extent_tree_szad_first)
 #define	extent_tree_szad_insert JEMALLOC_N(extent_tree_szad_insert)
@@ -273,14 +291,11 @@
 #define	huge_ralloc_no_move JEMALLOC_N(huge_ralloc_no_move)
 #define	huge_salloc JEMALLOC_N(huge_salloc)
 #define	iaalloc JEMALLOC_N(iaalloc)
+#define	ialloc JEMALLOC_N(ialloc)
 #define	iallocztm JEMALLOC_N(iallocztm)
-#define	icalloc JEMALLOC_N(icalloc)
-#define	icalloct JEMALLOC_N(icalloct)
+#define	iarena_cleanup JEMALLOC_N(iarena_cleanup)
 #define	idalloc JEMALLOC_N(idalloc)
-#define	idalloct JEMALLOC_N(idalloct)
 #define	idalloctm JEMALLOC_N(idalloctm)
-#define	imalloc JEMALLOC_N(imalloc)
-#define	imalloct JEMALLOC_N(imalloct)
 #define	in_valgrind JEMALLOC_N(in_valgrind)
 #define	index2size JEMALLOC_N(index2size)
 #define	index2size_compute JEMALLOC_N(index2size_compute)
@@ -304,7 +319,11 @@
 #define	jemalloc_prefork JEMALLOC_N(jemalloc_prefork)
 #define	large_maxclass JEMALLOC_N(large_maxclass)
 #define	lg_floor JEMALLOC_N(lg_floor)
+#define	lg_prof_sample JEMALLOC_N(lg_prof_sample)
 #define	malloc_cprintf JEMALLOC_N(malloc_cprintf)
+#define	malloc_mutex_assert_not_owner JEMALLOC_N(malloc_mutex_assert_not_owner)
+#define	malloc_mutex_assert_owner JEMALLOC_N(malloc_mutex_assert_owner)
+#define	malloc_mutex_boot JEMALLOC_N(malloc_mutex_boot)
 #define	malloc_mutex_init JEMALLOC_N(malloc_mutex_init)
 #define	malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock)
 #define	malloc_mutex_postfork_child JEMALLOC_N(malloc_mutex_postfork_child)
@@ -326,11 +345,13 @@
 #define	map_bias JEMALLOC_N(map_bias)
 #define	map_misc_offset JEMALLOC_N(map_misc_offset)
 #define	mb_write JEMALLOC_N(mb_write)
-#define	mutex_boot JEMALLOC_N(mutex_boot)
+#define	narenas_auto JEMALLOC_N(narenas_auto)
 #define	narenas_tdata_cleanup JEMALLOC_N(narenas_tdata_cleanup)
 #define	narenas_total_get JEMALLOC_N(narenas_total_get)
 #define	ncpus JEMALLOC_N(ncpus)
 #define	nhbins JEMALLOC_N(nhbins)
+#define	nhclasses JEMALLOC_N(nhclasses)
+#define	nlclasses JEMALLOC_N(nlclasses)
 #define	nstime_add JEMALLOC_N(nstime_add)
 #define	nstime_compare JEMALLOC_N(nstime_compare)
 #define	nstime_copy JEMALLOC_N(nstime_copy)
@@ -373,6 +394,7 @@
 #define	opt_xmalloc JEMALLOC_N(opt_xmalloc)
 #define	opt_zero JEMALLOC_N(opt_zero)
 #define	p2rz JEMALLOC_N(p2rz)
+#define	pages_boot JEMALLOC_N(pages_boot)
 #define	pages_commit JEMALLOC_N(pages_commit)
 #define	pages_decommit JEMALLOC_N(pages_decommit)
 #define	pages_map JEMALLOC_N(pages_map)
@@ -384,6 +406,7 @@
 #define	pow2_ceil_zu JEMALLOC_N(pow2_ceil_zu)
 #define	prng_lg_range JEMALLOC_N(prng_lg_range)
 #define	prng_range JEMALLOC_N(prng_range)
+#define	prof_active JEMALLOC_N(prof_active)
 #define	prof_active_get JEMALLOC_N(prof_active_get)
 #define	prof_active_get_unlocked JEMALLOC_N(prof_active_get_unlocked)
 #define	prof_active_set JEMALLOC_N(prof_active_set)
@@ -393,6 +416,7 @@
 #define	prof_boot0 JEMALLOC_N(prof_boot0)
 #define	prof_boot1 JEMALLOC_N(prof_boot1)
 #define	prof_boot2 JEMALLOC_N(prof_boot2)
+#define	prof_bt_count JEMALLOC_N(prof_bt_count)
 #define	prof_dump_header JEMALLOC_N(prof_dump_header)
 #define	prof_dump_open JEMALLOC_N(prof_dump_open)
 #define	prof_free JEMALLOC_N(prof_free)
@@ -410,7 +434,8 @@
 #define	prof_mdump JEMALLOC_N(prof_mdump)
 #define	prof_postfork_child JEMALLOC_N(prof_postfork_child)
 #define	prof_postfork_parent JEMALLOC_N(prof_postfork_parent)
-#define	prof_prefork JEMALLOC_N(prof_prefork)
+#define	prof_prefork0 JEMALLOC_N(prof_prefork0)
+#define	prof_prefork1 JEMALLOC_N(prof_prefork1)
 #define	prof_realloc JEMALLOC_N(prof_realloc)
 #define	prof_reset JEMALLOC_N(prof_reset)
 #define	prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update)
@@ -419,6 +444,7 @@
 #define	prof_tctx_reset JEMALLOC_N(prof_tctx_reset)
 #define	prof_tctx_set JEMALLOC_N(prof_tctx_set)
 #define	prof_tdata_cleanup JEMALLOC_N(prof_tdata_cleanup)
+#define	prof_tdata_count JEMALLOC_N(prof_tdata_count)
 #define	prof_tdata_get JEMALLOC_N(prof_tdata_get)
 #define	prof_tdata_init JEMALLOC_N(prof_tdata_init)
 #define	prof_tdata_reinit JEMALLOC_N(prof_tdata_reinit)
@@ -470,8 +496,6 @@
 #define	tcache_alloc_large JEMALLOC_N(tcache_alloc_large)
 #define	tcache_alloc_small JEMALLOC_N(tcache_alloc_small)
 #define	tcache_alloc_small_hard JEMALLOC_N(tcache_alloc_small_hard)
-#define	tcache_arena_associate JEMALLOC_N(tcache_arena_associate)
-#define	tcache_arena_dissociate JEMALLOC_N(tcache_arena_dissociate)
 #define	tcache_arena_reassociate JEMALLOC_N(tcache_arena_reassociate)
 #define	tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large)
 #define	tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small)
@@ -506,38 +530,83 @@
 #define	ticker_ticks JEMALLOC_N(ticker_ticks)
 #define	tsd_arena_get JEMALLOC_N(tsd_arena_get)
 #define	tsd_arena_set JEMALLOC_N(tsd_arena_set)
+#define	tsd_arenap_get JEMALLOC_N(tsd_arenap_get)
+#define	tsd_arenas_tdata_bypass_get JEMALLOC_N(tsd_arenas_tdata_bypass_get)
+#define	tsd_arenas_tdata_bypass_set JEMALLOC_N(tsd_arenas_tdata_bypass_set)
+#define	tsd_arenas_tdata_bypassp_get JEMALLOC_N(tsd_arenas_tdata_bypassp_get)
+#define	tsd_arenas_tdata_get JEMALLOC_N(tsd_arenas_tdata_get)
+#define	tsd_arenas_tdata_set JEMALLOC_N(tsd_arenas_tdata_set)
+#define	tsd_arenas_tdatap_get JEMALLOC_N(tsd_arenas_tdatap_get)
 #define	tsd_boot JEMALLOC_N(tsd_boot)
 #define	tsd_boot0 JEMALLOC_N(tsd_boot0)
 #define	tsd_boot1 JEMALLOC_N(tsd_boot1)
 #define	tsd_booted JEMALLOC_N(tsd_booted)
+#define	tsd_booted_get JEMALLOC_N(tsd_booted_get)
 #define	tsd_cleanup JEMALLOC_N(tsd_cleanup)
 #define	tsd_cleanup_wrapper JEMALLOC_N(tsd_cleanup_wrapper)
 #define	tsd_fetch JEMALLOC_N(tsd_fetch)
 #define	tsd_get JEMALLOC_N(tsd_get)
-#define	tsd_wrapper_get JEMALLOC_N(tsd_wrapper_get)
-#define	tsd_wrapper_set JEMALLOC_N(tsd_wrapper_set)
+#define	tsd_iarena_get JEMALLOC_N(tsd_iarena_get)
+#define	tsd_iarena_set JEMALLOC_N(tsd_iarena_set)
+#define	tsd_iarenap_get JEMALLOC_N(tsd_iarenap_get)
 #define	tsd_initialized JEMALLOC_N(tsd_initialized)
 #define	tsd_init_check_recursion JEMALLOC_N(tsd_init_check_recursion)
 #define	tsd_init_finish JEMALLOC_N(tsd_init_finish)
 #define	tsd_init_head JEMALLOC_N(tsd_init_head)
+#define	tsd_narenas_tdata_get JEMALLOC_N(tsd_narenas_tdata_get)
+#define	tsd_narenas_tdata_set JEMALLOC_N(tsd_narenas_tdata_set)
+#define	tsd_narenas_tdatap_get JEMALLOC_N(tsd_narenas_tdatap_get)
+#define	tsd_wrapper_get JEMALLOC_N(tsd_wrapper_get)
+#define	tsd_wrapper_set JEMALLOC_N(tsd_wrapper_set)
 #define	tsd_nominal JEMALLOC_N(tsd_nominal)
 #define	tsd_prof_tdata_get JEMALLOC_N(tsd_prof_tdata_get)
 #define	tsd_prof_tdata_set JEMALLOC_N(tsd_prof_tdata_set)
+#define	tsd_prof_tdatap_get JEMALLOC_N(tsd_prof_tdatap_get)
 #define	tsd_quarantine_get JEMALLOC_N(tsd_quarantine_get)
 #define	tsd_quarantine_set JEMALLOC_N(tsd_quarantine_set)
+#define	tsd_quarantinep_get JEMALLOC_N(tsd_quarantinep_get)
 #define	tsd_set JEMALLOC_N(tsd_set)
 #define	tsd_tcache_enabled_get JEMALLOC_N(tsd_tcache_enabled_get)
 #define	tsd_tcache_enabled_set JEMALLOC_N(tsd_tcache_enabled_set)
+#define	tsd_tcache_enabledp_get JEMALLOC_N(tsd_tcache_enabledp_get)
 #define	tsd_tcache_get JEMALLOC_N(tsd_tcache_get)
 #define	tsd_tcache_set JEMALLOC_N(tsd_tcache_set)
+#define	tsd_tcachep_get JEMALLOC_N(tsd_tcachep_get)
 #define	tsd_thread_allocated_get JEMALLOC_N(tsd_thread_allocated_get)
 #define	tsd_thread_allocated_set JEMALLOC_N(tsd_thread_allocated_set)
+#define	tsd_thread_allocatedp_get JEMALLOC_N(tsd_thread_allocatedp_get)
 #define	tsd_thread_deallocated_get JEMALLOC_N(tsd_thread_deallocated_get)
 #define	tsd_thread_deallocated_set JEMALLOC_N(tsd_thread_deallocated_set)
+#define	tsd_thread_deallocatedp_get JEMALLOC_N(tsd_thread_deallocatedp_get)
 #define	tsd_tls JEMALLOC_N(tsd_tls)
 #define	tsd_tsd JEMALLOC_N(tsd_tsd)
+#define	tsd_tsdn JEMALLOC_N(tsd_tsdn)
+#define	tsd_witness_fork_get JEMALLOC_N(tsd_witness_fork_get)
+#define	tsd_witness_fork_set JEMALLOC_N(tsd_witness_fork_set)
+#define	tsd_witness_forkp_get JEMALLOC_N(tsd_witness_forkp_get)
+#define	tsd_witnesses_get JEMALLOC_N(tsd_witnesses_get)
+#define	tsd_witnesses_set JEMALLOC_N(tsd_witnesses_set)
+#define	tsd_witnessesp_get JEMALLOC_N(tsd_witnessesp_get)
+#define	tsdn_fetch JEMALLOC_N(tsdn_fetch)
+#define	tsdn_null JEMALLOC_N(tsdn_null)
+#define	tsdn_tsd JEMALLOC_N(tsdn_tsd)
 #define	u2rz JEMALLOC_N(u2rz)
 #define	valgrind_freelike_block JEMALLOC_N(valgrind_freelike_block)
 #define	valgrind_make_mem_defined JEMALLOC_N(valgrind_make_mem_defined)
 #define	valgrind_make_mem_noaccess JEMALLOC_N(valgrind_make_mem_noaccess)
 #define	valgrind_make_mem_undefined JEMALLOC_N(valgrind_make_mem_undefined)
+#define	witness_assert_lockless JEMALLOC_N(witness_assert_lockless)
+#define	witness_assert_not_owner JEMALLOC_N(witness_assert_not_owner)
+#define	witness_assert_owner JEMALLOC_N(witness_assert_owner)
+#define	witness_fork_cleanup JEMALLOC_N(witness_fork_cleanup)
+#define	witness_init JEMALLOC_N(witness_init)
+#define	witness_lock JEMALLOC_N(witness_lock)
+#define	witness_lock_error JEMALLOC_N(witness_lock_error)
+#define	witness_lockless_error JEMALLOC_N(witness_lockless_error)
+#define	witness_not_owner_error JEMALLOC_N(witness_not_owner_error)
+#define	witness_owner_error JEMALLOC_N(witness_owner_error)
+#define	witness_postfork_child JEMALLOC_N(witness_postfork_child)
+#define	witness_postfork_parent JEMALLOC_N(witness_postfork_parent)
+#define	witness_prefork JEMALLOC_N(witness_prefork)
+#define	witness_unlock JEMALLOC_N(witness_unlock)
+#define	witnesses_cleanup JEMALLOC_N(witnesses_cleanup)
diff --git a/include/jemalloc/internal/private_unnamespace.h b/include/jemalloc/internal/private_unnamespace.h
index e620fc7..b5a79bd 100644
--- a/include/jemalloc/internal/private_unnamespace.h
+++ b/include/jemalloc/internal/private_unnamespace.h
@@ -5,10 +5,12 @@
 #undef arena_basic_stats_merge
 #undef arena_bin_index
 #undef arena_bin_info
-#undef arena_bitselm_get
+#undef arena_bitselm_get_const
+#undef arena_bitselm_get_mutable
 #undef arena_boot
 #undef arena_choose
 #undef arena_choose_hard
+#undef arena_choose_impl
 #undef arena_chunk_alloc_huge
 #undef arena_chunk_cache_maybe_insert
 #undef arena_chunk_cache_maybe_remove
@@ -34,6 +36,7 @@
 #undef arena_dss_prec_get
 #undef arena_dss_prec_set
 #undef arena_get
+#undef arena_ichoose
 #undef arena_init
 #undef arena_lg_dirty_mult_default_get
 #undef arena_lg_dirty_mult_default_set
@@ -60,7 +63,8 @@
 #undef arena_mapbits_unallocated_size_get
 #undef arena_mapbits_unallocated_size_set
 #undef arena_mapbits_unzeroed_get
-#undef arena_mapbitsp_get
+#undef arena_mapbitsp_get_const
+#undef arena_mapbitsp_get_mutable
 #undef arena_mapbitsp_read
 #undef arena_mapbitsp_write
 #undef arena_maxrun
@@ -69,7 +73,8 @@
 #undef arena_metadata_allocated_get
 #undef arena_metadata_allocated_sub
 #undef arena_migrate
-#undef arena_miscelm_get
+#undef arena_miscelm_get_const
+#undef arena_miscelm_get_mutable
 #undef arena_miscelm_to_pageind
 #undef arena_miscelm_to_rpages
 #undef arena_new
@@ -81,7 +86,10 @@
 #undef arena_palloc
 #undef arena_postfork_child
 #undef arena_postfork_parent
-#undef arena_prefork
+#undef arena_prefork0
+#undef arena_prefork1
+#undef arena_prefork2
+#undef arena_prefork3
 #undef arena_prof_accum
 #undef arena_prof_accum_impl
 #undef arena_prof_accum_locked
@@ -97,6 +105,7 @@
 #undef arena_ralloc_no_move
 #undef arena_rd_to_miscelm
 #undef arena_redzone_corruption
+#undef arena_reset
 #undef arena_run_regind
 #undef arena_run_to_miscelm
 #undef arena_salloc
@@ -123,6 +132,11 @@
 #undef atomic_sub_uint32
 #undef atomic_sub_uint64
 #undef atomic_sub_z
+#undef atomic_write_p
+#undef atomic_write_u
+#undef atomic_write_uint32
+#undef atomic_write_uint64
+#undef atomic_write_z
 #undef base_alloc
 #undef base_boot
 #undef base_postfork_child
@@ -148,7 +162,6 @@
 #undef chunk_alloc_mmap
 #undef chunk_alloc_wrapper
 #undef chunk_boot
-#undef chunk_dalloc_arena
 #undef chunk_dalloc_cache
 #undef chunk_dalloc_mmap
 #undef chunk_dalloc_wrapper
@@ -168,7 +181,6 @@
 #undef chunk_postfork_child
 #undef chunk_postfork_parent
 #undef chunk_prefork
-#undef chunk_purge_arena
 #undef chunk_purge_wrapper
 #undef chunk_register
 #undef chunks_rtree
@@ -200,6 +212,8 @@
 #undef extent_node_addr_set
 #undef extent_node_arena_get
 #undef extent_node_arena_set
+#undef extent_node_committed_get
+#undef extent_node_committed_set
 #undef extent_node_dirty_insert
 #undef extent_node_dirty_linkage_init
 #undef extent_node_dirty_remove
@@ -210,6 +224,8 @@
 #undef extent_node_size_set
 #undef extent_node_zeroed_get
 #undef extent_node_zeroed_set
+#undef extent_tree_ad_destroy
+#undef extent_tree_ad_destroy_recurse
 #undef extent_tree_ad_empty
 #undef extent_tree_ad_first
 #undef extent_tree_ad_insert
@@ -227,6 +243,8 @@
 #undef extent_tree_ad_reverse_iter_recurse
 #undef extent_tree_ad_reverse_iter_start
 #undef extent_tree_ad_search
+#undef extent_tree_szad_destroy
+#undef extent_tree_szad_destroy_recurse
 #undef extent_tree_szad_empty
 #undef extent_tree_szad_first
 #undef extent_tree_szad_insert
@@ -273,14 +291,11 @@
 #undef huge_ralloc_no_move
 #undef huge_salloc
 #undef iaalloc
+#undef ialloc
 #undef iallocztm
-#undef icalloc
-#undef icalloct
+#undef iarena_cleanup
 #undef idalloc
-#undef idalloct
 #undef idalloctm
-#undef imalloc
-#undef imalloct
 #undef in_valgrind
 #undef index2size
 #undef index2size_compute
@@ -304,7 +319,11 @@
 #undef jemalloc_prefork
 #undef large_maxclass
 #undef lg_floor
+#undef lg_prof_sample
 #undef malloc_cprintf
+#undef malloc_mutex_assert_not_owner
+#undef malloc_mutex_assert_owner
+#undef malloc_mutex_boot
 #undef malloc_mutex_init
 #undef malloc_mutex_lock
 #undef malloc_mutex_postfork_child
@@ -326,11 +345,13 @@
 #undef map_bias
 #undef map_misc_offset
 #undef mb_write
-#undef mutex_boot
+#undef narenas_auto
 #undef narenas_tdata_cleanup
 #undef narenas_total_get
 #undef ncpus
 #undef nhbins
+#undef nhclasses
+#undef nlclasses
 #undef nstime_add
 #undef nstime_compare
 #undef nstime_copy
@@ -373,6 +394,7 @@
 #undef opt_xmalloc
 #undef opt_zero
 #undef p2rz
+#undef pages_boot
 #undef pages_commit
 #undef pages_decommit
 #undef pages_map
@@ -384,6 +406,7 @@
 #undef pow2_ceil_zu
 #undef prng_lg_range
 #undef prng_range
+#undef prof_active
 #undef prof_active_get
 #undef prof_active_get_unlocked
 #undef prof_active_set
@@ -393,6 +416,7 @@
 #undef prof_boot0
 #undef prof_boot1
 #undef prof_boot2
+#undef prof_bt_count
 #undef prof_dump_header
 #undef prof_dump_open
 #undef prof_free
@@ -410,7 +434,8 @@
 #undef prof_mdump
 #undef prof_postfork_child
 #undef prof_postfork_parent
-#undef prof_prefork
+#undef prof_prefork0
+#undef prof_prefork1
 #undef prof_realloc
 #undef prof_reset
 #undef prof_sample_accum_update
@@ -419,6 +444,7 @@
 #undef prof_tctx_reset
 #undef prof_tctx_set
 #undef prof_tdata_cleanup
+#undef prof_tdata_count
 #undef prof_tdata_get
 #undef prof_tdata_init
 #undef prof_tdata_reinit
@@ -470,8 +496,6 @@
 #undef tcache_alloc_large
 #undef tcache_alloc_small
 #undef tcache_alloc_small_hard
-#undef tcache_arena_associate
-#undef tcache_arena_dissociate
 #undef tcache_arena_reassociate
 #undef tcache_bin_flush_large
 #undef tcache_bin_flush_small
@@ -506,38 +530,83 @@
 #undef ticker_ticks
 #undef tsd_arena_get
 #undef tsd_arena_set
+#undef tsd_arenap_get
+#undef tsd_arenas_tdata_bypass_get
+#undef tsd_arenas_tdata_bypass_set
+#undef tsd_arenas_tdata_bypassp_get
+#undef tsd_arenas_tdata_get
+#undef tsd_arenas_tdata_set
+#undef tsd_arenas_tdatap_get
 #undef tsd_boot
 #undef tsd_boot0
 #undef tsd_boot1
 #undef tsd_booted
+#undef tsd_booted_get
 #undef tsd_cleanup
 #undef tsd_cleanup_wrapper
 #undef tsd_fetch
 #undef tsd_get
-#undef tsd_wrapper_get
-#undef tsd_wrapper_set
+#undef tsd_iarena_get
+#undef tsd_iarena_set
+#undef tsd_iarenap_get
 #undef tsd_initialized
 #undef tsd_init_check_recursion
 #undef tsd_init_finish
 #undef tsd_init_head
+#undef tsd_narenas_tdata_get
+#undef tsd_narenas_tdata_set
+#undef tsd_narenas_tdatap_get
+#undef tsd_wrapper_get
+#undef tsd_wrapper_set
 #undef tsd_nominal
 #undef tsd_prof_tdata_get
 #undef tsd_prof_tdata_set
+#undef tsd_prof_tdatap_get
 #undef tsd_quarantine_get
 #undef tsd_quarantine_set
+#undef tsd_quarantinep_get
 #undef tsd_set
 #undef tsd_tcache_enabled_get
 #undef tsd_tcache_enabled_set
+#undef tsd_tcache_enabledp_get
 #undef tsd_tcache_get
 #undef tsd_tcache_set
+#undef tsd_tcachep_get
 #undef tsd_thread_allocated_get
 #undef tsd_thread_allocated_set
+#undef tsd_thread_allocatedp_get
 #undef tsd_thread_deallocated_get
 #undef tsd_thread_deallocated_set
+#undef tsd_thread_deallocatedp_get
 #undef tsd_tls
 #undef tsd_tsd
+#undef tsd_tsdn
+#undef tsd_witness_fork_get
+#undef tsd_witness_fork_set
+#undef tsd_witness_forkp_get
+#undef tsd_witnesses_get
+#undef tsd_witnesses_set
+#undef tsd_witnessesp_get
+#undef tsdn_fetch
+#undef tsdn_null
+#undef tsdn_tsd
 #undef u2rz
 #undef valgrind_freelike_block
 #undef valgrind_make_mem_defined
 #undef valgrind_make_mem_noaccess
 #undef valgrind_make_mem_undefined
+#undef witness_assert_lockless
+#undef witness_assert_not_owner
+#undef witness_assert_owner
+#undef witness_fork_cleanup
+#undef witness_init
+#undef witness_lock
+#undef witness_lock_error
+#undef witness_lockless_error
+#undef witness_not_owner_error
+#undef witness_owner_error
+#undef witness_postfork_child
+#undef witness_postfork_parent
+#undef witness_prefork
+#undef witness_unlock
+#undef witnesses_cleanup
diff --git a/include/jemalloc/jemalloc.h b/include/jemalloc/jemalloc.h
index bfc95ec..5ed13cb 100644
--- a/include/jemalloc/jemalloc.h
+++ b/include/jemalloc/jemalloc.h
@@ -94,20 +94,20 @@ extern "C" {
 #include <limits.h>
 #include <strings.h>
 
-#define	JEMALLOC_VERSION "4.1.0-4-g33184bf69813087bf1885b0993685f9d03320c69"
+#define	JEMALLOC_VERSION "4.2.1-0-g3de035335255d553bdb344c32ffdb603816195d8"
 #define	JEMALLOC_VERSION_MAJOR 4
-#define	JEMALLOC_VERSION_MINOR 1
-#define	JEMALLOC_VERSION_BUGFIX 0
-#define	JEMALLOC_VERSION_NREV 4
-#define	JEMALLOC_VERSION_GID "33184bf69813087bf1885b0993685f9d03320c69"
+#define	JEMALLOC_VERSION_MINOR 2
+#define	JEMALLOC_VERSION_BUGFIX 1
+#define	JEMALLOC_VERSION_NREV 0
+#define	JEMALLOC_VERSION_GID "3de035335255d553bdb344c32ffdb603816195d8"
 
 #  define MALLOCX_LG_ALIGN(la)	((int)(la))
 #  if LG_SIZEOF_PTR == 2
-#    define MALLOCX_ALIGN(a)	((int)(ffs(a)-1))
+#    define MALLOCX_ALIGN(a)	((int)(ffs((int)(a))-1))
 #  else
 #    define MALLOCX_ALIGN(a)						\
-       ((int)(((a) < (size_t)INT_MAX) ? ffs((int)(a))-1 :		\
-       ffs((int)((a)>>32))+31))
+       ((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 :	\
+       ffs((int)(((size_t)(a))>>32))+31))
 #  endif
 #  define MALLOCX_ZERO	((int)0x40)
 /*
@@ -119,7 +119,7 @@ extern "C" {
 /*
  * Bias arena index bits so that 0 encodes "use an automatically chosen arena".
  */
-#  define MALLOCX_ARENA(a)	((int)(((a)+1) << 20))
+#  define MALLOCX_ARENA(a)	((((int)(a))+1) << 20)
 
 #if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW)
 #  define JEMALLOC_CXX_THROW throw()
diff --git a/include/jemalloc/jemalloc_macros.h b/include/jemalloc/jemalloc_macros.h
index d4a3794..04d143f 100644
--- a/include/jemalloc/jemalloc_macros.h
+++ b/include/jemalloc/jemalloc_macros.h
@@ -4,20 +4,20 @@
 #include <limits.h>
 #include <strings.h>
 
-#define	JEMALLOC_VERSION "4.1.0-4-g33184bf69813087bf1885b0993685f9d03320c69"
+#define	JEMALLOC_VERSION "4.2.1-0-g3de035335255d553bdb344c32ffdb603816195d8"
 #define	JEMALLOC_VERSION_MAJOR 4
-#define	JEMALLOC_VERSION_MINOR 1
-#define	JEMALLOC_VERSION_BUGFIX 0
-#define	JEMALLOC_VERSION_NREV 4
-#define	JEMALLOC_VERSION_GID "33184bf69813087bf1885b0993685f9d03320c69"
+#define	JEMALLOC_VERSION_MINOR 2
+#define	JEMALLOC_VERSION_BUGFIX 1
+#define	JEMALLOC_VERSION_NREV 0
+#define	JEMALLOC_VERSION_GID "3de035335255d553bdb344c32ffdb603816195d8"
 
 #  define MALLOCX_LG_ALIGN(la)	((int)(la))
 #  if LG_SIZEOF_PTR == 2
-#    define MALLOCX_ALIGN(a)	((int)(ffs(a)-1))
+#    define MALLOCX_ALIGN(a)	((int)(ffs((int)(a))-1))
 #  else
 #    define MALLOCX_ALIGN(a)						\
-       ((int)(((a) < (size_t)INT_MAX) ? ffs((int)(a))-1 :		\
-       ffs((int)((a)>>32))+31))
+       ((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 :	\
+       ffs((int)(((size_t)(a))>>32))+31))
 #  endif
 #  define MALLOCX_ZERO	((int)0x40)
 /*
@@ -29,7 +29,7 @@
 /*
  * Bias arena index bits so that 0 encodes "use an automatically chosen arena".
  */
-#  define MALLOCX_ARENA(a)	((int)(((a)+1) << 20))
+#  define MALLOCX_ARENA(a)	((((int)(a))+1) << 20)
 
 #if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW)
 #  define JEMALLOC_CXX_THROW throw()
diff --git a/src/android_je_iterate.c b/src/android_je_iterate.c
index 3fbbdc9..7aa4099 100644
--- a/src/android_je_iterate.c
+++ b/src/android_je_iterate.c
@@ -101,7 +101,7 @@ static void je_iterate_chunk(arena_chunk_t *chunk,
       void *rpages;
 
       size = arena_mapbits_large_size_get(chunk, pageind);
-      rpages = arena_miscelm_to_rpages(arena_miscelm_get(chunk, pageind));
+      rpages = arena_miscelm_to_rpages(arena_miscelm_get_mutable(chunk, pageind));
       callback((uintptr_t)rpages, size, arg);
     } else {
       /* Run of small allocations */
@@ -110,7 +110,7 @@ static void je_iterate_chunk(arena_chunk_t *chunk,
 
       assert(arena_mapbits_small_runind_get(chunk, pageind) == pageind);
       binind = arena_mapbits_binind_get(chunk, pageind);
-      run = &arena_miscelm_get(chunk, pageind)->run;
+      run = &arena_miscelm_get_mutable(chunk, pageind)->run;
       assert(run->binind == binind);
       size = arena_bin_info[binind].run_size;
 
diff --git a/src/android_je_mallinfo.c b/src/android_je_mallinfo.c
index 1637edd..802a3eb 100644
--- a/src/android_je_mallinfo.c
+++ b/src/android_je_mallinfo.c
@@ -21,25 +21,25 @@ struct mallinfo je_mallinfo() {
   struct mallinfo mi;
   memset(&mi, 0, sizeof(mi));
 
-  malloc_mutex_lock(&arenas_lock);
+  malloc_mutex_lock(TSDN_NULL, &arenas_lock);
   for (unsigned i = 0; i < narenas_auto; i++) {
     if (arenas[i] != NULL) {
-      malloc_mutex_lock(&arenas[i]->lock);
+      malloc_mutex_lock(TSDN_NULL, &arenas[i]->lock);
       mi.hblkhd += arenas[i]->stats.mapped;
       mi.uordblks += arenas[i]->stats.allocated_large;
       mi.uordblks += arenas[i]->stats.allocated_huge;
-      malloc_mutex_unlock(&arenas[i]->lock);
+      malloc_mutex_unlock(TSDN_NULL, &arenas[i]->lock);
 
       for (unsigned j = 0; j < NBINS; j++) {
         arena_bin_t* bin = &arenas[i]->bins[j];
 
-        malloc_mutex_lock(&bin->lock);
+        malloc_mutex_lock(TSDN_NULL, &bin->lock);
         mi.uordblks += arena_bin_info[j].reg_size * bin->stats.curregs;
-        malloc_mutex_unlock(&bin->lock);
+        malloc_mutex_unlock(TSDN_NULL, &bin->lock);
       }
     }
   }
-  malloc_mutex_unlock(&arenas_lock);
+  malloc_mutex_unlock(TSDN_NULL, &arenas_lock);
   mi.fordblks = mi.hblkhd - mi.uordblks;
   mi.usmblks = mi.hblkhd;
   return mi;
@@ -57,25 +57,25 @@ struct mallinfo __mallinfo_arena_info(size_t aidx) {
   struct mallinfo mi;
   memset(&mi, 0, sizeof(mi));
 
-  malloc_mutex_lock(&arenas_lock);
+  malloc_mutex_lock(TSDN_NULL, &arenas_lock);
   if (aidx < narenas_auto) {
     if (arenas[aidx] != NULL) {
-      malloc_mutex_lock(&arenas[aidx]->lock);
+      malloc_mutex_lock(TSDN_NULL, &arenas[aidx]->lock);
       mi.hblkhd = arenas[aidx]->stats.mapped;
       mi.ordblks = arenas[aidx]->stats.allocated_large;
       mi.uordblks = arenas[aidx]->stats.allocated_huge;
-      malloc_mutex_unlock(&arenas[aidx]->lock);
+      malloc_mutex_unlock(TSDN_NULL, &arenas[aidx]->lock);
 
       for (unsigned j = 0; j < NBINS; j++) {
         arena_bin_t* bin = &arenas[aidx]->bins[j];
 
-        malloc_mutex_lock(&bin->lock);
+        malloc_mutex_lock(TSDN_NULL, &bin->lock);
         mi.fsmblks += arena_bin_info[j].reg_size * bin->stats.curregs;
-        malloc_mutex_unlock(&bin->lock);
+        malloc_mutex_unlock(TSDN_NULL, &bin->lock);
       }
     }
   }
-  malloc_mutex_unlock(&arenas_lock);
+  malloc_mutex_unlock(TSDN_NULL, &arenas_lock);
   return mi;
 }
 
@@ -83,18 +83,18 @@ struct mallinfo __mallinfo_bin_info(size_t aidx, size_t bidx) {
   struct mallinfo mi;
   memset(&mi, 0, sizeof(mi));
 
-  malloc_mutex_lock(&arenas_lock);
+  malloc_mutex_lock(TSDN_NULL, &arenas_lock);
   if (aidx < narenas_auto && bidx < NBINS) {
     if (arenas[aidx] != NULL) {
       arena_bin_t* bin = &arenas[aidx]->bins[bidx];
 
-      malloc_mutex_lock(&bin->lock);
+      malloc_mutex_lock(TSDN_NULL, &bin->lock);
       mi.ordblks = arena_bin_info[bidx].reg_size * bin->stats.curregs;
       mi.uordblks = bin->stats.nmalloc;
       mi.fordblks = bin->stats.ndalloc;
-      malloc_mutex_unlock(&bin->lock);
+      malloc_mutex_unlock(TSDN_NULL, &bin->lock);
     }
   }
-  malloc_mutex_unlock(&arenas_lock);
+  malloc_mutex_unlock(TSDN_NULL, &arenas_lock);
   return mi;
 }
diff --git a/test/include/test/jemalloc_test.h b/test/include/test/jemalloc_test.h
index 8348ac5..ec3a599 100644
--- a/test/include/test/jemalloc_test.h
+++ b/test/include/test/jemalloc_test.h
@@ -19,39 +19,6 @@
 #  include <pthread.h>
 #endif
 
-/******************************************************************************/
-/*
- * Define always-enabled assertion macros, so that test assertions execute even
- * if assertions are disabled in the library code.  These definitions must
- * exist prior to including "jemalloc/internal/util.h".
- */
-#define	assert(e) do {							\
-	if (!(e)) {							\
-		malloc_printf(						\
-		    "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n",	\
-		    __FILE__, __LINE__, #e);				\
-		abort();						\
-	}								\
-} while (0)
-
-#define	not_reached() do {						\
-	malloc_printf(							\
-	    "<jemalloc>: %s:%d: Unreachable code reached\n",		\
-	    __FILE__, __LINE__);					\
-	abort();							\
-} while (0)
-
-#define	not_implemented() do {						\
-	malloc_printf("<jemalloc>: %s:%d: Not implemented\n",		\
-	    __FILE__, __LINE__);					\
-	abort();							\
-} while (0)
-
-#define	assert_not_implemented(e) do {					\
-	if (!(e))							\
-		not_implemented();					\
-} while (0)
-
 #include "test/jemalloc_test_defs.h"
 
 #ifdef JEMALLOC_OSSPIN
@@ -86,6 +53,14 @@
 #  include "jemalloc/internal/jemalloc_internal_defs.h"
 #  include "jemalloc/internal/jemalloc_internal_macros.h"
 
+static const bool config_debug =
+#ifdef JEMALLOC_DEBUG
+    true
+#else
+    false
+#endif
+    ;
+
 #  define JEMALLOC_N(n) je_##n
 #  include "jemalloc/internal/private_namespace.h"
 
@@ -149,3 +124,40 @@
 #include "test/thd.h"
 #define	MEXP 19937
 #include "test/SFMT.h"
+
+/******************************************************************************/
+/*
+ * Define always-enabled assertion macros, so that test assertions execute even
+ * if assertions are disabled in the library code.
+ */
+#undef assert
+#undef not_reached
+#undef not_implemented
+#undef assert_not_implemented
+
+#define	assert(e) do {							\
+	if (!(e)) {							\
+		malloc_printf(						\
+		    "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n",	\
+		    __FILE__, __LINE__, #e);				\
+		abort();						\
+	}								\
+} while (0)
+
+#define	not_reached() do {						\
+	malloc_printf(							\
+	    "<jemalloc>: %s:%d: Unreachable code reached\n",		\
+	    __FILE__, __LINE__);					\
+	abort();							\
+} while (0)
+
+#define	not_implemented() do {						\
+	malloc_printf("<jemalloc>: %s:%d: Not implemented\n",		\
+	    __FILE__, __LINE__);					\
+	abort();							\
+} while (0)
+
+#define	assert_not_implemented(e) do {					\
+	if (!(e))							\
+		not_implemented();					\
+} while (0)