Fix/refactor tcaches synchronization. am: f674f43893 am: 412112bb89

am: 50e982afd2

Change-Id: I35c4bb99f921ec53e6b4d3f7339c158a5884e984

6 files changed, 110 insertions, 38 deletions

diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index c1c6c40..14c8fe2 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -529,6 +529,9 @@ tcache_flush
 tcache_get
 tcache_get_hard
 tcache_maxclass
+tcache_prefork
+tcache_postfork_child
+tcache_postfork_parent
 tcache_salloc
 tcache_stats_merge
 tcaches
diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h
index 4d0a8ff..da4f54f 100644
--- a/include/jemalloc/internal/tcache.h
+++ b/include/jemalloc/internal/tcache.h
@@ -161,6 +161,9 @@ bool	tcaches_create(tsd_t *tsd, unsigned *r_ind);
 void	tcaches_flush(tsd_t *tsd, unsigned ind);
 void	tcaches_destroy(tsd_t *tsd, unsigned ind);
 bool	tcache_boot(tsdn_t *tsdn);
+void tcache_prefork(tsdn_t *tsdn);
+void tcache_postfork_parent(tsdn_t *tsdn);
+void tcache_postfork_child(tsdn_t *tsdn);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h
index cdf15d7..63bc22a 100644
--- a/include/jemalloc/internal/witness.h
+++ b/include/jemalloc/internal/witness.h
@@ -14,19 +14,20 @@ typedef int witness_comp_t (const witness_t *, const witness_t *);
 
 #define	WITNESS_RANK_INIT		1U
 #define	WITNESS_RANK_CTL		1U
-#define	WITNESS_RANK_ARENAS		2U
+#define WITNESS_RANK_TCACHES		2U
+#define	WITNESS_RANK_ARENAS		3U
 
-#define	WITNESS_RANK_PROF_DUMP		3U
-#define	WITNESS_RANK_PROF_BT2GCTX	4U
-#define	WITNESS_RANK_PROF_TDATAS	5U
-#define	WITNESS_RANK_PROF_TDATA		6U
-#define	WITNESS_RANK_PROF_GCTX		7U
+#define	WITNESS_RANK_PROF_DUMP		4U
+#define	WITNESS_RANK_PROF_BT2GCTX	5U
+#define	WITNESS_RANK_PROF_TDATAS	6U
+#define	WITNESS_RANK_PROF_TDATA		7U
+#define	WITNESS_RANK_PROF_GCTX		8U
 
-#define	WITNESS_RANK_ARENA		8U
-#define	WITNESS_RANK_ARENA_CHUNKS	9U
-#define	WITNESS_RANK_ARENA_NODE_CACHE	10
+#define	WITNESS_RANK_ARENA		9U
+#define	WITNESS_RANK_ARENA_CHUNKS	10U
+#define	WITNESS_RANK_ARENA_NODE_CACHE	11U
 
-#define	WITNESS_RANK_BASE		11U
+#define	WITNESS_RANK_BASE		12U
 
 #define	WITNESS_RANK_LEAF		0xffffffffU
 #define	WITNESS_RANK_ARENA_BIN		WITNESS_RANK_LEAF
diff --git a/src/ctl.c b/src/ctl.c
index bc78b20..1e62e2d 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -1476,7 +1476,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 	if (!config_tcache)
 		return (ENOENT);
 
-	malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
 	READONLY();
 	if (tcaches_create(tsd, &tcache_ind)) {
 		ret = EFAULT;
@@ -1486,8 +1485,7 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 
 	ret = 0;
 label_return:
-	malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
-	return (ret);
+	return ret;
 }
 
 static int
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 11aebd9..72f561f 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -2839,6 +2839,7 @@ _malloc_prefork(void)
 	witness_prefork(tsd);
 	/* Acquire all mutexes in a safe order. */
 	ctl_prefork(tsd_tsdn(tsd));
+	tcache_prefork(tsd_tsdn(tsd));
 	malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
 	prof_prefork0(tsd_tsdn(tsd));
 	for (i = 0; i < 3; i++) {
@@ -2898,6 +2899,7 @@ _malloc_postfork(void)
 	}
 	prof_postfork_parent(tsd_tsdn(tsd));
 	malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
+	tcache_postfork_parent(tsd_tsdn(tsd));
 	ctl_postfork_parent(tsd_tsdn(tsd));
 }
 
@@ -2922,6 +2924,7 @@ jemalloc_postfork_child(void)
 	}
 	prof_postfork_child(tsd_tsdn(tsd));
 	malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
+	tcache_postfork_child(tsd_tsdn(tsd));
 	ctl_postfork_child(tsd_tsdn(tsd));
 }
 
diff --git a/src/tcache.c b/src/tcache.c
index 21540ff..e3b04be 100755
--- a/src/tcache.c
+++ b/src/tcache.c
@@ -21,6 +21,9 @@ static unsigned		tcaches_past;
 /* Head of singly linked list tracking available tcaches elements. */
 static tcaches_t	*tcaches_avail;
 
+/* Protects tcaches{,_past,_avail}. */
+static malloc_mutex_t	tcaches_mtx;
+
 /******************************************************************************/
 
 size_t
@@ -444,29 +447,56 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
 	}
 }
 
+static bool
+tcaches_create_prep(tsd_t *tsd) {
+	bool err;
+
+	malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
+
+	if (tcaches == NULL) {
+		tcaches = base_alloc(tsd_tsdn(tsd), sizeof(tcache_t *) *
+		    (MALLOCX_TCACHE_MAX+1));
+		if (tcaches == NULL) {
+			err = true;
+			goto label_return;
+		}
+	}
+
+	if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) {
+		err = true;
+		goto label_return;
+	}
+
+	err = false;
+label_return:
+	malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
+	return err;
+}
+
 bool
-tcaches_create(tsd_t *tsd, unsigned *r_ind)
-{
+tcaches_create(tsd_t *tsd, unsigned *r_ind) {
+	bool err;
 	arena_t *arena;
 	tcache_t *tcache;
 	tcaches_t *elm;
 
-	if (tcaches == NULL) {
-		tcaches = base_alloc(tsd_tsdn(tsd), sizeof(tcache_t *) *
-		    (MALLOCX_TCACHE_MAX+1));
-		if (tcaches == NULL)
-			return (true);
+	if (tcaches_create_prep(tsd)) {
+		err = true;
+		goto label_return;
 	}
 
-	if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX)
-		return (true);
 	arena = arena_ichoose(tsd, NULL);
-	if (unlikely(arena == NULL))
-		return (true);
+	if (unlikely(arena == NULL)) {
+		err = true;
+		goto label_return;
+	}
 	tcache = tcache_create(tsd_tsdn(tsd), arena);
-	if (tcache == NULL)
-		return (true);
+	if (tcache == NULL) {
+		err = true;
+		goto label_return;
+	}
 
+	malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
 	if (tcaches_avail != NULL) {
 		elm = tcaches_avail;
 		tcaches_avail = tcaches_avail->next;
@@ -478,41 +508,50 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind)
 		*r_ind = tcaches_past;
 		tcaches_past++;
 	}
+	malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
 
-	return (false);
+	err = false;
+label_return:
+	malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &tcaches_mtx);
+	return err;
 }
 
 static void
-tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm)
-{
+tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) {
+	malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx);
 
-	if (elm->tcache == NULL)
+	if (elm->tcache == NULL) {
 		return;
+	}
 	tcache_destroy(tsd, elm->tcache);
 	elm->tcache = NULL;
 }
 
 void
-tcaches_flush(tsd_t *tsd, unsigned ind)
-{
-
+tcaches_flush(tsd_t *tsd, unsigned ind) {
+	malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
 	tcaches_elm_flush(tsd, &tcaches[ind]);
+	malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
 }
 
 void
-tcaches_destroy(tsd_t *tsd, unsigned ind)
-{
-	tcaches_t *elm = &tcaches[ind];
+tcaches_destroy(tsd_t *tsd, unsigned ind) {
+	tcaches_t *elm;
+
+	malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
+	elm = &tcaches[ind];
 	tcaches_elm_flush(tsd, elm);
 	elm->next = tcaches_avail;
 	tcaches_avail = elm;
+	malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
 }
 
 bool
-tcache_boot(tsdn_t *tsdn)
-{
+tcache_boot(tsdn_t *tsdn) {
 	unsigned i;
 
+	cassert(config_tcache);
+
 	/*
 	 * If necessary, clamp opt_lg_tcache_max, now that large_maxclass is
 	 * known.
@@ -524,6 +563,10 @@ tcache_boot(tsdn_t *tsdn)
 	else
 		tcache_maxclass = (ZU(1) << opt_lg_tcache_max);
 
+	if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES)) {
+		return true;
+	}
+
 	nhbins = size2index(tcache_maxclass) + 1;
 
 	/* Initialize tcache_bin_info. */
@@ -553,3 +596,24 @@ tcache_boot(tsdn_t *tsdn)
 
 	return (false);
 }
+
+void
+tcache_prefork(tsdn_t *tsdn) {
+	if (!config_prof && opt_tcache) {
+		malloc_mutex_prefork(tsdn, &tcaches_mtx);
+	}
+}
+
+void
+tcache_postfork_parent(tsdn_t *tsdn) {
+	if (!config_prof && opt_tcache) {
+		malloc_mutex_postfork_parent(tsdn, &tcaches_mtx);
+	}
+}
+
+void
+tcache_postfork_child(tsdn_t *tsdn) {
+	if (!config_prof && opt_tcache) {
+		malloc_mutex_postfork_child(tsdn, &tcaches_mtx);
+	}
+}