diff options
author | Qi Wang <interwq@gwu.edu> | 2017-03-29 20:18:02 (GMT) |
---|---|---|
committer | Qi Wang <interwq@gmail.com> | 2017-04-04 07:19:21 (GMT) |
commit | 9ed84b0d458a22e1d98f071f8fb5efb2de24998e (patch) | |
tree | efa706cc6fd5a018ed5b1fba6ecd7f0bc601dca8 | |
parent | 5bf800a54247c5752053831e15f7b132bf9fddbf (diff) | |
download | jemalloc-9ed84b0d458a22e1d98f071f8fb5efb2de24998e.zip jemalloc-9ed84b0d458a22e1d98f071f8fb5efb2de24998e.tar.gz jemalloc-9ed84b0d458a22e1d98f071f8fb5efb2de24998e.tar.bz2 |
Add init function support to tsd members.
This will facilitate embedding tcache into tsd, which will require proper
initialization cannot be done via the static initializer. Make tsd->rtree_ctx
to be initialized via rtree_ctx_data_init().
-rw-r--r-- | include/jemalloc/internal/private_symbols.txt | 2 | ||||
-rw-r--r-- | include/jemalloc/internal/rtree_externs.h | 1 | ||||
-rw-r--r-- | include/jemalloc/internal/rtree_types.h | 14 | ||||
-rw-r--r-- | include/jemalloc/internal/tsd_externs.h | 1 | ||||
-rw-r--r-- | include/jemalloc/internal/tsd_inlines.h | 9 | ||||
-rw-r--r-- | include/jemalloc/internal/tsd_structs.h | 34 | ||||
-rw-r--r-- | src/rtree.c | 11 | ||||
-rw-r--r-- | src/tsd.c | 19 | ||||
-rw-r--r-- | test/unit/rtree.c | 16 |
9 files changed, 74 insertions, 33 deletions
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 5d03f5d..1cced60 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -424,6 +424,7 @@ prof_thread_name_set psz2ind psz2u rtree_clear +rtree_ctx_data_init rtree_delete rtree_extent_read rtree_extent_szind_read @@ -520,6 +521,7 @@ tsd_booted tsd_booted_get tsd_cleanup tsd_cleanup_wrapper +tsd_data_init tsd_fetch tsd_fetch_impl tsd_get diff --git a/include/jemalloc/internal/rtree_externs.h b/include/jemalloc/internal/rtree_externs.h index 5145c12..482f6ba 100644 --- a/include/jemalloc/internal/rtree_externs.h +++ b/include/jemalloc/internal/rtree_externs.h @@ -43,5 +43,6 @@ void rtree_leaf_elm_witness_access(tsdn_t *tsdn, const rtree_t *rtree, const rtree_leaf_elm_t *elm); void rtree_leaf_elm_witness_release(tsdn_t *tsdn, const rtree_t *rtree, const rtree_leaf_elm_t *elm); +bool rtree_ctx_data_init(rtree_ctx_t *ctx); #endif /* JEMALLOC_INTERNAL_RTREE_EXTERNS_H */ diff --git a/include/jemalloc/internal/rtree_types.h b/include/jemalloc/internal/rtree_types.h index de3893b..e604156 100644 --- a/include/jemalloc/internal/rtree_types.h +++ b/include/jemalloc/internal/rtree_types.h @@ -38,6 +38,9 @@ typedef struct rtree_s rtree_t; # define RTREE_LEAF_COMPACT #endif +/* Needed for initialization only. */ +#define RTREE_LEAFKEY_INVALID ((uintptr_t)1) + /* * Number of leafkey/leaf pairs to cache. Each entry supports an entire leaf, * so the cache hit rate is typically high even with a small number of entries. @@ -51,12 +54,13 @@ typedef struct rtree_s rtree_t; * the tree nodes, and the cache will itself suffer cache misses if made overly * large, not to mention the cost of linear search. */ -#define RTREE_CTX_NCACHE 8 +#define RTREE_CTX_NCACHE 8 -/* Static initializer for rtree_ctx_t. */ -#define RTREE_CTX_INITIALIZER { \ - {{0, NULL} /* C initializes all trailing elements to NULL. */} \ -} +/* + * Zero initializer required for tsd initialization only. Proper initialization + * done via rtree_ctx_data_init(). + */ +#define RTREE_CTX_ZERO_INITIALIZER {{{0}}} /* * Maximum number of concurrently acquired elements per thread. This controls diff --git a/include/jemalloc/internal/tsd_externs.h b/include/jemalloc/internal/tsd_externs.h index 87ebaf2..9b88a56 100644 --- a/include/jemalloc/internal/tsd_externs.h +++ b/include/jemalloc/internal/tsd_externs.h @@ -14,5 +14,6 @@ void *tsd_init_check_recursion(tsd_init_head_t *head, void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); #endif void tsd_cleanup(void *arg); +bool tsd_data_init(void *arg); #endif /* JEMALLOC_INTERNAL_TSD_EXTERNS_H */ diff --git a/include/jemalloc/internal/tsd_inlines.h b/include/jemalloc/internal/tsd_inlines.h index 96de406..7d57b7d 100644 --- a/include/jemalloc/internal/tsd_inlines.h +++ b/include/jemalloc/internal/tsd_inlines.h @@ -8,7 +8,7 @@ tsd_t *tsd_fetch_impl(bool init); tsd_t *tsd_fetch(void); tsdn_t *tsd_tsdn(tsd_t *tsd); bool tsd_nominal(tsd_t *tsd); -#define O(n, t, gs, c) \ +#define O(n, t, gs, i, c) \ t *tsd_##n##p_get(tsd_t *tsd); \ t tsd_##n##_get(tsd_t *tsd); \ void tsd_##n##_set(tsd_t *tsd, t n); @@ -39,9 +39,11 @@ tsd_fetch_impl(bool init) { tsd->state = tsd_state_nominal; /* Trigger cleanup handler registration. */ tsd_set(tsd); + tsd_data_init(tsd); } else if (tsd->state == tsd_state_purgatory) { tsd->state = tsd_state_reincarnated; tsd_set(tsd); + tsd_data_init(tsd); } else { assert(tsd->state == tsd_state_reincarnated); } @@ -76,7 +78,7 @@ tsd_##n##_set(tsd_t *tsd, t n) { \ tsd->n = n; \ } #define MALLOC_TSD_getset_no(n, t) -#define O(n, t, gs, c) \ +#define O(n, t, gs, i, c) \ JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get(tsd_t *tsd) { \ return &tsd->n; \ @@ -121,8 +123,7 @@ tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback) { * return a pointer to it. */ if (unlikely(tsdn_null(tsdn))) { - static const rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; - memcpy(fallback, &rtree_ctx, sizeof(rtree_ctx_t)); + rtree_ctx_data_init(fallback); return fallback; } return tsd_rtree_ctx(tsdn_tsd(tsdn)); diff --git a/include/jemalloc/internal/tsd_structs.h b/include/jemalloc/internal/tsd_structs.h index 722b966..b4ac09f 100644 --- a/include/jemalloc/internal/tsd_structs.h +++ b/include/jemalloc/internal/tsd_structs.h @@ -15,23 +15,23 @@ struct tsd_init_head_s { #endif #define MALLOC_TSD \ -/* O(name, type, [gs]et, cleanup) */ \ - O(tcache, tcache_t *, yes, yes) \ - O(thread_allocated, uint64_t, yes, no) \ - O(thread_deallocated, uint64_t, yes, no) \ - O(prof_tdata, prof_tdata_t *, yes, yes) \ - O(iarena, arena_t *, yes, yes) \ - O(arena, arena_t *, yes, yes) \ - O(arenas_tdata, arena_tdata_t *,yes, yes) \ - O(narenas_tdata, unsigned, yes, no) \ - O(arenas_tdata_bypass, bool, no, no) \ +/* O(name, type, [gs]et, init, cleanup) */ \ + O(tcache, tcache_t *, yes, no, yes) \ + O(thread_allocated, uint64_t, yes, no, no) \ + O(thread_deallocated, uint64_t, yes, no, no) \ + O(prof_tdata, prof_tdata_t *, yes, no, yes) \ + O(iarena, arena_t *, yes, no, yes) \ + O(arena, arena_t *, yes, no, yes) \ + O(arenas_tdata, arena_tdata_t *,yes, no, yes) \ + O(narenas_tdata, unsigned, yes, no, no) \ + O(arenas_tdata_bypass, bool, no, no, no) \ O(tcache_enabled, tcache_enabled_t, \ - yes, no) \ - O(rtree_ctx, rtree_ctx_t, no, no) \ - O(witnesses, witness_list_t, no, yes) \ + yes, no, no) \ + O(rtree_ctx, rtree_ctx_t, no, yes, no) \ + O(witnesses, witness_list_t, no, no, yes) \ O(rtree_leaf_elm_witnesses, rtree_leaf_elm_witness_tsd_t, \ - no, no) \ - O(witness_fork, bool, yes, no) \ + no, no, no) \ + O(witness_fork, bool, yes, no, no) #define TSD_INITIALIZER { \ tsd_state_uninitialized, \ @@ -45,7 +45,7 @@ struct tsd_init_head_s { 0, \ false, \ tcache_enabled_default, \ - RTREE_CTX_INITIALIZER, \ + RTREE_CTX_ZERO_INITIALIZER, \ ql_head_initializer(witnesses), \ RTREE_ELM_WITNESS_TSD_INITIALIZER, \ false \ @@ -53,7 +53,7 @@ struct tsd_init_head_s { struct tsd_s { tsd_state_t state; -#define O(n, t, gs, c) \ +#define O(n, t, gs, i, c) \ t n; MALLOC_TSD #undef O diff --git a/src/rtree.c b/src/rtree.c index a07380f..b2c6824 100644 --- a/src/rtree.c +++ b/src/rtree.c @@ -424,3 +424,14 @@ rtree_leaf_elm_witness_release(tsdn_t *tsdn, const rtree_t *rtree, witness_unlock(tsdn, witness); rtree_leaf_elm_witness_dalloc(tsdn_tsd(tsdn), witness, elm); } + +bool +rtree_ctx_data_init(rtree_ctx_t *ctx) { + for (unsigned i = 0; i < RTREE_CTX_NCACHE; i++) { + rtree_ctx_cache_elm_t *cache = &ctx->cache[i]; + cache->leafkey = RTREE_LEAFKEY_INVALID; + cache->leaf = NULL; + } + + return false; +} @@ -60,6 +60,23 @@ malloc_tsd_cleanup_register(bool (*f)(void)) { ncleanups++; } +bool +tsd_data_init(void *arg) { + tsd_t *tsd = (tsd_t *)arg; +#define MALLOC_TSD_init_yes(n, t) \ + if (n##_data_init(&tsd->n)) { \ + return true; \ + } +#define MALLOC_TSD_init_no(n, t) +#define O(n, t, gs, i, c) \ + MALLOC_TSD_init_##i(n, t) +MALLOC_TSD +#undef MALLOC_TSD_init_yes +#undef MALLOC_TSD_init_no +#undef O + return false; +} + void tsd_cleanup(void *arg) { tsd_t *tsd = (tsd_t *)arg; @@ -72,7 +89,7 @@ tsd_cleanup(void *arg) { #define MALLOC_TSD_cleanup_yes(n, t) \ n##_cleanup(tsd); #define MALLOC_TSD_cleanup_no(n, t) -#define O(n, t, gs, c) \ +#define O(n, t, gs, i, c) \ MALLOC_TSD_cleanup_##c(n, t) MALLOC_TSD #undef MALLOC_TSD_cleanup_yes diff --git a/test/unit/rtree.c b/test/unit/rtree.c index 7a25c47..3c5b2df 100644 --- a/test/unit/rtree.c +++ b/test/unit/rtree.c @@ -68,7 +68,8 @@ TEST_BEGIN(test_rtree_read_empty) { tsdn = tsdn_fetch(); rtree_t *rtree = &test_rtree; - rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; + rtree_ctx_t rtree_ctx; + rtree_ctx_data_init(&rtree_ctx); assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure"); assert_ptr_null(rtree_extent_read(tsdn, rtree, &rtree_ctx, PAGE, false), "rtree_extent_read() should return NULL for empty tree"); @@ -89,7 +90,8 @@ typedef struct { static void * thd_start(void *varg) { thd_start_arg_t *arg = (thd_start_arg_t *)varg; - rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; + rtree_ctx_t rtree_ctx; + rtree_ctx_data_init(&rtree_ctx); sfmt_t *sfmt; extent_t *extent; tsdn_t *tsdn; @@ -173,7 +175,8 @@ TEST_BEGIN(test_rtree_extrema) { tsdn_t *tsdn = tsdn_fetch(); rtree_t *rtree = &test_rtree; - rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; + rtree_ctx_t rtree_ctx; + rtree_ctx_data_init(&rtree_ctx); assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure"); assert_false(rtree_write(tsdn, rtree, &rtree_ctx, PAGE, &extent_a, @@ -207,8 +210,8 @@ TEST_BEGIN(test_rtree_bits) { extent_state_active, false, false); rtree_t *rtree = &test_rtree; - rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; - + rtree_ctx_t rtree_ctx; + rtree_ctx_data_init(&rtree_ctx); assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure"); for (unsigned i = 0; i < sizeof(keys)/sizeof(uintptr_t); i++) { @@ -240,7 +243,8 @@ TEST_BEGIN(test_rtree_random) { tsdn_t *tsdn = tsdn_fetch(); uintptr_t keys[NSET]; rtree_t *rtree = &test_rtree; - rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER; + rtree_ctx_t rtree_ctx; + rtree_ctx_data_init(&rtree_ctx); extent_t extent; extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0, |