summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/jemalloc/internal/ctl.h2
-rw-r--r--include/jemalloc/internal/jemalloc_internal.h.in21
-rw-r--r--src/ctl.c149
3 files changed, 106 insertions, 66 deletions
diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h
index 4d4f304..dfb1e8e 100644
--- a/include/jemalloc/internal/ctl.h
+++ b/include/jemalloc/internal/ctl.h
@@ -61,7 +61,7 @@ struct ctl_stats_s {
size_t mapped;
size_t retained;
unsigned narenas;
- ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */
+ ctl_arena_stats_t *arenas[1 << MALLOCX_ARENA_BITS];
};
#endif /* JEMALLOC_H_STRUCTS */
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index 11a2736..991c541 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -208,11 +208,18 @@ typedef unsigned szind_t;
*
* aaaaaaaa aaaatttt tttttttt 0znnnnnn
*/
-#define MALLOCX_ARENA_MASK ((int)~0xfffff)
-#define MALLOCX_ARENA_MAX 0xffe
-#define MALLOCX_TCACHE_MASK ((int)~0xfff000ffU)
-#define MALLOCX_TCACHE_MAX 0xffd
-#define MALLOCX_LG_ALIGN_MASK ((int)0x3f)
+#define MALLOCX_ARENA_BITS 12
+#define MALLOCX_TCACHE_BITS 12
+#define MALLOCX_LG_ALIGN_BITS 6
+#define MALLOCX_ARENA_SHIFT 20
+#define MALLOCX_TCACHE_SHIFT 8
+#define MALLOCX_ARENA_MASK \
+ (((1 << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT)
+#define MALLOCX_ARENA_MAX ((1 << MALLOCX_ARENA_BITS) - 2)
+#define MALLOCX_TCACHE_MASK \
+ (((1 << MALLOCX_TCACHE_BITS) - 1) << MALLOCX_TCACHE_SHIFT)
+#define MALLOCX_TCACHE_MAX ((1 << MALLOCX_TCACHE_BITS) - 3)
+#define MALLOCX_LG_ALIGN_MASK ((1 << MALLOCX_LG_ALIGN_BITS) - 1)
/* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */
#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \
(ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))
@@ -222,9 +229,9 @@ typedef unsigned szind_t;
((bool)(flags & MALLOCX_ZERO))
#define MALLOCX_TCACHE_GET(flags) \
- (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> 8)) - 2)
+ (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> MALLOCX_TCACHE_SHIFT)) - 2)
#define MALLOCX_ARENA_GET(flags) \
- (((unsigned)(((unsigned)flags) >> 20)) - 1)
+ (((unsigned)(((unsigned)flags) >> MALLOCX_ARENA_SHIFT)) - 1)
/* Smallest size class to support. */
#define TINY_MIN (1U << LG_TINY_MIN)
diff --git a/src/ctl.c b/src/ctl.c
index 872da80..d5b384c 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -6,12 +6,12 @@
/*
* ctl_mtx protects the following:
- * - ctl_stats.*
+ * - ctl_stats->*
*/
static malloc_mutex_t ctl_mtx;
static bool ctl_initialized;
static uint64_t ctl_epoch;
-static ctl_stats_t ctl_stats;
+static ctl_stats_t *ctl_stats;
/******************************************************************************/
/* Helpers for named and indexed nodes. */
@@ -455,7 +455,7 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
a = 0;
break;
default:
- if (compat && i == ctl_stats.narenas) {
+ if (compat && i == ctl_stats->narenas) {
/*
* Provide deprecated backward compatibility for
* accessing the merged stats at index narenas rather
@@ -463,7 +463,7 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
* removal in 6.0.0.
*/
a = 0;
- } else if (validate && i >= ctl_stats.narenas)
+ } else if (validate && i >= ctl_stats->narenas)
a = UINT_MAX;
else {
/*
@@ -471,8 +471,8 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
* more than one past the range of indices that have
* initialized stats.
*/
- assert(i < ctl_stats.narenas || (!validate && i ==
- ctl_stats.narenas));
+ assert(i < ctl_stats->narenas || (!validate && i ==
+ ctl_stats->narenas));
a = (unsigned)i + 1;
}
break;
@@ -482,10 +482,31 @@ stats_arenas_i2a_impl(size_t i, bool compat, bool validate)
}
static ctl_arena_stats_t *
-stats_arenas_i(size_t i)
+stats_arenas_i_impl(tsdn_t *tsdn, size_t i, bool compat, bool init)
{
+ ctl_arena_stats_t *ret;
+
+ assert(!compat || !init);
+
+ ret = ctl_stats->arenas[stats_arenas_i2a_impl(i, compat, false)];
+ if (init && ret == NULL) {
+ ret = (ctl_arena_stats_t *)base_alloc(tsdn, b0get(),
+ sizeof(ctl_arena_stats_t), QUANTUM);
+ if (ret == NULL)
+ return (NULL);
+ ctl_stats->arenas[stats_arenas_i2a_impl(i, compat, false)] =
+ ret;
+ }
+
+ return (ret);
+}
- return (&ctl_stats.arenas[stats_arenas_i2a_impl(i, true, false)]);
+static ctl_arena_stats_t *
+stats_arenas_i(size_t i)
+{
+ ctl_arena_stats_t *ret = stats_arenas_i_impl(TSDN_NULL, i, true, false);
+ assert(ret != NULL);
+ return (ret);
}
static void
@@ -609,25 +630,15 @@ ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, unsigned i)
static bool
ctl_grow(tsdn_t *tsdn, extent_hooks_t *extent_hooks)
{
- ctl_arena_stats_t *astats;
- /* Initialize new arena. */
- if (arena_init(tsdn, ctl_stats.narenas, extent_hooks) == NULL)
+ /* Trigger stats allocation. */
+ if (stats_arenas_i_impl(tsdn, ctl_stats->narenas, false, true) == NULL)
return (true);
- /* Allocate extended arena stats. */
- astats = (ctl_arena_stats_t *)a0malloc((ctl_stats.narenas + 2) *
- sizeof(ctl_arena_stats_t));
- if (astats == NULL)
+ /* Initialize new arena. */
+ if (arena_init(tsdn, ctl_stats->narenas, extent_hooks) == NULL)
return (true);
-
- /* Initialize the new astats element. */
- memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) *
- sizeof(ctl_arena_stats_t));
- memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t));
- a0dalloc(ctl_stats.arenas);
- ctl_stats.arenas = astats;
- ctl_stats.narenas++;
+ ctl_stats->narenas++;
return (false);
}
@@ -637,7 +648,7 @@ ctl_refresh(tsdn_t *tsdn)
{
unsigned i;
ctl_arena_stats_t *sstats = stats_arenas_i(MALLCTL_ARENAS_ALL);
- VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas);
+ VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats->narenas);
/*
* Clear sum stats, since they will be merged into by
@@ -645,10 +656,10 @@ ctl_refresh(tsdn_t *tsdn)
*/
ctl_arena_clear(sstats);
- for (i = 0; i < ctl_stats.narenas; i++)
+ for (i = 0; i < ctl_stats->narenas; i++)
tarenas[i] = arena_get(tsdn, i, false);
- for (i = 0; i < ctl_stats.narenas; i++) {
+ for (i = 0; i < ctl_stats->narenas; i++) {
ctl_arena_stats_t *astats = stats_arenas_i(i);
bool initialized = (tarenas[i] != NULL);
@@ -658,14 +669,14 @@ ctl_refresh(tsdn_t *tsdn)
}
if (config_stats) {
- ctl_stats.allocated = sstats->allocated_small +
+ ctl_stats->allocated = sstats->allocated_small +
sstats->astats.allocated_large;
- ctl_stats.active = (sstats->pactive << LG_PAGE);
- ctl_stats.metadata = sstats->astats.base +
+ ctl_stats->active = (sstats->pactive << LG_PAGE);
+ ctl_stats->metadata = sstats->astats.base +
sstats->astats.internal;
- ctl_stats.resident = sstats->astats.resident;
- ctl_stats.mapped = sstats->astats.mapped;
- ctl_stats.retained = sstats->astats.retained;
+ ctl_stats->resident = sstats->astats.resident;
+ ctl_stats->mapped = sstats->astats.mapped;
+ ctl_stats->retained = sstats->astats.retained;
}
ctl_epoch++;
@@ -678,20 +689,41 @@ ctl_init(tsdn_t *tsdn)
malloc_mutex_lock(tsdn, &ctl_mtx);
if (!ctl_initialized) {
+ ctl_arena_stats_t *sstats;
+ unsigned i;
+
/*
- * Allocate space for one extra arena stats element, which
- * contains summed stats across all arenas.
+ * Allocate demand-zeroed space for pointers to the full range
+ * of supported arena indices.
*/
- ctl_stats.narenas = narenas_total_get();
- ctl_stats.arenas = (ctl_arena_stats_t *)a0malloc(
- (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t));
- if (ctl_stats.arenas == NULL) {
+ if (ctl_stats == NULL) {
+ ctl_stats = (ctl_stats_t *)base_alloc(tsdn, b0get(),
+ sizeof(ctl_stats_t), QUANTUM);
+ if (ctl_stats == NULL) {
+ ret = true;
+ goto label_return;
+ }
+ }
+
+ /*
+ * Allocate space for the current full range of arenas here
+ * rather than doing it lazily elsewhere, in order to limit when
+ * OOM-caused errors can occur.
+ */
+ if ((sstats = stats_arenas_i_impl(tsdn, MALLCTL_ARENAS_ALL,
+ false, true)) == NULL) {
ret = true;
goto label_return;
}
- memset(ctl_stats.arenas, 0, (ctl_stats.narenas + 1) *
- sizeof(ctl_arena_stats_t));
- stats_arenas_i(MALLCTL_ARENAS_ALL)->initialized = true;
+ sstats->initialized = true;
+
+ ctl_stats->narenas = narenas_total_get();
+ for (i = 0; i < ctl_stats->narenas; i++) {
+ if (stats_arenas_i_impl(tsdn, i, false, true) == NULL) {
+ ret = true;
+ goto label_return;
+ }
+ }
ctl_epoch = 0;
ctl_refresh(tsdn);
@@ -1428,7 +1460,7 @@ arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all)
malloc_mutex_lock(tsdn, &ctl_mtx);
{
- unsigned narenas = ctl_stats.narenas;
+ unsigned narenas = ctl_stats->narenas;
/*
* Access via index narenas is deprecated, and scheduled for
@@ -1511,7 +1543,7 @@ arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
arena_ind = (unsigned)mib[1];
if (config_debug) {
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
- assert(arena_ind < ctl_stats.narenas);
+ assert(arena_ind < ctl_stats->narenas);
malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
}
assert(arena_ind >= opt_narenas);
@@ -1563,7 +1595,8 @@ arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
* Access via index narenas is deprecated, and scheduled for removal in
* 6.0.0.
*/
- if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind == ctl_stats.narenas) {
+ if (arena_ind == MALLCTL_ARENAS_ALL || arena_ind ==
+ ctl_stats->narenas) {
if (dss_prec != dss_prec_limit &&
extent_dss_prec_set(dss_prec)) {
ret = EFAULT;
@@ -1664,7 +1697,7 @@ arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
const ctl_named_node_t *ret;
malloc_mutex_lock(tsdn, &ctl_mtx);
- if (i > ctl_stats.narenas && i != MALLCTL_ARENAS_ALL) {
+ if (i > ctl_stats->narenas && i != MALLCTL_ARENAS_ALL) {
ret = NULL;
goto label_return;
}
@@ -1690,7 +1723,7 @@ arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
ret = EINVAL;
goto label_return;
}
- narenas = ctl_stats.narenas;
+ narenas = ctl_stats->narenas;
READ(narenas, unsigned);
ret = 0;
@@ -1708,13 +1741,13 @@ arenas_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
READONLY();
- if (*oldlenp != ctl_stats.narenas * sizeof(bool)) {
+ if (*oldlenp != ctl_stats->narenas * sizeof(bool)) {
ret = EINVAL;
- nread = (*oldlenp < ctl_stats.narenas * sizeof(bool))
- ? (unsigned)(*oldlenp / sizeof(bool)) : ctl_stats.narenas;
+ nread = (*oldlenp < ctl_stats->narenas * sizeof(bool))
+ ? (unsigned)(*oldlenp / sizeof(bool)) : ctl_stats->narenas;
} else {
ret = 0;
- nread = ctl_stats.narenas;
+ nread = ctl_stats->narenas;
}
for (i = 0; i < nread; i++)
@@ -1795,7 +1828,7 @@ arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
ret = EAGAIN;
goto label_return;
}
- narenas = ctl_stats.narenas - 1;
+ narenas = ctl_stats->narenas - 1;
READ(narenas, unsigned);
ret = 0;
@@ -1932,12 +1965,12 @@ CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t)
/******************************************************************************/
-CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t)
-CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t)
-CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats.metadata, size_t)
-CTL_RO_CGEN(config_stats, stats_resident, ctl_stats.resident, size_t)
-CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t)
-CTL_RO_CGEN(config_stats, stats_retained, ctl_stats.retained, size_t)
+CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t)
+CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t)
+CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t)
+CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t)
+CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t)
+CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t)
CTL_RO_GEN(stats_arenas_i_dss, stats_arenas_i(mib[2])->dss, const char *)
CTL_RO_GEN(stats_arenas_i_decay_time, stats_arenas_i(mib[2])->decay_time,
@@ -2036,7 +2069,7 @@ stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i)
malloc_mutex_lock(tsdn, &ctl_mtx);
a = stats_arenas_i2a_impl(i, true, true);
- if (a == UINT_MAX || !ctl_stats.arenas[a].initialized) {
+ if (a == UINT_MAX || !ctl_stats->arenas[a]->initialized) {
ret = NULL;
goto label_return;
}
generated by cgit v0.12 at 2025-08-30 22:38:30 (GMT)