summaryrefslogtreecommitdiffstats
path: root/include/jemalloc/internal
diff options
context:
space:
mode:
Diffstat (limited to 'include/jemalloc/internal')
-rw-r--r--include/jemalloc/internal/arena.h33
-rw-r--r--include/jemalloc/internal/jemalloc_internal.h.in21
-rw-r--r--include/jemalloc/internal/private_namespace.h6
-rw-r--r--include/jemalloc/internal/prof.h14
-rw-r--r--include/jemalloc/internal/quarantine.h43
-rw-r--r--include/jemalloc/internal/tcache.h4
6 files changed, 89 insertions, 32 deletions
diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h
index 8fdee93..f2c18f4 100644
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -463,9 +463,9 @@ void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind,
size_t runind, size_t binind, size_t flags);
void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
size_t unzeroed);
-void arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes);
-void arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes);
-void arena_prof_accum(arena_t *arena, uint64_t accumbytes);
+bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes);
+bool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes);
+bool arena_prof_accum(arena_t *arena, uint64_t accumbytes);
size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits);
size_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
@@ -663,7 +663,7 @@ arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
*mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed;
}
-JEMALLOC_INLINE void
+JEMALLOC_INLINE bool
arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes)
{
@@ -672,33 +672,40 @@ arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes)
arena->prof_accumbytes += accumbytes;
if (arena->prof_accumbytes >= prof_interval) {
- prof_idump();
arena->prof_accumbytes -= prof_interval;
+ return (true);
}
+ return (false);
}
-JEMALLOC_INLINE void
+JEMALLOC_INLINE bool
arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes)
{
cassert(config_prof);
if (prof_interval == 0)
- return;
- arena_prof_accum_impl(arena, accumbytes);
+ return (false);
+ return (arena_prof_accum_impl(arena, accumbytes));
}
-JEMALLOC_INLINE void
+JEMALLOC_INLINE bool
arena_prof_accum(arena_t *arena, uint64_t accumbytes)
{
cassert(config_prof);
if (prof_interval == 0)
- return;
- malloc_mutex_lock(&arena->lock);
- arena_prof_accum_impl(arena, accumbytes);
- malloc_mutex_unlock(&arena->lock);
+ return (false);
+
+ {
+ bool ret;
+
+ malloc_mutex_lock(&arena->lock);
+ ret = arena_prof_accum_impl(arena, accumbytes);
+ malloc_mutex_unlock(&arena->lock);
+ return (ret);
+ }
}
JEMALLOC_ALWAYS_INLINE size_t
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index c606c12..50d84ca 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -287,7 +287,7 @@ static const bool config_ivsalloc =
# ifdef __powerpc__
# define LG_QUANTUM 4
# endif
-# ifdef __s390x__
+# ifdef __s390__
# define LG_QUANTUM 4
# endif
# ifdef __SH4__
@@ -440,15 +440,18 @@ static const bool config_ivsalloc =
} while (0)
#else
#define RUNNING_ON_VALGRIND ((unsigned)0)
-#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
-#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB)
-#define VALGRIND_FREELIKE_BLOCK(addr, rzB)
-#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len)
-#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len)
-#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero)
+#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \
+ do {} while (0)
+#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \
+ do {} while (0)
+#define VALGRIND_FREELIKE_BLOCK(addr, rzB) do {} while (0)
+#define VALGRIND_MAKE_MEM_NOACCESS(_qzz_addr, _qzz_len) do {} while (0)
+#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len) do {} while (0)
+#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len) do {} while (0)
+#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do {} while (0)
#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \
- old_rzsize, zero)
-#define JEMALLOC_VALGRIND_FREE(ptr, rzsize)
+ old_rzsize, zero) do {} while (0)
+#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do {} while (0)
#endif
#include "jemalloc/internal/util.h"
diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h
index 903fb4d..65de316 100644
--- a/include/jemalloc/internal/private_namespace.h
+++ b/include/jemalloc/internal/private_namespace.h
@@ -306,7 +306,13 @@
#define prof_tdata_tsd_get_wrapper JEMALLOC_N(prof_tdata_tsd_get_wrapper)
#define prof_tdata_tsd_set JEMALLOC_N(prof_tdata_tsd_set)
#define quarantine JEMALLOC_N(quarantine)
+#define quarantine_alloc_hook JEMALLOC_N(quarantine_alloc_hook)
#define quarantine_boot JEMALLOC_N(quarantine_boot)
+#define quarantine_booted JEMALLOC_N(quarantine_booted)
+#define quarantine_cleanup JEMALLOC_N(quarantine_cleanup)
+#define quarantine_init JEMALLOC_N(quarantine_init)
+#define quarantine_tls JEMALLOC_N(quarantine_tls)
+#define quarantine_tsd JEMALLOC_N(quarantine_tsd)
#define quarantine_tsd_boot JEMALLOC_N(quarantine_tsd_boot)
#define quarantine_tsd_cleanup_wrapper JEMALLOC_N(quarantine_tsd_cleanup_wrapper)
#define quarantine_tsd_get JEMALLOC_N(quarantine_tsd_get)
diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h
index 47f22ad..119a5b1 100644
--- a/include/jemalloc/internal/prof.h
+++ b/include/jemalloc/internal/prof.h
@@ -237,7 +237,7 @@ void prof_postfork_child(void);
\
assert(size == s2u(size)); \
\
- prof_tdata = prof_tdata_get(); \
+ prof_tdata = prof_tdata_get(true); \
if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) { \
if (prof_tdata != NULL) \
ret = (prof_thr_cnt_t *)(uintptr_t)1U; \
@@ -286,7 +286,7 @@ void prof_postfork_child(void);
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *)
-prof_tdata_t *prof_tdata_get(void);
+prof_tdata_t *prof_tdata_get(bool create);
void prof_sample_threshold_update(prof_tdata_t *prof_tdata);
prof_ctx_t *prof_ctx_get(const void *ptr);
void prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
@@ -304,17 +304,15 @@ malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL,
prof_tdata_cleanup)
JEMALLOC_INLINE prof_tdata_t *
-prof_tdata_get(void)
+prof_tdata_get(bool create)
{
prof_tdata_t *prof_tdata;
cassert(config_prof);
prof_tdata = *prof_tdata_tsd_get();
- if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) {
- if (prof_tdata == NULL)
- prof_tdata = prof_tdata_init();
- }
+ if (create && prof_tdata == NULL)
+ prof_tdata = prof_tdata_init();
return (prof_tdata);
}
@@ -397,7 +395,7 @@ prof_sample_accum_update(size_t size)
/* Sampling logic is unnecessary if the interval is 1. */
assert(opt_lg_prof_sample != 0);
- prof_tdata = *prof_tdata_tsd_get();
+ prof_tdata = prof_tdata_get(false);
if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
return (true);
diff --git a/include/jemalloc/internal/quarantine.h b/include/jemalloc/internal/quarantine.h
index 38f3d69..16f677f 100644
--- a/include/jemalloc/internal/quarantine.h
+++ b/include/jemalloc/internal/quarantine.h
@@ -1,6 +1,9 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
+typedef struct quarantine_obj_s quarantine_obj_t;
+typedef struct quarantine_s quarantine_t;
+
/* Default per thread quarantine size if valgrind is enabled. */
#define JEMALLOC_VALGRIND_QUARANTINE_DEFAULT (ZU(1) << 24)
@@ -8,17 +11,57 @@
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
+struct quarantine_obj_s {
+ void *ptr;
+ size_t usize;
+};
+
+struct quarantine_s {
+ size_t curbytes;
+ size_t curobjs;
+ size_t first;
+#define LG_MAXOBJS_INIT 10
+ size_t lg_maxobjs;
+ quarantine_obj_t objs[1]; /* Dynamically sized ring buffer. */
+};
+
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
+quarantine_t *quarantine_init(size_t lg_maxobjs);
void quarantine(void *ptr);
+void quarantine_cleanup(void *arg);
bool quarantine_boot(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
+#ifndef JEMALLOC_ENABLE_INLINE
+malloc_tsd_protos(JEMALLOC_ATTR(unused), quarantine, quarantine_t *)
+
+void quarantine_alloc_hook(void);
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_QUARANTINE_C_))
+malloc_tsd_externs(quarantine, quarantine_t *)
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, quarantine, quarantine_t *, NULL,
+ quarantine_cleanup)
+
+JEMALLOC_ALWAYS_INLINE void
+quarantine_alloc_hook(void)
+{
+ quarantine_t *quarantine;
+
+ assert(config_fill && opt_quarantine);
+
+ quarantine = *quarantine_tsd_get();
+ if (quarantine == NULL)
+ quarantine_init(LG_MAXOBJS_INIT);
+}
+#endif
+
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h
index 71900c2..ba36204 100644
--- a/include/jemalloc/internal/tcache.h
+++ b/include/jemalloc/internal/tcache.h
@@ -320,8 +320,8 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
}
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
memset(ret, 0, size);
- VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
}
+ VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
if (config_stats)
tbin->tstats.nrequests++;
@@ -371,8 +371,8 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
} else {
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
memset(ret, 0, size);
- VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
}
+ VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
if (config_stats)
tbin->tstats.nrequests++;
generated by cgit v0.12 at 2025-12-30 03:25:06 (GMT)