Convert extent_t's usize to szind.

Rather than storing usize only for large (and prof-promoted)
allocations, store the size class index for allocations that reside
within the extent, such that the size class index is valid for all
extents that contain extant allocations, and invalid otherwise (mainly
to make debugging simpler).

7 files changed, 92 insertions, 88 deletions

diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h
index b718451..0d4aff3 100644
--- a/include/jemalloc/internal/arena_inlines_b.h
+++ b/include/jemalloc/internal/arena_inlines_b.h
@@ -147,15 +147,15 @@ arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
 			    extent, ptr);
 		}
 	} else {
-		size_t usize = extent_usize_get(extent);
+		szind_t szind = extent_szind_get(extent);
 
-		if (likely(tcache != NULL) && usize <= tcache_maxclass) {
-			if (config_prof && unlikely(usize <= SMALL_MAXCLASS)) {
+		if (likely(tcache != NULL) && szind < nhbins) {
+			if (config_prof && unlikely(szind < NBINS)) {
 				arena_dalloc_promoted(tsdn, extent, ptr,
 				    tcache, slow_path);
 			} else {
 				tcache_dalloc_large(tsdn_tsd(tsdn), tcache,
-				    ptr, usize, slow_path);
+				    ptr, szind, slow_path);
 			}
 		} else {
 			large_dalloc(tsdn, extent);
@@ -169,25 +169,25 @@ arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
 	assert(!tsdn_null(tsdn) || tcache == NULL);
 	assert(ptr != NULL);
 
+	szind_t szind = size2index(size);
 	if (likely(extent_slab_get(extent))) {
 		/* Small allocation. */
 		if (likely(tcache != NULL)) {
-			szind_t binind = size2index(size);
-			assert(binind == extent_slab_data_get(extent)->binind);
-			tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, binind,
+			assert(szind == extent_slab_data_get(extent)->binind);
+			tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
 			    slow_path);
 		} else {
 			arena_dalloc_small(tsdn, extent_arena_get(extent),
 			    extent, ptr);
 		}
 	} else {
-		if (likely(tcache != NULL) && size <= tcache_maxclass) {
-			if (config_prof && unlikely(size <= SMALL_MAXCLASS)) {
+		if (likely(tcache != NULL) && szind < nhbins) {
+			if (config_prof && unlikely(szind < NBINS)) {
 				arena_dalloc_promoted(tsdn, extent, ptr,
 				    tcache, slow_path);
 			} else {
 				tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
-				    size, slow_path);
+				    szind, slow_path);
 			}
 		} else {
 			large_dalloc(tsdn, extent);
diff --git a/include/jemalloc/internal/extent_externs.h b/include/jemalloc/internal/extent_externs.h
index e8f632f..68c49a1 100644
--- a/include/jemalloc/internal/extent_externs.h
+++ b/include/jemalloc/internal/extent_externs.h
@@ -4,15 +4,15 @@
 extern rtree_t			extents_rtree;
 extern const extent_hooks_t	extent_hooks_default;
 
-extent_t	*extent_alloc(tsdn_t *tsdn, arena_t *arena);
-void	extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
+extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
+void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
 
-extent_hooks_t	*extent_hooks_get(arena_t *arena);
-extent_hooks_t	*extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks);
+extent_hooks_t *extent_hooks_get(arena_t *arena);
+extent_hooks_t *extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks);
 
 #ifdef JEMALLOC_JET
-size_t	extent_size_quantize_floor(size_t size);
-size_t	extent_size_quantize_ceil(size_t size);
+size_t extent_size_quantize_floor(size_t size);
+size_t extent_size_quantize_ceil(size_t size);
 #endif
 
 ph_proto(, extent_heap_, extent_heap_t, extent_t)
@@ -23,8 +23,8 @@ extent_state_t extents_state_get(const extents_t *extents);
 size_t extents_npages_get(extents_t *extents);
 extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr,
-    size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit,
-    bool slab);
+    size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
+    bool *zero, bool *commit);
 void extents_dalloc(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent);
 extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena,
@@ -32,32 +32,32 @@ extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena,
 void extents_prefork(tsdn_t *tsdn, extents_t *extents);
 void extents_postfork_parent(tsdn_t *tsdn, extents_t *extents);
 void extents_postfork_child(tsdn_t *tsdn, extents_t *extents);
-extent_t	*extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
-    size_t alignment, bool *zero, bool *commit, bool slab);
-void	extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
-bool	extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
+extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
+    extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad,
+    size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit);
+void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
+bool extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent);
-void	extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
+void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent);
-bool	extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
+bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
     size_t length);
-bool	extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
+bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
     size_t length);
-bool	extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
+bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
     size_t length);
-bool	extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
+bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
     size_t length);
-extent_t	*extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
+extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
-    size_t usize_a, size_t size_b, size_t usize_b);
-bool	extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
+    szind_t szind_a, size_t size_b, szind_t szind_b);
+bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
     extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b);
 
-bool	extent_boot(void);
+bool extent_boot(void);
 
 #endif /* JEMALLOC_INTERNAL_EXTENT_EXTERNS_H */
diff --git a/include/jemalloc/internal/extent_inlines.h b/include/jemalloc/internal/extent_inlines.h
index 989c0d1..549c8f2 100644
--- a/include/jemalloc/internal/extent_inlines.h
+++ b/include/jemalloc/internal/extent_inlines.h
@@ -2,37 +2,38 @@
 #define JEMALLOC_INTERNAL_EXTENT_INLINES_H
 
 #ifndef JEMALLOC_ENABLE_INLINE
-extent_t	*extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent);
-arena_t	*extent_arena_get(const extent_t *extent);
-void	*extent_base_get(const extent_t *extent);
-void	*extent_addr_get(const extent_t *extent);
-size_t	extent_size_get(const extent_t *extent);
-size_t	extent_usize_get(const extent_t *extent);
-void	*extent_before_get(const extent_t *extent);
-void	*extent_last_get(const extent_t *extent);
-void	*extent_past_get(const extent_t *extent);
-size_t	extent_sn_get(const extent_t *extent);
-extent_state_t	extent_state_get(const extent_t *extent);
-bool	extent_zeroed_get(const extent_t *extent);
-bool	extent_committed_get(const extent_t *extent);
-bool	extent_slab_get(const extent_t *extent);
-arena_slab_data_t	*extent_slab_data_get(extent_t *extent);
-const arena_slab_data_t	*extent_slab_data_get_const(const extent_t *extent);
-prof_tctx_t	*extent_prof_tctx_get(const extent_t *extent);
-void	extent_arena_set(extent_t *extent, arena_t *arena);
-void	extent_addr_set(extent_t *extent, void *addr);
-void	extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
-void	extent_size_set(extent_t *extent, size_t size);
-void	extent_usize_set(extent_t *extent, size_t usize);
-void	extent_sn_set(extent_t *extent, size_t sn);
-void	extent_state_set(extent_t *extent, extent_state_t state);
-void	extent_zeroed_set(extent_t *extent, bool zeroed);
-void	extent_committed_set(extent_t *extent, bool committed);
-void	extent_slab_set(extent_t *extent, bool slab);
-void	extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx);
-void	extent_init(extent_t *extent, arena_t *arena, void *addr,
-    size_t size, size_t usize, size_t sn, extent_state_t state, bool zeroed,
-    bool committed, bool slab);
+extent_t *extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent);
+arena_t *extent_arena_get(const extent_t *extent);
+void *extent_base_get(const extent_t *extent);
+void *extent_addr_get(const extent_t *extent);
+size_t extent_size_get(const extent_t *extent);
+szind_t extent_szind_get(const extent_t *extent);
+size_t extent_usize_get(const extent_t *extent);
+void *extent_before_get(const extent_t *extent);
+void *extent_last_get(const extent_t *extent);
+void *extent_past_get(const extent_t *extent);
+size_t extent_sn_get(const extent_t *extent);
+extent_state_t extent_state_get(const extent_t *extent);
+bool extent_zeroed_get(const extent_t *extent);
+bool extent_committed_get(const extent_t *extent);
+bool extent_slab_get(const extent_t *extent);
+arena_slab_data_t *extent_slab_data_get(extent_t *extent);
+const arena_slab_data_t *extent_slab_data_get_const(const extent_t *extent);
+prof_tctx_t *extent_prof_tctx_get(const extent_t *extent);
+void extent_arena_set(extent_t *extent, arena_t *arena);
+void extent_addr_set(extent_t *extent, void *addr);
+void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
+void extent_size_set(extent_t *extent, size_t size);
+void extent_szind_set(extent_t *extent, szind_t szind);
+void extent_sn_set(extent_t *extent, size_t sn);
+void extent_state_set(extent_t *extent, extent_state_t state);
+void extent_zeroed_set(extent_t *extent, bool zeroed);
+void extent_committed_set(extent_t *extent, bool committed);
+void extent_slab_set(extent_t *extent, bool slab);
+void extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx);
+void extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
+    bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed,
+    bool committed);
 void extent_list_init(extent_list_t *list);
 extent_t *extent_list_first(const extent_list_t *list);
 extent_t *extent_list_last(const extent_list_t *list);
@@ -40,9 +41,9 @@ void extent_list_append(extent_list_t *list, extent_t *extent);
 void extent_list_replace(extent_list_t *list, extent_t *to_remove,
     extent_t *to_insert);
 void extent_list_remove(extent_list_t *list, extent_t *extent);
-int	extent_sn_comp(const extent_t *a, const extent_t *b);
-int	extent_ad_comp(const extent_t *a, const extent_t *b);
-int	extent_snad_comp(const extent_t *a, const extent_t *b);
+int extent_sn_comp(const extent_t *a, const extent_t *b);
+int extent_ad_comp(const extent_t *a, const extent_t *b);
+int extent_snad_comp(const extent_t *a, const extent_t *b);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_))
@@ -79,10 +80,15 @@ extent_size_get(const extent_t *extent) {
 	return extent->e_size;
 }
 
+JEMALLOC_INLINE szind_t
+extent_szind_get(const extent_t *extent) {
+	assert(extent->e_szind < NSIZES); /* Never call when "invalid". */
+	return extent->e_szind;
+}
+
 JEMALLOC_INLINE size_t
 extent_usize_get(const extent_t *extent) {
-	assert(!extent->e_slab);
-	return extent->e_usize;
+	return index2size(extent_szind_get(extent));
 }
 
 JEMALLOC_INLINE void *
@@ -180,8 +186,9 @@ extent_size_set(extent_t *extent, size_t size) {
 }
 
 JEMALLOC_INLINE void
-extent_usize_set(extent_t *extent, size_t usize) {
-	extent->e_usize = usize;
+extent_szind_set(extent_t *extent, szind_t szind) {
+	assert(szind <= NSIZES); /* NSIZES means "invalid". */
+	extent->e_szind = szind;
 }
 
 JEMALLOC_INLINE void
@@ -216,19 +223,19 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) {
 
 JEMALLOC_INLINE void
 extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
-    size_t usize, size_t sn, extent_state_t state, bool zeroed, bool committed,
-    bool slab) {
+    bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed,
+    bool committed) {
 	assert(addr == PAGE_ADDR2BASE(addr) || !slab);
 
 	extent_arena_set(extent, arena);
 	extent_addr_set(extent, addr);
 	extent_size_set(extent, size);
-	extent_usize_set(extent, usize);
+	extent_slab_set(extent, slab);
+	extent_szind_set(extent, szind);
 	extent_sn_set(extent, sn);
 	extent_state_set(extent, state);
 	extent_zeroed_set(extent, zeroed);
 	extent_committed_set(extent, committed);
-	extent_slab_set(extent, slab);
 	if (config_prof) {
 		extent_prof_tctx_set(extent, NULL);
 	}
diff --git a/include/jemalloc/internal/extent_structs.h b/include/jemalloc/internal/extent_structs.h
index 001b7c1..82cfa58 100644
--- a/include/jemalloc/internal/extent_structs.h
+++ b/include/jemalloc/internal/extent_structs.h
@@ -20,10 +20,12 @@ struct extent_s {
 	size_t			e_size;
 
 	/*
-	 * Usable size, typically smaller than extent size due to large_pad or
+	 * Usable size class index for allocations residing in this extent,
+	 * regardless of whether the extent is a slab.  Extent size and usable
+	 * size often differ even for non-slabs, either due to large_pad or
 	 * promotion of sampled small regions.
 	 */
-	size_t			e_usize;
+	szind_t			e_szind;
 
 	/*
 	 * Serial number (potentially non-unique).
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index 97b41bb..b184380 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -536,8 +536,6 @@ void	jemalloc_postfork_child(void);
 #include "jemalloc/internal/witness_inlines.h"
 #include "jemalloc/internal/mutex_inlines.h"
 #include "jemalloc/internal/rtree_inlines.h"
-#include "jemalloc/internal/extent_inlines.h"
-#include "jemalloc/internal/base_inlines.h"
 
 #ifndef JEMALLOC_ENABLE_INLINE
 pszind_t	psz2ind(size_t psz);
@@ -565,7 +563,6 @@ ticker_t	*decay_ticker_get(tsd_t *tsd, unsigned ind);
 malloc_cpuid_t	malloc_getcpu(void);
 unsigned	percpu_arena_choose(void);
 unsigned	percpu_arena_ind_limit(void);
-
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
@@ -882,8 +879,6 @@ percpu_arena_ind_limit(void) {
 	}
 }
 
-
-
 JEMALLOC_INLINE arena_tdata_t *
 arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) {
 	arena_tdata_t *tdata;
@@ -938,6 +933,8 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) {
 }
 #endif
 
+#include "jemalloc/internal/extent_inlines.h"
+#include "jemalloc/internal/base_inlines.h"
 #include "jemalloc/internal/bitmap_inlines.h"
 /*
  * Include portions of arena code interleaved with tcache code in order to
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index 5ca7281..d68f6b6 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -191,8 +191,9 @@ extent_snad_comp
 extent_split_wrapper
 extent_state_get
 extent_state_set
+extent_szind_get
+extent_szind_set
 extent_usize_get
-extent_usize_set
 extent_zeroed_get
 extent_zeroed_set
 extents_alloc
diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h
index a90107f..fd7e176 100644
--- a/include/jemalloc/internal/tcache_inlines.h
+++ b/include/jemalloc/internal/tcache_inlines.h
@@ -15,7 +15,7 @@ void	*tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
 void	tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr,
     szind_t binind, bool slow_path);
 void	tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr,
-    size_t size, bool slow_path);
+    szind_t binind, bool slow_path);
 tcache_t	*tcaches_get(tsd_t *tsd, unsigned ind);
 #endif
 
@@ -271,19 +271,16 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
 }
 
 JEMALLOC_ALWAYS_INLINE void
-tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size,
+tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
     bool slow_path) {
-	szind_t binind;
 	tcache_bin_t *tbin;
 	tcache_bin_info_t *tbin_info;
 
 	assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SMALL_MAXCLASS);
 	assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_maxclass);
 
-	binind = size2index(size);
-
 	if (slow_path && config_fill && unlikely(opt_junk_free)) {
-		large_dalloc_junk(ptr, size);
+		large_dalloc_junk(ptr, index2size(binind));
 	}
 
 	tbin = &tcache->tbins[binind];