From 48be46ec1f3f8010570165daa1da4bf9961f3a83 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 1 Feb 2022 15:05:18 +0000 Subject: bpo-46072: Add some object layout and allocation stats (GH-31051) --- Include/internal/pycore_code.h | 12 ++++++++++++ Objects/dictobject.c | 12 ++++++++++++ Objects/obmalloc.c | 4 ++++ Python/specialize.c | 12 ++++++++++++ Tools/scripts/summarize_stats.py | 12 +++++++++++- 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 68b536f..45c7752 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -305,9 +305,19 @@ typedef struct _call_stats { uint64_t pyeval_calls; } CallStats; +typedef struct _object_stats { + uint64_t allocations; + uint64_t frees; + uint64_t new_values; + uint64_t dict_materialized_on_request; + uint64_t dict_materialized_new_key; + uint64_t dict_materialized_too_big; +} ObjectStats; + typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; + ObjectStats object_stats; } PyStats; extern PyStats _py_stats; @@ -316,6 +326,7 @@ extern PyStats _py_stats; #define STAT_DEC(opname, name) _py_stats.opcode_stats[opname].specialization.name-- #define OPCODE_EXE_INC(opname) _py_stats.opcode_stats[opname].execution_count++ #define CALL_STAT_INC(name) _py_stats.call_stats.name++ +#define OBJECT_STAT_INC(name) _py_stats.object_stats.name++ void _Py_PrintSpecializationStats(int to_file); @@ -326,6 +337,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define STAT_DEC(opname, name) ((void)0) #define OPCODE_EXE_INC(opname) ((void)0) #define CALL_STAT_INC(name) ((void)0) +#define OBJECT_STAT_INC(name) ((void)0) #endif diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 39be189..0ad0f0b 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -114,6 +114,7 @@ As a consequence of this, split keys have a maximum size of 16. #include "Python.h" #include "pycore_bitutils.h" // _Py_bit_length #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_code.h" // stats #include "pycore_dict.h" // PyDictKeysObject #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -4990,6 +4991,7 @@ _PyObject_InitializeDict(PyObject *obj) return 0; } if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + OBJECT_STAT_INC(new_values); return init_inline_values(obj, tp); } PyObject *dict; @@ -5033,6 +5035,7 @@ _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) { assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); + OBJECT_STAT_INC(dict_materialized_on_request); return make_dict_from_instance_attributes(keys, values); } @@ -5051,6 +5054,14 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyErr_SetObject(PyExc_AttributeError, name); return -1; } +#ifdef Py_STATS + if (shared_keys_usable_size(keys) > 14) { + OBJECT_STAT_INC(dict_materialized_too_big); + } + else { + OBJECT_STAT_INC(dict_materialized_new_key); + } +#endif PyObject *dict = make_dict_from_instance_attributes(keys, values); if (dict == NULL) { return -1; @@ -5183,6 +5194,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) PyObject **dictptr = _PyObject_ManagedDictPointer(obj); if (*values_ptr) { assert(*dictptr == NULL); + OBJECT_STAT_INC(dict_materialized_on_request); *dictptr = dict = make_dict_from_instance_attributes(CACHED_KEYS(tp), *values_ptr); if (dict != NULL) { *values_ptr = NULL; diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index e3df7e8..bad4dc0 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_pymem.h" // _PyTraceMalloc_Config +#include "pycore_code.h" // stats #include #include // malloc() @@ -695,6 +696,7 @@ PyObject_Malloc(size_t size) /* see PyMem_RawMalloc() */ if (size > (size_t)PY_SSIZE_T_MAX) return NULL; + OBJECT_STAT_INC(allocations); return _PyObject.malloc(_PyObject.ctx, size); } @@ -704,6 +706,7 @@ PyObject_Calloc(size_t nelem, size_t elsize) /* see PyMem_RawMalloc() */ if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) return NULL; + OBJECT_STAT_INC(allocations); return _PyObject.calloc(_PyObject.ctx, nelem, elsize); } @@ -719,6 +722,7 @@ PyObject_Realloc(void *ptr, size_t new_size) void PyObject_Free(void *ptr) { + OBJECT_STAT_INC(frees); _PyObject.free(_PyObject.ctx, ptr); } diff --git a/Python/specialize.c b/Python/specialize.c index aec94d9..5771a41 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -172,9 +172,21 @@ print_call_stats(FILE *out, CallStats *stats) } static void +print_object_stats(FILE *out, ObjectStats *stats) +{ + fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations); + fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees); + fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values); + fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request); + fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); + fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); +} + +static void print_stats(FILE *out, PyStats *stats) { print_spec_stats(out, stats->opcode_stats); print_call_stats(out, &stats->call_stats); + print_object_stats(out, &stats->object_stats); } void diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 34cbad5..2761dcb 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -105,7 +105,17 @@ def main(): total += value for key, value in stats.items(): if "Calls to" in key: - print(f"{key}: {value} {100*value/total:0.1f}%") + print(f" {key}: {value} {100*value/total:0.1f}%") + print("Object stats:") + total = stats.get("Object new values") + for key, value in stats.items(): + if key.startswith("Object"): + if "materialize" in key: + print(f" {key}: {value} {100*value/total:0.1f}%") + else: + print(f" {key}: {value}") + total = 0 + if __name__ == "__main__": main() -- cgit v0.12