summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Include/internal/pycore_freelist_state.h2
-rw-r--r--Include/internal/pycore_long.h2
-rw-r--r--Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst1
-rw-r--r--Objects/longobject.c78
-rw-r--r--Objects/object.c1
-rw-r--r--Python/bytecodes.c25
-rw-r--r--Python/executor_cases.c.h24
-rw-r--r--Python/generated_cases.c.h24
8 files changed, 102 insertions, 55 deletions
diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h
index 4e04cf4..a1a94c1 100644
--- a/Include/internal/pycore_freelist_state.h
+++ b/Include/internal/pycore_freelist_state.h
@@ -14,6 +14,7 @@ extern "C" {
# define Py_dicts_MAXFREELIST 80
# define Py_dictkeys_MAXFREELIST 80
# define Py_floats_MAXFREELIST 100
+# define Py_ints_MAXFREELIST 100
# define Py_slices_MAXFREELIST 1
# define Py_contexts_MAXFREELIST 255
# define Py_async_gens_MAXFREELIST 80
@@ -35,6 +36,7 @@ struct _Py_freelist {
struct _Py_freelists {
struct _Py_freelist floats;
+ struct _Py_freelist ints;
struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
struct _Py_freelist lists;
struct _Py_freelist dicts;
diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h
index 196b415..8bead00 100644
--- a/Include/internal/pycore_long.h
+++ b/Include/internal/pycore_long.h
@@ -55,6 +55,8 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp);
/* other API */
+PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self);
+
#define _PyLong_SMALL_INTS _Py_SINGLETON(small_ints)
// _PyLong_GetZero() and _PyLong_GetOne() must always be available
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst
new file mode 100644
index 0000000..fd15709
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst
@@ -0,0 +1 @@
+Increase performance of :class:`int` by adding a freelist for compact ints.
diff --git a/Objects/longobject.c b/Objects/longobject.c
index 4aa3568..96d59f5 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -6,6 +6,7 @@
#include "pycore_bitutils.h" // _Py_popcount32()
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_call.h" // _PyObject_MakeTpCall
+#include "pycore_freelist.h" // _Py_FREELIST_FREE, _Py_FREELIST_POP
#include "pycore_long.h" // _Py_SmallInts
#include "pycore_object.h" // _PyObject_Init()
#include "pycore_runtime.h" // _PY_NSMALLPOSINTS
@@ -42,7 +43,7 @@ static inline void
_Py_DECREF_INT(PyLongObject *op)
{
assert(PyLong_CheckExact(op));
- _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free);
+ _Py_DECREF_SPECIALIZED((PyObject *)op, _PyLong_ExactDealloc);
}
static inline int
@@ -220,15 +221,18 @@ _PyLong_FromMedium(sdigit x)
{
assert(!IS_SMALL_INT(x));
assert(is_medium_int(x));
- /* We could use a freelist here */
- PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject));
+
+ PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints);
if (v == NULL) {
- PyErr_NoMemory();
- return NULL;
+ v = PyObject_Malloc(sizeof(PyLongObject));
+ if (v == NULL) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ _PyObject_Init((PyObject*)v, &PyLong_Type);
}
digit abs_x = x < 0 ? -x : x;
_PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1);
- _PyObject_Init((PyObject*)v, &PyLong_Type);
v->long_value.ob_digit[0] = abs_x;
return (PyObject*)v;
}
@@ -3611,24 +3615,60 @@ long_richcompare(PyObject *self, PyObject *other, int op)
Py_RETURN_RICHCOMPARE(result, 0, op);
}
+static inline int
+compact_int_is_small(PyObject *self)
+{
+ PyLongObject *pylong = (PyLongObject *)self;
+ assert(_PyLong_IsCompact(pylong));
+ stwodigits ival = medium_value(pylong);
+ if (IS_SMALL_INT(ival)) {
+ PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival);
+ if (pylong == small_pylong) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
+void
+_PyLong_ExactDealloc(PyObject *self)
+{
+ assert(PyLong_CheckExact(self));
+ if (_PyLong_IsCompact((PyLongObject *)self)) {
+ #ifndef Py_GIL_DISABLED
+ if (compact_int_is_small(self)) {
+ // See PEP 683, section Accidental De-Immortalizing for details
+ _Py_SetImmortal(self);
+ return;
+ }
+ #endif
+ _Py_FREELIST_FREE(ints, self, PyObject_Free);
+ return;
+ }
+ PyObject_Free(self);
+}
+
static void
long_dealloc(PyObject *self)
{
- /* This should never get called, but we also don't want to SEGV if
- * we accidentally decref small Ints out of existence. Instead,
- * since small Ints are immortal, re-set the reference count.
- */
- PyLongObject *pylong = (PyLongObject*)self;
- if (pylong && _PyLong_IsCompact(pylong)) {
- stwodigits ival = medium_value(pylong);
- if (IS_SMALL_INT(ival)) {
- PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival);
- if (pylong == small_pylong) {
- _Py_SetImmortal(self);
- return;
- }
+ assert(self);
+ if (_PyLong_IsCompact((PyLongObject *)self)) {
+ if (compact_int_is_small(self)) {
+ /* This should never get called, but we also don't want to SEGV if
+ * we accidentally decref small Ints out of existence. Instead,
+ * since small Ints are immortal, re-set the reference count.
+ *
+ * See PEP 683, section Accidental De-Immortalizing for details
+ */
+ _Py_SetImmortal(self);
+ return;
+ }
+ if (PyLong_CheckExact(self)) {
+ _Py_FREELIST_FREE(ints, self, PyObject_Free);
+ return;
}
}
+
Py_TYPE(self)->tp_free(self);
}
diff --git a/Objects/object.c b/Objects/object.c
index c64675b..d584414 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -936,6 +936,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization)
clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree);
}
clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free);
+ clear_freelist(&freelists->ints, is_finalization, free_object);
}
/*
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index d0e4c2b..f0eb540 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -26,6 +26,7 @@
#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_range.h" // _PyRangeIterObject
+#include "pycore_long.h" // _PyLong_ExactDealloc()
#include "pycore_setobject.h" // _PySet_NextEntry()
#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs
#include "pycore_tuple.h" // _PyTuple_ITEMS()
@@ -514,8 +515,8 @@ dummy_func(
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
INPUTS_DEAD();
ERROR_IF(res_o == NULL, error);
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -527,8 +528,8 @@ dummy_func(
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
INPUTS_DEAD();
ERROR_IF(res_o == NULL, error);
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -540,8 +541,8 @@ dummy_func(
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
INPUTS_DEAD();
ERROR_IF(res_o == NULL, error);
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -801,7 +802,7 @@ dummy_func(
assert(res_o != NULL);
Py_INCREF(res_o);
#endif
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
DEAD(sub_st);
PyStackRef_CLOSE(list_st);
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -821,7 +822,7 @@ dummy_func(
DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c);
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
DEAD(sub_st);
PyStackRef_CLOSE(str_st);
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -842,7 +843,7 @@ dummy_func(
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
assert(res_o != NULL);
Py_INCREF(res_o);
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
DEAD(sub_st);
PyStackRef_CLOSE(tuple_st);
res = PyStackRef_FromPyObjectSteal(res_o);
@@ -959,7 +960,7 @@ dummy_func(
assert(old_value != NULL);
UNLOCK_OBJECT(list); // unlock before decrefs!
Py_DECREF(old_value);
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
DEAD(sub_st);
PyStackRef_CLOSE(list_st);
}
@@ -2476,9 +2477,9 @@ dummy_func(
Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o);
// 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
int sign_ish = COMPARISON_BIT(ileft, iright);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
DEAD(left);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
DEAD(right);
res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 18f1977..19ba67a 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -626,8 +626,8 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) JUMP_TO_ERROR();
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -646,8 +646,8 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) JUMP_TO_ERROR();
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -666,8 +666,8 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) JUMP_TO_ERROR();
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -1000,7 +1000,7 @@
assert(res_o != NULL);
Py_INCREF(res_o);
#endif
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
PyStackRef_CLOSE(list_st);
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -1042,7 +1042,7 @@
}
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
PyStackRef_CLOSE(str_st);
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -1081,7 +1081,7 @@
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
assert(res_o != NULL);
Py_INCREF(res_o);
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
PyStackRef_CLOSE(tuple_st);
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -1264,7 +1264,7 @@
assert(old_value != NULL);
UNLOCK_OBJECT(list); // unlock before decrefs!
Py_DECREF(old_value);
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
PyStackRef_CLOSE(list_st);
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
@@ -3075,8 +3075,8 @@
Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o);
// 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
int sign_ish = COMPARISON_BIT(ileft, iright);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
stack_pointer[-2] = res;
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index fc0f555..51227c9 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -118,8 +118,8 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -285,8 +285,8 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -356,8 +356,8 @@
PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
STAT_INC(BINARY_OP, hit);
PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
if (res_o == NULL) goto pop_2_error;
res = PyStackRef_FromPyObjectSteal(res_o);
}
@@ -590,7 +590,7 @@
assert(res_o != NULL);
Py_INCREF(res_o);
#endif
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
PyStackRef_CLOSE(list_st);
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -622,7 +622,7 @@
DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
PyStackRef_CLOSE(str_st);
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -654,7 +654,7 @@
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
assert(res_o != NULL);
Py_INCREF(res_o);
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
PyStackRef_CLOSE(tuple_st);
res = PyStackRef_FromPyObjectSteal(res_o);
stack_pointer[-2] = res;
@@ -3333,8 +3333,8 @@
Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o);
// 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg
int sign_ish = COMPARISON_BIT(ileft, iright);
- PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free);
- PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False;
// It's always a bool, so we don't care about oparg & 16.
}
@@ -7721,7 +7721,7 @@
assert(old_value != NULL);
UNLOCK_OBJECT(list); // unlock before decrefs!
Py_DECREF(old_value);
- PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
+ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
PyStackRef_CLOSE(list_st);
stack_pointer += -3;
assert(WITHIN_STACK_BOUNDS());
generated by cgit v0.12 at 2024-12-29 11:55:39 (GMT)