From 8d59eb1b66c51b2b918da9881c57d07d08df43b7 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Tue, 15 Oct 2019 14:00:16 +0200 Subject: bpo-37961, tracemalloc: add Traceback.total_nframe (GH-15545) Add a total_nframe field to the traces collected by the tracemalloc module. This field indicates the original number of frames before it was truncated. --- Doc/library/tracemalloc.rst | 15 ++++ Lib/test/test_tracemalloc.py | 90 +++++++++++----------- Lib/tracemalloc.py | 26 +++++-- .../2019-08-27-10-30-44.bpo-37961.4nm0zZ.rst | 2 + Modules/_tracemalloc.c | 45 +++++++---- 5 files changed, 114 insertions(+), 64 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-08-27-10-30-44.bpo-37961.4nm0zZ.rst diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 2d327c0..e423470 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -313,6 +313,9 @@ Functions frames. By default, a trace of a memory block only stores the most recent frame: the limit is ``1``. *nframe* must be greater or equal to ``1``. + You can still read the original number of total frames that composed the + traceback by looking at the :attr:`Traceback.total_nframe` attribute. + Storing more than ``1`` frame is only useful to compute statistics grouped by ``'traceback'`` or to compute cumulative statistics: see the :meth:`Snapshot.compare_to` and :meth:`Snapshot.statistics` methods. @@ -659,6 +662,9 @@ Traceback When a snapshot is taken, tracebacks of traces are limited to :func:`get_traceback_limit` frames. See the :func:`take_snapshot` function. + The original number of frames of the traceback is stored in the + :attr:`Traceback.total_nframe` attribute. That allows to know if a traceback + has been truncated by the traceback limit. The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback` instance. @@ -666,6 +672,15 @@ Traceback .. versionchanged:: 3.7 Frames are now sorted from the oldest to the most recent, instead of most recent to oldest. + .. attribute:: total_nframe + + Total number of frames that composed the traceback before truncation. + This attribute can be set to ``None`` if the information is not + available. + + .. versionchanged:: 3.9 + The :attr:`Traceback.total_nframe` attribute was added. + .. method:: format(limit=None, most_recent_first=False) Format the traceback as a list of lines with newlines. Use the diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 4b9bf4e..7283d9c 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -36,7 +36,7 @@ def allocate_bytes(size): bytes_len = (size - EMPTY_STRING_SIZE) frames = get_frames(nframe, 1) data = b'x' * bytes_len - return data, tracemalloc.Traceback(frames) + return data, tracemalloc.Traceback(frames, min(len(frames), nframe)) def create_snapshots(): traceback_limit = 2 @@ -45,27 +45,27 @@ def create_snapshots(): # traceback_frames) tuples. traceback_frames is a tuple of (filename, # line_number) tuples. raw_traces = [ - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), - (1, 2, (('a.py', 5), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4)), 3), - (2, 66, (('b.py', 1),)), + (2, 66, (('b.py', 1),), 1), - (3, 7, (('', 0),)), + (3, 7, (('', 0),), 1), ] snapshot = tracemalloc.Snapshot(raw_traces, traceback_limit) raw_traces2 = [ - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), - (2, 2, (('a.py', 5), ('b.py', 4))), - (2, 5000, (('a.py', 5), ('b.py', 4))), + (2, 2, (('a.py', 5), ('b.py', 4)), 3), + (2, 5000, (('a.py', 5), ('b.py', 4)), 3), - (4, 400, (('c.py', 578),)), + (4, 400, (('c.py', 578),), 1), ] snapshot2 = tracemalloc.Snapshot(raw_traces2, traceback_limit) @@ -125,7 +125,7 @@ class TestTracemallocEnabled(unittest.TestCase): nframe = tracemalloc.get_traceback_limit() frames = get_frames(nframe, -3) - obj_traceback = tracemalloc.Traceback(frames) + obj_traceback = tracemalloc.Traceback(frames, min(len(frames), nframe)) traceback = tracemalloc.get_object_traceback(obj) self.assertIsNotNone(traceback) @@ -167,7 +167,7 @@ class TestTracemallocEnabled(unittest.TestCase): trace = self.find_trace(traces, obj_traceback) self.assertIsInstance(trace, tuple) - domain, size, traceback = trace + domain, size, traceback, length = trace self.assertEqual(size, obj_size) self.assertEqual(traceback, obj_traceback._frames) @@ -197,8 +197,8 @@ class TestTracemallocEnabled(unittest.TestCase): trace1 = self.find_trace(traces, obj1_traceback) trace2 = self.find_trace(traces, obj2_traceback) - domain1, size1, traceback1 = trace1 - domain2, size2, traceback2 = trace2 + domain1, size1, traceback1, length1 = trace1 + domain2, size2, traceback2, length2 = trace2 self.assertIs(traceback2, traceback1) def test_get_traced_memory(self): @@ -259,6 +259,9 @@ class TestTracemallocEnabled(unittest.TestCase): # take a snapshot snapshot = tracemalloc.take_snapshot() + # This can vary + self.assertGreater(snapshot.traces[1].traceback.total_nframe, 10) + # write on disk snapshot.dump(support.TESTFN) self.addCleanup(support.unlink, support.TESTFN) @@ -321,7 +324,7 @@ class TestSnapshot(unittest.TestCase): maxDiff = 4000 def test_create_snapshot(self): - raw_traces = [(0, 5, (('a.py', 2),))] + raw_traces = [(0, 5, (('a.py', 2),), 10)] with contextlib.ExitStack() as stack: stack.enter_context(patch.object(tracemalloc, 'is_tracing', @@ -336,6 +339,7 @@ class TestSnapshot(unittest.TestCase): self.assertEqual(len(snapshot.traces), 1) trace = snapshot.traces[0] self.assertEqual(trace.size, 5) + self.assertEqual(trace.traceback.total_nframe, 10) self.assertEqual(len(trace.traceback), 1) self.assertEqual(trace.traceback[0].filename, 'a.py') self.assertEqual(trace.traceback[0].lineno, 2) @@ -351,11 +355,11 @@ class TestSnapshot(unittest.TestCase): # exclude b.py snapshot3 = snapshot.filter_traces((filter1,)) self.assertEqual(snapshot3.traces._traces, [ - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (1, 2, (('a.py', 5), ('b.py', 4))), - (3, 7, (('', 0),)), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (1, 2, (('a.py', 5), ('b.py', 4)), 3), + (3, 7, (('', 0),), 1), ]) # filter_traces() must not touch the original snapshot @@ -364,10 +368,10 @@ class TestSnapshot(unittest.TestCase): # only include two lines of a.py snapshot4 = snapshot3.filter_traces((filter2, filter3)) self.assertEqual(snapshot4.traces._traces, [ - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (1, 2, (('a.py', 5), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (1, 2, (('a.py', 5), ('b.py', 4)), 3), ]) # No filter: just duplicate the snapshot @@ -388,21 +392,21 @@ class TestSnapshot(unittest.TestCase): # exclude a.py of domain 1 snapshot3 = snapshot.filter_traces((filter1,)) self.assertEqual(snapshot3.traces._traces, [ - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (2, 66, (('b.py', 1),)), - (3, 7, (('', 0),)), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (2, 66, (('b.py', 1),), 1), + (3, 7, (('', 0),), 1), ]) # include domain 1 snapshot3 = snapshot.filter_traces((filter1,)) self.assertEqual(snapshot3.traces._traces, [ - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (2, 66, (('b.py', 1),)), - (3, 7, (('', 0),)), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (2, 66, (('b.py', 1),), 1), + (3, 7, (('', 0),), 1), ]) def test_filter_traces_domain_filter(self): @@ -413,17 +417,17 @@ class TestSnapshot(unittest.TestCase): # exclude domain 2 snapshot3 = snapshot.filter_traces((filter1,)) self.assertEqual(snapshot3.traces._traces, [ - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (0, 10, (('a.py', 2), ('b.py', 4))), - (1, 2, (('a.py', 5), ('b.py', 4))), - (2, 66, (('b.py', 1),)), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (0, 10, (('a.py', 2), ('b.py', 4)), 3), + (1, 2, (('a.py', 5), ('b.py', 4)), 3), + (2, 66, (('b.py', 1),), 1), ]) # include domain 2 snapshot3 = snapshot.filter_traces((filter2,)) self.assertEqual(snapshot3.traces._traces, [ - (3, 7, (('', 0),)), + (3, 7, (('', 0),), 1), ]) def test_snapshot_group_by_line(self): diff --git a/Lib/tracemalloc.py b/Lib/tracemalloc.py index 80b521c..69b4170 100644 --- a/Lib/tracemalloc.py +++ b/Lib/tracemalloc.py @@ -182,15 +182,20 @@ class Traceback(Sequence): Sequence of Frame instances sorted from the oldest frame to the most recent frame. """ - __slots__ = ("_frames",) + __slots__ = ("_frames", '_total_nframe') - def __init__(self, frames): + def __init__(self, frames, total_nframe=None): Sequence.__init__(self) # frames is a tuple of frame tuples: see Frame constructor for the # format of a frame tuple; it is reversed, because _tracemalloc # returns frames sorted from most recent to oldest, but the # Python API expects oldest to most recent self._frames = tuple(reversed(frames)) + self._total_nframe = total_nframe + + @property + def total_nframe(self): + return self._total_nframe def __len__(self): return len(self._frames) @@ -221,7 +226,12 @@ class Traceback(Sequence): return str(self[0]) def __repr__(self): - return "" % (tuple(self),) + s = "" + return s def format(self, limit=None, most_recent_first=False): lines = [] @@ -280,7 +290,7 @@ class Trace: @property def traceback(self): - return Traceback(self._trace[2]) + return Traceback(*self._trace[2:]) def __eq__(self, other): if not isinstance(other, Trace): @@ -378,7 +388,7 @@ class Filter(BaseFilter): return self._match_frame(filename, lineno) def _match(self, trace): - domain, size, traceback = trace + domain, size, traceback, total_nframe = trace res = self._match_traceback(traceback) if self.domain is not None: if self.inclusive: @@ -398,7 +408,7 @@ class DomainFilter(BaseFilter): return self._domain def _match(self, trace): - domain, size, traceback = trace + domain, size, traceback, total_nframe = trace return (domain == self.domain) ^ (not self.inclusive) @@ -475,7 +485,7 @@ class Snapshot: tracebacks = {} if not cumulative: for trace in self.traces._traces: - domain, size, trace_traceback = trace + domain, size, trace_traceback, total_nframe = trace try: traceback = tracebacks[trace_traceback] except KeyError: @@ -496,7 +506,7 @@ class Snapshot: else: # cumulative statistics for trace in self.traces._traces: - domain, size, trace_traceback = trace + domain, size, trace_traceback, total_nframe = trace for frame in trace_traceback: try: traceback = tracebacks[frame] diff --git a/Misc/NEWS.d/next/Library/2019-08-27-10-30-44.bpo-37961.4nm0zZ.rst b/Misc/NEWS.d/next/Library/2019-08-27-10-30-44.bpo-37961.4nm0zZ.rst new file mode 100644 index 0000000..ebfff77 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-08-27-10-30-44.bpo-37961.4nm0zZ.rst @@ -0,0 +1,2 @@ +Add a ``total_nframe`` field to the traces collected by the tracemalloc module. +This field indicates the original number of frames before it was truncated. diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 26d7600..211c6fb 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -78,15 +78,20 @@ __attribute__((packed)) typedef struct { Py_uhash_t hash; - int nframe; + /* Number of frames stored */ + uint16_t nframe; + /* Total number of frames the traceback had */ + uint16_t total_nframe; frame_t frames[1]; } traceback_t; #define TRACEBACK_SIZE(NFRAME) \ (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME - 1)) -#define MAX_NFRAME \ - ((INT_MAX - (int)sizeof(traceback_t)) / (int)sizeof(frame_t) + 1) +/* The maximum number of frames is either: + - The maximum number of frames we can store in `traceback_t.nframe` + - The maximum memory size_t we can allocate */ +static const unsigned long MAX_NFRAME = Py_MIN(UINT16_MAX, ((SIZE_MAX - sizeof(traceback_t)) / sizeof(frame_t) + 1)); static PyObject *unknown_filename = NULL; @@ -308,6 +313,9 @@ hashtable_compare_traceback(_Py_hashtable_t *ht, const void *pkey, if (traceback1->nframe != traceback2->nframe) return 0; + if (traceback1->total_nframe != traceback2->total_nframe) + return 0; + for (i=0; i < traceback1->nframe; i++) { frame1 = &traceback1->frames[i]; frame2 = &traceback2->frames[i]; @@ -416,6 +424,7 @@ traceback_hash(traceback_t *traceback) /* the cast might truncate len; that doesn't change hash stability */ mult += (Py_uhash_t)(82520UL + len + len); } + x ^= traceback->total_nframe; x += 97531UL; return x; } @@ -436,11 +445,13 @@ traceback_get_frames(traceback_t *traceback) } for (pyframe = tstate->frame; pyframe != NULL; pyframe = pyframe->f_back) { - tracemalloc_get_frame(pyframe, &traceback->frames[traceback->nframe]); - assert(traceback->frames[traceback->nframe].filename != NULL); - traceback->nframe++; - if (traceback->nframe == _Py_tracemalloc_config.max_nframe) - break; + if (traceback->nframe < _Py_tracemalloc_config.max_nframe) { + tracemalloc_get_frame(pyframe, &traceback->frames[traceback->nframe]); + assert(traceback->frames[traceback->nframe].filename != NULL); + traceback->nframe++; + } + if (traceback->total_nframe < UINT16_MAX) + traceback->total_nframe++; } } @@ -456,6 +467,7 @@ traceback_new(void) /* get frames */ traceback = tracemalloc_traceback; traceback->nframe = 0; + traceback->total_nframe = 0; traceback_get_frames(traceback); if (traceback->nframe == 0) return &tracemalloc_empty_traceback; @@ -1001,6 +1013,7 @@ tracemalloc_init(void) PyUnicode_InternInPlace(&unknown_filename); tracemalloc_empty_traceback.nframe = 1; + tracemalloc_empty_traceback.total_nframe = 1; /* borrowed reference */ tracemalloc_empty_traceback.frames[0].filename = unknown_filename; tracemalloc_empty_traceback.frames[0].lineno = 0; @@ -1046,10 +1059,10 @@ tracemalloc_start(int max_nframe) PyMemAllocatorEx alloc; size_t size; - if (max_nframe < 1 || max_nframe > MAX_NFRAME) { + if (max_nframe < 1 || (unsigned long) max_nframe > MAX_NFRAME) { PyErr_Format(PyExc_ValueError, - "the number of frames must be in range [1; %i]", - (int)MAX_NFRAME); + "the number of frames must be in range [1; %lu]", + MAX_NFRAME); return -1; } @@ -1062,7 +1075,6 @@ tracemalloc_start(int max_nframe) return 0; } - assert(1 <= max_nframe && max_nframe <= MAX_NFRAME); _Py_tracemalloc_config.max_nframe = max_nframe; /* allocate a buffer to store a new traceback */ @@ -1234,7 +1246,7 @@ trace_to_pyobject(unsigned int domain, trace_t *trace, PyObject *trace_obj = NULL; PyObject *obj; - trace_obj = PyTuple_New(3); + trace_obj = PyTuple_New(4); if (trace_obj == NULL) return NULL; @@ -1259,6 +1271,13 @@ trace_to_pyobject(unsigned int domain, trace_t *trace, } PyTuple_SET_ITEM(trace_obj, 2, obj); + obj = PyLong_FromUnsignedLong(trace->traceback->total_nframe); + if (obj == NULL) { + Py_DECREF(trace_obj); + return NULL; + } + PyTuple_SET_ITEM(trace_obj, 3, obj); + return trace_obj; } -- cgit v0.12