diff options
31 files changed, 1305 insertions, 18 deletions
@@ -56,6 +56,7 @@ libpython*.dylib *.profclang? *.profraw *.dyn +Include/pydtrace_probes.h Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index de65950..593341c 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -28,4 +28,5 @@ Currently, the HOWTOs are: argparse.rst ipaddress.rst clinic.rst + instrumentation.rst diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst new file mode 100644 index 0000000..2b21224 --- /dev/null +++ b/Doc/howto/instrumentation.rst @@ -0,0 +1,411 @@ +.. _instrumentation: + +=============================================== +Instrumenting CPython with DTrace and SystemTap +=============================================== + +:author: David Malcolm +:author: Łukasz Langa + +DTrace and SystemTap are monitoring tools, each providing a way to inspect +what the processes on a computer system are doing. They both use +domain-specific languages allowing a user to write scripts which: + + - filter which processes are to be observed + - gather data from the processes of interest + - generate reports on the data + +As of Python 3.6, CPython can be built with embedded "markers", also +known as "probes", that can be observed by a DTrace or SystemTap script, +making it easier to monitor what the CPython processes on a system are +doing. + +.. I'm using ".. code-block:: c" for SystemTap scripts, as "c" is syntactically + the closest match that Sphinx supports + +.. impl-detail:: + + DTrace markers are implementation details of the CPython interpreter. + No guarantees are made about probe compatibility between versions of + CPython. DTrace scripts can stop working or work incorrectly without + warning when changing CPython versions. + + +Enabling the static markers +--------------------------- + +macOS comes with built-in support for DTrace. On Linux, in order to +build CPython with the embedded markers for SystemTap, the SystemTap +development tools must be installed. + +On a Linux machine, this can be done via:: + + yum install systemtap-sdt-devel + +or:: + + sudo apt-get install systemtap-sdt-dev + + +CPython must then be configured `--with-dtrace`:: + + checking for --with-dtrace... yes + +On macOS, you can list available DTrace probes by running a Python +process in the background and listing all probes made available by the +Python provider:: + + $ python3.6 -q & + $ sudo dtrace -l -P python$! # or: dtrace -l -m python3.6 + + ID PROVIDER MODULE FUNCTION NAME + 29564 python18035 python3.6 _PyEval_EvalFrameDefault function-entry + 29565 python18035 python3.6 dtrace_function_entry function-entry + 29566 python18035 python3.6 _PyEval_EvalFrameDefault function-return + 29567 python18035 python3.6 dtrace_function_return function-return + 29568 python18035 python3.6 collect gc-done + 29569 python18035 python3.6 collect gc-start + 29570 python18035 python3.6 _PyEval_EvalFrameDefault line + 29571 python18035 python3.6 maybe_dtrace_line line + +On Linux, you can verify if the SystemTap static markers are present in +the built binary by seeing if it contains a ".note.stapsdt" section. + +.. code-block:: bash + + $ readelf -S ./python | grep .note.stapsdt + [30] .note.stapsdt NOTE 0000000000000000 00308d78 + +If you've built Python as a shared library (with --enable-shared), you +need to look instead within the shared library. For example: + +.. code-block:: bash + + $ readelf -S libpython3.3dm.so.1.0 | grep .note.stapsdt + [29] .note.stapsdt NOTE 0000000000000000 00365b68 + +Sufficiently modern readelf can print the metadata: + +.. code-block:: bash + + $ readelf -n ./python + + Displaying notes found at file offset 0x00000254 with length 0x00000020: + Owner Data size Description + GNU 0x00000010 NT_GNU_ABI_TAG (ABI version tag) + OS: Linux, ABI: 2.6.32 + + Displaying notes found at file offset 0x00000274 with length 0x00000024: + Owner Data size Description + GNU 0x00000014 NT_GNU_BUILD_ID (unique build ID bitstring) + Build ID: df924a2b08a7e89f6e11251d4602022977af2670 + + Displaying notes found at file offset 0x002d6c30 with length 0x00000144: + Owner Data size Description + stapsdt 0x00000031 NT_STAPSDT (SystemTap probe descriptors) + Provider: python + Name: gc__start + Location: 0x00000000004371c3, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6bf6 + Arguments: -4@%ebx + stapsdt 0x00000030 NT_STAPSDT (SystemTap probe descriptors) + Provider: python + Name: gc__done + Location: 0x00000000004374e1, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6bf8 + Arguments: -8@%rax + stapsdt 0x00000045 NT_STAPSDT (SystemTap probe descriptors) + Provider: python + Name: function__entry + Location: 0x000000000053db6c, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6be8 + Arguments: 8@%rbp 8@%r12 -4@%eax + stapsdt 0x00000046 NT_STAPSDT (SystemTap probe descriptors) + Provider: python + Name: function__return + Location: 0x000000000053dba8, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6bea + Arguments: 8@%rbp 8@%r12 -4@%eax + +The above metadata contains information for SystemTap describing how it +can patch strategically-placed machine code instructions to enable the +tracing hooks used by a SystemTap script. + + +Static DTrace probes +-------------------- + +The following example DTrace script can be used to show the call/return +hierarchy of a Python script, only tracing within the invocation of +a function called "start". In other words, import-time function +invocations are not going to be listed: + +.. code-block:: c + + self int indent; + + python$target:::function-entry + /copyinstr(arg1) == "start"/ + { + self->trace = 1; + } + + python$target:::function-entry + /self->trace/ + { + printf("%d\t%*s:", timestamp, 15, probename); + printf("%*s", self->indent, ""); + printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); + self->indent++; + } + + python$target:::function-return + /self->trace/ + { + self->indent--; + printf("%d\t%*s:", timestamp, 15, probename); + printf("%*s", self->indent, ""); + printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); + } + + python$target:::function-return + /copyinstr(arg1) == "start"/ + { + self->trace = 0; + } + +It can be invoked like this: + +.. code-block:: bash + + $ sudo dtrace -q -s call_stack.d -c "python3.6 script.py" + +The output looks like this:: + + 156641360502280 function-entry:call_stack.py:start:23 + 156641360518804 function-entry: call_stack.py:function_1:1 + 156641360532797 function-entry: call_stack.py:function_3:9 + 156641360546807 function-return: call_stack.py:function_3:10 + 156641360563367 function-return: call_stack.py:function_1:2 + 156641360578365 function-entry: call_stack.py:function_2:5 + 156641360591757 function-entry: call_stack.py:function_1:1 + 156641360605556 function-entry: call_stack.py:function_3:9 + 156641360617482 function-return: call_stack.py:function_3:10 + 156641360629814 function-return: call_stack.py:function_1:2 + 156641360642285 function-return: call_stack.py:function_2:6 + 156641360656770 function-entry: call_stack.py:function_3:9 + 156641360669707 function-return: call_stack.py:function_3:10 + 156641360687853 function-entry: call_stack.py:function_4:13 + 156641360700719 function-return: call_stack.py:function_4:14 + 156641360719640 function-entry: call_stack.py:function_5:18 + 156641360732567 function-return: call_stack.py:function_5:21 + 156641360747370 function-return:call_stack.py:start:28 + + +Static SystemTap markers +------------------------ + +The low-level way to use the SystemTap integration is to use the static +markers directly. This requires you to explicitly state the binary file +containing them. + +For example, this SystemTap script can be used to show the call/return +hierarchy of a Python script: + +.. code-block:: c + + probe process('python').mark("function__entry") { + filename = user_string($arg1); + funcname = user_string($arg2); + lineno = $arg3; + + printf("%s => %s in %s:%d\\n", + thread_indent(1), funcname, filename, lineno); + } + + probe process('python').mark("function__return") { + filename = user_string($arg1); + funcname = user_string($arg2); + lineno = $arg3; + + printf("%s <= %s in %s:%d\\n", + thread_indent(-1), funcname, filename, lineno); + } + +It can be invoked like this: + +.. code-block:: bash + + $ stap \ + show-call-hierarchy.stp \ + -c ./python test.py + +The output looks like this:: + + 11408 python(8274): => __contains__ in Lib/_abcoll.py:362 + 11414 python(8274): => __getitem__ in Lib/os.py:425 + 11418 python(8274): => encode in Lib/os.py:490 + 11424 python(8274): <= encode in Lib/os.py:493 + 11428 python(8274): <= __getitem__ in Lib/os.py:426 + 11433 python(8274): <= __contains__ in Lib/_abcoll.py:366 + +where the columns are: + + - time in microseconds since start of script + + - name of executable + + - PID of process + +and the remainder indicates the call/return hierarchy as the script executes. + +For a `--enable-shared` build of CPython, the markers are contained within the +libpython shared library, and the probe's dotted path needs to reflect this. For +example, this line from the above example:: + + probe process('python').mark("function__entry") { + +should instead read:: + + probe process('python').library("libpython3.6dm.so.1.0").mark("function__entry") { + +(assuming a debug build of CPython 3.6) + + +Available static markers +------------------------ + +.. I'm reusing the "c:function" type for markers + +.. c:function:: function__entry(str filename, str funcname, int lineno) + + This marker indicates that execution of a Python function has begun. + It is only triggered for pure-Python (bytecode) functions. + + The filename, function name, and line number are provided back to the + tracing script as positional arguments, which must be accessed using + `$arg1`, `$arg2`, `$arg3`: + + * `$arg1` : `(const char *)` filename, accessible using `user_string($arg1)` + + * `$arg2` : `(const char *)` function name, accessible using + `user_string($arg2)` + + * `$arg3` : `int` line number + +.. c:function:: function__return(str filename, str funcname, int lineno) + + This marker is the converse of `function__entry`, and indicates that + execution of a Python function has ended (either via ``return``, or + via an exception). It is only triggered for pure-Python (bytecode) + functions. + + The arguments are the same as for `function__entry` + +.. c:function:: line(str filename, str funcname, int lineno) + + This marker indicates a Python line is about to be executed. It is + the equivalent of line-by-line tracing with a Python profiler. It is + not triggered within C functions. + + The arguments are the same as for `function__entry`. + +.. c:function:: gc__start(int generation) + + Fires when the Python interpreter starts a garbage collection cycle. + `arg0` is the generation to scan, like :func:`gc.collect()`. + +.. c:function:: gc__done(long collected) + + Fires when the Python interpreter finishes a garbage collection + cycle. `arg0` is the number of collected objects. + + +SystemTap Tapsets +----------------- + +The higher-level way to use the SystemTap integration is to use a "tapset": +SystemTap's equivalent of a library, which hides some of the lower-level +details of the static markers. + +Here is a tapset file, based on a non-shared build of CPython: + +.. code-block:: c + + /* + Provide a higher-level wrapping around the function__entry and + function__return markers: + \*/ + probe python.function.entry = process("python").mark("function__entry") + { + filename = user_string($arg1); + funcname = user_string($arg2); + lineno = $arg3; + frameptr = $arg4 + } + probe python.function.return = process("python").mark("function__return") + { + filename = user_string($arg1); + funcname = user_string($arg2); + lineno = $arg3; + frameptr = $arg4 + } + +If this file is installed in SystemTap's tapset directory (e.g. +`/usr/share/systemtap/tapset`), then these additional probepoints become +available: + +.. c:function:: python.function.entry(str filename, str funcname, int lineno, frameptr) + + This probe point indicates that execution of a Python function has begun. + It is only triggered for pure-python (bytecode) functions. + +.. c:function:: python.function.return(str filename, str funcname, int lineno, frameptr) + + This probe point is the converse of `python.function.return`, and indicates + that execution of a Python function has ended (either via ``return``, or + via an exception). It is only triggered for pure-python (bytecode) functions. + + +Examples +-------- +This SystemTap script uses the tapset above to more cleanly implement the +example given above of tracing the Python function-call hierarchy, without +needing to directly name the static markers: + +.. code-block:: c + + probe python.function.entry + { + printf("%s => %s in %s:%d\n", + thread_indent(1), funcname, filename, lineno); + } + + probe python.function.return + { + printf("%s <= %s in %s:%d\n", + thread_indent(-1), funcname, filename, lineno); + } + + +The following script uses the tapset above to provide a top-like view of all +running CPython code, showing the top 20 most frequently-entered bytecode +frames, each second, across the whole system: + +.. code-block:: c + + global fn_calls; + + probe python.function.entry + { + fn_calls[pid(), filename, funcname, lineno] += 1; + } + + probe timer.ms(1000) { + printf("\033[2J\033[1;1H") /* clear screen \*/ + printf("%6s %80s %6s %30s %6s\n", + "PID", "FILENAME", "LINE", "FUNCTION", "CALLS") + foreach ([pid, filename, funcname, lineno] in fn_calls- limit 20) { + printf("%6d %80s %6d %30s %6d\n", + pid, filename, lineno, funcname, + fn_calls[pid, filename, funcname, lineno]); + } + delete fn_calls; + } + diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index d3dd337..7b4d0b3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -407,6 +407,31 @@ Example of fatal error on buffer overflow using (Contributed by Victor Stinner in :issue:`26516` and :issue:`26564`.) +DTrace and SystemTap probing support +------------------------------------ + +Python can now be built ``--with-dtrace`` which enables static markers +for the following events in the interpreter: + +* function call/return + +* garbage collection started/finished + +* line of code executed. + +This can be used to instrument running interpreters in production, +without the need to recompile specific debug builds or providing +application-specific profiling/debugging code. + +More details in:ref:`instrumentation`. + +The current implementation is tested on Linux and macOS. Additional +markers may be added in the future. + +(Contributed by Łukasz Langa in :issue:`21590`, based on patches by +Jesús Cea Avión, David Malcolm, and Nikhil Benesch.) + + .. _whatsnew-deforder: PEP 520: Preserving Class Attribute Definition Order diff --git a/Include/pydtrace.d b/Include/pydtrace.d new file mode 100644 index 0000000..8836055 --- /dev/null +++ b/Include/pydtrace.d @@ -0,0 +1,19 @@ +/* Python DTrace provider */ + +provider python { + probe function__entry(const char *, const char *, int); + probe function__return(const char *, const char *, int); + probe instance__new__start(const char *, const char *); + probe instance__new__done(const char *, const char *); + probe instance__delete__start(const char *, const char *); + probe instance__delete__done(const char *, const char *); + probe line(const char *, const char *, int); + probe gc__start(int); + probe gc__done(long); +}; + +#pragma D attributes Evolving/Evolving/Common provider python provider +#pragma D attributes Evolving/Evolving/Common provider python module +#pragma D attributes Evolving/Evolving/Common provider python function +#pragma D attributes Evolving/Evolving/Common provider python name +#pragma D attributes Evolving/Evolving/Common provider python args diff --git a/Include/pydtrace.h b/Include/pydtrace.h new file mode 100644 index 0000000..4c06d0e --- /dev/null +++ b/Include/pydtrace.h @@ -0,0 +1,47 @@ +/* Static DTrace probes interface */ + +#ifndef Py_DTRACE_H +#define Py_DTRACE_H + +#ifdef WITH_DTRACE + +#include "pydtrace_probes.h" + +/* pydtrace_probes.h, on systems with DTrace, is auto-generated to include + `PyDTrace_{PROBE}` and `PyDTrace_{PROBE}_ENABLED()` macros for every probe + defined in pydtrace_provider.d. + + Calling these functions must be guarded by a `PyDTrace_{PROBE}_ENABLED()` + check to minimize performance impact when probing is off. For example: + + if (PyDTrace_FUNCTION_ENTRY_ENABLED()) + PyDTrace_FUNCTION_ENTRY(f); +*/ + +#else + +/* Without DTrace, compile to nothing. */ + +#define PyDTrace_LINE(arg0, arg1, arg2, arg3) do ; while (0) +#define PyDTrace_FUNCTION_ENTRY(arg0, arg1, arg2) do ; while (0) +#define PyDTrace_FUNCTION_RETURN(arg0, arg1, arg2) do ; while (0) +#define PyDTrace_GC_START(arg0) do ; while (0) +#define PyDTrace_GC_DONE(arg0) do ; while (0) +#define PyDTrace_INSTANCE_NEW_START(arg0) do ; while (0) +#define PyDTrace_INSTANCE_NEW_DONE(arg0) do ; while (0) +#define PyDTrace_INSTANCE_DELETE_START(arg0) do ; while (0) +#define PyDTrace_INSTANCE_DELETE_DONE(arg0) do ; while (0) + +#define PyDTrace_LINE_ENABLED() (0) +#define PyDTrace_FUNCTION_ENTRY_ENABLED() (0) +#define PyDTrace_FUNCTION_RETURN_ENABLED() (0) +#define PyDTrace_GC_START_ENABLED() (0) +#define PyDTrace_GC_DONE_ENABLED() (0) +#define PyDTrace_INSTANCE_NEW_START_ENABLED() (0) +#define PyDTrace_INSTANCE_NEW_DONE_ENABLED() (0) +#define PyDTrace_INSTANCE_DELETE_START_ENABLED() (0) +#define PyDTrace_INSTANCE_DELETE_DONE_ENABLED() (0) + +#endif /* !WITH_DTRACE */ + +#endif /* !Py_DTRACE_H */ diff --git a/Lib/test/dtracedata/assert_usable.d b/Lib/test/dtracedata/assert_usable.d new file mode 100644 index 0000000..0b2d4da --- /dev/null +++ b/Lib/test/dtracedata/assert_usable.d @@ -0,0 +1,5 @@ +BEGIN +{ + printf("probe: success\n"); + exit(0); +} diff --git a/Lib/test/dtracedata/assert_usable.stp b/Lib/test/dtracedata/assert_usable.stp new file mode 100644 index 0000000..88e7e68 --- /dev/null +++ b/Lib/test/dtracedata/assert_usable.stp @@ -0,0 +1,5 @@ +probe begin +{ + println("probe: success") + exit () +} diff --git a/Lib/test/dtracedata/call_stack.d b/Lib/test/dtracedata/call_stack.d new file mode 100644 index 0000000..450e939 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.d @@ -0,0 +1,31 @@ +self int indent; + +python$target:::function-entry +/copyinstr(arg1) == "start"/ +{ + self->trace = 1; +} + +python$target:::function-entry +/self->trace/ +{ + printf("%d\t%*s:", timestamp, 15, probename); + printf("%*s", self->indent, ""); + printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); + self->indent++; +} + +python$target:::function-return +/self->trace/ +{ + self->indent--; + printf("%d\t%*s:", timestamp, 15, probename); + printf("%*s", self->indent, ""); + printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); +} + +python$target:::function-return +/copyinstr(arg1) == "start"/ +{ + self->trace = 0; +} diff --git a/Lib/test/dtracedata/call_stack.d.expected b/Lib/test/dtracedata/call_stack.d.expected new file mode 100644 index 0000000..27849d1 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.d.expected @@ -0,0 +1,18 @@ + function-entry:call_stack.py:start:23 + function-entry: call_stack.py:function_1:1 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 +function-return: call_stack.py:function_1:2 + function-entry: call_stack.py:function_2:5 + function-entry: call_stack.py:function_1:1 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 +function-return: call_stack.py:function_1:2 +function-return: call_stack.py:function_2:6 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 + function-entry: call_stack.py:function_4:13 +function-return: call_stack.py:function_4:14 + function-entry: call_stack.py:function_5:18 +function-return: call_stack.py:function_5:21 +function-return:call_stack.py:start:28 diff --git a/Lib/test/dtracedata/call_stack.py b/Lib/test/dtracedata/call_stack.py new file mode 100644 index 0000000..ee9f3ae --- /dev/null +++ b/Lib/test/dtracedata/call_stack.py @@ -0,0 +1,30 @@ +def function_1(): + function_3(1, 2) + +# Check stacktrace +def function_2(): + function_1() + +# CALL_FUNCTION_VAR +def function_3(dummy, dummy2): + pass + +# CALL_FUNCTION_KW +def function_4(**dummy): + return 1 + return 2 # unreachable + +# CALL_FUNCTION_VAR_KW +def function_5(dummy, dummy2, **dummy3): + if False: + return 7 + return 8 + +def start(): + function_1() + function_2() + function_3(1, 2) + function_4(test=42) + function_5(*(1, 2), **{"test": 42}) + +start() diff --git a/Lib/test/dtracedata/call_stack.stp b/Lib/test/dtracedata/call_stack.stp new file mode 100644 index 0000000..54082c2 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.stp @@ -0,0 +1,41 @@ +global tracing + +function basename:string(path:string) +{ + last_token = token = tokenize(path, "/"); + while (token != "") { + last_token = token; + token = tokenize("", "/"); + } + return last_token; +} + +probe process.mark("function__entry") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 1; + } +} + +probe process.mark("function__entry"), process.mark("function__return") +{ + filename = user_string($arg1); + funcname = user_string($arg2); + lineno = $arg3; + + if (tracing) { + printf("%d\t%s:%s:%s:%d\n", gettimeofday_us(), $$name, + basename(filename), funcname, lineno); + } +} + +probe process.mark("function__return") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 0; + } +} diff --git a/Lib/test/dtracedata/call_stack.stp.expected b/Lib/test/dtracedata/call_stack.stp.expected new file mode 100644 index 0000000..32cf396 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.stp.expected @@ -0,0 +1,14 @@ +function__entry:call_stack.py:start:23 +function__entry:call_stack.py:function_1:1 +function__return:call_stack.py:function_1:2 +function__entry:call_stack.py:function_2:5 +function__entry:call_stack.py:function_1:1 +function__return:call_stack.py:function_1:2 +function__return:call_stack.py:function_2:6 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 +function__entry:call_stack.py:function_4:13 +function__return:call_stack.py:function_4:14 +function__entry:call_stack.py:function_5:18 +function__return:call_stack.py:function_5:21 +function__return:call_stack.py:start:28 diff --git a/Lib/test/dtracedata/gc.d b/Lib/test/dtracedata/gc.d new file mode 100644 index 0000000..4d91487 --- /dev/null +++ b/Lib/test/dtracedata/gc.d @@ -0,0 +1,18 @@ +python$target:::function-entry +/copyinstr(arg1) == "start"/ +{ + self->trace = 1; +} + +python$target:::gc-start, +python$target:::gc-done +/self->trace/ +{ + printf("%d\t%s:%ld\n", timestamp, probename, arg0); +} + +python$target:::function-return +/copyinstr(arg1) == "start"/ +{ + self->trace = 0; +} diff --git a/Lib/test/dtracedata/gc.d.expected b/Lib/test/dtracedata/gc.d.expected new file mode 100644 index 0000000..8e5ac2a --- /dev/null +++ b/Lib/test/dtracedata/gc.d.expected @@ -0,0 +1,8 @@ +gc-start:0 +gc-done:0 +gc-start:1 +gc-done:0 +gc-start:2 +gc-done:0 +gc-start:2 +gc-done:1 diff --git a/Lib/test/dtracedata/gc.py b/Lib/test/dtracedata/gc.py new file mode 100644 index 0000000..144a783 --- /dev/null +++ b/Lib/test/dtracedata/gc.py @@ -0,0 +1,13 @@ +import gc + +def start(): + gc.collect(0) + gc.collect(1) + gc.collect(2) + l = [] + l.append(l) + del l + gc.collect(2) + +gc.collect() +start() diff --git a/Lib/test/dtracedata/gc.stp b/Lib/test/dtracedata/gc.stp new file mode 100644 index 0000000..162c6d3 --- /dev/null +++ b/Lib/test/dtracedata/gc.stp @@ -0,0 +1,26 @@ +global tracing + +probe process.mark("function__entry") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 1; + } +} + +probe process.mark("gc__start"), process.mark("gc__done") +{ + if (tracing) { + printf("%d\t%s:%ld\n", gettimeofday_us(), $$name, $arg1); + } +} + +probe process.mark("function__return") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 0; + } +} diff --git a/Lib/test/dtracedata/gc.stp.expected b/Lib/test/dtracedata/gc.stp.expected new file mode 100644 index 0000000..7e6e622 --- /dev/null +++ b/Lib/test/dtracedata/gc.stp.expected @@ -0,0 +1,8 @@ +gc__start:0 +gc__done:0 +gc__start:1 +gc__done:0 +gc__start:2 +gc__done:0 +gc__start:2 +gc__done:1 diff --git a/Lib/test/dtracedata/instance.py b/Lib/test/dtracedata/instance.py new file mode 100644 index 0000000..f142137 --- /dev/null +++ b/Lib/test/dtracedata/instance.py @@ -0,0 +1,24 @@ +import gc + +class old_style_class(): + pass +class new_style_class(object): + pass + +a = old_style_class() +del a +gc.collect() +b = new_style_class() +del b +gc.collect() + +a = old_style_class() +del old_style_class +gc.collect() +b = new_style_class() +del new_style_class +gc.collect() +del a +gc.collect() +del b +gc.collect() diff --git a/Lib/test/dtracedata/line.d b/Lib/test/dtracedata/line.d new file mode 100644 index 0000000..03f22db --- /dev/null +++ b/Lib/test/dtracedata/line.d @@ -0,0 +1,7 @@ +python$target:::line +/(copyinstr(arg1)=="test_line")/ +{ + printf("%d\t%s:%s:%s:%d\n", timestamp, + probename, basename(copyinstr(arg0)), + copyinstr(arg1), arg2); +} diff --git a/Lib/test/dtracedata/line.d.expected b/Lib/test/dtracedata/line.d.expected new file mode 100644 index 0000000..9b16ce7 --- /dev/null +++ b/Lib/test/dtracedata/line.d.expected @@ -0,0 +1,20 @@ +line:line.py:test_line:2 +line:line.py:test_line:3 +line:line.py:test_line:4 +line:line.py:test_line:5 +line:line.py:test_line:6 +line:line.py:test_line:7 +line:line.py:test_line:8 +line:line.py:test_line:9 +line:line.py:test_line:10 +line:line.py:test_line:11 +line:line.py:test_line:4 +line:line.py:test_line:5 +line:line.py:test_line:6 +line:line.py:test_line:7 +line:line.py:test_line:8 +line:line.py:test_line:10 +line:line.py:test_line:11 +line:line.py:test_line:4 +line:line.py:test_line:12 +line:line.py:test_line:13 diff --git a/Lib/test/dtracedata/line.py b/Lib/test/dtracedata/line.py new file mode 100644 index 0000000..0930ff3 --- /dev/null +++ b/Lib/test/dtracedata/line.py @@ -0,0 +1,17 @@ +def test_line(): + a = 1 + print('# Preamble', a) + for i in range(2): + a = i + b = i+2 + c = i+3 + if c < 4: + a = c + d = a + b +c + print('#', a, b, c, d) + a = 1 + print('# Epilogue', a) + + +if __name__ == '__main__': + test_line() diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py new file mode 100644 index 0000000..ca239b3 --- /dev/null +++ b/Lib/test/test_dtrace.py @@ -0,0 +1,178 @@ +import dis +import os.path +import re +import subprocess +import sys +import types +import unittest + +from test.support import findfile, run_unittest + + +def abspath(filename): + return os.path.abspath(findfile(filename, subdir="dtracedata")) + + +def normalize_trace_output(output): + """Normalize DTrace output for comparison. + + DTrace keeps a per-CPU buffer, and when showing the fired probes, buffers + are concatenated. So if the operating system moves our thread around, the + straight result can be "non-causal". So we add timestamps to the probe + firing, sort by that field, then strip it from the output""" + + # When compiling with '--with-pydebug', strip '[# refs]' debug output. + output = re.sub(r"\[[0-9]+ refs\]", "", output) + try: + result = [ + row.split("\t") + for row in output.splitlines() + if row and not row.startswith('#') + ] + result.sort(key=lambda row: int(row[0])) + result = [row[1] for row in result] + return "\n".join(result) + except (IndexError, ValueError): + raise AssertionError( + "tracer produced unparseable output:\n{}".format(output) + ) + + +class TraceBackend: + EXTENSION = None + COMMAND = None + COMMAND_ARGS = [] + + def run_case(self, name, optimize_python=None): + actual_output = normalize_trace_output(self.trace_python( + script_file=abspath(name + self.EXTENSION), + python_file=abspath(name + ".py"), + optimize_python=optimize_python)) + + with open(abspath(name + self.EXTENSION + ".expected")) as f: + expected_output = f.read().rstrip() + + return (expected_output, actual_output) + + def generate_trace_command(self, script_file, subcommand=None): + command = self.COMMAND + [script_file] + if subcommand: + command += ["-c", subcommand] + return command + + def trace(self, script_file, subcommand=None): + command = self.generate_trace_command(script_file, subcommand) + stdout, _ = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True).communicate() + return stdout + + def trace_python(self, script_file, python_file, optimize_python=None): + python_flags = [] + if optimize_python: + python_flags.extend(["-O"] * optimize_python) + subcommand = " ".join([sys.executable] + python_flags + [python_file]) + return self.trace(script_file, subcommand) + + def assert_usable(self): + try: + output = self.trace(abspath("assert_usable" + self.EXTENSION)) + output = output.strip() + except FileNotFoundError as fnfe: + output = str(fnfe) + if output != "probe: success": + raise unittest.SkipTest( + "{}(1) failed: {}".format(self.COMMAND[0], output) + ) + + +class DTraceBackend(TraceBackend): + EXTENSION = ".d" + COMMAND = ["dtrace", "-q", "-s"] + + +class SystemTapBackend(TraceBackend): + EXTENSION = ".stp" + COMMAND = ["stap", "-g"] + + +class TraceTests(unittest.TestCase): + # unittest.TestCase options + maxDiff = None + + # TraceTests options + backend = None + optimize_python = 0 + + @classmethod + def setUpClass(self): + self.backend.assert_usable() + + def run_case(self, name): + actual_output, expected_output = self.backend.run_case( + name, optimize_python=self.optimize_python) + self.assertEqual(actual_output, expected_output) + + def test_function_entry_return(self): + self.run_case("call_stack") + + def test_verify_call_opcodes(self): + """Ensure our call stack test hits all function call opcodes""" + + opcodes = set(["CALL_FUNCTION", "CALL_FUNCTION_EX", "CALL_FUNCTION_KW"]) + + with open(abspath("call_stack.py")) as f: + code_string = f.read() + + def get_function_instructions(funcname): + # Recompile with appropriate optimization setting + code = compile(source=code_string, + filename="<string>", + mode="exec", + optimize=self.optimize_python) + + for c in code.co_consts: + if isinstance(c, types.CodeType) and c.co_name == funcname: + return dis.get_instructions(c) + return [] + + for instruction in get_function_instructions('start'): + opcodes.discard(instruction.opname) + + self.assertEqual(set(), opcodes) + + def test_gc(self): + self.run_case("gc") + + def test_line(self): + self.run_case("line") + + +class DTraceNormalTests(TraceTests): + backend = DTraceBackend() + optimize_python = 0 + + +class DTraceOptimizedTests(TraceTests): + backend = DTraceBackend() + optimize_python = 2 + + +class SystemTapNormalTests(TraceTests): + backend = SystemTapBackend() + optimize_python = 0 + + +class SystemTapOptimizedTests(TraceTests): + backend = SystemTapBackend() + optimize_python = 2 + + +def test_main(): + run_unittest(DTraceNormalTests, DTraceOptimizedTests, SystemTapNormalTests, + SystemTapOptimizedTests) + + +if __name__ == '__main__': + test_main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 4445b24..908ca52 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -48,6 +48,10 @@ PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ LLVM_PROF_FILE=@LLVM_PROF_FILE@ LLVM_PROF_ERR=@LLVM_PROF_ERR@ +DTRACE= @DTRACE@ +DFLAGS= @DFLAGS@ +DTRACE_HEADERS= @DTRACE_HEADERS@ +DTRACE_OBJS= @DTRACE_OBJS@ GNULD= @GNULD@ @@ -315,7 +319,7 @@ OPCODE_H_DIR= $(srcdir)/Include OPCODE_H_SCRIPT= $(srcdir)/Tools/scripts/generate_opcode_h.py OPCODE_H= $(OPCODE_H_DIR)/opcode.h OPCODE_H_GEN= $(PYTHON_FOR_GEN) $(OPCODE_H_SCRIPT) $(srcdir)/Lib/opcode.py $(OPCODE_H) -# + ########################################################################## # AST AST_H_DIR= Include @@ -391,7 +395,8 @@ PYTHON_OBJS= \ Python/$(DYNLOADFILE) \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ - $(THREADOBJ) + $(THREADOBJ) \ + $(DTRACE_OBJS) ########################################################################## @@ -451,6 +456,15 @@ LIBRARY_OBJS= \ $(LIBRARY_OBJS_OMIT_FROZEN) \ Python/frozen.o +########################################################################## +# DTrace + +# On some systems, object files that reference DTrace probes need to be modified +# in-place by dtrace(1). +DTRACE_DEPS = \ + Python/ceval.o +# XXX: should gcmodule, etc. be here, too? + ######################################################################### # Rules @@ -852,6 +866,18 @@ Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h Python/frozen.o: Python/importlib.h Python/importlib_external.h +# Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to +# follow our naming conventions. dtrace(1) uses the output filename to generate +# an include guard, so we can't use a pipeline to transform its output. +Include/pydtrace_probes.h: $(srcdir)/Include/pydtrace.d + $(DTRACE) $(DFLAGS) -o $@ -h -s $< + : sed in-place edit with POSIX-only tools + sed 's/PYTHON_/PyDTrace_/' $@ > $@.tmp + mv $@.tmp $@ + +Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) + $(DTRACE) $(DFLAGS) -o $@ -G -s $< $(DTRACE_DEPS) + Objects/typeobject.o: Objects/typeslots.inc Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py $(PYTHON_FOR_GEN) $(srcdir)/Objects/typeslots.py < $(srcdir)/Include/typeslots.h Objects/typeslots.inc @@ -918,6 +944,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/pycapsule.h \ $(srcdir)/Include/pyctype.h \ $(srcdir)/Include/pydebug.h \ + $(srcdir)/Include/pydtrace.h \ $(srcdir)/Include/pyerrors.h \ $(srcdir)/Include/pyfpe.h \ $(srcdir)/Include/pyhash.h \ @@ -949,7 +976,8 @@ PYTHON_HEADERS= \ $(srcdir)/Include/weakrefobject.h \ pyconfig.h \ $(PARSER_HEADERS) \ - $(AST_H) + $(AST_H) \ + $(DTRACE_HEADERS) $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) @@ -1158,6 +1186,7 @@ LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ test/audiodata \ test/capath test/data \ test/cjkencodings test/decimaltestdata test/xmltestdata \ + test/dtracedata \ test/eintrdata \ test/imghdrdata \ test/libregrtest \ @@ -1569,6 +1598,7 @@ clean: pycremoval -rm -f Lib/lib2to3/*Grammar*.pickle -rm -f Programs/_testembed Programs/_freeze_importlib -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' + -rm -f Include/pydtrace_probes.h profile-removal: find . -name '*.gc??' -exec rm -f {} ';' @@ -120,6 +120,7 @@ Ben Bell Thomas Bellman Alexander “Саша” Belopolsky Eli Bendersky +Nikhil Benesch David Benjamin Oscar Benjamin Andrew Bennetts @@ -329,6 +329,8 @@ Build make time when --with-optimizations is enabled. Also improve our ability to find the llvm-profdata tool on MacOS and some Linuxes. +- Issue #21590: Support for DTrace and SystemTap probes. + - Issue #26307: The profile-opt build now applys PGO to the built-in modules. - Issue #26539: Add the --with-optimizations flag to turn on LTO and PGO build diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 07950a6..2575d96 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -25,6 +25,7 @@ #include "Python.h" #include "frameobject.h" /* for PyFrame_ClearFreeList */ +#include "pydtrace.h" #include "pytime.h" /* for _PyTime_GetMonotonicClock() */ /* Get an object's GC head */ @@ -925,6 +926,9 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, PySys_WriteStderr("\n"); } + if (PyDTrace_GC_START_ENABLED()) + PyDTrace_GC_START(generation); + /* update collection and allocation counters */ if (generation+1 < NUM_GENERATIONS) generations[generation+1].count += 1; @@ -1069,6 +1073,10 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, stats->collections++; stats->collected += m; stats->uncollectable += n; + + if (PyDTrace_GC_DONE_ENABLED()) + PyDTrace_GC_DONE(n+m); + return n+m; } diff --git a/Python/ceval.c b/Python/ceval.c index d3bd8b5..a396e81 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -15,6 +15,7 @@ #include "dictobject.h" #include "frameobject.h" #include "opcode.h" +#include "pydtrace.h" #include "setobject.h" #include "structmember.h" @@ -50,6 +51,9 @@ static void call_exc_trace(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *); static int maybe_call_line_trace(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *, int *, int *, int *); +static void maybe_dtrace_line(PyFrameObject *, int *, int *, int *); +static void dtrace_function_entry(PyFrameObject *); +static void dtrace_function_return(PyFrameObject *); static PyObject * cmp_outcome(int, PyObject *, PyObject *); static PyObject * import_name(PyFrameObject *, PyObject *, PyObject *, PyObject *); @@ -822,7 +826,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) #ifdef LLTRACE #define FAST_DISPATCH() \ { \ - if (!lltrace && !_Py_TracingPossible) { \ + if (!lltrace && !_Py_TracingPossible && !PyDTrace_LINE_ENABLED()) { \ f->f_lasti = INSTR_OFFSET(); \ NEXTOPARG(); \ goto *opcode_targets[opcode]; \ @@ -832,7 +836,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) #else #define FAST_DISPATCH() \ { \ - if (!_Py_TracingPossible) { \ + if (!_Py_TracingPossible && !PyDTrace_LINE_ENABLED()) { \ f->f_lasti = INSTR_OFFSET(); \ NEXTOPARG(); \ goto *opcode_targets[opcode]; \ @@ -1042,6 +1046,9 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) } } + if (PyDTrace_FUNCTION_ENTRY_ENABLED()) + dtrace_function_entry(f); + co = f->f_code; names = co->co_names; consts = co->co_consts; @@ -1162,6 +1169,9 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) fast_next_opcode: f->f_lasti = INSTR_OFFSET(); + if (PyDTrace_LINE_ENABLED()) + maybe_dtrace_line(f, &instr_lb, &instr_ub, &instr_prev); + /* line-by-line tracing support */ if (_Py_TracingPossible && @@ -3620,6 +3630,8 @@ fast_yield: /* pop frame */ exit_eval_frame: + if (PyDTrace_FUNCTION_RETURN_ENABLED()) + dtrace_function_return(f); Py_LeaveRecursiveCall(); f->f_executing = 0; tstate->frame = f->f_back; @@ -5415,3 +5427,65 @@ _PyEval_RequestCodeExtraIndex(freefunc free) tstate->co_extra_freefuncs[new_index] = free; return new_index; } + +static void +dtrace_function_entry(PyFrameObject *f) +{ + char* filename; + char* funcname; + int lineno; + + filename = PyUnicode_AsUTF8(f->f_code->co_filename); + funcname = PyUnicode_AsUTF8(f->f_code->co_name); + lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); + + PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); +} + +static void +dtrace_function_return(PyFrameObject *f) +{ + char* filename; + char* funcname; + int lineno; + + filename = PyUnicode_AsUTF8(f->f_code->co_filename); + funcname = PyUnicode_AsUTF8(f->f_code->co_name); + lineno = PyCode_Addr2Line(f->f_code, f->f_lasti); + + PyDTrace_FUNCTION_RETURN(filename, funcname, lineno); +} + +/* DTrace equivalent of maybe_call_line_trace. */ +static void +maybe_dtrace_line(PyFrameObject *frame, + int *instr_lb, int *instr_ub, int *instr_prev) +{ + int line = frame->f_lineno; + char *co_filename, *co_name; + + /* If the last instruction executed isn't in the current + instruction window, reset the window. + */ + if (frame->f_lasti < *instr_lb || frame->f_lasti >= *instr_ub) { + PyAddrPair bounds; + line = _PyCode_CheckLineNumber(frame->f_code, frame->f_lasti, + &bounds); + *instr_lb = bounds.ap_lower; + *instr_ub = bounds.ap_upper; + } + /* If the last instruction falls at the start of a line or if + it represents a jump backwards, update the frame's line + number and call the trace function. */ + if (frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev) { + frame->f_lineno = line; + co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename); + if (!co_filename) + co_filename = "?"; + co_name = PyUnicode_AsUTF8(frame->f_code->co_name); + if (!co_name) + co_name = "?"; + PyDTrace_LINE(co_filename, co_name, line); + } + *instr_prev = frame->f_lasti; +} @@ -642,6 +642,10 @@ TRUE MACHDEP_OBJS DYNLOADFILE DLINCLDIR +DTRACE_OBJS +DTRACE_HEADERS +DFLAGS +DTRACE THREADOBJ LDLAST USE_THREAD_MODULE @@ -713,6 +717,7 @@ MULTIARCH ac_ct_CXX MAINCC CXX +SED GREP CPP OBJEXT @@ -780,7 +785,6 @@ infodir docdir oldincludedir includedir -runstatedir localstatedir sharedstatedir sysconfdir @@ -832,6 +836,7 @@ enable_ipv6 with_doc_strings with_pymalloc with_valgrind +with_dtrace with_fpectl with_libm with_libc @@ -890,7 +895,6 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' -runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1143,15 +1147,6 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; - -runstatedir | --runstatedir | --runstatedi | --runstated \ - | --runstate | --runstat | --runsta | --runst | --runs \ - | --run | --ru | --r) - ac_prev=runstatedir ;; - -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ - | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ - | --run=* | --ru=* | --r=*) - runstatedir=$ac_optarg ;; - -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1289,7 +1284,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir runstatedir + libdir localedir mandir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1442,7 +1437,6 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] - --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1535,6 +1529,7 @@ Optional Packages: --with(out)-doc-strings disable/enable documentation strings --with(out)-pymalloc disable/enable specialized mallocs --with-valgrind Enable Valgrind support + --with(out)-dtrace disable/enable DTrace support --with-fpectl enable SIGFPE catching --with-libm=STRING math library --with-libc=STRING C library @@ -4581,6 +4576,75 @@ $as_echo "$ac_cv_path_GREP" >&6; } GREP="$ac_cv_path_GREP" +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a sed that does not truncate output" >&5 +$as_echo_n "checking for a sed that does not truncate output... " >&6; } +if ${ac_cv_path_SED+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_script=s/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ + for ac_i in 1 2 3 4 5 6 7; do + ac_script="$ac_script$as_nl$ac_script" + done + echo "$ac_script" 2>/dev/null | sed 99q >conftest.sed + { ac_script=; unset ac_script;} + if test -z "$SED"; then + ac_path_SED_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_prog in sed gsed; do + for ac_exec_ext in '' $ac_executable_extensions; do + ac_path_SED="$as_dir/$ac_prog$ac_exec_ext" + as_fn_executable_p "$ac_path_SED" || continue +# Check for GNU ac_path_SED and select it if it is found. + # Check for GNU $ac_path_SED +case `"$ac_path_SED" --version 2>&1` in +*GNU*) + ac_cv_path_SED="$ac_path_SED" ac_path_SED_found=:;; +*) + ac_count=0 + $as_echo_n 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + $as_echo '' >> "conftest.nl" + "$ac_path_SED" -f conftest.sed < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + as_fn_arith $ac_count + 1 && ac_count=$as_val + if test $ac_count -gt ${ac_path_SED_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_SED="$ac_path_SED" + ac_path_SED_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break + done + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; +esac + + $ac_path_SED_found && break 3 + done + done + done +IFS=$as_save_IFS + if test -z "$ac_cv_path_SED"; then + as_fn_error $? "no acceptable sed could be found in \$PATH" "$LINENO" 5 + fi +else + ac_cv_path_SED=$SED +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_SED" >&5 +$as_echo "$ac_cv_path_SED" >&6; } + SED="$ac_cv_path_SED" + rm -f conftest.sed + @@ -10864,6 +10928,102 @@ fi OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT" fi +# Check for DTrace support +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-dtrace" >&5 +$as_echo_n "checking for --with-dtrace... " >&6; } + +# Check whether --with-dtrace was given. +if test "${with_dtrace+set}" = set; then : + withval=$with_dtrace; +else + with_dtrace=no +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_dtrace" >&5 +$as_echo "$with_dtrace" >&6; } + + + + + +DTRACE= +DFLAGS= +DTRACE_HEADERS= +DTRACE_OBJS= + +if test "$with_dtrace" = "yes" +then + # Extract the first word of "dtrace", so it can be a program name with args. +set dummy dtrace; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_DTRACE+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $DTRACE in + [\\/]* | ?:[\\/]*) + ac_cv_path_DTRACE="$DTRACE" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_DTRACE="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_path_DTRACE" && ac_cv_path_DTRACE="not found" + ;; +esac +fi +DTRACE=$ac_cv_path_DTRACE +if test -n "$DTRACE"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DTRACE" >&5 +$as_echo "$DTRACE" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + if test "$DTRACE" = "not found"; then + as_fn_error $? "dtrace command not found on \$PATH" "$LINENO" 5 + fi + +$as_echo "#define WITH_DTRACE 1" >>confdefs.h + + DTRACE_HEADERS="Include/pydtrace_probes.h" + + # On OS X, DTrace providers do not need to be explicitly compiled and + # linked into the binary. Correspondingly, dtrace(1) is missing the ELF + # generation flag '-G'. We check for presence of this flag, rather than + # hardcoding support by OS, in the interest of robustness. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether DTrace probes require linking" >&5 +$as_echo_n "checking whether DTrace probes require linking... " >&6; } +if ${ac_cv_dtrace_link+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_cv_dtrace_link=no + echo 'BEGIN' > conftest.d + "$DTRACE" -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ + ac_cv_dtrace_link=yes + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_dtrace_link" >&5 +$as_echo "$ac_cv_dtrace_link" >&6; } + if test "$ac_cv_dtrace_link" = "yes"; then + DTRACE_OBJS="Python/pydtrace.o" + fi +fi + # -I${DLINCLDIR} is added to the compile rule for importdl.o DLINCLDIR=. diff --git a/configure.ac b/configure.ac index 269c41e..b1af05e 100644 --- a/configure.ac +++ b/configure.ac @@ -694,6 +694,7 @@ fi AC_PROG_CC AC_PROG_CPP AC_PROG_GREP +AC_PROG_SED AC_SUBST(CXX) AC_SUBST(MAINCC) @@ -3245,6 +3246,47 @@ if test "$with_valgrind" != no; then OPT="-DDYNAMIC_ANNOTATIONS_ENABLED=1 $OPT" fi +# Check for DTrace support +AC_MSG_CHECKING(for --with-dtrace) +AC_ARG_WITH(dtrace, + AC_HELP_STRING(--with(out)-dtrace, [disable/enable DTrace support]),, + with_dtrace=no) +AC_MSG_RESULT($with_dtrace) + +AC_SUBST(DTRACE) +AC_SUBST(DFLAGS) +AC_SUBST(DTRACE_HEADERS) +AC_SUBST(DTRACE_OBJS) +DTRACE= +DFLAGS= +DTRACE_HEADERS= +DTRACE_OBJS= + +if test "$with_dtrace" = "yes" +then + AC_PATH_PROG(DTRACE, [dtrace], [not found]) + if test "$DTRACE" = "not found"; then + AC_MSG_ERROR([dtrace command not found on \$PATH]) + fi + AC_DEFINE(WITH_DTRACE, 1, [Define if you want to compile in DTrace support]) + DTRACE_HEADERS="Include/pydtrace_probes.h" + + # On OS X, DTrace providers do not need to be explicitly compiled and + # linked into the binary. Correspondingly, dtrace(1) is missing the ELF + # generation flag '-G'. We check for presence of this flag, rather than + # hardcoding support by OS, in the interest of robustness. + AC_CACHE_CHECK([whether DTrace probes require linking], + [ac_cv_dtrace_link], [dnl + ac_cv_dtrace_link=no + echo 'BEGIN' > conftest.d + "$DTRACE" -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ + ac_cv_dtrace_link=yes + ]) + if test "$ac_cv_dtrace_link" = "yes"; then + DTRACE_OBJS="Python/pydtrace.o" + fi +fi + # -I${DLINCLDIR} is added to the compile rule for importdl.o AC_SUBST(DLINCLDIR) DLINCLDIR=. diff --git a/pyconfig.h.in b/pyconfig.h.in index 7682c48..d3f61f7 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1370,6 +1370,9 @@ /* Define if you want documentation strings in extension modules */ #undef WITH_DOC_STRINGS +/* Define if you want to compile in DTrace support */ +#undef WITH_DTRACE + /* Define if you want to use the new-style (Openstep, Rhapsody, MacOS) dynamic linker (dyld) instead of the old-style (NextStep) dynamic linker (rld). Dyld is necessary to support frameworks. */ |