From 6be848922bc0f4c632c255c39de82a45b6480286 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Jan 2022 19:24:28 +0100 Subject: bpo-44133: Link Python executable with object files (GH-30556) When Python is built without --enable-shared, the "python" program is now linked to object files, rather than being linked to the Python library (libpython.a), to make sure that all symbols are exported. Previously, the linker omitted some symbols like the Py_FrozenMain() function. When Python is configured with --without-static-libpython, the Python static library (libpython.a) is no longer built. * Check --without-static-libpython earlier in configure.ac * Add LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variables to Makefile. * test_capi now ensures that the "Py_FrozenMain" symbol is exported. --- Lib/test/test_capi.py | 18 ++++ Makefile.pre.in | 11 ++- .../Build/2022-01-12-13-34-52.bpo-44133.HYCNXb.rst | 5 + .../Build/2022-01-12-13-42-16.bpo-44133.NgyNAh.rst | 2 + configure | 105 +++++++++++---------- configure.ac | 75 ++++++++------- 6 files changed, 131 insertions(+), 85 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2022-01-12-13-34-52.bpo-44133.HYCNXb.rst create mode 100644 Misc/NEWS.d/next/Build/2022-01-12-13-42-16.bpo-44133.NgyNAh.rst diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 99263bf..9f21785 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -643,6 +643,24 @@ class CAPITest(unittest.TestCase): expected = compile(code, "", "exec") self.assertEqual(result.co_consts, expected.co_consts) + def test_export_symbols(self): + # bpo-44133: Ensure that the "Py_FrozenMain" and + # "PyThread_get_thread_native_id" symbols are exported by the Python + # (directly by the binary, or via by the Python dynamic library). + ctypes = import_helper.import_module('ctypes') + names = ['PyThread_get_thread_native_id'] + + # Python/frozenmain.c fails to build on Windows when the symbols are + # missing: + # - PyWinFreeze_ExeInit + # - PyWinFreeze_ExeTerm + # - PyInitFrozenExtensions + if os.name != 'nt': + names.append('Py_FrozenMain') + for name in names: + with self.subTest(name=name): + self.assertTrue(hasattr(ctypes.pythonapi, name)) + class TestPendingCalls(unittest.TestCase): diff --git a/Makefile.pre.in b/Makefile.pre.in index 41b123a..a84badcd 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -265,6 +265,7 @@ DLLLIBRARY= @DLLLIBRARY@ LDLIBRARYDIR= @LDLIBRARYDIR@ INSTSONAME= @INSTSONAME@ LIBRARY_DEPS= @LIBRARY_DEPS@ +LINK_PYTHON_DEPS=@LINK_PYTHON_DEPS@ PY_ENABLE_SHARED= @PY_ENABLE_SHARED@ STATIC_LIBPYTHON= @STATIC_LIBPYTHON@ @@ -526,6 +527,8 @@ LIBRARY_OBJS= \ Modules/getpath.o \ Python/frozen.o +LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@ + ########################################################################## # DTrace @@ -721,8 +724,8 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --srcdir $(srcdir) # Build the interpreter -$(BUILDPYTHON): Programs/python.o $(LIBRARY_DEPS) - $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) +$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) platform: $(BUILDPYTHON) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -965,8 +968,8 @@ regen-test-frozenmain: $(BUILDPYTHON) # using Programs/freeze_test_frozenmain.py $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Programs/freeze_test_frozenmain.py Programs/test_frozenmain.h -Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS) - $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) +Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) ############################################################################ # "Bootstrap Python" used to run deepfreeze.py diff --git a/Misc/NEWS.d/next/Build/2022-01-12-13-34-52.bpo-44133.HYCNXb.rst b/Misc/NEWS.d/next/Build/2022-01-12-13-34-52.bpo-44133.HYCNXb.rst new file mode 100644 index 0000000..7c2a48a --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-01-12-13-34-52.bpo-44133.HYCNXb.rst @@ -0,0 +1,5 @@ +When Python is built without :option:`--enable-shared`, the ``python`` +program is now linked to object files, rather than being linked to the Python +static library (libpython.a), to make sure that all symbols are exported. +Previously, the linker omitted some symbols like the :c:func:`Py_FrozenMain` +function. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2022-01-12-13-42-16.bpo-44133.NgyNAh.rst b/Misc/NEWS.d/next/Build/2022-01-12-13-42-16.bpo-44133.NgyNAh.rst new file mode 100644 index 0000000..3542850 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-01-12-13-42-16.bpo-44133.NgyNAh.rst @@ -0,0 +1,2 @@ +When Python is configured with :option:`--without-static-libpython`, the Python +static library (libpython.a) is no longer built. Patch by Victor Stinner. diff --git a/configure b/configure index 127b350..ccbf4fc 100755 --- a/configure +++ b/configure @@ -775,8 +775,6 @@ MODULE__IO_TRUE MODULES_SETUP_STDLIB MODULE_BUILDTYPE TEST_MODULES -LIBRARY_DEPS -STATIC_LIBPYTHON OPENSSL_RPATH OPENSSL_LDFLAGS OPENSSL_LIBS @@ -877,6 +875,10 @@ READELF ARFLAGS ac_ct_AR AR +LINK_PYTHON_OBJS +LINK_PYTHON_DEPS +LIBRARY_DEPS +STATIC_LIBPYTHON GNULD EXPORTSFROM EXPORTSYMS @@ -1007,6 +1009,7 @@ with_cxx_main with_emscripten_target with_suffix enable_shared +with_static_libpython enable_profiling with_pydebug with_trace_refs @@ -1048,7 +1051,6 @@ with_openssl_rpath with_ssl_default_suites with_builtin_hashlib_hashes with_experimental_isolated_subinterpreters -with_static_libpython enable_test_modules ' ac_precious_vars='build_alias @@ -1758,6 +1760,9 @@ Optional Packages: Emscripten platform --with-suffix=SUFFIX set executable suffix to SUFFIX (default is empty, yes is mapped to '.exe') + --without-static-libpython + do not build libpythonMAJOR.MINOR.a and do not + install python.o (default is yes) --with-pydebug build with Py_DEBUG defined (default is no) --with-trace-refs enable tracing references for debugging purpose (default is no) @@ -1840,9 +1845,6 @@ Optional Packages: --with-experimental-isolated-subinterpreters better isolate subinterpreters, experimental build mode (default is no) - --without-static-libpython - do not build libpythonMAJOR.MINOR.a and do not - install python.o (default is yes) Some influential environment variables: PKG_CONFIG path to pkg-config utility @@ -6428,6 +6430,30 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_shared" >&5 $as_echo "$enable_shared" >&6; } +# --with-static-libpython +STATIC_LIBPYTHON=1 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-static-libpython" >&5 +$as_echo_n "checking for --with-static-libpython... " >&6; } + +# Check whether --with-static-libpython was given. +if test "${with_static_libpython+set}" = set; then : + withval=$with_static_libpython; +if test "$withval" = no +then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; }; + STATIC_LIBPYTHON=0 +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; }; +fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +fi + + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --enable-profiling" >&5 $as_echo_n "checking for --enable-profiling... " >&6; } # Check whether --enable-profiling was given. @@ -6550,6 +6576,31 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 $as_echo "$LDLIBRARY" >&6; } +# LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable +LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' +LINK_PYTHON_DEPS='$(LIBRARY_DEPS)' +if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then + LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS" + if test "$STATIC_LIBPYTHON" = 1; then + LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS" + fi + # Link Python program to the shared library + LINK_PYTHON_OBJS='$(BLDLIBRARY)' +else + if test "$STATIC_LIBPYTHON" = 0; then + # Build Python needs object files but don't need to build + # Python static library + LINK_PYTHON_DEPS="$LIBRARY_DEPS \$(LIBRARY_OBJS)" + fi + LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS" + # Link Python program to object files + LINK_PYTHON_OBJS='$(LIBRARY_OBJS)' +fi + + + + +# ar program if test -n "$ac_tool_prefix"; then for ac_prog in ar aal @@ -21213,48 +21264,6 @@ $as_echo "no" >&6; } fi -# --with-static-libpython -STATIC_LIBPYTHON=1 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-static-libpython" >&5 -$as_echo_n "checking for --with-static-libpython... " >&6; } - -# Check whether --with-static-libpython was given. -if test "${with_static_libpython+set}" = set; then : - withval=$with_static_libpython; -if test "$withval" = no -then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; }; - STATIC_LIBPYTHON=0 -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; }; -fi -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -fi - -LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' -if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then - LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS" - if test "$STATIC_LIBPYTHON" = 1; then - LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS" - fi -else - LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS" -fi - -case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser) : - LIBRARY_DEPS="$LIBRARY_DEPS \$(WASM_STDLIB)" ;; #( - *) : - ;; -esac - - - - # Check whether to disable test modules. Once set, setup.py will not build # test extension modules and "make install" will not install test suites. { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --disable-test-modules" >&5 diff --git a/configure.ac b/configure.ac index e5ebf7b..89041b2 100644 --- a/configure.ac +++ b/configure.ac @@ -1231,6 +1231,23 @@ then fi AC_MSG_RESULT($enable_shared) +# --with-static-libpython +STATIC_LIBPYTHON=1 +AC_MSG_CHECKING(for --with-static-libpython) +AC_ARG_WITH(static-libpython, + AS_HELP_STRING([--without-static-libpython], + [do not build libpythonMAJOR.MINOR.a and do not install python.o (default is yes)]), +[ +if test "$withval" = no +then + AC_MSG_RESULT(no); + STATIC_LIBPYTHON=0 +else + AC_MSG_RESULT(yes); +fi], +[AC_MSG_RESULT(yes)]) +AC_SUBST(STATIC_LIBPYTHON) + AC_MSG_CHECKING(for --enable-profiling) AC_ARG_ENABLE(profiling, AS_HELP_STRING([--enable-profiling], [enable C-level code profiling with gprof (default is no)])) @@ -1336,6 +1353,31 @@ fi AC_MSG_RESULT($LDLIBRARY) +# LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable +LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' +LINK_PYTHON_DEPS='$(LIBRARY_DEPS)' +if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then + LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS" + if test "$STATIC_LIBPYTHON" = 1; then + LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS" + fi + # Link Python program to the shared library + LINK_PYTHON_OBJS='$(BLDLIBRARY)' +else + if test "$STATIC_LIBPYTHON" = 0; then + # Build Python needs object files but don't need to build + # Python static library + LINK_PYTHON_DEPS="$LIBRARY_DEPS \$(LIBRARY_OBJS)" + fi + LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS" + # Link Python program to object files + LINK_PYTHON_OBJS='$(LIBRARY_OBJS)' +fi +AC_SUBST(LIBRARY_DEPS) +AC_SUBST(LINK_PYTHON_DEPS) +AC_SUBST(LINK_PYTHON_OBJS) + +# ar program AC_SUBST(AR) AC_CHECK_TOOLS(AR, ar aal, ar) @@ -6273,39 +6315,6 @@ else fi], [AC_MSG_RESULT(no)]) -# --with-static-libpython -STATIC_LIBPYTHON=1 -AC_MSG_CHECKING(for --with-static-libpython) -AC_ARG_WITH(static-libpython, - AS_HELP_STRING([--without-static-libpython], - [do not build libpythonMAJOR.MINOR.a and do not install python.o (default is yes)]), -[ -if test "$withval" = no -then - AC_MSG_RESULT(no); - STATIC_LIBPYTHON=0 -else - AC_MSG_RESULT(yes); -fi], -[AC_MSG_RESULT(yes)]) -LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' -if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then - LIBRARY_DEPS="\$(LDLIBRARY) $LIBRARY_DEPS" - if test "$STATIC_LIBPYTHON" = 1; then - LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS" - fi -else - LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS" -fi - -dnl browser needs a WASM assets stdlib bundle -AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser], [LIBRARY_DEPS="$LIBRARY_DEPS \$(WASM_STDLIB)"], -) - -AC_SUBST(STATIC_LIBPYTHON) -AC_SUBST(LIBRARY_DEPS) - # Check whether to disable test modules. Once set, setup.py will not build # test extension modules and "make install" will not install test suites. AC_MSG_CHECKING(for --disable-test-modules) -- cgit v0.12