From f0150ac94a85c863ec1dcb58b9e33ed7ce465ec8 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 29 Oct 2021 18:49:57 +0300 Subject: bpo-45548: Some test modules must be built as shared libs (GH-29268) Some test cases don't work when test modules are static extensions. Add dependency on Modules/config.c to trigger a rebuild whenever a module build type is changed. ``makesetup`` puts shared extensions into ``Modules/`` directory. Create symlinks from pybuilddir so the extensions can be imported. Note: It is not possible to use the content of pybuilddir.txt as a build target. Makefile evaluates target variables in the first pass. The pybuilddir.txt file does not exist at that point. --- Doc/whatsnew/3.11.rst | 7 +++++++ Makefile.pre.in | 13 +++++++++++-- .../next/Build/2021-10-28-14-47-22.bpo-45548.mdCBxB.rst | 4 ++++ Modules/Setup | 10 ++++++---- Modules/makesetup | 3 ++- setup.py | 17 +++++++++++------ 6 files changed, 41 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2021-10-28-14-47-22.bpo-45548.mdCBxB.rst diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 21ad466..156bfbd 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -507,6 +507,13 @@ Build Changes except empty tuple singleton. (Contributed by Christian Heimes in :issue:`45522`) +* ``Modules/Setup`` and ``Modules/makesetup`` have been improved and tied up. + Extension modules can now be built through ``makesetup``. All except some + test modules can be linked statically into main binary or library. + (Contributed by Brett Cannon and Christian Heimes in :issue:`45548`, + :issue:`45570`, :issue:`45571`, and :issue:`43974`.) + + C API Changes ============= diff --git a/Makefile.pre.in b/Makefile.pre.in index 60acc16..322800a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -705,8 +705,17 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) fi -oldsharedmods: $(SHAREDMODS) - +# create relative links from build/lib.platform/egg.so to Modules/egg.so +# pybuilddir.txt is created too late. We cannot use it in Makefile +# targets. ln --relative is not portable. +oldsharedmods: $(SHAREDMODS) pybuilddir.txt + @target=`cat pybuilddir.txt`; \ + $(MKDIR_P) $$target; \ + for mod in X $(SHAREDMODS); do \ + if test $$mod != X; then \ + $(LN) -sf ../../$$mod $$target/`basename $$mod`; \ + fi; \ + done Makefile Modules/config.c: Makefile.pre \ $(srcdir)/Modules/config.c.in \ diff --git a/Misc/NEWS.d/next/Build/2021-10-28-14-47-22.bpo-45548.mdCBxB.rst b/Misc/NEWS.d/next/Build/2021-10-28-14-47-22.bpo-45548.mdCBxB.rst new file mode 100644 index 0000000..e6ccd52 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-10-28-14-47-22.bpo-45548.mdCBxB.rst @@ -0,0 +1,4 @@ +``Modules/Setup`` and ``Modules/makesetup`` have been improved. The +``Setup`` file now contains working rules for all extensions. Outdated +comments have been removed. Rules defined by ``makesetup`` track +dependencies correctly. diff --git a/Modules/Setup b/Modules/Setup index b4eae38..57584f3 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -297,14 +297,16 @@ xxsubtype xxsubtype.c # Required for the test suite to pass! #_xxsubinterpreters _xxsubinterpretersmodule.c #_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c -#_ctypes_test _ctypes/_ctypes_test.c #_testbuffer _testbuffer.c -#_testimportmultiple _testimportmultiple.c #_testinternalcapi _testinternalcapi.c -#_testmultiphase _testmultiphase.c + +# Some testing modules MUST be built as shared libraries. #*shared* -#_testcapi _testcapimodule.c # CANNOT be statically compiled! +#_ctypes_test _ctypes/_ctypes_test.c +#_testcapi _testcapimodule.c +#_testimportmultiple _testimportmultiple.c +#_testmultiphase _testmultiphase.c # --- # Uncommenting the following line tells makesetup that all following modules diff --git a/Modules/makesetup b/Modules/makesetup index 7547315..543992c 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -241,7 +241,8 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | cc="$cc \$(PY_BUILTIN_MODULE_CFLAGS)";; esac mods_upper=$(echo $mods | tr '[a-z]' '[A-Z]') - rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(PYTHON_HEADERS); $cc $cpps -c $src -o $obj" + # force rebuild when header file or module build flavor (static/shared) is changed + rule="$obj: $src \$(MODULE_${mods_upper}_DEPS) \$(PYTHON_HEADERS) Modules/config.c; $cc $cpps -c $src -o $obj" echo "$rule" >>$rulesf done case $doconfig in diff --git a/setup.py b/setup.py index f32dd4c..6fa8a67 100644 --- a/setup.py +++ b/setup.py @@ -426,12 +426,13 @@ class PyBuildExt(build_ext): # re-compile extensions if a header file has been changed ext.depends.extend(headers) - def remove_configured_extensions(self): + def handle_configured_extensions(self): # The sysconfig variables built by makesetup that list the already # built modules and the disabled modules as configured by the Setup # files. - sysconf_built = sysconfig.get_config_var('MODBUILT_NAMES').split() - sysconf_dis = sysconfig.get_config_var('MODDISABLED_NAMES').split() + sysconf_built = set(sysconfig.get_config_var('MODBUILT_NAMES').split()) + sysconf_shared = set(sysconfig.get_config_var('MODSHARED_NAMES').split()) + sysconf_dis = set(sysconfig.get_config_var('MODDISABLED_NAMES').split()) mods_built = [] mods_disabled = [] @@ -449,11 +450,15 @@ class PyBuildExt(build_ext): mods_configured] # Remove the shared libraries built by a previous build. for ext in mods_configured: + # Don't remove shared extensions which have been built + # by Modules/Setup + if ext.name in sysconf_shared: + continue fullpath = self.get_ext_fullpath(ext.name) - if os.path.exists(fullpath): + if os.path.lexists(fullpath): os.unlink(fullpath) - return (mods_built, mods_disabled) + return mods_built, mods_disabled def set_compiler_executables(self): # When you run "make CC=altcc" or something similar, you really want @@ -478,7 +483,7 @@ class PyBuildExt(build_ext): self.remove_disabled() self.update_sources_depends() - mods_built, mods_disabled = self.remove_configured_extensions() + mods_built, mods_disabled = self.handle_configured_extensions() self.set_compiler_executables() if LIST_MODULE_NAMES: -- cgit v0.12