From 28b7c05131765ebdf2efa106c861b7b08008f1fc Mon Sep 17 00:00:00 2001 From: "doko@ubuntu.com" Date: Thu, 21 Mar 2013 15:02:16 -0700 Subject: - Issue #13150: sysconfig no longer parses the Makefile and config.h files when imported, instead doing it at build time. This makes importing sysconfig faster and reduces Python startup time by 20%. --- Lib/distutils/sysconfig.py | 63 +++----------------------------------- Lib/pprint.py | 5 +++- Lib/sysconfig.py | 75 ++++++++++++++++++++++++++++++++++++++++++++-- Makefile.pre.in | 12 ++++++-- Misc/NEWS | 4 +++ 5 files changed, 93 insertions(+), 66 deletions(-) diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 250ef38..0c726d9 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -387,66 +387,11 @@ _config_vars = None def _init_posix(): """Initialize the module as appropriate for POSIX systems.""" - g = {} - # load the installed Makefile: - try: - filename = get_makefile_filename() - parse_makefile(filename, g) - except IOError, msg: - my_msg = "invalid Python installation: unable to open %s" % filename - if hasattr(msg, "strerror"): - my_msg = my_msg + " (%s)" % msg.strerror - - raise DistutilsPlatformError(my_msg) - - # load the installed pyconfig.h: - try: - filename = get_config_h_filename() - parse_config_h(file(filename), g) - except IOError, msg: - my_msg = "invalid Python installation: unable to open %s" % filename - if hasattr(msg, "strerror"): - my_msg = my_msg + " (%s)" % msg.strerror - - raise DistutilsPlatformError(my_msg) - - # On AIX, there are wrong paths to the linker scripts in the Makefile - # -- these paths are relative to the Python source, but when installed - # the scripts are in another directory. - if python_build: - g['LDSHARED'] = g['BLDSHARED'] - - elif get_python_version() < '2.1': - # The following two branches are for 1.5.2 compatibility. - if sys.platform == 'aix4': # what about AIX 3.x ? - # Linker script is in the config directory, not in Modules as the - # Makefile says. - python_lib = get_python_lib(standard_lib=1) - ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix') - python_exp = os.path.join(python_lib, 'config', 'python.exp') - - g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp) - - elif sys.platform == 'beos': - # Linker script is in the config directory. In the Makefile it is - # relative to the srcdir, which after installation no longer makes - # sense. - python_lib = get_python_lib(standard_lib=1) - linkerscript_path = string.split(g['LDSHARED'])[0] - linkerscript_name = os.path.basename(linkerscript_path) - linkerscript = os.path.join(python_lib, 'config', - linkerscript_name) - - # XXX this isn't the right place to do this: adding the Python - # library to the link, if needed, should be in the "build_ext" - # command. (It's also needed for non-MS compilers on Windows, and - # it's taken care of for them by the 'build_ext.get_libraries()' - # method.) - g['LDSHARED'] = ("%s -L%s/lib -lpython%s" % - (linkerscript, PREFIX, get_python_version())) - + # _sysconfigdata is generated at build time, see the sysconfig module + from _sysconfigdata import build_time_vars global _config_vars - _config_vars = g + _config_vars = {} + _config_vars.update(build_time_vars) def _init_nt(): diff --git a/Lib/pprint.py b/Lib/pprint.py index 910283e..330099d 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -37,7 +37,10 @@ saferepr() import sys as _sys import warnings -from cStringIO import StringIO as _StringIO +try: + from cStringIO import StringIO as _StringIO +except ImportError: + from StringIO import StringIO as _StringIO __all__ = ["pprint","pformat","isreadable","isrecursive","saferepr", "PrettyPrinter"] diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index d74ca39..aa69351 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -278,9 +278,10 @@ def _get_makefile_filename(): return os.path.join(_PROJECT_BASE, "Makefile") return os.path.join(get_path('platstdlib'), "config", "Makefile") - -def _init_posix(vars): - """Initialize the module as appropriate for POSIX systems.""" +def _generate_posix_vars(): + """Generate the Python module containing build-time variables.""" + import pprint + vars = {} # load the installed Makefile: makefile = _get_makefile_filename() try: @@ -308,6 +309,49 @@ def _init_posix(vars): if _PYTHON_BUILD: vars['LDSHARED'] = vars['BLDSHARED'] + # There's a chicken-and-egg situation on OS X with regards to the + # _sysconfigdata module after the changes introduced by #15298: + # get_config_vars() is called by get_platform() as part of the + # `make pybuilddir.txt` target -- which is a precursor to the + # _sysconfigdata.py module being constructed. Unfortunately, + # get_config_vars() eventually calls _init_posix(), which attempts + # to import _sysconfigdata, which we won't have built yet. In order + # for _init_posix() to work, if we're on Darwin, just mock up the + # _sysconfigdata module manually and populate it with the build vars. + # This is more than sufficient for ensuring the subsequent call to + # get_platform() succeeds. + name = '_sysconfigdata' + if 'darwin' in sys.platform: + import imp + module = imp.new_module(name) + module.build_time_vars = vars + sys.modules[name] = module + + pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version[:3]) + if hasattr(sys, "gettotalrefcount"): + pybuilddir += '-pydebug' + try: + os.makedirs(pybuilddir) + except OSError: + pass + destfile = os.path.join(pybuilddir, name + '.py') + + with open(destfile, 'wb') as f: + f.write('# system configuration generated and used by' + ' the sysconfig module\n') + f.write('build_time_vars = ') + pprint.pprint(vars, stream=f) + + # Create file used for sys.path fixup -- see Modules/getpath.c + with open('pybuilddir.txt', 'w') as f: + f.write(pybuilddir) + +def _init_posix(vars): + """Initialize the module as appropriate for POSIX systems.""" + # _sysconfigdata is generated at build time, see _generate_posix_vars() + from _sysconfigdata import build_time_vars + vars.update(build_time_vars) + def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories @@ -565,3 +609,28 @@ def get_platform(): def get_python_version(): return _PY_VERSION_SHORT + + +def _print_dict(title, data): + for index, (key, value) in enumerate(sorted(data.items())): + if index == 0: + print '%s: ' % (title) + print '\t%s = "%s"' % (key, value) + + +def _main(): + """Display all information sysconfig detains.""" + if '--generate-posix-vars' in sys.argv: + _generate_posix_vars() + return + print 'Platform: "%s"' % get_platform() + print 'Python version: "%s"' % get_python_version() + print 'Current installation scheme: "%s"' % _get_default_scheme() + print + _print_dict('Paths', get_paths()) + print + _print_dict('Variables', get_config_vars()) + + +if __name__ == '__main__': + _main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 93fb0cf..05b09d5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -437,15 +437,20 @@ $(BUILDPYTHON): Modules/python.o $(LIBRARY) $(LDLIBRARY) Modules/python.o \ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) -platform: $(BUILDPYTHON) +platform: $(BUILDPYTHON) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print get_platform()+"-"+sys.version[0:3]' >platform +# Create build directory and generate the sysconfig build-time data there. +# pybuilddir.txt contains the name of the build dir and is used for +# sys.path fixup -- see Modules/getpath.c. +pybuilddir.txt: $(BUILDPYTHON) + $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars # Build the shared modules # Under GNU make, MAKEFLAGS are sorted and normalized; the 's' for # -s, --silent or --quiet is always the first char. # Under BSD make, MAKEFLAGS might be " -s -v x=y". -sharedmods: $(BUILDPYTHON) +sharedmods: $(BUILDPYTHON) pybuilddir.txt @case "$$MAKEFLAGS" in \ *\ -s*|s*) quiet="-q";; \ *) quiet="";; \ @@ -955,7 +960,7 @@ libinstall: build_all $(srcdir)/Lib/$(PLATDIR) $(srcdir)/Modules/xxmodule.c else true; \ fi; \ done - @for i in $(srcdir)/Lib/*.py $(srcdir)/Lib/*.doc $(srcdir)/Lib/*.egg-info ; \ + @for i in $(srcdir)/Lib/*.py `cat pybuilddir.txt`/_sysconfigdata.py $(srcdir)/Lib/*.doc $(srcdir)/Lib/*.egg-info ; \ do \ if test -x $$i; then \ $(INSTALL_SCRIPT) $$i $(DESTDIR)$(LIBDEST); \ @@ -1133,6 +1138,7 @@ sharedinstall: sharedmods --install-scripts=$(BINDIR) \ --install-platlib=$(DESTSHARED) \ --root=$(DESTDIR)/ + -rm $(DESTDIR)$(DESTSHARED)/_sysconfigdata.py* # Here are a couple of targets for MacOSX again, to install a full # framework-based Python. frameworkinstall installs everything, the diff --git a/Misc/NEWS b/Misc/NEWS index 1f3601c..5c689f7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -216,6 +216,10 @@ Core and Builtins Library ------- +- Issue #13150: sysconfig no longer parses the Makefile and config.h files + when imported, instead doing it at build time. This makes importing + sysconfig faster and reduces Python startup time by 20%. + - Issue #10212: cStringIO and struct.unpack support new buffer objects. - Issue #12098: multiprocessing on Windows now starts child processes -- cgit v0.12