From 6cb4343d0d374a1b0ba01390ccef5b8868a5e9fc Mon Sep 17 00:00:00 2001 From: "doko@ubuntu.com" Date: Mon, 8 Apr 2013 21:20:09 +0200 Subject: - Issue #13150, #17512: sysconfig no longer parses the Makefile and config.h files when imported, instead doing it at build time. This makes importing sysconfig faster and reduces Python startup time by 20%. --- .bzrignore | 1 + .hgignore | 1 + Lib/distutils/sysconfig.py | 63 +++----------------------------------- Lib/pprint.py | 5 +++- Lib/site.py | 15 ---------- Lib/sysconfig.py | 75 ++++++++++++++++++++++++++++++++++++++++++++-- Makefile.pre.in | 13 ++++++-- Misc/NEWS | 4 +++ Modules/getpath.c | 23 +++++++++++--- 9 files changed, 115 insertions(+), 85 deletions(-) diff --git a/.bzrignore b/.bzrignore index 2527052..abca6b1 100644 --- a/.bzrignore +++ b/.bzrignore @@ -11,6 +11,7 @@ python build Makefile.pre platform +pybuilddir.txt pyconfig.h libpython*.a libpython*.so* diff --git a/.hgignore b/.hgignore index de9bd5d..c51c53c 100644 --- a/.hgignore +++ b/.hgignore @@ -35,6 +35,7 @@ Parser/pgen$ Parser/pgen.stamp$ ^core ^python-gdb.py +^pybuilddir.txt syntax: glob python.exe-gdb.py diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 250ef38..0c726d9 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -387,66 +387,11 @@ _config_vars = None def _init_posix(): """Initialize the module as appropriate for POSIX systems.""" - g = {} - # load the installed Makefile: - try: - filename = get_makefile_filename() - parse_makefile(filename, g) - except IOError, msg: - my_msg = "invalid Python installation: unable to open %s" % filename - if hasattr(msg, "strerror"): - my_msg = my_msg + " (%s)" % msg.strerror - - raise DistutilsPlatformError(my_msg) - - # load the installed pyconfig.h: - try: - filename = get_config_h_filename() - parse_config_h(file(filename), g) - except IOError, msg: - my_msg = "invalid Python installation: unable to open %s" % filename - if hasattr(msg, "strerror"): - my_msg = my_msg + " (%s)" % msg.strerror - - raise DistutilsPlatformError(my_msg) - - # On AIX, there are wrong paths to the linker scripts in the Makefile - # -- these paths are relative to the Python source, but when installed - # the scripts are in another directory. - if python_build: - g['LDSHARED'] = g['BLDSHARED'] - - elif get_python_version() < '2.1': - # The following two branches are for 1.5.2 compatibility. - if sys.platform == 'aix4': # what about AIX 3.x ? - # Linker script is in the config directory, not in Modules as the - # Makefile says. - python_lib = get_python_lib(standard_lib=1) - ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix') - python_exp = os.path.join(python_lib, 'config', 'python.exp') - - g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp) - - elif sys.platform == 'beos': - # Linker script is in the config directory. In the Makefile it is - # relative to the srcdir, which after installation no longer makes - # sense. - python_lib = get_python_lib(standard_lib=1) - linkerscript_path = string.split(g['LDSHARED'])[0] - linkerscript_name = os.path.basename(linkerscript_path) - linkerscript = os.path.join(python_lib, 'config', - linkerscript_name) - - # XXX this isn't the right place to do this: adding the Python - # library to the link, if needed, should be in the "build_ext" - # command. (It's also needed for non-MS compilers on Windows, and - # it's taken care of for them by the 'build_ext.get_libraries()' - # method.) - g['LDSHARED'] = ("%s -L%s/lib -lpython%s" % - (linkerscript, PREFIX, get_python_version())) - + # _sysconfigdata is generated at build time, see the sysconfig module + from _sysconfigdata import build_time_vars global _config_vars - _config_vars = g + _config_vars = {} + _config_vars.update(build_time_vars) def _init_nt(): diff --git a/Lib/pprint.py b/Lib/pprint.py index 910283e..330099d 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -37,7 +37,10 @@ saferepr() import sys as _sys import warnings -from cStringIO import StringIO as _StringIO +try: + from cStringIO import StringIO as _StringIO +except ImportError: + from StringIO import StringIO as _StringIO __all__ = ["pprint","pformat","isreadable","isrecursive","saferepr", "PrettyPrinter"] diff --git a/Lib/site.py b/Lib/site.py index 54e5154..f1b0ae8 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -114,18 +114,6 @@ def removeduppaths(): sys.path[:] = L return known_paths -# XXX This should not be part of site.py, since it is needed even when -# using the -S option for Python. See http://www.python.org/sf/586680 -def addbuilddir(): - """Append ./build/lib. in case we're running in the build dir - (especially for Guido :-)""" - from sysconfig import get_platform - s = "build/lib.%s-%.3s" % (get_platform(), sys.version) - if hasattr(sys, 'gettotalrefcount'): - s += '-pydebug' - s = os.path.join(os.path.dirname(sys.path.pop()), s) - sys.path.append(s) - def _init_pathinfo(): """Return a set containing all existing directory entries from sys.path""" @@ -537,9 +525,6 @@ def main(): abs__file__() known_paths = removeduppaths() - if (os.name == "posix" and sys.path and - os.path.basename(sys.path[-1]) == "Modules"): - addbuilddir() if ENABLE_USER_SITE is None: ENABLE_USER_SITE = check_enableusersite() known_paths = addusersitepackages(known_paths) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index d74ca39..aa69351 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -278,9 +278,10 @@ def _get_makefile_filename(): return os.path.join(_PROJECT_BASE, "Makefile") return os.path.join(get_path('platstdlib'), "config", "Makefile") - -def _init_posix(vars): - """Initialize the module as appropriate for POSIX systems.""" +def _generate_posix_vars(): + """Generate the Python module containing build-time variables.""" + import pprint + vars = {} # load the installed Makefile: makefile = _get_makefile_filename() try: @@ -308,6 +309,49 @@ def _init_posix(vars): if _PYTHON_BUILD: vars['LDSHARED'] = vars['BLDSHARED'] + # There's a chicken-and-egg situation on OS X with regards to the + # _sysconfigdata module after the changes introduced by #15298: + # get_config_vars() is called by get_platform() as part of the + # `make pybuilddir.txt` target -- which is a precursor to the + # _sysconfigdata.py module being constructed. Unfortunately, + # get_config_vars() eventually calls _init_posix(), which attempts + # to import _sysconfigdata, which we won't have built yet. In order + # for _init_posix() to work, if we're on Darwin, just mock up the + # _sysconfigdata module manually and populate it with the build vars. + # This is more than sufficient for ensuring the subsequent call to + # get_platform() succeeds. + name = '_sysconfigdata' + if 'darwin' in sys.platform: + import imp + module = imp.new_module(name) + module.build_time_vars = vars + sys.modules[name] = module + + pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version[:3]) + if hasattr(sys, "gettotalrefcount"): + pybuilddir += '-pydebug' + try: + os.makedirs(pybuilddir) + except OSError: + pass + destfile = os.path.join(pybuilddir, name + '.py') + + with open(destfile, 'wb') as f: + f.write('# system configuration generated and used by' + ' the sysconfig module\n') + f.write('build_time_vars = ') + pprint.pprint(vars, stream=f) + + # Create file used for sys.path fixup -- see Modules/getpath.c + with open('pybuilddir.txt', 'w') as f: + f.write(pybuilddir) + +def _init_posix(vars): + """Initialize the module as appropriate for POSIX systems.""" + # _sysconfigdata is generated at build time, see _generate_posix_vars() + from _sysconfigdata import build_time_vars + vars.update(build_time_vars) + def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories @@ -565,3 +609,28 @@ def get_platform(): def get_python_version(): return _PY_VERSION_SHORT + + +def _print_dict(title, data): + for index, (key, value) in enumerate(sorted(data.items())): + if index == 0: + print '%s: ' % (title) + print '\t%s = "%s"' % (key, value) + + +def _main(): + """Display all information sysconfig detains.""" + if '--generate-posix-vars' in sys.argv: + _generate_posix_vars() + return + print 'Platform: "%s"' % get_platform() + print 'Python version: "%s"' % get_python_version() + print 'Current installation scheme: "%s"' % _get_default_scheme() + print + _print_dict('Paths', get_paths()) + print + _print_dict('Variables', get_config_vars()) + + +if __name__ == '__main__': + _main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 93fb0cf..9d55550 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -437,15 +437,20 @@ $(BUILDPYTHON): Modules/python.o $(LIBRARY) $(LDLIBRARY) Modules/python.o \ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) -platform: $(BUILDPYTHON) +platform: $(BUILDPYTHON) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print get_platform()+"-"+sys.version[0:3]' >platform +# Create build directory and generate the sysconfig build-time data there. +# pybuilddir.txt contains the name of the build dir and is used for +# sys.path fixup -- see Modules/getpath.c. +pybuilddir.txt: $(BUILDPYTHON) + $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars # Build the shared modules # Under GNU make, MAKEFLAGS are sorted and normalized; the 's' for # -s, --silent or --quiet is always the first char. # Under BSD make, MAKEFLAGS might be " -s -v x=y". -sharedmods: $(BUILDPYTHON) +sharedmods: $(BUILDPYTHON) pybuilddir.txt @case "$$MAKEFLAGS" in \ *\ -s*|s*) quiet="-q";; \ *) quiet="";; \ @@ -955,7 +960,7 @@ libinstall: build_all $(srcdir)/Lib/$(PLATDIR) $(srcdir)/Modules/xxmodule.c else true; \ fi; \ done - @for i in $(srcdir)/Lib/*.py $(srcdir)/Lib/*.doc $(srcdir)/Lib/*.egg-info ; \ + @for i in $(srcdir)/Lib/*.py `cat pybuilddir.txt`/_sysconfigdata.py $(srcdir)/Lib/*.doc $(srcdir)/Lib/*.egg-info ; \ do \ if test -x $$i; then \ $(INSTALL_SCRIPT) $$i $(DESTDIR)$(LIBDEST); \ @@ -1133,6 +1138,7 @@ sharedinstall: sharedmods --install-scripts=$(BINDIR) \ --install-platlib=$(DESTSHARED) \ --root=$(DESTDIR)/ + -rm $(DESTDIR)$(DESTSHARED)/_sysconfigdata.py* # Here are a couple of targets for MacOSX again, to install a full # framework-based Python. frameworkinstall installs everything, the @@ -1288,6 +1294,7 @@ distclean: clobber Modules/Setup Modules/Setup.local Modules/Setup.config \ Modules/ld_so_aix Modules/python.exp Misc/python.pc -rm -f python*-gdb.py + -rm -f pybuilddir.txt find $(srcdir) '(' -name '*.fdc' -o -name '*~' \ -o -name '[@,#]*' -o -name '*.old' \ -o -name '*.orig' -o -name '*.rej' \ diff --git a/Misc/NEWS b/Misc/NEWS index e2ed81c..57e0cc6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -14,6 +14,10 @@ Core and Builtins Library ------- +- Issue #13150, #17512: sysconfig no longer parses the Makefile and config.h + files when imported, instead doing it at build time. This makes importing + sysconfig faster and reduces Python startup time by 20%. + - Issue #13163: Rename operands in smtplib.SMTP._get_socket to correct names; fixes otherwise misleading output in tracebacks and when when debug is on. diff --git a/Modules/getpath.c b/Modules/getpath.c index 9faafa3..de96d47 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -335,12 +335,27 @@ search_for_exec_prefix(char *argv0_path, char *home) return 1; } - /* Check to see if argv[0] is in the build directory */ + /* Check to see if argv[0] is in the build directory. "pybuilddir.txt" + is written by setup.py and contains the relative path to the location + of shared library modules. */ strcpy(exec_prefix, argv0_path); - joinpath(exec_prefix, "Modules/Setup"); + joinpath(exec_prefix, "pybuilddir.txt"); if (isfile(exec_prefix)) { - reduce(exec_prefix); - return -1; + FILE *f = fopen(exec_prefix, "r"); + if (f == NULL) + errno = 0; + else { + char rel_builddir_path[MAXPATHLEN+1]; + size_t n; + n = fread(rel_builddir_path, 1, MAXPATHLEN, f); + rel_builddir_path[n] = '\0'; + fclose(f); + if (n >= 0) { + strcpy(exec_prefix, argv0_path); + joinpath(exec_prefix, rel_builddir_path); + return -1; + } + } } /* Search from argv0_path, until root is found */ -- cgit v0.12