Issue #13590: OS X Xcode 4 - improve support for universal extension modules

    In particular, fix extension module build failures when trying to use
    32-bit-only installer Pythons on systems with Xcode 4 (currently
    OS X 10.8, 10.7, and optionally 10.6).
    * Backport 3.3.0 fixes to 2.7 branch (for release in 2.7.4)
    * Since Xcode 4 removes ppc support, extension module builds now
      check for ppc compiler support and by default remove ppc and
      ppc64 archs when they are not available.
    * Extension module builds now revert to using system installed
      headers and libs (/usr and /System/Library) if the SDK used
      to build the interpreter is not installed or has moved.
    * Try to avoid building extension modules with deprecated
      and problematic Apple llvm-gcc compiler.  If original compiler
      is not available, use clang instead by default.

4 files changed, 60 insertions, 242 deletions

diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py
index 4b193b2..daa4dc7 100644
--- a/Lib/distutils/sysconfig.py
+++ b/Lib/distutils/sysconfig.py
@@ -141,7 +141,7 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
             "I don't know where Python installs its library "
             "on platform '%s'" % os.name)
 
-_USE_CLANG = None
+
 
 def customize_compiler(compiler):
     """Do any platform-specific customization of a CCompiler instance.
@@ -150,6 +150,21 @@ def customize_compiler(compiler):
     varies across Unices and is stored in Python's Makefile.
     """
     if compiler.compiler_type == "unix":
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
+
         (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \
             get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
                             'CCSHARED', 'LDSHARED', 'SO', 'AR',
@@ -157,36 +172,7 @@ def customize_compiler(compiler):
 
         newcc = None
         if 'CC' in os.environ:
-            newcc = os.environ['CC']
-        elif sys.platform == 'darwin' and cc == 'gcc-4.2':
-            # Issue #13590:
-            #       Since Apple removed gcc-4.2 in Xcode 4.2, we can no
-            #       longer assume it is available for extension module builds.
-            #       If Python was built with gcc-4.2, check first to see if
-            #       it is available on this system; if not, try to use clang
-            #       instead unless the caller explicitly set CC.
-            global _USE_CLANG
-            if _USE_CLANG is None:
-                from distutils import log
-                from subprocess import Popen, PIPE
-                p = Popen("! type gcc-4.2 && type clang && exit 2",
-                                shell=True, stdout=PIPE, stderr=PIPE)
-                p.wait()
-                if p.returncode == 2:
-                    _USE_CLANG = True
-                    log.warn("gcc-4.2 not found, using clang instead")
-                else:
-                    _USE_CLANG = False
-            if _USE_CLANG:
-                newcc = 'clang'
-        if newcc:
-            # On OS X, if CC is overridden, use that as the default
-            #       command for LDSHARED as well
-            if (sys.platform == 'darwin'
-                    and 'LDSHARED' not in os.environ
-                    and ldshared.startswith(cc)):
-                ldshared = newcc + ldshared[len(cc):]
-            cc = newcc
+            cc = os.environ['CC']
         if 'CXX' in os.environ:
             cxx = os.environ['CXX']
         if 'LDSHARED' in os.environ:
@@ -518,66 +504,11 @@ def get_config_vars(*args):
         _config_vars['prefix'] = PREFIX
         _config_vars['exec_prefix'] = EXEC_PREFIX
 
+        # OS X platforms require special customization to handle
+        # multi-architecture, multi-os-version installers
         if sys.platform == 'darwin':
-            kernel_version = os.uname()[2] # Kernel version (8.4.3)
-            major_version = int(kernel_version.split('.')[0])
-
-            if major_version < 8:
-                # On Mac OS X before 10.4, check if -arch and -isysroot
-                # are in CFLAGS or LDFLAGS and remove them if they are.
-                # This is needed when building extensions on a 10.3 system
-                # using a universal build of python.
-                for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-                    flags = _config_vars[key]
-                    flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                    flags = re.sub('-isysroot [^ \t]*', ' ', flags)
-                    _config_vars[key] = flags
-
-            else:
-
-                # Allow the user to override the architecture flags using
-                # an environment variable.
-                # NOTE: This name was introduced by Apple in OSX 10.5 and
-                # is used by several scripting languages distributed with
-                # that OS release.
-
-                if 'ARCHFLAGS' in os.environ:
-                    arch = os.environ['ARCHFLAGS']
-                    for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                        flags = _config_vars[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                        flags = flags + ' ' + arch
-                        _config_vars[key] = flags
-
-                # If we're on OSX 10.5 or later and the user tries to
-                # compiles an extension using an SDK that is not present
-                # on the current machine it is better to not use an SDK
-                # than to fail.
-                #
-                # The major usecase for this is users using a Python.org
-                # binary installer  on OSX 10.6: that installer uses
-                # the 10.4u SDK, but that SDK is not installed by default
-                # when you install Xcode.
-                #
-                m = re.search('-isysroot\s+(\S+)', _config_vars['CFLAGS'])
-                if m is not None:
-                    sdk = m.group(1)
-                    if not os.path.exists(sdk):
-                        for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                             # a number of derived variables. These need to be
-                             # patched up as well.
-                            'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                            flags = _config_vars[key]
-                            flags = re.sub('-isysroot\s+\S+(\s|$)', ' ', flags)
-                            _config_vars[key] = flags
+            import _osx_support
+            _osx_support.customize_config_vars(_config_vars)
 
     if args:
         vals = []
diff --git a/Lib/distutils/tests/test_sysconfig.py b/Lib/distutils/tests/test_sysconfig.py
index 49570c4..c064d2b 100644
--- a/Lib/distutils/tests/test_sysconfig.py
+++ b/Lib/distutils/tests/test_sysconfig.py
@@ -72,6 +72,35 @@ class SysconfigTestCase(support.EnvironGuard,
                              'OTHER': 'foo'})
 
 
+    def test_sysconfig_module(self):
+        import sysconfig as global_sysconfig
+        self.assertEqual(global_sysconfig.get_config_var('CFLAGS'), sysconfig.get_config_var('CFLAGS'))
+        self.assertEqual(global_sysconfig.get_config_var('LDFLAGS'), sysconfig.get_config_var('LDFLAGS'))
+
+    @unittest.skipIf(sysconfig.get_config_var('CUSTOMIZED_OSX_COMPILER'),'compiler flags customized')
+    def test_sysconfig_compiler_vars(self):
+        # On OS X, binary installers support extension module building on
+        # various levels of the operating system with differing Xcode
+        # configurations.  This requires customization of some of the
+        # compiler configuration directives to suit the environment on
+        # the installed machine.  Some of these customizations may require
+        # running external programs and, so, are deferred until needed by
+        # the first extension module build.  With Python 3.3, only
+        # the Distutils version of sysconfig is used for extension module
+        # builds, which happens earlier in the Distutils tests.  This may
+        # cause the following tests to fail since no tests have caused
+        # the global version of sysconfig to call the customization yet.
+        # The solution for now is to simply skip this test in this case.
+        # The longer-term solution is to only have one version of sysconfig.
+
+        import sysconfig as global_sysconfig
+        if sysconfig.get_config_var('CUSTOMIZED_OSX_COMPILER'):
+            return
+        self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED'))
+        self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC'))
+
+
+
 def test_suite():
     suite = unittest.TestSuite()
     suite.addTest(unittest.makeSuite(SysconfigTestCase))
diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py
index c49ac9b..2aa1cb1 100644
--- a/Lib/distutils/unixccompiler.py
+++ b/Lib/distutils/unixccompiler.py
@@ -26,6 +26,9 @@ from distutils.errors import \
      DistutilsExecError, CompileError, LibError, LinkError
 from distutils import log
 
+if sys.platform == 'darwin':
+    import _osx_support
+
 # XXX Things not currently handled:
 #   * optimization/debug/warning flags; we just use whatever's in Python's
 #     Makefile and live with it.  Is this adequate?  If not, we might
@@ -41,68 +44,6 @@ from distutils import log
 #     should just happily stuff them into the preprocessor/compiler/linker
 #     options and carry on.
 
-def _darwin_compiler_fixup(compiler_so, cc_args):
-    """
-    This function will strip '-isysroot PATH' and '-arch ARCH' from the
-    compile flags if the user has specified one them in extra_compile_flags.
-
-    This is needed because '-arch ARCH' adds another architecture to the
-    build, without a way to remove an architecture. Furthermore GCC will
-    barf if multiple '-isysroot' arguments are present.
-    """
-    stripArch = stripSysroot = 0
-
-    compiler_so = list(compiler_so)
-    kernel_version = os.uname()[2] # 8.4.3
-    major_version = int(kernel_version.split('.')[0])
-
-    if major_version < 8:
-        # OSX before 10.4.0, these don't support -arch and -isysroot at
-        # all.
-        stripArch = stripSysroot = True
-    else:
-        stripArch = '-arch' in cc_args
-        stripSysroot = '-isysroot' in cc_args
-
-    if stripArch or 'ARCHFLAGS' in os.environ:
-        while 1:
-            try:
-                index = compiler_so.index('-arch')
-                # Strip this argument and the next one:
-                del compiler_so[index:index+2]
-            except ValueError:
-                break
-
-    if 'ARCHFLAGS' in os.environ and not stripArch:
-        # User specified different -arch flags in the environ,
-        # see also distutils.sysconfig
-        compiler_so = compiler_so + os.environ['ARCHFLAGS'].split()
-
-    if stripSysroot:
-        try:
-            index = compiler_so.index('-isysroot')
-            # Strip this argument and the next one:
-            del compiler_so[index:index+2]
-        except ValueError:
-            pass
-
-    # Check if the SDK that is used during compilation actually exists,
-    # the universal build requires the usage of a universal SDK and not all
-    # users have that installed by default.
-    sysroot = None
-    if '-isysroot' in cc_args:
-        idx = cc_args.index('-isysroot')
-        sysroot = cc_args[idx+1]
-    elif '-isysroot' in compiler_so:
-        idx = compiler_so.index('-isysroot')
-        sysroot = compiler_so[idx+1]
-
-    if sysroot and not os.path.isdir(sysroot):
-        log.warn("Compiling with an SDK that doesn't seem to exist: %s",
-                sysroot)
-        log.warn("Please check your Xcode installation")
-
-    return compiler_so
 
 class UnixCCompiler(CCompiler):
 
@@ -172,7 +113,8 @@ class UnixCCompiler(CCompiler):
     def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts):
         compiler_so = self.compiler_so
         if sys.platform == 'darwin':
-            compiler_so = _darwin_compiler_fixup(compiler_so, cc_args + extra_postargs)
+            compiler_so = _osx_support.compiler_fixup(compiler_so,
+                                                    cc_args + extra_postargs)
         try:
             self.spawn(compiler_so + cc_args + [src, '-o', obj] +
                        extra_postargs)
@@ -251,7 +193,7 @@ class UnixCCompiler(CCompiler):
                     linker[i] = self.compiler_cxx[i]
 
                 if sys.platform == 'darwin':
-                    linker = _darwin_compiler_fixup(linker, ld_args)
+                    linker = _osx_support.compiler_fixup(linker, ld_args)
 
                 self.spawn(linker + ld_args)
             except DistutilsExecError, msg:
diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py
index 0c24e8c..5279411 100644
--- a/Lib/distutils/util.py
+++ b/Lib/distutils/util.py
@@ -93,94 +93,10 @@ def get_platform ():
         if m:
             release = m.group()
     elif osname[:6] == "darwin":
-        #
-        # For our purposes, we'll assume that the system version from
-        # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set
-        # to. This makes the compatibility story a bit more sane because the
-        # machine is going to compile and link as if it were
-        # MACOSX_DEPLOYMENT_TARGET.
-        from distutils.sysconfig import get_config_vars
-        cfgvars = get_config_vars()
-
-        macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET')
-
-        if 1:
-            # Always calculate the release of the running machine,
-            # needed to determine if we can build fat binaries or not.
-
-            macrelease = macver
-            # Get the system version. Reading this plist is a documented
-            # way to get the system version (see the documentation for
-            # the Gestalt Manager)
-            try:
-                f = open('/System/Library/CoreServices/SystemVersion.plist')
-            except IOError:
-                # We're on a plain darwin box, fall back to the default
-                # behaviour.
-                pass
-            else:
-                try:
-                    m = re.search(
-                            r'<key>ProductUserVisibleVersion</key>\s*' +
-                            r'<string>(.*?)</string>', f.read())
-                    if m is not None:
-                        macrelease = '.'.join(m.group(1).split('.')[:2])
-                    # else: fall back to the default behaviour
-                finally:
-                    f.close()
-
-        if not macver:
-            macver = macrelease
-
-        if macver:
-            from distutils.sysconfig import get_config_vars
-            release = macver
-            osname = "macosx"
-
-            if (macrelease + '.') >= '10.4.' and \
-                    '-arch' in get_config_vars().get('CFLAGS', '').strip():
-                # The universal build will build fat binaries, but not on
-                # systems before 10.4
-                #
-                # Try to detect 4-way universal builds, those have machine-type
-                # 'universal' instead of 'fat'.
-
-                machine = 'fat'
-                cflags = get_config_vars().get('CFLAGS')
-
-                archs = re.findall('-arch\s+(\S+)', cflags)
-                archs = tuple(sorted(set(archs)))
-
-                if len(archs) == 1:
-                    machine = archs[0]
-                elif archs == ('i386', 'ppc'):
-                    machine = 'fat'
-                elif archs == ('i386', 'x86_64'):
-                    machine = 'intel'
-                elif archs == ('i386', 'ppc', 'x86_64'):
-                    machine = 'fat3'
-                elif archs == ('ppc64', 'x86_64'):
-                    machine = 'fat64'
-                elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'):
-                    machine = 'universal'
-                else:
-                    raise ValueError(
-                       "Don't know machine value for archs=%r"%(archs,))
-
-            elif machine == 'i386':
-                # On OSX the machine type returned by uname is always the
-                # 32-bit variant, even if the executable architecture is
-                # the 64-bit variant
-                if sys.maxint >= 2**32:
-                    machine = 'x86_64'
-
-            elif machine in ('PowerPC', 'Power_Macintosh'):
-                # Pick a sane name for the PPC architecture.
-                machine = 'ppc'
-
-                # See 'i386' case
-                if sys.maxint >= 2**32:
-                    machine = 'ppc64'
+        import _osx_support, distutils.sysconfig
+        osname, release, machine = _osx_support.get_platform_osx(
+                                        distutils.sysconfig.get_config_vars(),
+                                        osname, release, machine)
 
     return "%s-%s-%s" % (osname, release, machine)