[3.8] bpo-41100: Support macOS 11 Big Sur and Apple Silicon Macs (#25806)

* bpo-41100: Support macOS 11 and Apple Silicon on Python 3.8

This is a partial backport of bpo-41100 changes `e8b1c038b14b5fc8120aab62c9bf5fb840274cb6` and `96d906b144e6e6aa96c5ffebecbcc5d38034bbda` for Python 3.8. We introduce the ability to build Python from source for `arm64` on macOS, but we do not make a promise of support. This allows us to omit support for Universal2 binaries as well as weak-linking of symbols from the macOS SDK based on the deployment target, which are larger changes much more difficult to merge.

This also includes a backport of subsequent bpo-42688 change `7e729978fa08a360cbf936dc215ba7dd25a06a08` to fix build errors with external `libffi`.

* bpo-41116: Ensure system supplied libraries are found on macOS 11 (GH-23301) (GH-23455)

On macOS system provided libraries are in a shared library cache
and not at their usual location. This PR teaches distutils to search
in the SDK, even if there was no "-sysroot" argument in
the compiler flags.
(cherry picked from commit 404a719b5127602c1a948f8e189ab61cd3f147d8)

* bpo-42504: fix for MACOSX_DEPLOYMENT_TARGET=11 (GH-23556)

macOS releases numbering has changed as of macOS 11 Big Sur.  Previously, major releases were of the form 10.x, 10.x+1, 10.x+2, etc; as of Big Sur, they are now x, x+1, etc, so, for example, 10.15, 10.15.1, ..., 10.15.7, 11, 11.0.1, 11.1, ..., 12, 12.1, etc. Allow Python to build with single-digit deployment target values. Patch provided by FX Coudert.
(cherry picked from commit 5291639e611dc3f55a34666036f2c3424648ba50)

* bpo-42504: Ensure that get_config_var('MACOSX_DEPLOYMENT_TARGET') is a string (GH-24341) (GH-24410)

* bpo-42504: Ensure that get_config_var('MACOSX_DEPLOYMENT_TARGET') is a string
(cherry picked from commit 49926cf2bcc8b44d9b8f148d81979ada191dd9d5)

Co-authored-by: Ronald Oussoren <ronaldoussoren@mac.com>
Co-authored-by: FX Coudert <fxcoudert@gmail.com>
Co-authored-by: Max Bélanger <aeromax@gmail.com>

4 files changed, 108 insertions, 28 deletions

diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
index 8e1fdd8..b07def1 100755
--- a/Mac/BuildScript/build-installer.py
+++ b/Mac/BuildScript/build-installer.py
@@ -116,7 +116,8 @@ WORKDIR = "/tmp/_py"
 DEPSRC = os.path.join(WORKDIR, 'third-party')
 DEPSRC = os.path.expanduser('~/Universal/other-sources')
 
-universal_opts_map = { '32-bit': ('i386', 'ppc',),
+universal_opts_map = { 'universal2': ('arm64', 'x86_64'),
+                       '32-bit': ('i386', 'ppc',),
                        '64-bit': ('x86_64', 'ppc64',),
                        'intel':  ('i386', 'x86_64'),
                        'intel-32':  ('i386',),
@@ -124,6 +125,7 @@ universal_opts_map = { '32-bit': ('i386', 'ppc',),
                        '3-way':  ('ppc', 'i386', 'x86_64'),
                        'all':    ('i386', 'ppc', 'x86_64', 'ppc64',) }
 default_target_map = {
+        'universal2': '10.9',
         '64-bit': '10.5',
         '3-way': '10.5',
         'intel': '10.5',
@@ -151,6 +153,9 @@ DEPTARGET = '10.5'
 def getDeptargetTuple():
     return tuple([int(n) for n in DEPTARGET.split('.')[0:2]])
 
+def getBuildTuple():
+    return tuple([int(n) for n in platform.mac_ver()[0].split('.')[0:2]])
+
 def getTargetCompilers():
     target_cc_map = {
         '10.4': ('gcc-4.0', 'g++-4.0'),
@@ -190,6 +195,34 @@ EXPECTED_SHARED_LIBS = {}
 def internalTk():
     return getDeptargetTuple() >= (10, 6)
 
+# Do we use 8.6.8 when building our own copy
+# of Tcl/Tk or a modern version.
+#   We use the old version when buildin on
+#   old versions of macOS due to build issues.
+def useOldTk():
+    return getBuildTuple() < (10, 15)
+
+
+def tweak_tcl_build(basedir, archList):
+    with open("Makefile", "r") as fp:
+        contents = fp.readlines()
+
+    # For reasons I don't understand the tcl configure script
+    # decides that some stdlib symbols aren't present, before
+    # deciding that strtod is broken.
+    new_contents = []
+    for line in contents:
+        if line.startswith("COMPAT_OBJS"):
+            # note: the space before strtod.o is intentional,
+            # the detection of a broken strtod results in
+            # "fixstrod.o" on this line.
+            for nm in ("strstr.o", "strtoul.o", " strtod.o"):
+                line = line.replace(nm, "")
+        new_contents.append(line)
+
+    with open("Makefile", "w") as fp:
+        fp.writelines(new_contents)
+
 # List of names of third party software built with this installer.
 # The names will be inserted into the rtf version of the License.
 THIRD_PARTY_LIBS = []
@@ -219,11 +252,26 @@ def library_recipes():
     ])
 
     if internalTk():
+        if useOldTk():
+            tcl_tk_ver='8.6.8'
+            tcl_checksum='81656d3367af032e0ae6157eff134f89'
+
+            tk_checksum='5e0faecba458ee1386078fb228d008ba'
+            tk_patches = ['tk868_on_10_8_10_9.patch']
+
+        else:
+            tcl_tk_ver='8.6.11'
+            tcl_checksum='8a4c004f48984a03a7747e9ba06e4da4'
+
+            tk_checksum='c7ee71a2d05bba78dfffd76528dc17c6'
+            tk_patches = [ ]
+
+
         result.extend([
           dict(
-              name="Tcl 8.6.8",
-              url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl8.6.8-src.tar.gz",
-              checksum='81656d3367af032e0ae6157eff134f89',
+              name="Tcl %s"%(tcl_tk_ver,),
+              url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl%s-src.tar.gz"%(tcl_tk_ver,),
+              checksum=tcl_checksum,
               buildDir="unix",
               configure_pre=[
                     '--enable-shared',
@@ -231,18 +279,17 @@ def library_recipes():
                     '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),),
               ],
               useLDFlags=False,
+              buildrecipe=tweak_tcl_build,
               install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{
                   "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')),
                   "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())),
                   },
               ),
           dict(
-              name="Tk 8.6.8",
-              url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk8.6.8-src.tar.gz",
-              checksum='5e0faecba458ee1386078fb228d008ba',
-              patches=[
-                  "tk868_on_10_8_10_9.patch",
-                   ],
+              name="Tk %s"%(tcl_tk_ver,),
+              url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk%s-src.tar.gz"%(tcl_tk_ver,),
+              checksum=tk_checksum,
+              patches=tk_patches,
               buildDir="unix",
               configure_pre=[
                     '--enable-aqua',
@@ -537,8 +584,8 @@ def checkEnvironment():
     Check that we're running on a supported system.
     """
 
-    if sys.version_info[0:2] < (2, 5):
-        fatal("This script must be run with Python 2.5 (or later)")
+    if sys.version_info[0:2] < (2, 7):
+        fatal("This script must be run with Python 2.7 (or later)")
 
     if platform.system() != 'Darwin':
         fatal("This script should be run on a macOS 10.5 (or later) system")
@@ -606,9 +653,6 @@ def checkEnvironment():
         base_path = base_path + ':' + OLD_DEVELOPER_TOOLS
     os.environ['PATH'] = base_path
     print("Setting default PATH: %s"%(os.environ['PATH']))
-    # Ensure we have access to sphinx-build.
-    # You may have to create a link in /usr/bin for it.
-    runCommand('sphinx-build --version')
 
 def parseOptions(args=None):
     """
@@ -801,6 +845,7 @@ def build_universal_openssl(basedir, archList):
         arch_opts = {
             "i386": ["darwin-i386-cc"],
             "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"],
+            "arm64": ["darwin64-arm64-cc"],
             "ppc": ["darwin-ppc-cc"],
             "ppc64": ["darwin64-ppc-cc"],
         }
@@ -1566,8 +1611,39 @@ def buildDMG():
     if os.path.exists(outdir):
         shutil.rmtree(outdir)
 
+    # We used to use the deployment target as the last characters of the
+    # installer file name. With the introduction of weaklinked installer
+    # variants, we may have two variants with the same file name, i.e.
+    # both ending in '10.9'.  To avoid this, we now use the major/minor
+    # version numbers of the macOS version we are building on.
+    # Also, as of macOS 11, operating system version numbering has
+    # changed from three components to two, i.e.
+    #   10.14.1, 10.14.2, ...
+    #   10.15.1, 10.15.2, ...
+    #   11.1, 11.2, ...
+    #   12.1, 12.2, ...
+    # (A further twist is that, when running on macOS 11, binaries built
+    # on older systems may be shown an operating system version of 10.16
+    # instead of 11.  We should not run into that situation here.)
+    # Also we should use "macos" instead of "macosx" going forward.
+    #
+    # To maintain compability for legacy variants, the file name for
+    # builds on macOS 10.15 and earlier remains:
+    #   python-3.x.y-macosx10.z.{dmg->pkg}
+    #   e.g. python-3.9.4-macosx10.9.{dmg->pkg}
+    # and for builds on macOS 11+:
+    #   python-3.x.y-macosz.{dmg->pkg}
+    #   e.g. python-3.9.4-macos11.{dmg->pkg}
+
+    build_tuple = getBuildTuple()
+    if build_tuple[0] < 11:
+        os_name = 'macosx'
+        build_system_version = '%s.%s' % build_tuple
+    else:
+        os_name = 'macos'
+        build_system_version = str(build_tuple[0])
     imagepath = os.path.join(outdir,
-                    'python-%s-macosx%s'%(getFullVersion(),DEPTARGET))
+                    'python-%s-%s%s'%(getFullVersion(),os_name,build_system_version))
     if INCLUDE_TIMESTAMP:
         imagepath = imagepath + '-%04d-%02d-%02d'%(time.localtime()[:3])
     imagepath = imagepath + '.dmg'
diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf
index b070506..cc08304 100644
--- a/Mac/BuildScript/resources/ReadMe.rtf
+++ b/Mac/BuildScript/resources/ReadMe.rtf
@@ -1,4 +1,4 @@
-{\rtf1\ansi\ansicpg1252\cocoartf2513
+{\rtf1\ansi\ansicpg1252\cocoartf2580
 \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fmodern\fcharset0 CourierNewPSMT;
 \f3\fswiss\fcharset0 Helvetica-Oblique;}
 {\colortbl;\red255\green255\blue255;}
@@ -10,16 +10,12 @@
 \
 \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
 
-\f1\b \cf0 \ul \ulc0 macOS 11 Big Sur not fully supported\
+\f1\b \cf0 \ul \ulc0 macOS 11 (Big Sur) and Apple Silicon Mac support [new in 3.8.10]\
 
 \f0\b0 \ulnone \
-Python 3.8.7 is not yet fully supported on macOS 11 Big Sur.  It will install on macOS 11 Big Sur and will run on Apple Silicon Macs using Rosetta 2 translation.  But a few features do not work correctly, most noticeably those involving searching for system libraries such as 
-\f2 ctypes.util.find_library()
-\f0  and in 
-\f2 Distutils
-\f0 .  Python 3.9.1 or later provides full support for Big Sur and Apple Silicon Macs, including building natively on Apple Silicon Macs and support for 
+As of Python 3.8.10, Python is now supported on macOS 11 Big Sur.  The binaries included in this installer install on macOS releases from macOS 10.9 through macOS 11 Big Sur and will run on Apple Silicon Macs using Rosetta 2 translation.  Starting with Python 3.9.1, we provide a new 
 \f2 universal2
-\f0  binaries\
+\f0  installer variant which includes binaries that run natively on both Apple Silicon Macs and Intel-based Macs. \
 \
 
 \f1\b \ul Certificate verification and OpenSSL\
diff --git a/Mac/README.rst b/Mac/README.rst
index ec7d873..f63a424 100644
--- a/Mac/README.rst
+++ b/Mac/README.rst
@@ -120,6 +120,8 @@ support ppc (Xcode 4 on 10.6 and later systems).  The flavor can be specified
 using the configure option ``--with-universal-archs=VALUE``. The following
 values are available:
 
+  * ``universal2``: ``arm64``, ``x86_64``
+
   * ``intel``:	  ``i386``, ``x86_64``
 
   * ``intel-32``: ``i386``
@@ -155,6 +157,8 @@ following combinations of SDKs and universal-archs flavors are available:
 
   * 10.15 and later SDKs support ``intel-64`` only
 
+  * 11.0 and later SDKs support ``universal2``
+
 The makefile for a framework build will also install ``python3.x-32``
 binaries when the universal architecture includes at least one 32-bit
 architecture (that is, for all flavors but ``64-bit`` and ``intel-64``).
diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c
index c8bd3ba..78813e8 100644
--- a/Mac/Tools/pythonw.c
+++ b/Mac/Tools/pythonw.c
@@ -95,9 +95,6 @@ setup_spawnattr(posix_spawnattr_t* spawnattr)
     size_t count;
     cpu_type_t cpu_types[1];
     short flags = 0;
-#ifdef __LP64__
-    int   ch;
-#endif
 
     if ((errno = posix_spawnattr_init(spawnattr)) != 0) {
         err(2, "posix_spawnattr_int");
@@ -119,10 +116,16 @@ setup_spawnattr(posix_spawnattr_t* spawnattr)
 
 #elif defined(__ppc__)
     cpu_types[0] = CPU_TYPE_POWERPC;
+
 #elif defined(__i386__)
     cpu_types[0] = CPU_TYPE_X86;
+
+#elif defined(__arm64__)
+    cpu_types[0] = CPU_TYPE_ARM64;
+
 #else
 #       error "Unknown CPU"
+
 #endif
 
     if (posix_spawnattr_setbinpref_np(spawnattr, count,
@@ -220,7 +223,8 @@ main(int argc, char **argv) {
     /* We're weak-linking to posix-spawnv to ensure that
      * an executable build on 10.5 can work on 10.4.
      */
-    if (posix_spawn != NULL) {
+
+    if (&posix_spawn != NULL) {
         posix_spawnattr_t spawnattr = NULL;
 
         setup_spawnattr(&spawnattr);