bpo-41100: Support macOS 11 and Apple Silicon (GH-22855)

Co-authored-by:  Lawrence D’Anna <lawrence_danna@apple.com>

* Add support for macOS 11 and Apple Silicon (aka arm64)
   
  As a side effect of this work use the system copy of libffi on macOS, and remove the vendored copy

* Support building on recent versions of macOS while deploying to older versions

  This allows building installers on macOS 11 while still supporting macOS 10.9.

4 files changed, 115 insertions, 5 deletions

diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
index 2548b21..0e76d3c 100755
--- a/Mac/BuildScript/build-installer.py
+++ b/Mac/BuildScript/build-installer.py
@@ -116,7 +116,8 @@ WORKDIR = "/tmp/_py"
 DEPSRC = os.path.join(WORKDIR, 'third-party')
 DEPSRC = os.path.expanduser('~/Universal/other-sources')
 
-universal_opts_map = { '32-bit': ('i386', 'ppc',),
+universal_opts_map = { 'universal2': ('arm64', 'x86_64'),
+                       '32-bit': ('i386', 'ppc',),
                        '64-bit': ('x86_64', 'ppc64',),
                        'intel':  ('i386', 'x86_64'),
                        'intel-32':  ('i386',),
@@ -124,6 +125,7 @@ universal_opts_map = { '32-bit': ('i386', 'ppc',),
                        '3-way':  ('ppc', 'i386', 'x86_64'),
                        'all':    ('i386', 'ppc', 'x86_64', 'ppc64',) }
 default_target_map = {
+        'universal2': '10.9',
         '64-bit': '10.5',
         '3-way': '10.5',
         'intel': '10.5',
@@ -190,6 +192,27 @@ EXPECTED_SHARED_LIBS = {}
 def internalTk():
     return getDeptargetTuple() >= (10, 6)
 
+
+def tweak_tcl_build(basedir, archList):
+    with open("Makefile", "r") as fp:
+        contents = fp.readlines()
+
+    # For reasons I don't understand the tcl configure script
+    # decides that some stdlib symbols aren't present, before
+    # deciding that strtod is broken.
+    new_contents = []
+    for line in contents:
+        if line.startswith("COMPAT_OBJS"):
+            # note: the space before strtod.o is intentional,
+            # the detection of a broken strtod results in
+            # "fixstrod.o" on this line.
+            for nm in ("strstr.o", "strtoul.o", " strtod.o"):
+                line = line.replace(nm, "")
+        new_contents.append(line)
+
+    with open("Makefile", "w") as fp:
+        fp.writelines(new_contents)
+
 # List of names of third party software built with this installer.
 # The names will be inserted into the rtf version of the License.
 THIRD_PARTY_LIBS = []
@@ -215,6 +238,9 @@ def library_recipes():
               buildrecipe=build_universal_openssl,
               configure=None,
               install=None,
+              patches=[
+                  "openssl-mac-arm64.patch",
+                   ],
           ),
     ])
 
@@ -231,6 +257,7 @@ def library_recipes():
                     '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),),
               ],
               useLDFlags=False,
+              buildrecipe=tweak_tcl_build,
               install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{
                   "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')),
                   "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())),
@@ -801,6 +828,7 @@ def build_universal_openssl(basedir, archList):
         arch_opts = {
             "i386": ["darwin-i386-cc"],
             "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"],
+            "arm64": ["darwin64-arm64-cc"],
             "ppc": ["darwin-ppc-cc"],
             "ppc64": ["darwin64-ppc-cc"],
         }
diff --git a/Mac/BuildScript/openssl-mac-arm64.patch b/Mac/BuildScript/openssl-mac-arm64.patch
new file mode 100644
index 0000000..11267fb
--- /dev/null
+++ b/Mac/BuildScript/openssl-mac-arm64.patch
@@ -0,0 +1,41 @@
+diff -ur openssl-1.1.1g-orig/Configurations/10-main.conf openssl-1.1.1g/Configurations/10-main.conf
+--- openssl-1.1.1g-orig/Configurations/10-main.conf	2020-04-21 14:22:39.000000000 +0200
++++ openssl-1.1.1g/Configurations/10-main.conf	2020-07-26 12:21:32.000000000 +0200
+@@ -1557,6 +1557,14 @@
+         bn_ops           => "SIXTY_FOUR_BIT_LONG",
+         perlasm_scheme   => "macosx",
+     },
++    "darwin64-arm64-cc" => {
++        inherit_from     => [ "darwin-common", asm("aarch64_asm") ],
++        CFLAGS           => add("-Wall"),
++        cflags           => add("-arch arm64"),
++        lib_cppflags     => add("-DL_ENDIAN"),
++        bn_ops           => "SIXTY_FOUR_BIT_LONG",
++        perlasm_scheme   => "ios64",
++    },
+ 
+ ##### GNU Hurd
+     "hurd-x86" => {
+diff -ur openssl-1.1.1g-orig/config openssl-1.1.1g/config
+--- openssl-1.1.1g-orig/config	2020-04-21 14:22:39.000000000 +0200
++++ openssl-1.1.1g/config	2020-07-26 12:21:59.000000000 +0200
+@@ -255,6 +255,9 @@
+ 		;;
+ 	    x86_64)
+ 		echo "x86_64-apple-darwin${VERSION}"
++                ;;
++	    arm64)
++		echo "arm64-apple-darwin${VERSION}"
+ 		;;
+ 	    *)
+ 		echo "i686-apple-darwin${VERSION}"
+@@ -497,6 +500,9 @@
+ 	else
+ 	    OUT="darwin64-x86_64-cc"
+ 	fi ;;
++  x86_64-apple-darwin*)
++	OUT="darwin64-arm64-cc"
++        ;;
+   armv6+7-*-iphoneos)
+ 	__CNF_CFLAGS="$__CNF_CFLAGS -arch armv6 -arch armv7"
+ 	__CNF_CXXFLAGS="$__CNF_CXXFLAGS -arch armv6 -arch armv7"
diff --git a/Mac/README.rst b/Mac/README.rst
index ec7d873..f3638aa 100644
--- a/Mac/README.rst
+++ b/Mac/README.rst
@@ -120,6 +120,8 @@ support ppc (Xcode 4 on 10.6 and later systems).  The flavor can be specified
 using the configure option ``--with-universal-archs=VALUE``. The following
 values are available:
 
+  * ``universal2``: ``arm64``, ``x86_64``
+
   * ``intel``:	  ``i386``, ``x86_64``
 
   * ``intel-32``: ``i386``
@@ -155,6 +157,8 @@ following combinations of SDKs and universal-archs flavors are available:
 
   * 10.15 and later SDKs support ``intel-64`` only
 
+  * 11.0 and later SDKs support ``universal2``
+
 The makefile for a framework build will also install ``python3.x-32``
 binaries when the universal architecture includes at least one 32-bit
 architecture (that is, for all flavors but ``64-bit`` and ``intel-64``).
@@ -352,6 +356,39 @@ A framework install also installs some applications in ``/Applications/Python X.
 And lastly a framework installation installs files in ``/usr/local/bin``, all of
 them symbolic links to files in ``/Library/Frameworks/Python.framework/Versions/X.Y/bin``.
 
+Weak linking support
+====================
+
+The CPython sources support building with the latest SDK while targetting deployment
+to macOS 10.9. This is done through weak linking of symbols introduced in macOS
+10.10 or later and checking for their availability at runtime.
+
+This requires the use of Apple's compiler toolchain on macOS 10.13 or later.
+
+The basic implementation pattern is:
+
+* ``HAVE_<FUNCTION>`` is a macro defined (or not) by the configure script
+
+* ``HAVE_<FUNCTION>_RUNTIME`` is a macro defined in the relevant source
+  files. This expands to a call to ``__builtin_available`` when using
+  a new enough Apple compiler, and to a true value otherwise.
+
+* Use ``HAVE_<FUNCTION>_RUNTIME`` before calling ``<function>``. This macro
+  *must* be used a the sole expression in an if statement::
+
+   if (HAVE_<FUNCTION>_RUNTIME) {
+     /* <function> is available */
+   }
+
+  Or:
+
+   if (HAVE_<FUNCTION>_RUNTIME) {} else {
+     /* <function> is not available */
+   }
+
+  Using other patterns (such as ``!HAVE_<FUNCTION>_RUNTIME``) is not supported
+  by Apple's compilers.
+
 
 Resources
 =========
diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c
index c8bd3ba..78813e8 100644
--- a/Mac/Tools/pythonw.c
+++ b/Mac/Tools/pythonw.c
@@ -95,9 +95,6 @@ setup_spawnattr(posix_spawnattr_t* spawnattr)
     size_t count;
     cpu_type_t cpu_types[1];
     short flags = 0;
-#ifdef __LP64__
-    int   ch;
-#endif
 
     if ((errno = posix_spawnattr_init(spawnattr)) != 0) {
         err(2, "posix_spawnattr_int");
@@ -119,10 +116,16 @@ setup_spawnattr(posix_spawnattr_t* spawnattr)
 
 #elif defined(__ppc__)
     cpu_types[0] = CPU_TYPE_POWERPC;
+
 #elif defined(__i386__)
     cpu_types[0] = CPU_TYPE_X86;
+
+#elif defined(__arm64__)
+    cpu_types[0] = CPU_TYPE_ARM64;
+
 #else
 #       error "Unknown CPU"
+
 #endif
 
     if (posix_spawnattr_setbinpref_np(spawnattr, count,
@@ -220,7 +223,8 @@ main(int argc, char **argv) {
     /* We're weak-linking to posix-spawnv to ensure that
      * an executable build on 10.5 can work on 10.4.
      */
-    if (posix_spawn != NULL) {
+
+    if (&posix_spawn != NULL) {
         posix_spawnattr_t spawnattr = NULL;
 
         setup_spawnattr(&spawnattr);