Issue #17128: Use private version of OpenSSL for 3.x OS X 10.5+ installers.

Among other issues, the Apple-supplied 0.9.7 libs for the 10.5 ABI cannot
verify newer SHA-256 certs as now used by python.org services.  Document
in the installer ReadMe some of the certificate management issues that
users now need to be more concerned with due to PEP 476's enabling cert
verification by default.  For now, continue to use the Apple-supplied
0.9.8 libs for the 10.6+ installer since they use Apple private APIs to
verify certificates using the system- and user-managed CA keychain stores.

1 files changed, 218 insertions, 27 deletions

diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
index 46e715a..d0c91ec 100755
--- a/Mac/BuildScript/build-installer.py
+++ b/Mac/BuildScript/build-installer.py
@@ -62,11 +62,16 @@ def shellQuote(value):
     return "'%s'"%(value.replace("'", "'\"'\"'"))
 
 def grepValue(fn, variable):
+    """
+    Return the unquoted value of a variable from a file..
+    QUOTED_VALUE='quotes'    -> str('quotes')
+    UNQUOTED_VALUE=noquotes  -> str('noquotes')
+    """
     variable = variable + '='
     for ln in open(fn, 'r'):
         if ln.startswith(variable):
             value = ln[len(variable):].strip()
-            return value[1:-1]
+            return value.strip("\"'")
     raise RuntimeError("Cannot find variable %s" % variable[:-1])
 
 _cache_getVersion = None
@@ -78,9 +83,6 @@ def getVersion():
             os.path.join(SRCDIR, 'configure'), 'PACKAGE_VERSION')
     return _cache_getVersion
 
-def getVersionTuple():
-    return tuple([int(n) for n in getVersion().split('.')])
-
 def getVersionMajorMinor():
     return tuple([int(n) for n in getVersion().split('.', 2)])
 
@@ -97,6 +99,9 @@ def getFullVersion():
             return _cache_getFullVersion
     raise RuntimeError("Cannot find full version??")
 
+FW_PREFIX = ["Library", "Frameworks", "Python.framework"]
+FW_VERSION_PREFIX = "--undefined--" # initialized in parseOptions
+
 # The directory we'll use to create the build (will be erased and recreated)
 WORKDIR = "/tmp/_py"
 
@@ -164,7 +169,7 @@ def getTargetCompilers():
 
 CC, CXX = getTargetCompilers()
 
-PYTHON_3 = getVersionTuple() >= (3, 0)
+PYTHON_3 = getVersionMajorMinor() >= (3, 0)
 
 USAGE = textwrap.dedent("""\
     Usage: build_python [options]
@@ -188,6 +193,10 @@ USAGE = textwrap.dedent("""\
 #                       '/Library/Frameworks/Tk.framework/Versions/8.5/Tk']
 EXPECTED_SHARED_LIBS = {}
 
+# List of names of third party software built with this installer.
+# The names will be inserted into the rtf version of the License.
+THIRD_PARTY_LIBS = []
+
 # Instructions for building libraries that are necessary for building a
 # batteries included python.
 #   [The recipes are defined here for convenience but instantiated later after
@@ -197,6 +206,49 @@ def library_recipes():
 
     LT_10_5 = bool(getDeptargetTuple() < (10, 5))
 
+    if getDeptargetTuple() < (10, 6):
+        # The OpenSSL libs shipped with OS X 10.5 and earlier are
+        # hopelessly out-of-date and do not include Apple's tie-in to
+        # the root certificates in the user and system keychains via TEA
+        # that was introduced in OS X 10.6.  Note that this applies to
+        # programs built and linked with a 10.5 SDK even when run on
+        # newer versions of OS X.
+        #
+        # Dealing with CAs is messy.  For now, just supply a
+        # local libssl and libcrypto for the older installer variants
+        # (e.g. the python.org 10.5+ 32-bit-only installer) that use the
+        # same default ssl certfile location as the system libs do:
+        #   /System/Library/OpenSSL/cert.pem
+        # Then at least TLS connections can be negotiated with sites that
+        # use sha-256 certs like python.org, assuming the proper CA certs
+        # have been supplied.  The default CA cert management issues for
+        # 10.5 and earlier builds are the same as before, other than it is
+        # now more obvious with cert checking enabled by default in the
+        # standard library.
+        #
+        # For builds with 10.6+ SDKs, continue to use the deprecated but
+        # less out-of-date Apple 0.9.8 libs for now.  While they are less
+        # secure than using an up-to-date 1.0.1 version, doing so
+        # avoids the big problems of forcing users to have to manage
+        # default CAs themselves, thanks to the Apple libs using private TEA
+        # APIs for cert validation from keychains if validation using the
+        # standard OpenSSL locations (/System/Library/OpenSSL, normally empty)
+        # fails.
+
+        result.extend([
+          dict(
+              name="OpenSSL 1.0.1j",
+              url="https://www.openssl.org/source/openssl-1.0.1j.tar.gz",
+              checksum='f7175c9cd3c39bb1907ac8bba9df8ed3',
+              patches=[
+                  "openssl_sdk_makedepend.patch",
+                   ],
+              buildrecipe=build_universal_openssl,
+              configure=None,
+              install=None,
+          ),
+        ])
+
 #   Disable for now
     if False:   # if getDeptargetTuple() > (10, 5):
         result.extend([
@@ -617,6 +669,7 @@ def parseOptions(args=None):
     """
     global WORKDIR, DEPSRC, SDKPATH, SRCDIR, DEPTARGET
     global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX
+    global FW_VERSION_PREFIX
 
     if args is None:
         args = sys.argv[1:]
@@ -676,19 +729,21 @@ def parseOptions(args=None):
 
     CC, CXX = getTargetCompilers()
 
-    print("Settings:")
-    print(" * Source directory:", SRCDIR)
-    print(" * Build directory: ", WORKDIR)
-    print(" * SDK location:    ", SDKPATH)
-    print(" * Third-party source:", DEPSRC)
-    print(" * Deployment target:", DEPTARGET)
-    print(" * Universal architectures:", ARCHLIST)
-    print(" * C compiler:", CC)
-    print(" * C++ compiler:", CXX)
+    FW_VERSION_PREFIX = FW_PREFIX[:] + ["Versions", getVersion()]
+
+    print("-- Settings:")
+    print("   * Source directory:    %s" % SRCDIR)
+    print("   * Build directory:     %s" % WORKDIR)
+    print("   * SDK location:        %s" % SDKPATH)
+    print("   * Third-party source:  %s" % DEPSRC)
+    print("   * Deployment target:   %s" % DEPTARGET)
+    print("   * Universal archs:     %s" % str(ARCHLIST))
+    print("   * C compiler:          %s" % CC)
+    print("   * C++ compiler:        %s" % CXX)
+    print("")
+    print(" -- Building a Python %s framework at patch level %s"
+                % (getVersion(), getFullVersion()))
     print("")
-
-
-
 
 def extractArchive(builddir, archiveName):
     """
@@ -780,6 +835,132 @@ def verifyThirdPartyFile(url, checksum, fname):
                 % (shellQuote(fname), checksum) ):
         fatal('MD5 checksum mismatch for file %s' % fname)
 
+def build_universal_openssl(basedir, archList):
+    """
+    Special case build recipe for universal build of openssl.
+
+    The upstream OpenSSL build system does not directly support
+    OS X universal builds.  We need to build each architecture
+    separately then lipo them together into fat libraries.
+    """
+
+    # OpenSSL fails to build with Xcode 2.5 (on OS X 10.4).
+    # If we are building on a 10.4.x or earlier system,
+    # unilaterally disable assembly code building to avoid the problem.
+    no_asm = int(platform.release().split(".")[0]) < 9
+
+    def build_openssl_arch(archbase, arch):
+        "Build one architecture of openssl"
+        arch_opts = {
+            "i386": ["darwin-i386-cc"],
+            "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"],
+            "ppc": ["darwin-ppc-cc"],
+            "ppc64": ["darwin64-ppc-cc"],
+        }
+        configure_opts = [
+            "no-krb5",
+            "no-idea",
+            "no-mdc2",
+            "no-rc5",
+            "no-zlib",
+            "enable-tlsext",
+            "no-ssl2",
+            "no-ssl3",
+            "no-ssl3-method",
+            # "enable-unit-test",
+            "shared",
+            "--install_prefix=%s"%shellQuote(archbase),
+            "--prefix=%s"%os.path.join("/", *FW_VERSION_PREFIX),
+            "--openssldir=/System/Library/OpenSSL",
+        ]
+        if no_asm:
+            configure_opts.append("no-asm")
+        runCommand(" ".join(["perl", "Configure"]
+                        + arch_opts[arch] + configure_opts))
+        runCommand("make depend OSX_SDK=%s" % SDKPATH)
+        runCommand("make all OSX_SDK=%s" % SDKPATH)
+        runCommand("make install_sw OSX_SDK=%s" % SDKPATH)
+        # runCommand("make test")
+        return
+
+    srcdir = os.getcwd()
+    universalbase = os.path.join(srcdir, "..",
+                        os.path.basename(srcdir) + "-universal")
+    os.mkdir(universalbase)
+    archbasefws = []
+    for arch in archList:
+        # fresh copy of the source tree
+        archsrc = os.path.join(universalbase, arch, "src")
+        shutil.copytree(srcdir, archsrc, symlinks=True)
+        # install base for this arch
+        archbase = os.path.join(universalbase, arch, "root")
+        os.mkdir(archbase)
+        # Python framework base within install_prefix:
+        # the build will install into this framework..
+        # This is to ensure that the resulting shared libs have
+        # the desired real install paths built into them.
+        archbasefw = os.path.join(archbase, *FW_VERSION_PREFIX)
+
+        # build one architecture
+        os.chdir(archsrc)
+        build_openssl_arch(archbase, arch)
+        os.chdir(srcdir)
+        archbasefws.append(archbasefw)
+
+    # copy arch-independent files from last build into the basedir framework
+    basefw = os.path.join(basedir, *FW_VERSION_PREFIX)
+    shutil.copytree(
+            os.path.join(archbasefw, "include", "openssl"),
+            os.path.join(basefw, "include", "openssl")
+            )
+
+    shlib_version_number = grepValue(os.path.join(archsrc, "Makefile"),
+            "SHLIB_VERSION_NUMBER")
+    #   e.g. -> "1.0.0"
+    libcrypto = "libcrypto.dylib"
+    libcrypto_versioned = libcrypto.replace(".", "."+shlib_version_number+".")
+    #   e.g. -> "libcrypto.1.0.0.dylib"
+    libssl = "libssl.dylib"
+    libssl_versioned = libssl.replace(".", "."+shlib_version_number+".")
+    #   e.g. -> "libssl.1.0.0.dylib"
+
+    try:
+        os.mkdir(os.path.join(basefw, "lib"))
+    except OSError:
+        pass
+
+    # merge the individual arch-dependent shared libs into a fat shared lib
+    archbasefws.insert(0, basefw)
+    for (lib_unversioned, lib_versioned) in [
+                (libcrypto, libcrypto_versioned),
+                (libssl, libssl_versioned)
+            ]:
+        runCommand("lipo -create -output " +
+                    " ".join(shellQuote(
+                            os.path.join(fw, "lib", lib_versioned))
+                                    for fw in archbasefws))
+        # and create an unversioned symlink of it
+        os.symlink(lib_versioned, os.path.join(basefw, "lib", lib_unversioned))
+
+    # Create links in the temp include and lib dirs that will be injected
+    # into the Python build so that setup.py can find them while building
+    # and the versioned links so that the setup.py post-build import test
+    # does not fail.
+    relative_path = os.path.join("..", "..", "..", *FW_VERSION_PREFIX)
+    for fn in [
+            ["include", "openssl"],
+            ["lib", libcrypto],
+            ["lib", libssl],
+            ["lib", libcrypto_versioned],
+            ["lib", libssl_versioned],
+        ]:
+        os.symlink(
+            os.path.join(relative_path, *fn),
+            os.path.join(basedir, "usr", "local", *fn)
+        )
+
+    return
+
 def buildRecipe(recipe, basedir, archList):
     """
     Build software using a recipe. This function does the
@@ -789,8 +970,10 @@ def buildRecipe(recipe, basedir, archList):
     curdir = os.getcwd()
 
     name = recipe['name']
+    THIRD_PARTY_LIBS.append(name)
     url = recipe['url']
     configure = recipe.get('configure', './configure')
+    buildrecipe = recipe.get('buildrecipe', None)
     install = recipe.get('install', 'make && make install DESTDIR=%s'%(
         shellQuote(basedir)))
 
@@ -888,8 +1071,13 @@ def buildRecipe(recipe, basedir, archList):
         print("Running configure for %s"%(name,))
         runCommand(' '.join(configure_args) + ' 2>&1')
 
-    print("Running install for %s"%(name,))
-    runCommand('{ ' + install + ' ;} 2>&1')
+    if buildrecipe is not None:
+        # call special-case build recipe, e.g. for openssl
+        buildrecipe(basedir, archList)
+
+    if install is not None:
+        print("Running install for %s"%(name,))
+        runCommand('{ ' + install + ' ;} 2>&1')
 
     print("Done %s"%(name,))
     print("")
@@ -1145,6 +1333,7 @@ def patchFile(inPath, outPath):
     data = data.replace('$MACOSX_DEPLOYMENT_TARGET', ''.join((DEPTARGET, ' or later')))
     data = data.replace('$ARCHITECTURES', ", ".join(universal_opts_map[UNIVERSALARCHS]))
     data = data.replace('$INSTALL_SIZE', installSize())
+    data = data.replace('$THIRD_PARTY_LIBS', "\\\n".join(THIRD_PARTY_LIBS))
 
     # This one is not handy as a template variable
     data = data.replace('$PYTHONFRAMEWORKINSTALLDIR', '/Library/Frameworks/Python.framework')
@@ -1327,8 +1516,6 @@ def buildInstaller():
         else:
             patchFile(os.path.join('resources', fn), os.path.join(rsrcDir, fn))
 
-    shutil.copy("../../LICENSE", os.path.join(rsrcDir, 'License.txt'))
-
 
 def installSize(clear=False, _saved=[]):
     if clear:
@@ -1436,12 +1623,14 @@ def main():
 
 
     # Prepare the applications folder
-    fn = os.path.join(WORKDIR, "_root", "Applications",
-                "Python %s"%(getVersion(),), "Update Shell Profile.command")
-    patchScript("scripts/postflight.patch-profile",  fn)
-
     folder = os.path.join(WORKDIR, "_root", "Applications", "Python %s"%(
         getVersion(),))
+    fn = os.path.join(folder, "License.rtf")
+    patchFile("resources/License.rtf",  fn)
+    fn = os.path.join(folder, "ReadMe.rtf")
+    patchFile("resources/ReadMe.rtf",  fn)
+    fn = os.path.join(folder, "Update Shell Profile.command")
+    patchScript("scripts/postflight.patch-profile",  fn)
     os.chmod(folder, STAT_0o755)
     setIcon(folder, "../Icons/Python Folder.icns")
 
@@ -1449,10 +1638,12 @@ def main():
     buildInstaller()
 
     # And copy the readme into the directory containing the installer
-    patchFile('resources/ReadMe.txt', os.path.join(WORKDIR, 'installer', 'ReadMe.txt'))
+    patchFile('resources/ReadMe.rtf',
+                os.path.join(WORKDIR, 'installer', 'ReadMe.rtf'))
 
     # Ditto for the license file.
-    shutil.copy('../../LICENSE', os.path.join(WORKDIR, 'installer', 'License.txt'))
+    patchFile('resources/License.rtf',
+                os.path.join(WORKDIR, 'installer', 'License.rtf'))
 
     fp = open(os.path.join(WORKDIR, 'installer', 'Build.txt'), 'w')
     fp.write("# BUILD INFO\n")