Issue #17128: Build OS X installer for 3.6 with private copy of OpenSSL.

Also provide a sample Install Certificates command script to install a
set of root certificates from the third-party certifi module.

4 files changed, 111 insertions, 82 deletions

diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
index ef93a6e..e983f54 100755
--- a/Mac/BuildScript/build-installer.py
+++ b/Mac/BuildScript/build-installer.py
@@ -101,6 +101,7 @@ def getFullVersion():
 
 FW_PREFIX = ["Library", "Frameworks", "Python.framework"]
 FW_VERSION_PREFIX = "--undefined--" # initialized in parseOptions
+FW_SSL_DIRECTORY = "--undefined--" # initialized in parseOptions
 
 # The directory we'll use to create the build (will be erased and recreated)
 WORKDIR = "/tmp/_py"
@@ -206,41 +207,11 @@ def library_recipes():
 
     LT_10_5 = bool(getDeptargetTuple() < (10, 5))
 
-    if not (10, 5) < getDeptargetTuple() < (10, 10):
-        # The OpenSSL libs shipped with OS X 10.5 and earlier are
-        # hopelessly out-of-date and do not include Apple's tie-in to
-        # the root certificates in the user and system keychains via TEA
-        # that was introduced in OS X 10.6.  Note that this applies to
-        # programs built and linked with a 10.5 SDK even when run on
-        # newer versions of OS X.
-        #
-        # Dealing with CAs is messy.  For now, just supply a
-        # local libssl and libcrypto for the older installer variants
-        # (e.g. the python.org 10.5+ 32-bit-only installer) that use the
-        # same default ssl certfile location as the system libs do:
-        #   /System/Library/OpenSSL/cert.pem
-        # Then at least TLS connections can be negotiated with sites that
-        # use sha-256 certs like python.org, assuming the proper CA certs
-        # have been supplied.  The default CA cert management issues for
-        # 10.5 and earlier builds are the same as before, other than it is
-        # now more obvious with cert checking enabled by default in the
-        # standard library.
-        #
-        # For builds with 10.6 through 10.9 SDKs,
-        # continue to use the deprecated but
-        # less out-of-date Apple 0.9.8 libs for now.  While they are less
-        # secure than using an up-to-date 1.0.1 version, doing so
-        # avoids the big problems of forcing users to have to manage
-        # default CAs themselves, thanks to the Apple libs using private TEA
-        # APIs for cert validation from keychains if validation using the
-        # standard OpenSSL locations (/System/Library/OpenSSL, normally empty)
-        # fails.
-        #
-        # Since Apple removed the header files for the deprecated system
-        # OpenSSL as of the Xcode 7 release (for OS X 10.10+), we do not
-        # have much choice but to build our own copy here, too.
+    # Since Apple removed the header files for the deprecated system
+    # OpenSSL as of the Xcode 7 release (for OS X 10.10+), we do not
+    # have much choice but to build our own copy here, too.
 
-        result.extend([
+    result.extend([
           dict(
               name="OpenSSL 1.0.2h",
               url="https://www.openssl.org/source/openssl-1.0.2h.tar.gz",
@@ -252,7 +223,7 @@ def library_recipes():
               configure=None,
               install=None,
           ),
-        ])
+    ])
 
 #   Disable for now
     if False:   # if getDeptargetTuple() > (10, 5):
@@ -676,6 +647,7 @@ def parseOptions(args=None):
     global WORKDIR, DEPSRC, SDKPATH, SRCDIR, DEPTARGET
     global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX
     global FW_VERSION_PREFIX
+    global FW_SSL_DIRECTORY
 
     if args is None:
         args = sys.argv[1:]
@@ -736,6 +708,7 @@ def parseOptions(args=None):
     CC, CXX = getTargetCompilers()
 
     FW_VERSION_PREFIX = FW_PREFIX[:] + ["Versions", getVersion()]
+    FW_SSL_DIRECTORY = FW_VERSION_PREFIX[:] + ["etc", "openssl"]
 
     print("-- Settings:")
     print("   * Source directory:    %s" % SRCDIR)
@@ -877,7 +850,7 @@ def build_universal_openssl(basedir, archList):
             "shared",
             "--install_prefix=%s"%shellQuote(archbase),
             "--prefix=%s"%os.path.join("/", *FW_VERSION_PREFIX),
-            "--openssldir=/System/Library/OpenSSL",
+            "--openssldir=%s"%os.path.join("/", *FW_SSL_DIRECTORY),
         ]
         if no_asm:
             configure_opts.append("no-asm")
@@ -1195,12 +1168,14 @@ def buildPython():
                 'Python.framework', 'Versions', getVersion(),
                 'lib'))))
 
-    path_to_lib = os.path.join(rootDir, 'Library', 'Frameworks',
-                                'Python.framework', 'Versions',
-                                version, 'lib', 'python%s'%(version,))
+    frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework')
+    frmDirVersioned = os.path.join(frmDir, 'Versions', version)
+    path_to_lib = os.path.join(frmDirVersioned, 'lib', 'python%s'%(version,))
+    # create directory for OpenSSL certificates
+    sslDir = os.path.join(frmDirVersioned, 'etc', 'openssl')
+    os.makedirs(sslDir)
 
     print("Fix file modes")
-    frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework')
     gid = grp.getgrnam('admin').gr_gid
 
     shared_lib_error = False
@@ -1642,6 +1617,8 @@ def main():
     patchFile("resources/ReadMe.rtf",  fn)
     fn = os.path.join(folder, "Update Shell Profile.command")
     patchScript("scripts/postflight.patch-profile",  fn)
+    fn = os.path.join(folder, "Install Certificates.command")
+    patchScript("resources/install_certificates.command",  fn)
     os.chmod(folder, STAT_0o755)
     setIcon(folder, "../Icons/Python Folder.icns")
 
diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf
index 1af2451..04dceaf 100644
--- a/Mac/BuildScript/resources/ReadMe.rtf
+++ b/Mac/BuildScript/resources/ReadMe.rtf
@@ -1,6 +1,7 @@
-{\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf460
+{\rtf1\ansi\ansicpg1252\cocoartf1504
 {\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fmodern\fcharset0 CourierNewPSMT;}
 {\colortbl;\red255\green255\blue255;}
+{\*\expandedcolortbl;\csgray\c100000;}
 \margl1440\margr1440\vieww13380\viewh14600\viewkind0
 \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
 
@@ -11,60 +12,56 @@
 \b \cf0 \ul \ulc0 Which installer variant should I use?
 \b0 \ulnone \
 \
-For the initial alpha releases of Python 3.6, Python.org provides only one installer variant for download: one that installs a 
+\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
+
+\b \cf0 **NEW**
+\b0  For Python 3.6, the python.org website now provides only one installer variant for download: one that installs a 
 \i 64-bit/32-bit Intel
 \i0  Python capable of running on 
 \i Mac OS X 10.6 (Snow Leopard)
-\i0  or later.  This will change prior to the beta releases of 3.6.0.  This ReadMe was installed with the 
+\i0  or later.  This ReadMe was installed with the 
 \i $MACOSX_DEPLOYMENT_TARGET
-\i0  variant.  By default, Python will automatically run in 64-bit mode if your system supports it.  Also see 
-\i Certificate verification and OpenSSL
-\i0  below.  The Pythons installed by this installer is built with private copies of some third-party libraries not included with or newer than those in OS X itself.  The list of these libraries varies by installer variant and is included at the end of the License.rtf file.
+\i0  variant.  By default, Python will automatically run in 64-bit mode if your system supports it.  The Python installed by this installer is built with private copies of some third-party libraries not included with or newer than those in OS X itself.  The list of these libraries is included at the end of the License.rtf file.
 \b \ul \
 \
-Update your version of Tcl/Tk to use IDLE or other Tk applications
-\b0 \ulnone \
-\
-To use IDLE or other programs that use the Tkinter graphical user interface toolkit, you need to install a newer third-party version of the 
-\i Tcl/Tk
-\i0  frameworks.  Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of 
-\i Tcl/Tk
-\i0  for this version of Python and of Mac OS X.  For the initial alpha releases of Python 3.6, the installer is linked with Tcl/Tk 8.5; this will change prior to the beta releases of 3.6.0.\
-
-\b \ul \
 Certificate verification and OpenSSL\
+\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
 
-\b0 \ulnone \
-Python 3.6 includes a number of network security enhancements that were released in Python 3.4.3 and Python 2.7.10.  {\field{\*\fldinst{HYPERLINK "https://www.python.org/dev/peps/pep-0476/"}}{\fldrslt PEP 476}} changes several standard library modules, like 
-\i httplib
-\i0 , 
-\i urllib
-\i0 , and 
-\i xmlrpclib
-\i0 , to by default verify certificates presented by servers over secure (TLS) connections.  The verification is performed by the OpenSSL libraries that Python is linked to.  Prior to 3.4.3, both python.org installers dynamically linked with Apple-supplied OpenSSL libraries shipped with OS X.  OS X provides a multiple level security framework that stores trust certificates in system and user keychains managed by the 
+\b0 \cf0 \ulnone \
+\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
+
+\b \cf0 **NEW**
+\b0  This variant of Python 3.6 now includes its own private copy of OpenSSL 1.0.2.  Unlike previous releases, the deprecated Apple-supplied OpenSSL libraries are no longer used.  This also means that the trust certificates in system and user keychains managed by the 
 \i Keychain Access 
 \i0 application and the 
 \i security
-\i0  command line utility.\
-\
-For OS X 10.6+, Apple also provides 
-\i OpenSSL
-\i0  
-\i 0.9.8 libraries
-\i0 .  Apple's 0.9.8 version includes an important additional feature: if a certificate cannot be verified using the manually administered certificates in 
-\f1 /System/Library/OpenSSL
-\f0 , the certificates managed by the system security framework In the user and system keychains are also consulted (using Apple private APIs).  For the initial alpha releases of Python 3.6, the 
-\i 64-bit/32-bit 10.6+ python.org variant
-\i0  continues to be dynamically linked with Apple's OpenSSL 0.9.8 since it was felt that the loss of the system-provided certificates and management tools outweighs the additional security features provided by newer versions of OpenSSL.  This will change prior to the beta releases of 3.6.0 as Apple has deprecated use of the system-supplied OpenSSL libraries.  If you do need features from newer versions of OpenSSL, there are third-party OpenSSL wrapper packages available through 
-\i PyPI
-\i0 .\
+\i0  command line utility are no longer used as defaults by the Python 
+\f1 ssl
+\f0  module.  For 3.6.0b1, a sample command script is included in 
+\f1 /Applications/Python 3.6
+\f0  to install a curated bundle of default root certificates from the third-party 
+\f1 certifi
+\f0  package ({\field{\*\fldinst{HYPERLINK "https://pypi.python.org/pypi/certifi"}}{\fldrslt https://pypi.python.org/pypi/certifi}}).  If you choose to use 
+\f1 certifi
+\f0 , you should consider subscribing to the{\field{\*\fldinst{HYPERLINK "https://certifi.io/en/latest/"}}{\fldrslt  project's email update service}} to be notified when the certificate bundle is updated.\
 \
 The bundled 
 \f1 pip
-\f0  included with the Python 3.6 installers has its own default certificate store for verifying download connections.\
+\f0  included with the Python 3.6 installer has its own default certificate store for verifying download connections.\
+\
+\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
+
+\b \cf0 \ul Update your version of Tcl/Tk to use IDLE or other Tk applications
+\b0 \ulnone \
 \
+To use IDLE or other programs that use the Tkinter graphical user interface toolkit, you need to install a newer third-party version of the 
+\i Tcl/Tk
+\i0  frameworks.  Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of 
+\i Tcl/Tk
+\i0  for this version of Python and of Mac OS X.  For the initial alpha releases of Python 3.6, the installer is still linked with Tcl/Tk 8.5; this will change prior to the beta 2 release of 3.6.0.\
 
-\b \ul Other changes\
+\b \ul \
+Other changes\
 
 \b0 \ulnone \
 For other changes in this release, see the 
diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf
index dfb75d8..3a9ab04 100644
--- a/Mac/BuildScript/resources/Welcome.rtf
+++ b/Mac/BuildScript/resources/Welcome.rtf
@@ -1,8 +1,9 @@
-{\rtf1\ansi\ansicpg1252\cocoartf1343\cocoasubrtf160
+{\rtf1\ansi\ansicpg1252\cocoartf1504
 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;}
 {\colortbl;\red255\green255\blue255;}
+{\*\expandedcolortbl;\csgray\c100000;}
 \paperw11905\paperh16837\margl1440\margr1440\vieww12200\viewh10880\viewkind0
-\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640
+\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0
 
 \f0\fs24 \cf0 This package will install 
 \b Python $FULL_VERSION
@@ -16,8 +17,14 @@
 \b IDLE
 \b0 .\
 \
+\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0
 
-\b IMPORTANT:
+\b \cf0 NEW:
+\b0   There are important changes in this release regarding network security and trust certificates.  Please see the ReadMe for more details.\
+\
+\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0
+
+\b \cf0 IMPORTANT:
 \b0  
 \b IDLE
 \b0  and other programs using the 
diff --git a/Mac/BuildScript/resources/install_certificates.command b/Mac/BuildScript/resources/install_certificates.command
new file mode 100755
index 0000000..1d2e2d8
--- /dev/null
+++ b/Mac/BuildScript/resources/install_certificates.command
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+/Library/Frameworks/Python.framework/Versions/@PYVER@/bin/python@PYVER@ << "EOF"
+
+# install_certifi.py
+#
+# sample script to install or update a set of default Root Certificates
+# for the ssl module.  Uses the certificates provided by the certifi package:
+#       https://pypi.python.org/pypi/certifi
+
+import os
+import os.path
+import ssl
+import stat
+import subprocess
+import sys
+
+STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
+             | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP
+             | stat.S_IROTH |                stat.S_IXOTH )
+
+def main():
+    openssl_dir, openssl_cafile = os.path.split(
+        ssl.get_default_verify_paths().openssl_cafile)
+
+    print(" -- pip install --upgrade certifi")
+    subprocess.check_call([sys.executable,
+        "-E", "-s", "-m", "pip", "install", "--upgrade", "certifi"])
+
+    import certifi
+
+    # change working directory to the default SSL directory
+    os.chdir(openssl_dir)
+    relpath_to_certifi_cafile = os.path.relpath(certifi.where())
+    print(" -- removing any existing file or link")
+    try:
+        os.remove(openssl_cafile)
+    except FileNotFoundError:
+        pass
+    print(" -- creating symlink to certifi certificate bundle")
+    os.symlink(relpath_to_certifi_cafile, openssl_cafile)
+    print(" -- setting permissions")
+    os.chmod(openssl_cafile, STAT_0o775)
+    print(" -- update complete")
+
+if __name__ == '__main__':
+    main()
+EOF