From 16f164e9b8b25b4c3e203249be44e910b0ee5827 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 22 Feb 2016 04:02:30 -0600 Subject: Issue #26268: Update the prepare_ssl.py script It can now handle OpenSSL versions 1.0.2e and greater, which don't include include files in include/. Note that sources prepared by this script no longer support the old project files for 2.7; you now have to have Perl available to use the old build_ssl.py script with sources from svn.python.org. --- Misc/NEWS | 4 ++ PCbuild/prepare_ssl.py | 165 +++++++++++++++++-------------------------------- 2 files changed, 59 insertions(+), 110 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 9de4fce..53dbcce 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -263,6 +263,10 @@ Build Windows ------- +- Issue #26268: Update the prepare_ssl.py script to handle OpenSSL releases + that don't include the contents of the include directory (that is, 1.0.2e + and later). + - Issue #26071: bdist_wininst created binaries fail to start and find 32bit Python diff --git a/PCbuild/prepare_ssl.py b/PCbuild/prepare_ssl.py index 4203dab..dea4d87 100644 --- a/PCbuild/prepare_ssl.py +++ b/PCbuild/prepare_ssl.py @@ -21,11 +21,11 @@ import os import re import sys -import shutil import subprocess +from shutil import copy # Find all "foo.exe" files on the PATH. -def find_all_on_path(filename, extras = None): +def find_all_on_path(filename, extras=None): entries = os.environ["PATH"].split(os.pathsep) ret = [] for p in entries: @@ -39,6 +39,7 @@ def find_all_on_path(filename, extras = None): ret.append(fname) return ret + # Find a suitable Perl installation for OpenSSL. # cygwin perl does *not* work. ActivePerl does. # Being a Perl dummy, the simplest way I can check is if the "Win32" package @@ -61,82 +62,47 @@ def find_working_perl(perls): print("NO perl interpreters were found on this machine at all!") print(" Please install ActivePerl and ensure it appears on your path") -def create_makefile64(makefile, m32): - """Create and fix makefile for 64bit - - Replace 32 with 64bit directories - """ - if not os.path.isfile(m32): - return - with open(m32) as fin: - with open(makefile, 'w') as fout: - for line in fin: - line = line.replace("=tmp32", "=tmp64") - line = line.replace("=out32", "=out64") - line = line.replace("=inc32", "=inc64") - # force 64 bit machine - line = line.replace("MKLIB=lib", "MKLIB=lib /MACHINE:X64") - line = line.replace("LFLAGS=", "LFLAGS=/MACHINE:X64 ") - # don't link against the lib on 64bit systems - line = line.replace("bufferoverflowu.lib", "") - fout.write(line) - os.unlink(m32) - -def create_asms(makefile): + +def create_asms(makefile, tmp_d): #create a custom makefile out of the provided one asm_makefile = os.path.splitext(makefile)[0] + '.asm.mak' - with open(makefile) as fin: - with open(asm_makefile, 'w') as fout: - for line in fin: - # Keep everything up to the install target (it's convenient) - if line.startswith('install: all'): - break - else: + with open(makefile) as fin, open(asm_makefile, 'w') as fout: + for line in fin: + # Keep everything up to the install target (it's convenient) + if line.startswith('install: all'): + break + fout.write(line) + asms = [] + for line in fin: + if '.asm' in line and line.strip().endswith('.pl'): + asms.append(line.split(':')[0]) + while line.strip(): fout.write(line) - asms = [] - for line in fin: - if '.asm' in line and line.strip().endswith('.pl'): - asms.append(line.split(':')[0]) - while line.strip(): - fout.write(line) - line = next(fin) - fout.write('\n') + line = next(fin) + fout.write('\n') - fout.write('asms: $(TMP_D) ') - fout.write(' '.join(asms)) - fout.write('\n') + fout.write('asms: $(TMP_D) ') + fout.write(' '.join(asms)) + fout.write('\n') + os.system('nmake /f {} PERL=perl TMP_D={} asms'.format(asm_makefile, tmp_d)) - os.system('nmake /f {} PERL=perl asms'.format(asm_makefile)) - os.unlink(asm_makefile) - - -def fix_makefile(makefile): - """Fix some stuff in all makefiles - """ - if not os.path.isfile(makefile): - return +def copy_includes(makefile, suffix): + dir = 'include'+suffix+'\\openssl' + os.makedirs(dir, exist_ok=True) copy_if_different = r'$(PERL) $(SRC_D)\util\copy-if-different.pl' with open(makefile) as fin: - lines = fin.readlines() - with open(makefile, 'w') as fout: - for line in lines: - if line.startswith("PERL="): - continue - if line.startswith("CP="): - line = "CP=copy\n" - if line.startswith("MKDIR="): - line = "MKDIR=mkdir\n" - if line.startswith("CFLAG="): - line = line.strip() - for algo in ("RC5", "MDC2", "IDEA"): - noalgo = " -DOPENSSL_NO_%s" % algo - if noalgo not in line: - line = line + noalgo - line = line + '\n' + for line in fin: if copy_if_different in line: - line = line.replace(copy_if_different, 'copy /Y') - fout.write(line) + perl, script, src, dest = line.split() + if not '$(INCO_D)' in dest: + continue + # We're in the root of the source tree + src = src.replace('$(SRC_D)', '.').strip('"') + dest = dest.strip('"').replace('$(INCO_D)', dir) + print('copying', src, 'to', dest) + copy(src, dest) + def run_configure(configure, do_script): print("perl Configure "+configure+" no-idea no-mdc2") @@ -144,61 +110,37 @@ def run_configure(configure, do_script): print(do_script) os.system(do_script) -def cmp(f1, f2): - bufsize = 1024 * 8 - with open(f1, 'rb') as fp1, open(f2, 'rb') as fp2: - while True: - b1 = fp1.read(bufsize) - b2 = fp2.read(bufsize) - if b1 != b2: - return False - if not b1: - return True - -def copy(src, dst): - if os.path.isfile(dst) and cmp(src, dst): - return - shutil.copy(src, dst) def prep(arch): + makefile_template = "ms\\nt{}.mak" + generated_makefile = makefile_template.format('') if arch == "x86": configure = "VC-WIN32" do_script = "ms\\do_nasm" - makefile="ms\\nt.mak" - m32 = makefile - dirsuffix = "32" + suffix = "32" elif arch == "amd64": configure = "VC-WIN64A" do_script = "ms\\do_win64a" - makefile = "ms\\nt64.mak" - m32 = makefile.replace('64', '') - dirsuffix = "64" + suffix = "64" #os.environ["VSEXTCOMP_USECL"] = "MS_OPTERON" else: raise ValueError('Unrecognized platform: %s' % arch) - # rebuild makefile when we do the role over from 32 to 64 build - if arch == "amd64" and os.path.isfile(m32) and not os.path.isfile(makefile): - os.unlink(m32) - - # If the ssl makefiles do not exist, we invoke Perl to generate them. - # Due to a bug in this script, the makefile sometimes ended up empty - # Force a regeneration if it is. - if not os.path.isfile(makefile) or os.path.getsize(makefile)==0: - print("Creating the makefiles...") - sys.stdout.flush() - run_configure(configure, do_script) - - if arch == "amd64": - create_makefile64(makefile, m32) - fix_makefile(makefile) - copy(r"crypto\buildinf.h", r"crypto\buildinf_%s.h" % arch) - copy(r"crypto\opensslconf.h", r"crypto\opensslconf_%s.h" % arch) - else: - print(makefile, 'already exists!') + print("Creating the makefiles...") + sys.stdout.flush() + # run configure, copy includes, create asms + run_configure(configure, do_script) + makefile = makefile_template.format(suffix) + try: + os.unlink(makefile) + except FileNotFoundError: + pass + os.rename(generated_makefile, makefile) + copy_includes(makefile, suffix) print('creating asms...') - create_asms(makefile) + create_asms(makefile, 'tmp'+suffix) + def main(): if len(sys.argv) == 1: @@ -229,6 +171,9 @@ def main(): print("Found a working perl at '%s'" % (perl,)) else: sys.exit(1) + if not find_all_on_path('nmake.exe'): + print('Could not find nmake.exe, try running env.bat') + sys.exit(1) sys.stdout.flush() # Put our working Perl at the front of our path -- cgit v0.12 From 6fe57ad229d46516c1ead1d37c015824b2428746 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 22 Feb 2016 04:08:51 -0600 Subject: Issue #26268: Update Windows builds to use OpenSSL 1.0.2f --- Misc/NEWS | 3 ++- PCbuild/_hashlib.vcxproj | 4 ++-- PCbuild/_ssl.vcxproj | 4 ++-- PCbuild/get_externals.bat | 2 +- PCbuild/openssl.props | 7 +++---- PCbuild/python.props | 4 +++- PCbuild/readme.txt | 2 +- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 53dbcce..6155edd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -258,7 +258,8 @@ Build - Issue #25696: Fix installation of Python on UNIX with make -j9. -- Issue #26268: Update OS X 10.5 installer to use OpenSSL 1.0.2f. +- Issue #26268: Update OS X 10.5 installer and Windows builds to use + OpenSSL 1.0.2f. Windows ------- diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index d82b266..5616a1e 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -61,7 +61,7 @@ - $(opensslDir)include;%(AdditionalIncludeDirectories) + $(opensslIncludeDir);%(AdditionalIncludeDirectories) ws2_32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies) @@ -87,4 +87,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_ssl.vcxproj b/PCbuild/_ssl.vcxproj index 8594a06..73c9571 100644 --- a/PCbuild/_ssl.vcxproj +++ b/PCbuild/_ssl.vcxproj @@ -61,7 +61,7 @@ - $(opensslDir)include;%(AdditionalIncludeDirectories) + $(opensslIncludeDir);%(AdditionalIncludeDirectories) ws2_32.lib;crypt32.lib;$(OutDir)libeay$(PyDebugExt).lib;$(OutDir)ssleay$(PyDebugExt).lib;%(AdditionalDependencies) @@ -91,4 +91,4 @@ - \ No newline at end of file + diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 0b3c08b..f6ed070 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.6 if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06 -if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2d +if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2f set libraries=%libraries% sqlite-3.8.11.0 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-core-8.6.4.2 if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.6.4.2 diff --git a/PCbuild/openssl.props b/PCbuild/openssl.props index d094e59..e35c0d9 100644 --- a/PCbuild/openssl.props +++ b/PCbuild/openssl.props @@ -17,9 +17,8 @@ - + - @@ -53,7 +52,7 @@ 4244;4267 - $(opensslDir);$(opensslDir)include;$(opensslDir)crypto;$(opensslDir)crypto\asn1;$(opensslDir)crypto\evp;$(opensslDir)crypto\modes + $(opensslDir);$(opensslIncludeDir);$(opensslDir)crypto;$(opensslDir)crypto\asn1;$(opensslDir)crypto\evp;$(opensslDir)crypto\modes $(_PreprocessorDefinitionList);%(PreprocessorDefinitions) @@ -74,4 +73,4 @@ $(nasm) -o "$(IntDir)%(NasmCompile.Filename).obj" "%(NasmCompile.FullPath)"' /> - \ No newline at end of file + diff --git a/PCbuild/python.props b/PCbuild/python.props index 4d8f603..8ad9a89 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -41,7 +41,9 @@ $(ExternalsDir)sqlite-3.8.11.0\ $(ExternalsDir)bzip2-1.0.6\ $(ExternalsDir)xz-5.0.5\ - $(ExternalsDir)openssl-1.0.2d\ + $(ExternalsDir)openssl-1.0.2f\ + $(opensslDir)include32 + $(opensslDir)include64 $(ExternalsDir)\nasm-2.11.06\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 09a996f..78ee0f9 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -169,7 +169,7 @@ _lzma Homepage: http://tukaani.org/xz/ _ssl - Python wrapper for version 1.0.2d of the OpenSSL secure sockets + Python wrapper for version 1.0.2f of the OpenSSL secure sockets library, which is built by ssl.vcxproj Homepage: http://www.openssl.org/ -- cgit v0.12