From f56d132deb9fff861439ed56ed7414d22e4e4bb9 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 17 Jan 2024 21:52:23 +0000 Subject: gh-112984 Update Windows build and installer for free-threaded builds (GH-113129) --- .github/workflows/build_msi.yml | 2 + .github/workflows/reusable-windows.yml | 4 +- Doc/using/win_install_freethreaded.png | Bin 0 -> 236250 bytes Doc/using/windows.rst | 64 ++- Lib/test/test_ctypes/test_loading.py | 2 +- Lib/test/test_launcher.py | 40 +- Lib/test/test_regrtest.py | 4 + Lib/test/test_venv.py | 16 +- Lib/venv/__init__.py | 258 ++++++----- .../2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst | 1 + PC/layout/main.py | 110 ++++- PC/layout/support/constants.py | 3 + PC/layout/support/nuspec.py | 9 +- PC/layout/support/options.py | 19 +- PC/pyconfig.h.in | 19 +- PC/venvlauncher.c | 510 +++++++++++++++++++++ PCbuild/_asyncio.vcxproj | 2 +- PCbuild/_bz2.vcxproj | 2 +- PCbuild/_ctypes.vcxproj | 2 +- PCbuild/_ctypes_test.vcxproj | 2 +- PCbuild/_decimal.vcxproj | 2 +- PCbuild/_elementtree.vcxproj | 2 +- PCbuild/_hashlib.vcxproj | 2 +- PCbuild/_lzma.vcxproj | 2 +- PCbuild/_multiprocessing.vcxproj | 2 +- PCbuild/_overlapped.vcxproj | 2 +- PCbuild/_queue.vcxproj | 2 +- PCbuild/_socket.vcxproj | 2 +- PCbuild/_sqlite3.vcxproj | 2 +- PCbuild/_ssl.vcxproj | 2 +- PCbuild/_testbuffer.vcxproj | 2 +- PCbuild/_testcapi.vcxproj | 2 +- PCbuild/_testclinic.vcxproj | 4 +- PCbuild/_testclinic_limited.vcxproj | 2 +- PCbuild/_testconsole.vcxproj | 2 +- PCbuild/_testimportmultiple.vcxproj | 2 +- PCbuild/_testinternalcapi.vcxproj | 2 +- PCbuild/_testmultiphase.vcxproj | 2 +- PCbuild/_testsinglephase.vcxproj | 2 +- PCbuild/_tkinter.vcxproj | 2 +- PCbuild/_uuid.vcxproj | 2 +- PCbuild/_wmi.vcxproj | 2 +- PCbuild/_zoneinfo.vcxproj | 2 +- PCbuild/pyexpat.vcxproj | 2 +- PCbuild/pyproject.props | 13 +- PCbuild/python.props | 70 ++- PCbuild/python.vcxproj | 5 +- PCbuild/python3dll.vcxproj | 2 +- PCbuild/pythonw.vcxproj | 1 + PCbuild/rt.bat | 7 +- PCbuild/select.vcxproj | 2 +- PCbuild/sqlite3.vcxproj | 2 +- PCbuild/unicodedata.vcxproj | 2 +- PCbuild/venvlauncher.vcxproj | 9 +- PCbuild/venvlauncher.vcxproj.filters | 2 +- PCbuild/venvwlauncher.vcxproj | 9 +- PCbuild/venvwlauncher.vcxproj.filters | 2 +- PCbuild/winsound.vcxproj | 2 +- PCbuild/xxlimited.vcxproj | 2 +- PCbuild/xxlimited_35.vcxproj | 2 +- Tools/msi/build.bat | 17 + Tools/msi/bundle/Default.thm | 1 + Tools/msi/bundle/Default.wxl | 1 + Tools/msi/bundle/bundle.targets | 1 + Tools/msi/bundle/bundle.wxs | 10 + Tools/msi/bundle/packagegroups/freethreaded.wxs | 62 +++ Tools/msi/freethreaded/freethreaded.wixproj | 20 + Tools/msi/freethreaded/freethreaded.wxs | 18 + Tools/msi/freethreaded/freethreaded_d.wixproj | 20 + Tools/msi/freethreaded/freethreaded_d.wxs | 16 + .../freethreaded/freethreaded_en-US.wxl_template | 8 + Tools/msi/freethreaded/freethreaded_files.wxs | 175 +++++++ Tools/msi/freethreaded/freethreaded_pdb.wixproj | 20 + Tools/msi/freethreaded/freethreaded_pdb.wxs | 15 + Tools/msi/lib/lib_files.wxs | 12 +- Tools/msi/msi.props | 33 +- 76 files changed, 1436 insertions(+), 244 deletions(-) create mode 100644 Doc/using/win_install_freethreaded.png create mode 100644 Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst create mode 100644 PC/venvlauncher.c create mode 100644 Tools/msi/bundle/packagegroups/freethreaded.wxs create mode 100644 Tools/msi/freethreaded/freethreaded.wixproj create mode 100644 Tools/msi/freethreaded/freethreaded.wxs create mode 100644 Tools/msi/freethreaded/freethreaded_d.wixproj create mode 100644 Tools/msi/freethreaded/freethreaded_d.wxs create mode 100644 Tools/msi/freethreaded/freethreaded_en-US.wxl_template create mode 100644 Tools/msi/freethreaded/freethreaded_files.wxs create mode 100644 Tools/msi/freethreaded/freethreaded_pdb.wixproj create mode 100644 Tools/msi/freethreaded/freethreaded_pdb.wxs diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml index 29282df..65d32c7 100644 --- a/.github/workflows/build_msi.yml +++ b/.github/workflows/build_msi.yml @@ -32,6 +32,8 @@ jobs: strategy: matrix: type: [x86, x64, arm64] + env: + IncludeFreethreaded: true steps: - uses: actions/checkout@v4 - name: Build CPython installer diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index ae27c10..c0209e0 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -20,7 +20,7 @@ jobs: - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci + run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }} build_win_amd64: name: 'build and test (x64)' @@ -37,7 +37,7 @@ jobs: - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci + run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }} build_win_arm64: name: 'build (arm64)' diff --git a/Doc/using/win_install_freethreaded.png b/Doc/using/win_install_freethreaded.png new file mode 100644 index 0000000..0aa01c1 Binary files /dev/null and b/Doc/using/win_install_freethreaded.png differ diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 598bf3c..2a0e7b4 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -307,6 +307,46 @@ settings and replace any that have been removed or modified. "Uninstall" will remove Python entirely, with the exception of the :ref:`launcher`, which has its own entry in Programs and Features. +.. _install-freethreaded-windows: + +Installing Free-threaded Binaries +--------------------------------- + +.. versionadded:: 3.13 (Experimental) + +.. note:: + + Everything described in this section is considered experimental, + and should be expected to change in future releases. + +To install pre-built binaries with free-threading enabled (see :pep:`703`), you +should select "Customize installation". The second page of options includes the +"Download free-threaded binaries" checkbox. + +.. image:: win_install_freethreaded.png + +Selecting this option will download and install additional binaries to the same +location as the main Python install. The main executable is called +``python3.13t.exe``, and other binaries either receive a ``t`` suffix or a full +ABI suffix. Python source files and bundled third-party dependencies are shared +with the main install. + +The free-threaded version is registered as a regular Python install with the +tag ``3.13t`` (with a ``-32`` or ``-arm64`` suffix as normal for those +platforms). This allows tools to discover it, and for the :ref:`launcher` to +support ``py.exe -3.13t``. Note that the launcher will interpret ``py.exe -3`` +(or a ``python3`` shebang) as "the latest 3.x install", which will prefer the +free-threaded binaries over the regular ones, while ``py.exe -3.13`` will not. +If you use the short style of option, you may prefer to not install the +free-threaded binaries at this time. + +To specify the install option at the command line, use +``Include_freethreaded=1``. See :ref:`install-layout-option` for instructions on +pre-emptively downloading the additional binaries for offline install. The +options to include debug symbols and binaries also apply to the free-threaded +builds. + +Free-threaded binaries are also available :ref:`on nuget.org `. .. _windows-store: @@ -450,9 +490,29 @@ automatically use the headers and import libraries in your build. The package information pages on nuget.org are `www.nuget.org/packages/python `_ -for the 64-bit version and `www.nuget.org/packages/pythonx86 -`_ for the 32-bit version. +for the 64-bit version, `www.nuget.org/packages/pythonx86 +`_ for the 32-bit version, and +`www.nuget.org/packages/pythonarm64 +`_ for the ARM64 version + +Free-threaded packages +---------------------- + +.. versionadded:: 3.13 (Experimental) + +.. note:: + Everything described in this section is considered experimental, + and should be expected to change in future releases. + +Packages containing free-threaded binaries are named +`python-freethreaded `_ +for the 64-bit version, `pythonx86-freethreaded +`_ for the 32-bit +version, and `pythonarm64-freethreaded +`_ for the ARM64 +version. These packages contain both the ``python3.13t.exe`` and +``python.exe`` entry points, both of which run free threaded. .. _windows-embeddable: diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index 22db97b..9f0547f 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -141,7 +141,7 @@ class LoaderTest(unittest.TestCase): def test_load_dll_with_flags(self): _sqlite3 = import_helper.import_module("_sqlite3") src = _sqlite3.__file__ - if src.lower().endswith("_d.pyd"): + if src.partition(".")[0].lower().endswith("_d"): ext = "_d.dll" else: ext = ".dll" diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index bcd4ed6..3da6173 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -19,8 +19,10 @@ import winreg PY_EXE = "py.exe" +DEBUG_BUILD = False if sys.executable.casefold().endswith("_d.exe".casefold()): PY_EXE = "py_d.exe" + DEBUG_BUILD = True # Registry data to create. On removal, everything beneath top-level names will # be deleted. @@ -232,7 +234,7 @@ class RunPyMixin: p.stdin.close() p.wait(10) out = p.stdout.read().decode("utf-8", "replace") - err = p.stderr.read().decode("ascii", "replace") + err = p.stderr.read().decode("ascii", "replace").replace("\uFFFD", "?") if p.returncode != expect_returncode and support.verbose and not allow_fail: print("++ COMMAND ++") print([self.py_exe, *args]) @@ -273,7 +275,7 @@ class RunPyMixin: def fake_venv(self): venv = Path.cwd() / "Scripts" venv.mkdir(exist_ok=True, parents=True) - venv_exe = (venv / Path(sys.executable).name) + venv_exe = (venv / ("python_d.exe" if DEBUG_BUILD else "python.exe")) venv_exe.touch() try: yield venv_exe, {"VIRTUAL_ENV": str(venv.parent)} @@ -521,6 +523,9 @@ class TestLauncher(unittest.TestCase, RunPyMixin): self.assertEqual(str(venv_exe), m.group(1)) break else: + if support.verbose: + print(data["stdout"]) + print(data["stderr"]) self.fail("did not find active venv path") data = self.run_py(["-0"], env=env) @@ -616,25 +621,29 @@ class TestLauncher(unittest.TestCase, RunPyMixin): self.assertEqual("True", data["SearchInfo.oldStyleTag"]) def test_search_path(self): - stem = Path(sys.executable).stem + exe = Path("arbitrary-exe-name.exe").absolute() + exe.touch() + self.addCleanup(exe.unlink) with self.py_ini(TEST_PY_DEFAULTS): - with self.script(f"#! /usr/bin/env {stem} -prearg") as script: + with self.script(f"#! /usr/bin/env {exe.stem} -prearg") as script: data = self.run_py( [script, "-postarg"], - env={"PATH": f"{Path(sys.executable).parent};{os.getenv('PATH')}"}, + env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, ) - self.assertEqual(f"{sys.executable} -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"{exe} -prearg {script} -postarg", data["stdout"].strip()) def test_search_path_exe(self): # Leave the .exe on the name to ensure we don't add it a second time - name = Path(sys.executable).name + exe = Path("arbitrary-exe-name.exe").absolute() + exe.touch() + self.addCleanup(exe.unlink) with self.py_ini(TEST_PY_DEFAULTS): - with self.script(f"#! /usr/bin/env {name} -prearg") as script: + with self.script(f"#! /usr/bin/env {exe.name} -prearg") as script: data = self.run_py( [script, "-postarg"], - env={"PATH": f"{Path(sys.executable).parent};{os.getenv('PATH')}"}, + env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, ) - self.assertEqual(f"{sys.executable} -prearg {script} -postarg", data["stdout"].strip()) + self.assertEqual(f"{exe} -prearg {script} -postarg", data["stdout"].strip()) def test_recursive_search_path(self): stem = self.get_py_exe().stem @@ -727,15 +736,18 @@ class TestLauncher(unittest.TestCase, RunPyMixin): data = self.run_py([script], expect_returncode=103) with self.fake_venv() as (venv_exe, env): - # Put a real Python (ourselves) on PATH as a distraction. + # Put a "normal" Python on PATH as a distraction. # The active VIRTUAL_ENV should be preferred when the name isn't an # exact match. - env["PATH"] = f"{Path(sys.executable).parent};{os.environ['PATH']}" + exe = Path(Path(venv_exe).name).absolute() + exe.touch() + self.addCleanup(exe.unlink) + env["PATH"] = f"{exe.parent};{os.environ['PATH']}" with self.script(f'#! /usr/bin/env {stem} arg1') as script: data = self.run_py([script], env=env) self.assertEqual(data["stdout"].strip(), f"{venv_exe} arg1 {script}") - with self.script(f'#! /usr/bin/env {Path(sys.executable).stem} arg1') as script: + with self.script(f'#! /usr/bin/env {exe.stem} arg1') as script: data = self.run_py([script], env=env) - self.assertEqual(data["stdout"].strip(), f"{sys.executable} arg1 {script}") + self.assertEqual(data["stdout"].strip(), f"{exe} arg1 {script}") diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e828941..89562fa 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -845,6 +845,8 @@ class ProgramsTestCase(BaseTestCase): test_args.append('-x64') # 64-bit build if not support.Py_DEBUG: test_args.append('+d') # Release build, use python.exe + if sysconfig.get_config_var("Py_GIL_DISABLED"): + test_args.append('--disable-gil') self.run_batch(script, *test_args, *self.tests) @unittest.skipUnless(sys.platform == 'win32', 'Windows only') @@ -862,6 +864,8 @@ class ProgramsTestCase(BaseTestCase): rt_args.append('-x64') # 64-bit build if support.Py_DEBUG: rt_args.append('-d') # Debug build, use python_d.exe + if sysconfig.get_config_var("Py_GIL_DISABLED"): + rt_args.append('--disable-gil') self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 218e756..6852625 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -223,8 +223,14 @@ class BasicTest(BaseTest): def test_upgrade_dependencies(self): builder = venv.EnvBuilder() - bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' + bin_path = 'bin' python_exe = os.path.split(sys.executable)[1] + if sys.platform == 'win32': + bin_path = 'Scripts' + if os.path.normcase(os.path.splitext(python_exe)[0]).endswith('_d'): + python_exe = 'python_d.exe' + else: + python_exe = 'python.exe' with tempfile.TemporaryDirectory() as fake_env_dir: expect_exe = os.path.normcase( os.path.join(fake_env_dir, bin_path, python_exe) @@ -283,7 +289,9 @@ class BasicTest(BaseTest): # build environment ('is_python_build()', str(sysconfig.is_python_build())), ('get_makefile_filename()', sysconfig.get_makefile_filename()), - ('get_config_h_filename()', sysconfig.get_config_h_filename())): + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call out, err = check_output(cmd, encoding='utf-8') @@ -315,7 +323,9 @@ class BasicTest(BaseTest): # build environment ('is_python_build()', str(sysconfig.is_python_build())), ('get_makefile_filename()', sysconfig.get_makefile_filename()), - ('get_config_h_filename()', sysconfig.get_config_h_filename())): + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): with self.subTest(call): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call out, err = check_output(cmd, encoding='utf-8') diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index d960bf3..f04ca8f 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -139,6 +139,11 @@ class EnvBuilder: 'check that your PATH environment variable is ' 'correctly set.') dirname, exename = os.path.split(os.path.abspath(executable)) + if sys.platform == 'win32': + # Always create the simplest name in the venv. It will either be a + # link back to executable, or a copy of the appropriate launcher + _d = '_d' if os.path.splitext(exename)[0].endswith('_d') else '' + exename = f'python{_d}.exe' context.executable = executable context.python_dir = dirname context.python_exe = exename @@ -222,67 +227,26 @@ class EnvBuilder: args = ' '.join(args) f.write(f'command = {sys.executable} -m venv {args}\n') - if os.name != 'nt': - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - force_copy = not self.symlinks - if not force_copy: - try: - if not os.path.islink(dst): # can't link to itself! - if relative_symlinks_ok: - assert os.path.dirname(src) == os.path.dirname(dst) - os.symlink(os.path.basename(src), dst) - else: - os.symlink(src, dst) - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - force_copy = True - if force_copy: - shutil.copyfile(src, dst) - else: - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - bad_src = os.path.lexists(src) and not os.path.exists(src) - if self.symlinks and not bad_src and not os.path.islink(dst): - try: + def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): + """ + Try symlinking a file, and if that fails, fall back to copying. + (Unused on Windows, because we can't just copy a failed symlink file: we + switch to a different set of files instead.) + """ + assert os.name != 'nt' + force_copy = not self.symlinks + if not force_copy: + try: + if not os.path.islink(dst): # can't link to itself! if relative_symlinks_ok: assert os.path.dirname(src) == os.path.dirname(dst) os.symlink(os.path.basename(src), dst) else: os.symlink(src, dst) - return - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - - # On Windows, we rewrite symlinks to our base python.exe into - # copies of venvlauncher.exe - basename, ext = os.path.splitext(os.path.basename(src)) - srcfn = os.path.join(os.path.dirname(__file__), - "scripts", - "nt", - basename + ext) - # Builds or venv's from builds need to remap source file - # locations, as we do not put them into Lib/venv/scripts - if sysconfig.is_python_build() or not os.path.isfile(srcfn): - if basename.endswith('_d'): - ext = '_d' + ext - basename = basename[:-2] - if basename == 'python': - basename = 'venvlauncher' - elif basename == 'pythonw': - basename = 'venvwlauncher' - src = os.path.join(os.path.dirname(src), basename + ext) - else: - src = srcfn - if not os.path.exists(src): - if not bad_src: - logger.warning('Unable to copy %r', src) - return - + except Exception: # may need to use a more specific exception + logger.warning('Unable to symlink %r to %r', src, dst) + force_copy = True + if force_copy: shutil.copyfile(src, dst) def create_git_ignore_file(self, context): @@ -298,22 +262,23 @@ class EnvBuilder: 'see https://docs.python.org/3/library/venv.html\n') file.write('*\n') - def setup_python(self, context): - """ - Set up a Python executable in the environment. + if os.name != 'nt': + def setup_python(self, context): + """ + Set up a Python executable in the environment. - :param context: The information for the environment creation request - being processed. - """ - binpath = context.bin_path - path = context.env_exe - copier = self.symlink_or_copy - dirname = context.python_dir - if os.name != 'nt': + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + path = context.env_exe + copier = self.symlink_or_copy + dirname = context.python_dir copier(context.executable, path) if not os.path.islink(path): os.chmod(path, 0o755) - for suffix in ('python', 'python3', f'python3.{sys.version_info[1]}'): + for suffix in ('python', 'python3', + f'python3.{sys.version_info[1]}'): path = os.path.join(binpath, suffix) if not os.path.exists(path): # Issue 18807: make copies if @@ -321,30 +286,105 @@ class EnvBuilder: copier(context.env_exe, path, relative_symlinks_ok=True) if not os.path.islink(path): os.chmod(path, 0o755) - else: - if self.symlinks: - # For symlinking, we need a complete copy of the root directory - # If symlinks fail, you'll get unnecessary copies of files, but - # we assume that if you've opted into symlinks on Windows then - # you know what you're doing. - suffixes = [ - f for f in os.listdir(dirname) if - os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') - ] - if sysconfig.is_python_build(): - suffixes = [ - f for f in suffixes if - os.path.normcase(f).startswith(('python', 'vcruntime')) - ] + + else: + def setup_python(self, context): + """ + Set up a Python executable in the environment. + + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + dirname = context.python_dir + exename = os.path.basename(context.env_exe) + exe_stem = os.path.splitext(exename)[0] + exe_d = '_d' if os.path.normcase(exe_stem).endswith('_d') else '' + if sysconfig.is_python_build(): + scripts = dirname + else: + scripts = os.path.join(os.path.dirname(__file__), + 'scripts', 'nt') + if not sysconfig.get_config_var("Py_GIL_DISABLED"): + python_exe = os.path.join(dirname, f'python{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlauncher{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlauncher{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } else: - suffixes = {'python.exe', 'python_d.exe', 'pythonw.exe', 'pythonw_d.exe'} - base_exe = os.path.basename(context.env_exe) - suffixes.add(base_exe) + exe_t = f'3.{sys.version_info[1]}t' + python_exe = os.path.join(dirname, f'python{exe_t}{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_t}{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlaunchert{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlaunchert{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + + do_copies = True + if self.symlinks: + do_copies = False + # For symlinking, we need all the DLLs to be available alongside + # the executables. + link_sources.update({ + f: os.path.join(dirname, f) for f in os.listdir(dirname) + if os.path.normcase(f).startswith(('python', 'vcruntime')) + and os.path.normcase(os.path.splitext(f)[1]) == '.dll' + }) + + to_unlink = [] + for dest, src in link_sources.items(): + dest = os.path.join(binpath, dest) + try: + os.symlink(src, dest) + to_unlink.append(dest) + except OSError: + logger.warning('Unable to symlink %r to %r', src, dst) + do_copies = True + for f in to_unlink: + try: + os.unlink(f) + except OSError: + logger.warning('Failed to clean up symlink %r', + f) + logger.warning('Retrying with copies') + break - for suffix in suffixes: - src = os.path.join(dirname, suffix) - if os.path.lexists(src): - copier(src, os.path.join(binpath, suffix)) + if do_copies: + for dest, src in copy_sources.items(): + dest = os.path.join(binpath, dest) + try: + shutil.copy2(src, dest) + except OSError: + logger.warning('Unable to copy %r to %r', src, dest) if sysconfig.is_python_build(): # copy init.tcl @@ -437,6 +477,14 @@ class EnvBuilder: """ binpath = context.bin_path plen = len(path) + if os.name == 'nt': + def skip_file(f): + f = os.path.normcase(f) + return (f.startswith(('python', 'venv')) + and f.endswith(('.exe', '.pdb'))) + else: + def skip_file(f): + return False for root, dirs, files in os.walk(path): if root == path: # at top-level, remove irrelevant dirs for d in dirs[:]: @@ -444,8 +492,7 @@ class EnvBuilder: dirs.remove(d) continue # ignore files in top level for f in files: - if (os.name == 'nt' and f.startswith('python') - and f.endswith(('.exe', '.pdb'))): + if skip_file(f): continue srcfile = os.path.join(root, f) suffix = root[plen:].split(os.sep)[2:] @@ -456,20 +503,25 @@ class EnvBuilder: if not os.path.exists(dstdir): os.makedirs(dstdir) dstfile = os.path.join(dstdir, f) + if os.name == 'nt' and srcfile.endswith(('.exe', '.pdb')): + shutil.copy2(srcfile, dstfile) + continue with open(srcfile, 'rb') as f: data = f.read() - if not srcfile.endswith(('.exe', '.pdb')): - try: - data = data.decode('utf-8') - data = self.replace_variables(data, context) - data = data.encode('utf-8') - except UnicodeError as e: - data = None - logger.warning('unable to copy script %r, ' - 'may be binary: %s', srcfile, e) - if data is not None: + try: + new_data = ( + self.replace_variables(data.decode('utf-8'), context) + .encode('utf-8') + ) + except UnicodeError as e: + logger.warning('unable to copy script %r, ' + 'may be binary: %s', srcfile, e) + continue + if new_data == data: + shutil.copy2(srcfile, dstfile) + else: with open(dstfile, 'wb') as f: - f.write(data) + f.write(new_data) shutil.copymode(srcfile, dstfile) def upgrade_dependencies(self, context): diff --git a/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst b/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst new file mode 100644 index 0000000..429cd5b --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-19-22-32-28.gh-issue-112984.F7kFMl.rst @@ -0,0 +1 @@ +Adds free-threaded binaries to Windows installer as an optional component. diff --git a/PC/layout/main.py b/PC/layout/main.py index accfd51..d176b27 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -41,7 +41,7 @@ TCLTK_FILES_ONLY = FileNameSet("turtle.py") VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip") -EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext", "vcruntime*") +EXCLUDE_FROM_DLLS = FileStemSet("python*", "pyshellext", "vcruntime*") EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle") EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt") EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*") @@ -126,9 +126,9 @@ def get_layout(ns): n = new_name or n src = ns.build / f if ns.debug and src not in REQUIRED_DLLS: - if not src.stem.endswith("_d"): + if not "_d." in src.name: src = src.parent / (src.stem + "_d" + src.suffix) - if not n.endswith("_d"): + if "_d." not in f: n += "_d" f = n + "." + x yield dest + n + "." + x, src @@ -141,17 +141,45 @@ def get_layout(ns): if lib.is_file(): yield "libs/" + n + ".lib", lib + source = "python.exe" + sourcew = "pythonw.exe" + alias = [ + "python", + "python{}".format(VER_MAJOR) if ns.include_alias3 else "", + "python{}".format(VER_DOT) if ns.include_alias3x else "", + ] + aliasw = [ + "pythonw", + "pythonw{}".format(VER_MAJOR) if ns.include_alias3 else "", + "pythonw{}".format(VER_DOT) if ns.include_alias3x else "", + ] if ns.include_appxmanifest: - yield from in_build("python_uwp.exe", new_name="python{}".format(VER_DOT)) - yield from in_build("pythonw_uwp.exe", new_name="pythonw{}".format(VER_DOT)) - # For backwards compatibility, but we don't reference these ourselves. - yield from in_build("python_uwp.exe", new_name="python") - yield from in_build("pythonw_uwp.exe", new_name="pythonw") + source = "python_uwp.exe" + sourcew = "pythonw_uwp.exe" + elif ns.include_freethreaded: + source = "python{}t.exe".format(VER_DOT) + sourcew = "pythonw{}t.exe".format(VER_DOT) + if not ns.include_alias: + alias = [] + aliasw = [] + alias.extend([ + "python{}t".format(VER_DOT), + "python{}t".format(VER_MAJOR) if ns.include_alias3 else None, + ]) + aliasw.extend([ + "pythonw{}t".format(VER_DOT), + "pythonw{}t".format(VER_MAJOR) if ns.include_alias3 else None, + ]) + + for a in filter(None, alias): + yield from in_build(source, new_name=a) + for a in filter(None, aliasw): + yield from in_build(sourcew, new_name=a) + + if ns.include_freethreaded: + yield from in_build(FREETHREADED_PYTHON_DLL_NAME) else: - yield from in_build("python.exe", new_name="python") - yield from in_build("pythonw.exe", new_name="pythonw") - - yield from in_build(PYTHON_DLL_NAME) + yield from in_build(PYTHON_DLL_NAME) if ns.include_launchers and ns.include_appxmanifest: if ns.include_pip: @@ -160,7 +188,10 @@ def get_layout(ns): yield from in_build("pythonw_uwp.exe", new_name="idle{}".format(VER_DOT)) if ns.include_stable: - yield from in_build(PYTHON_STABLE_DLL_NAME) + if ns.include_freethreaded: + yield from in_build(FREETHREADED_PYTHON_STABLE_DLL_NAME) + else: + yield from in_build(PYTHON_STABLE_DLL_NAME) found_any = False for dest, src in rglob(ns.build, "vcruntime*.dll"): @@ -171,16 +202,28 @@ def get_layout(ns): yield "LICENSE.txt", ns.build / "LICENSE.txt" - for dest, src in rglob(ns.build, ("*.pyd", "*.dll")): - if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: - continue - if src in EXCLUDE_FROM_PYDS: - continue + for dest, src in rglob(ns.build, "*.pyd"): + if ns.include_freethreaded: + if not src.match("*.cp*t-win*.pyd"): + continue + if bool(src.match("*_d.cp*.pyd")) != bool(ns.debug): + continue + else: + if src.match("*.cp*t-win*.pyd"): + continue + if bool(src.match("*_d.pyd")) != bool(ns.debug): + continue if src in TEST_PYDS_ONLY and not ns.include_tests: continue if src in TCLTK_PYDS_ONLY and not ns.include_tcltk: continue + yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") + for dest, src in rglob(ns.build, "*.dll"): + if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: + continue + if src in EXCLUDE_FROM_DLLS: + continue yield from in_build(src.name, dest="" if ns.flat_dlls else "DLLs/") if ns.zip_lib: @@ -191,8 +234,12 @@ def get_layout(ns): yield "Lib/{}".format(dest), src if ns.include_venv: - yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python") - yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw") + if ns.include_freethreaded: + yield from in_build("venvlaunchert.exe", "Lib/venv/scripts/nt/") + yield from in_build("venvwlaunchert.exe", "Lib/venv/scripts/nt/") + else: + yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/") + yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/") if ns.include_tools: @@ -208,7 +255,6 @@ def get_layout(ns): yield PYTHON_PTH_NAME, ns.temp / PYTHON_PTH_NAME if ns.include_dev: - for dest, src in rglob(ns.source / "Include", "**/*.h"): yield "include/{}".format(dest), src yield "include/pyconfig.h", ns.build / "pyconfig.h" @@ -552,7 +598,6 @@ def main(): ns.source = ns.source or (Path(__file__).resolve().parent.parent.parent) ns.build = ns.build or Path(sys.executable).parent - ns.temp = ns.temp or Path(tempfile.mkdtemp()) ns.doc_build = ns.doc_build or (ns.source / "Doc" / "build") if not ns.source.is_absolute(): ns.source = (Path.cwd() / ns.source).resolve() @@ -565,7 +610,12 @@ def main(): if ns.include_cat and not ns.include_cat.is_absolute(): ns.include_cat = (Path.cwd() / ns.include_cat).resolve() if not ns.arch: - ns.arch = "amd64" if sys.maxsize > 2 ** 32 else "win32" + if sys.winver.endswith("-arm64"): + ns.arch = "arm64" + elif sys.winver.endswith("-32"): + ns.arch = "win32" + else: + ns.arch = "amd64" if ns.copy and not ns.copy.is_absolute(): ns.copy = (Path.cwd() / ns.copy).resolve() @@ -574,6 +624,14 @@ def main(): if ns.catalog and not ns.catalog.is_absolute(): ns.catalog = (Path.cwd() / ns.catalog).resolve() + if not ns.temp: + # Put temp on a Dev Drive for speed if we're copying to one. + # If not, the regular temp dir will have to do. + if ns.copy and getattr(os.path, "isdevdrive", lambda d: False)(ns.copy): + ns.temp = ns.copy.with_name(ns.copy.name + "_temp") + else: + ns.temp = Path(tempfile.mkdtemp()) + configure_logger(ns) log_info( @@ -602,6 +660,12 @@ Catalog: {ns.catalog}""", log_warning("Assuming --include-tcltk to support --include-idle") ns.include_tcltk = True + if not (ns.include_alias or ns.include_alias3 or ns.include_alias3x): + if ns.include_freethreaded: + ns.include_alias3x = True + else: + ns.include_alias = True + try: generate_source_files(ns) files = list(get_layout(ns)) diff --git a/PC/layout/support/constants.py b/PC/layout/support/constants.py index 8195c3d..ae22aa1 100644 --- a/PC/layout/support/constants.py +++ b/PC/layout/support/constants.py @@ -39,3 +39,6 @@ PYTHON_PTH_NAME = "python{}{}._pth".format(VER_MAJOR, VER_MINOR) PYTHON_CHM_NAME = "python{}{}{}{}.chm".format( VER_MAJOR, VER_MINOR, VER_MICRO, VER_SUFFIX ) + +FREETHREADED_PYTHON_DLL_NAME = "python{}{}t.dll".format(VER_MAJOR, VER_MINOR) +FREETHREADED_PYTHON_STABLE_DLL_NAME = "python{}t.dll".format(VER_MAJOR) diff --git a/PC/layout/support/nuspec.py b/PC/layout/support/nuspec.py index dbcb713..a87e0be 100644 --- a/PC/layout/support/nuspec.py +++ b/PC/layout/support/nuspec.py @@ -24,6 +24,10 @@ NUSPEC_PLATFORM_DATA = dict( amd64=("64-bit", "python", "Python"), arm32=("ARM", "pythonarm", "Python (ARM)"), arm64=("ARM64", "pythonarm64", "Python (ARM64)"), + win32t=("32-bit free-threaded", "pythonx86-freethreaded", "Python (32-bit, free-threaded)"), + amd64t=("64-bit free-threaded", "python-freethreaded", "Python (free-threaded)"), + arm32t=("ARM free-threaded", "pythonarm-freethreaded", "Python (ARM, free-threaded)"), + arm64t=("ARM64 free-threaded", "pythonarm64-freethreaded", "Python (ARM64, free-threaded)"), ) if not NUSPEC_DATA["PYTHON_VERSION"]: @@ -58,7 +62,10 @@ NUSPEC_TEMPLATE = r""" def _get_nuspec_data_overrides(ns): - for k, v in zip(NUSPEC_PLATFORM_DATA["_keys"], NUSPEC_PLATFORM_DATA[ns.arch]): + arch = ns.arch + if ns.include_freethreaded: + arch += "t" + for k, v in zip(NUSPEC_PLATFORM_DATA["_keys"], NUSPEC_PLATFORM_DATA[arch]): ev = os.getenv("PYTHON_NUSPEC_" + k) if ev: yield k, ev diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py index 60256fb..f1a8eb0 100644 --- a/PC/layout/support/options.py +++ b/PC/layout/support/options.py @@ -32,6 +32,10 @@ OPTIONS = { "nuspec": {"help": "a python.nuspec file"}, "chm": {"help": "the CHM documentation"}, "html-doc": {"help": "the HTML documentation"}, + "freethreaded": {"help": "freethreaded binaries", "not-in-all": True}, + "alias": {"help": "aliased python.exe entry-point binaries"}, + "alias3": {"help": "aliased python3.exe entry-point binaries"}, + "alias3x": {"help": "aliased python3.x.exe entry-point binaries"}, } @@ -47,6 +51,8 @@ PRESETS = { "dev", "launchers", "appxmanifest", + "alias", + "alias3x", # XXX: Disabled for now "precompile", ], }, @@ -59,9 +65,10 @@ PRESETS = { "venv", "props", "nuspec", + "alias", ], }, - "iot": {"help": "Windows IoT Core", "options": ["stable", "pip"]}, + "iot": {"help": "Windows IoT Core", "options": ["alias", "stable", "pip"]}, "default": { "help": "development kit package", "options": [ @@ -74,11 +81,19 @@ PRESETS = { "dev", "symbols", "html-doc", + "alias", ], }, "embed": { "help": "embeddable package", - "options": ["stable", "zip-lib", "flat-dlls", "underpth", "precompile"], + "options": [ + "alias", + "stable", + "zip-lib", + "flat-dlls", + "underpth", + "precompile", + ], }, } diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h.in index d8f0a6b..8bbf877 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h.in @@ -94,6 +94,9 @@ WIN32 is still required for the locale module. #endif #endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ +/* Define to 1 if you want to disable the GIL */ +#undef Py_GIL_DISABLED + /* Compiler specific defines */ /* ------------------------------------------------------------------------*/ @@ -305,8 +308,16 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* not building the core - must be an ext */ # if defined(_MSC_VER) /* So MSVC users need not specify the .lib - file in their Makefile (other compilers are - generally taken care of by distutils.) */ + file in their Makefile */ +# if defined(Py_GIL_DISABLED) +# if defined(_DEBUG) +# pragma comment(lib,"python313t_d.lib") +# elif defined(Py_LIMITED_API) +# pragma comment(lib,"python3t.lib") +# else +# pragma comment(lib,"python313t.lib") +# endif /* _DEBUG */ +# else /* Py_GIL_DISABLED */ # if defined(_DEBUG) # pragma comment(lib,"python313_d.lib") # elif defined(Py_LIMITED_API) @@ -314,6 +325,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ # else # pragma comment(lib,"python313.lib") # endif /* _DEBUG */ +# endif /* Py_GIL_DISABLED */ # endif /* _MSC_VER */ # endif /* Py_BUILD_CORE */ #endif /* MS_COREDLL */ @@ -739,7 +751,4 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ /* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */ #define HAVE_X509_VERIFY_PARAM_SET1_HOST 1 -/* Define if you want to disable the GIL */ -#undef Py_GIL_DISABLED - #endif /* !Py_CONFIG_H */ diff --git a/PC/venvlauncher.c b/PC/venvlauncher.c new file mode 100644 index 0000000..fe97d32 --- /dev/null +++ b/PC/venvlauncher.c @@ -0,0 +1,510 @@ +/* + * venv redirector for Windows + * + * This launcher looks for a nearby pyvenv.cfg to find the correct home + * directory, and then launches the original Python executable from it. + * The name of this executable is passed as argv[0]. + */ + +#define __STDC_WANT_LIB_EXT1__ 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MS_WINDOWS +#include "patchlevel.h" + +#define MAXLEN PATHCCH_MAX_CCH +#define MSGSIZE 1024 + +#define RC_NO_STD_HANDLES 100 +#define RC_CREATE_PROCESS 101 +#define RC_NO_PYTHON 103 +#define RC_NO_MEMORY 104 +#define RC_NO_VENV_CFG 106 +#define RC_BAD_VENV_CFG 107 +#define RC_NO_COMMANDLINE 108 +#define RC_INTERNAL_ERROR 109 + +// This should always be defined when we build for real, +// but it's handy to have a definition for quick testing +#ifndef EXENAME +#define EXENAME L"python.exe" +#endif + +#ifndef CFGNAME +#define CFGNAME L"pyvenv.cfg" +#endif + +static FILE * log_fp = NULL; + +void +debug(wchar_t * format, ...) +{ + va_list va; + + if (log_fp != NULL) { + wchar_t buffer[MAXLEN]; + int r = 0; + va_start(va, format); + r = vswprintf_s(buffer, MAXLEN, format, va); + va_end(va); + + if (r <= 0) { + return; + } + fwprintf(log_fp, L"%ls\n", buffer); + while (r && isspace(buffer[r])) { + buffer[r--] = L'\0'; + } + if (buffer[0]) { + OutputDebugStringW(buffer); + } + } +} + + +void +formatWinerror(int rc, wchar_t * message, int size) +{ + FormatMessageW( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, rc, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + message, size, NULL); +} + + +void +winerror(int err, wchar_t * format, ... ) +{ + va_list va; + wchar_t message[MSGSIZE]; + wchar_t win_message[MSGSIZE]; + int len; + + if (err == 0) { + err = GetLastError(); + } + + va_start(va, format); + len = _vsnwprintf_s(message, MSGSIZE, _TRUNCATE, format, va); + va_end(va); + + formatWinerror(err, win_message, MSGSIZE); + if (len >= 0) { + _snwprintf_s(&message[len], MSGSIZE - len, _TRUNCATE, L": %ls", + win_message); + } + +#if !defined(_WINDOWS) + fwprintf(stderr, L"%ls\n", message); +#else + MessageBoxW(NULL, message, L"Python venv launcher is sorry to say ...", + MB_OK); +#endif +} + + +void +error(wchar_t * format, ... ) +{ + va_list va; + wchar_t message[MSGSIZE]; + + va_start(va, format); + _vsnwprintf_s(message, MSGSIZE, _TRUNCATE, format, va); + va_end(va); + +#if !defined(_WINDOWS) + fwprintf(stderr, L"%ls\n", message); +#else + MessageBoxW(NULL, message, L"Python venv launcher is sorry to say ...", + MB_OK); +#endif +} + + +bool +isEnvVarSet(const wchar_t *name) +{ + /* only looking for non-empty, which means at least one character + and the null terminator */ + return GetEnvironmentVariableW(name, NULL, 0) >= 2; +} + + +bool +join(wchar_t *buffer, size_t bufferLength, const wchar_t *fragment) +{ + if (SUCCEEDED(PathCchCombineEx(buffer, bufferLength, buffer, fragment, PATHCCH_ALLOW_LONG_PATHS))) { + return true; + } + return false; +} + + +bool +split_parent(wchar_t *buffer, size_t bufferLength) +{ + return SUCCEEDED(PathCchRemoveFileSpec(buffer, bufferLength)); +} + + +/* + * Path calculation + */ + +int +calculate_pyvenvcfg_path(wchar_t *pyvenvcfg_path, size_t maxlen) +{ + if (!pyvenvcfg_path) { + error(L"invalid buffer provided"); + return RC_INTERNAL_ERROR; + } + if ((DWORD)maxlen != maxlen) { + error(L"path buffer is too large"); + return RC_INTERNAL_ERROR; + } + if (!GetModuleFileNameW(NULL, pyvenvcfg_path, (DWORD)maxlen)) { + winerror(GetLastError(), L"failed to read executable directory"); + return RC_NO_COMMANDLINE; + } + // Remove 'python.exe' from our path + if (!split_parent(pyvenvcfg_path, maxlen)) { + error(L"failed to remove segment from '%ls'", pyvenvcfg_path); + return RC_NO_COMMANDLINE; + } + // Replace with 'pyvenv.cfg' + if (!join(pyvenvcfg_path, maxlen, CFGNAME)) { + error(L"failed to append '%ls' to '%ls'", CFGNAME, pyvenvcfg_path); + return RC_NO_MEMORY; + } + // If it exists, return + if (GetFileAttributesW(pyvenvcfg_path) != INVALID_FILE_ATTRIBUTES) { + return 0; + } + // Otherwise, remove 'pyvenv.cfg' and (probably) 'Scripts' + if (!split_parent(pyvenvcfg_path, maxlen) || + !split_parent(pyvenvcfg_path, maxlen)) { + error(L"failed to remove segments from '%ls'", pyvenvcfg_path); + return RC_NO_COMMANDLINE; + } + // Replace 'pyvenv.cfg' + if (!join(pyvenvcfg_path, maxlen, CFGNAME)) { + error(L"failed to append '%ls' to '%ls'", CFGNAME, pyvenvcfg_path); + return RC_NO_MEMORY; + } + // If it exists, return + if (GetFileAttributesW(pyvenvcfg_path) != INVALID_FILE_ATTRIBUTES) { + return 0; + } + // Otherwise, we fail + winerror(GetLastError(), L"failed to locate %ls", CFGNAME); + return RC_NO_VENV_CFG; +} + + +/* + * pyvenv.cfg parsing + */ + +static int +find_home_value(const char *buffer, DWORD maxlen, const char **start, DWORD *length) +{ + if (!buffer || !start || !length) { + error(L"invalid find_home_value parameters()"); + return 0; + } + for (const char *s = strstr(buffer, "home"); + s && ((ptrdiff_t)s - (ptrdiff_t)buffer) < maxlen; + s = strstr(s + 1, "\nhome") + ) { + if (*s == '\n') { + ++s; + } + for (int i = 4; i > 0 && *s; --i, ++s); + + while (*s && iswspace(*s)) { + ++s; + } + if (*s != L'=') { + continue; + } + + do { + ++s; + } while (*s && iswspace(*s)); + + *start = s; + char *nl = strchr(s, '\n'); + if (nl) { + while (nl != s && iswspace(nl[-1])) { + --nl; + } + *length = (DWORD)((ptrdiff_t)nl - (ptrdiff_t)s); + } else { + *length = (DWORD)strlen(s); + } + return 1; + } + return 0; +} + + +int +read_home(const wchar_t *pyvenv_cfg, wchar_t *home_path, size_t maxlen) +{ + HANDLE hFile = CreateFileW(pyvenv_cfg, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, 0, NULL); + + if (hFile == INVALID_HANDLE_VALUE) { + winerror(GetLastError(), L"failed to open '%ls'", pyvenv_cfg); + return RC_BAD_VENV_CFG; + } + + // 8192 characters ought to be enough for anyone + // (doubled compared to the old implementation!) + char buffer[8192]; + DWORD len; + if (!ReadFile(hFile, buffer, sizeof(buffer) - 1, &len, NULL)) { + winerror(GetLastError(), L"failed to read '%ls'", pyvenv_cfg); + CloseHandle(hFile); + return RC_BAD_VENV_CFG; + } + CloseHandle(hFile); + // Ensure null termination + buffer[len] = '\0'; + + char *home; + DWORD home_len; + if (!find_home_value(buffer, sizeof(buffer), &home, &home_len)) { + error(L"no home= specified in '%ls'", pyvenv_cfg); + return RC_BAD_VENV_CFG; + } + + if ((DWORD)maxlen != maxlen) { + maxlen = 8192; + } + len = MultiByteToWideChar(CP_UTF8, 0, home, home_len, home_path, (DWORD)maxlen); + if (!len) { + winerror(GetLastError(), L"failed to decode home setting in '%ls'", pyvenv_cfg); + return RC_BAD_VENV_CFG; + } + home_path[len] = L'\0'; + + return 0; +} + + +int +locate_python(wchar_t *path, size_t maxlen) +{ + if (!join(path, maxlen, EXENAME)) { + error(L"failed to append %ls to '%ls'", EXENAME, path); + return RC_NO_MEMORY; + } + + if (GetFileAttributesW(path) == INVALID_FILE_ATTRIBUTES) { + winerror(GetLastError(), L"did not find executable at '%ls'", path); + return RC_NO_PYTHON; + } + + return 0; +} + + +int +smuggle_path() +{ + wchar_t buffer[MAXLEN]; + // We could use argv[0], but that may be wrong in certain rare cases (if the + // user is doing something weird like symlinks to venv redirectors), and + // what we _really_ want is the directory of the venv. We always copy the + // redirectors, so if we've made the venv, this will be correct. + DWORD len = GetModuleFileNameW(NULL, buffer, MAXLEN); + if (!len) { + winerror(GetLastError(), L"Failed to get own executable path"); + return RC_INTERNAL_ERROR; + } + buffer[len] = L'\0'; + debug(L"Setting __PYVENV_LAUNCHER__ = '%s'", buffer); + + if (!SetEnvironmentVariableW(L"__PYVENV_LAUNCHER__", buffer)) { + winerror(GetLastError(), L"Failed to set launcher environment"); + return RC_INTERNAL_ERROR; + } + + return 0; +} + +/* + * Process creation + */ + +static BOOL +safe_duplicate_handle(HANDLE in, HANDLE * pout, const wchar_t *name) +{ + BOOL ok; + HANDLE process = GetCurrentProcess(); + DWORD rc; + + *pout = NULL; + ok = DuplicateHandle(process, in, process, pout, 0, TRUE, + DUPLICATE_SAME_ACCESS); + if (!ok) { + rc = GetLastError(); + if (rc == ERROR_INVALID_HANDLE) { + debug(L"DuplicateHandle(%ls) returned ERROR_INVALID_HANDLE\n", name); + ok = TRUE; + } + else { + debug(L"DuplicateHandle(%ls) returned %d\n", name, rc); + } + } + return ok; +} + +static BOOL WINAPI +ctrl_c_handler(DWORD code) +{ + return TRUE; /* We just ignore all control events. */ +} + +static int +launch(const wchar_t *executable, wchar_t *cmdline) +{ + HANDLE job; + JOBOBJECT_EXTENDED_LIMIT_INFORMATION info; + DWORD rc; + BOOL ok; + STARTUPINFOW si; + PROCESS_INFORMATION pi; + +#if defined(_WINDOWS) + /* + When explorer launches a Windows (GUI) application, it displays + the "app starting" (the "pointer + hourglass") cursor for a number + of seconds, or until the app does something UI-ish (eg, creating a + window, or fetching a message). As this launcher doesn't do this + directly, that cursor remains even after the child process does these + things. We avoid that by doing a simple post+get message. + See http://bugs.python.org/issue17290 + */ + MSG msg; + + PostMessage(0, 0, 0, 0); + GetMessage(&msg, 0, 0, 0); +#endif + + debug(L"run_child: about to run '%ls' with '%ls'\n", executable, cmdline); + job = CreateJobObject(NULL, NULL); + ok = QueryInformationJobObject(job, JobObjectExtendedLimitInformation, + &info, sizeof(info), &rc); + if (!ok || (rc != sizeof(info)) || !job) { + winerror(GetLastError(), L"Job information querying failed"); + return RC_CREATE_PROCESS; + } + info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | + JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK; + ok = SetInformationJobObject(job, JobObjectExtendedLimitInformation, &info, + sizeof(info)); + if (!ok) { + winerror(GetLastError(), L"Job information setting failed"); + return RC_CREATE_PROCESS; + } + memset(&si, 0, sizeof(si)); + GetStartupInfoW(&si); + ok = safe_duplicate_handle(GetStdHandle(STD_INPUT_HANDLE), &si.hStdInput, L"stdin"); + if (!ok) { + return RC_NO_STD_HANDLES; + } + ok = safe_duplicate_handle(GetStdHandle(STD_OUTPUT_HANDLE), &si.hStdOutput, L"stdout"); + if (!ok) { + return RC_NO_STD_HANDLES; + } + ok = safe_duplicate_handle(GetStdHandle(STD_ERROR_HANDLE), &si.hStdError, L"stderr"); + if (!ok) { + return RC_NO_STD_HANDLES; + } + + ok = SetConsoleCtrlHandler(ctrl_c_handler, TRUE); + if (!ok) { + winerror(GetLastError(), L"control handler setting failed"); + return RC_CREATE_PROCESS; + } + + si.dwFlags = STARTF_USESTDHANDLES; + ok = CreateProcessW(executable, cmdline, NULL, NULL, TRUE, + 0, NULL, NULL, &si, &pi); + if (!ok) { + winerror(GetLastError(), L"Unable to create process using '%ls'", cmdline); + return RC_CREATE_PROCESS; + } + AssignProcessToJobObject(job, pi.hProcess); + CloseHandle(pi.hThread); + WaitForSingleObjectEx(pi.hProcess, INFINITE, FALSE); + ok = GetExitCodeProcess(pi.hProcess, &rc); + if (!ok) { + winerror(GetLastError(), L"Failed to get exit code of process"); + return RC_CREATE_PROCESS; + } + debug(L"child process exit code: %d", rc); + return rc; +} + + +int +process(int argc, wchar_t ** argv) +{ + int exitCode; + wchar_t pyvenvcfg_path[MAXLEN]; + wchar_t home_path[MAXLEN]; + + if (isEnvVarSet(L"PYLAUNCHER_DEBUG")) { + setvbuf(stderr, (char *)NULL, _IONBF, 0); + log_fp = stderr; + } + + exitCode = calculate_pyvenvcfg_path(pyvenvcfg_path, MAXLEN); + if (exitCode) return exitCode; + + exitCode = read_home(pyvenvcfg_path, home_path, MAXLEN); + if (exitCode) return exitCode; + + exitCode = locate_python(home_path, MAXLEN); + if (exitCode) return exitCode; + + // We do not update argv[0] to point at the target runtime, and so we do not + // pass through our original argv[0] in an environment variable. + //exitCode = smuggle_path(); + //if (exitCode) return exitCode; + + exitCode = launch(home_path, GetCommandLineW()); + return exitCode; +} + + +#if defined(_WINDOWS) + +int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, + LPWSTR lpstrCmd, int nShow) +{ + return process(__argc, __wargv); +} + +#else + +int cdecl wmain(int argc, wchar_t ** argv) +{ + return process(argc, argv); +} + +#endif diff --git a/PCbuild/_asyncio.vcxproj b/PCbuild/_asyncio.vcxproj index ed1e1bc..76b0ffd 100644 --- a/PCbuild/_asyncio.vcxproj +++ b/PCbuild/_asyncio.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_bz2.vcxproj b/PCbuild/_bz2.vcxproj index 3fe95fb..e0dc6ec 100644 --- a/PCbuild/_bz2.vcxproj +++ b/PCbuild/_bz2.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_ctypes.vcxproj b/PCbuild/_ctypes.vcxproj index 253da31..63d5fa4 100644 --- a/PCbuild/_ctypes.vcxproj +++ b/PCbuild/_ctypes.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_ctypes_test.vcxproj b/PCbuild/_ctypes_test.vcxproj index 8a01e74..9735473 100644 --- a/PCbuild/_ctypes_test.vcxproj +++ b/PCbuild/_ctypes_test.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_decimal.vcxproj b/PCbuild/_decimal.vcxproj index 0916f1a..490d7df 100644 --- a/PCbuild/_decimal.vcxproj +++ b/PCbuild/_decimal.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 8da5244..8c9c0e4 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index 6dad818..2cd2052 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_lzma.vcxproj b/PCbuild/_lzma.vcxproj index fe076a6..40107d4 100644 --- a/PCbuild/_lzma.vcxproj +++ b/PCbuild/_lzma.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_multiprocessing.vcxproj b/PCbuild/_multiprocessing.vcxproj index 77b6bfc..a65397f 100644 --- a/PCbuild/_multiprocessing.vcxproj +++ b/PCbuild/_multiprocessing.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_overlapped.vcxproj b/PCbuild/_overlapped.vcxproj index 9e60d3b..224bf05 100644 --- a/PCbuild/_overlapped.vcxproj +++ b/PCbuild/_overlapped.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_queue.vcxproj b/PCbuild/_queue.vcxproj index 8065b23..80a1c3c 100644 --- a/PCbuild/_queue.vcxproj +++ b/PCbuild/_queue.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_socket.vcxproj b/PCbuild/_socket.vcxproj index 78fa4d6..41af089 100644 --- a/PCbuild/_socket.vcxproj +++ b/PCbuild/_socket.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index 57c7413..9ae0a0f 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_ssl.vcxproj b/PCbuild/_ssl.vcxproj index 226ff50..d4e1aff 100644 --- a/PCbuild/_ssl.vcxproj +++ b/PCbuild/_ssl.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testbuffer.vcxproj b/PCbuild/_testbuffer.vcxproj index 917d7ae..4e721e8 100644 --- a/PCbuild/_testbuffer.vcxproj +++ b/PCbuild/_testbuffer.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 1c15541..6911aac 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testclinic.vcxproj b/PCbuild/_testclinic.vcxproj index e319b3c..ef98133 100644 --- a/PCbuild/_testclinic.vcxproj +++ b/PCbuild/_testclinic.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) @@ -107,4 +107,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_testclinic_limited.vcxproj b/PCbuild/_testclinic_limited.vcxproj index b00b2be..183a550 100644 --- a/PCbuild/_testclinic_limited.vcxproj +++ b/PCbuild/_testclinic_limited.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testconsole.vcxproj b/PCbuild/_testconsole.vcxproj index 5d7e14e..69d312b 100644 --- a/PCbuild/_testconsole.vcxproj +++ b/PCbuild/_testconsole.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testimportmultiple.vcxproj b/PCbuild/_testimportmultiple.vcxproj index 6d80d57..c35ac83 100644 --- a/PCbuild/_testimportmultiple.vcxproj +++ b/PCbuild/_testimportmultiple.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index 558f66c..a825cac 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testmultiphase.vcxproj b/PCbuild/_testmultiphase.vcxproj index 430eb52..e730fe3 100644 --- a/PCbuild/_testmultiphase.vcxproj +++ b/PCbuild/_testmultiphase.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_testsinglephase.vcxproj b/PCbuild/_testsinglephase.vcxproj index fb4bcd9..bf4dabf 100644 --- a/PCbuild/_testsinglephase.vcxproj +++ b/PCbuild/_testsinglephase.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_tkinter.vcxproj b/PCbuild/_tkinter.vcxproj index 30cedcb..117488a 100644 --- a/PCbuild/_tkinter.vcxproj +++ b/PCbuild/_tkinter.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_uuid.vcxproj b/PCbuild/_uuid.vcxproj index 2437b7e..50d81cc 100644 --- a/PCbuild/_uuid.vcxproj +++ b/PCbuild/_uuid.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_wmi.vcxproj b/PCbuild/_wmi.vcxproj index c1914a3..22fa896 100644 --- a/PCbuild/_wmi.vcxproj +++ b/PCbuild/_wmi.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/_zoneinfo.vcxproj b/PCbuild/_zoneinfo.vcxproj index 6e6389c..47b5bfa 100644 --- a/PCbuild/_zoneinfo.vcxproj +++ b/PCbuild/_zoneinfo.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index 001f8af..dc9161a 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 16ad91e..fd5fbc9 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -24,6 +24,13 @@ false + + $(TargetName)$(TargetExt) + <_TargetNameSep>$(TargetNameExt.LastIndexOf(`.`)) + $(TargetNameExt.Substring(0, $(_TargetNameSep))) + $(TargetNameExt.Substring($(_TargetNameSep))) + + <_VCToolsVersion>$([System.Version]::Parse(`$(VCToolsVersion)`).Major).$([System.Version]::Parse(`$(VCToolsVersion)`).Minor) @@ -38,7 +45,7 @@ <_PlatformPreprocessorDefinition>_WIN32; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition) - <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)"; + <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)$(PyDebugExt)"; @@ -158,8 +165,8 @@ public override bool Execute() { - - + + $([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\)) $(PySourcePath)\ - + $(PySourcePath)PCbuild\win32\ $(Py_OutDir)\win32\ @@ -52,7 +52,7 @@ $(PySourcePath)PCbuild\$(ArchName)\ $(BuildPath)\ $(BuildPath)instrumented\ - + ..\\.. ..\\..\\.. @@ -84,22 +84,19 @@ _d - + -test - + -32 -arm32 -arm64 - - - $(BuildPath)python$(PyDebugExt).exe true - + true @@ -141,7 +138,7 @@ @@ -223,22 +220,55 @@ )) )) $([msbuild]::Add($(Field3Value), 9000)) - + + + python$(MajorVersionNumber).$(MinorVersionNumber)t + python + $(BuildPath)$(PyExeName)$(PyDebugExt).exe + pythonw$(MajorVersionNumber).$(MinorVersionNumber)t + pythonw + - python$(MajorVersionNumber)$(MinorVersionNumber)$(PyDebugExt) + python$(MajorVersionNumber)$(MinorVersionNumber)t$(PyDebugExt) + python$(MajorVersionNumber)$(MinorVersionNumber)$(PyDebugExt) - python3$(PyDebugExt) + python3t + python3 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win32 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win_arm32 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win_arm64 .cp$(MajorVersionNumber)$(MinorVersionNumber)-win_amd64 - + $(MajorVersionNumber).$(MinorVersionNumber)$(PyArchExt)$(PyTestExt) + + + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win32 + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win_arm32 + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win_arm64 + .cp$(MajorVersionNumber)$(MinorVersionNumber)t-win_amd64 + + + $(MajorVersionNumber).$(MinorVersionNumber)t$(PyArchExt)$(PyTestExt) + + + + + .pyd - + + + $(FreethreadedPydTag) + + + $(PydTag).pyd + + + $(FreethreadedSysWinVer) + + diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index fdf573a..1e5ab87 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -72,6 +72,7 @@ + $(PyExeName) Application false MultiByte @@ -132,7 +133,7 @@ +"$(OutDir)$(PyExeName)$(PyDebugExt).exe" "$(PySourcePath)PC\validate_ucrtbase.py" $(UcrtName)' ContinueOnError="true" /> @@ -145,7 +146,7 @@ set PYTHONPATH=$(PySourcePath)Lib @echo Running $(Configuration)^|$(Platform) interpreter... @setlocal @set PYTHONHOME=$(PySourcePath) -@"$(OutDir)python$(PyDebugExt).exe" %* +@"$(OutDir)$(PyExeName)$(PyDebugExt).exe" %* <_ExistingContent Condition="Exists('$(PySourcePath)python.bat')">$([System.IO.File]::ReadAllText('$(PySourcePath)python.bat')) diff --git a/PCbuild/python3dll.vcxproj b/PCbuild/python3dll.vcxproj index ec22e6f..235ea1c 100644 --- a/PCbuild/python3dll.vcxproj +++ b/PCbuild/python3dll.vcxproj @@ -70,12 +70,12 @@ {885D4898-D08D-4091-9C40-C700CFE3FC5A} python3dll Win32Proj - python3 false + $(Py3DllName) DynamicLibrary diff --git a/PCbuild/pythonw.vcxproj b/PCbuild/pythonw.vcxproj index 31f2130..d6cf0c9 100644 --- a/PCbuild/pythonw.vcxproj +++ b/PCbuild/pythonw.vcxproj @@ -73,6 +73,7 @@ + $(PyWExeName) Application false diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 332ba5e..293f99a 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -9,6 +9,7 @@ rem -q runs the tests just once, and without deleting .pyc files. rem -p or -win32, -x64, -arm32, -arm64 rem Run the specified architecture of python (or python_d if -d rem was specified). If omitted, uses %PREFIX% if set or 64-bit. +rem --disable-gil Run free-threaded build. rem All leading instances of these switches are shifted off, and rem whatever remains (up to 9 arguments) is passed to regrtest.py. rem For example, @@ -29,6 +30,7 @@ rem rt -u "network,largefile" setlocal set pcbuild=%~dp0 +set pyname=python set suffix= set qmode= set dashO= @@ -39,15 +41,18 @@ set exe= if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts +rem HACK: Need some way to infer the version number in this script +if "%1"=="--disable-gil" (set pyname=python3.13t) & shift & goto CheckOpts if "%1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts if "%1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts +if "%1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts if "%1"=="-arm64" (set prefix=%pcbuild%arm64) & shift & goto CheckOpts if "%1"=="-arm32" (set prefix=%pcbuild%arm32) & shift & goto CheckOpts if "%1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts if not defined prefix set prefix=%pcbuild%amd64 -set exe=%prefix%\python%suffix%.exe +set exe=%prefix%\%pyname%%suffix%.exe set cmd="%exe%" %dashO% -m test %regrtestargs% if defined qmode goto Qmode diff --git a/PCbuild/select.vcxproj b/PCbuild/select.vcxproj index 750a713..d7448fd 100644 --- a/PCbuild/select.vcxproj +++ b/PCbuild/select.vcxproj @@ -78,7 +78,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/sqlite3.vcxproj b/PCbuild/sqlite3.vcxproj index c502d51..6bcc4e9 100644 --- a/PCbuild/sqlite3.vcxproj +++ b/PCbuild/sqlite3.vcxproj @@ -69,12 +69,12 @@ {A1A295E5-463C-437F-81CA-1F32367685DA} sqlite3 - .pyd false + $(PyStdlibPydExt) DynamicLibrary NotSet diff --git a/PCbuild/unicodedata.vcxproj b/PCbuild/unicodedata.vcxproj index addef75..781f938 100644 --- a/PCbuild/unicodedata.vcxproj +++ b/PCbuild/unicodedata.vcxproj @@ -79,7 +79,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/venvlauncher.vcxproj b/PCbuild/venvlauncher.vcxproj index 123e84e..1193e03 100644 --- a/PCbuild/venvlauncher.vcxproj +++ b/PCbuild/venvlauncher.vcxproj @@ -69,12 +69,13 @@ {494BAC80-A60C-43A9-99E7-ACB691CE2C4D} venvlauncher - venvlauncher false + venvlauncher + $(TargetName)t Application MultiByte @@ -91,19 +92,19 @@ - _CONSOLE;VENV_REDIRECT;%(PreprocessorDefinitions) + EXENAME=L"$(PyExeName)$(PyDebugExt).exe";_CONSOLE;%(PreprocessorDefinitions) MultiThreaded PY_ICON;%(PreprocessorDefinitions) - version.lib;%(AdditionalDependencies) + pathcch.lib;%(AdditionalDependencies) Console - + diff --git a/PCbuild/venvlauncher.vcxproj.filters b/PCbuild/venvlauncher.vcxproj.filters index ec13936..56a0f00 100644 --- a/PCbuild/venvlauncher.vcxproj.filters +++ b/PCbuild/venvlauncher.vcxproj.filters @@ -19,7 +19,7 @@ - + Source Files diff --git a/PCbuild/venvwlauncher.vcxproj b/PCbuild/venvwlauncher.vcxproj index b8504d5..1b61718 100644 --- a/PCbuild/venvwlauncher.vcxproj +++ b/PCbuild/venvwlauncher.vcxproj @@ -69,12 +69,13 @@ {FDB84CBB-2FB6-47C8-A2D6-091E0833239D} venvwlauncher - venvwlauncher false + venvwlauncher + $(TargetName)t Application MultiByte @@ -91,19 +92,19 @@ - _WINDOWS;VENV_REDIRECT;%(PreprocessorDefinitions) + EXENAME=L"$(PyExeName)$(PyDebugExt).exe";_WINDOWS;%(PreprocessorDefinitions) MultiThreaded PYW_ICON;%(PreprocessorDefinitions) - version.lib;%(AdditionalDependencies) + pathcch.lib;%(AdditionalDependencies) Windows - + diff --git a/PCbuild/venvwlauncher.vcxproj.filters b/PCbuild/venvwlauncher.vcxproj.filters index 8addc13..61a5143 100644 --- a/PCbuild/venvwlauncher.vcxproj.filters +++ b/PCbuild/venvwlauncher.vcxproj.filters @@ -9,7 +9,7 @@ - + Source Files diff --git a/PCbuild/winsound.vcxproj b/PCbuild/winsound.vcxproj index 32cedc9..c26029b 100644 --- a/PCbuild/winsound.vcxproj +++ b/PCbuild/winsound.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/xxlimited.vcxproj b/PCbuild/xxlimited.vcxproj index 1c776fb..093e692 100644 --- a/PCbuild/xxlimited.vcxproj +++ b/PCbuild/xxlimited.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/PCbuild/xxlimited_35.vcxproj b/PCbuild/xxlimited_35.vcxproj index dd830b3..3f4d446 100644 --- a/PCbuild/xxlimited_35.vcxproj +++ b/PCbuild/xxlimited_35.vcxproj @@ -80,7 +80,7 @@ - .pyd + $(PyStdlibPydExt) diff --git a/Tools/msi/build.bat b/Tools/msi/build.bat index b9aab88..2fe8a47 100644 --- a/Tools/msi/build.bat +++ b/Tools/msi/build.bat @@ -22,6 +22,9 @@ if "%~1" EQU "--no-test-marker" (set BUILDTEST=) && shift && goto CheckOpts if "%~1" EQU "--test-marker" (set BUILDTEST=--test-marker) && shift && goto CheckOpts if "%~1" EQU "--pack" (set BUILDPACK=1) && shift && goto CheckOpts if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts +rem %IncludeFreethreaded% is recognised by the MSI build, but not the regular build. +rem We use it to build twice and then build the installer with its extra option +if /I "%~1" EQU "--disable-gil" (set IncludeFreethreaded=true) && shift && goto CheckOpts if not defined BUILDX86 if not defined BUILDX64 if not defined BUILDARM64 (set BUILDX86=1) && (set BUILDX64=1) @@ -44,6 +47,20 @@ if errorlevel 1 exit /B %ERRORLEVEL% if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -e %REBUILD% %BUILDTEST% if errorlevel 1 exit /B %ERRORLEVEL% +if /I "%IncludeFreethreaded%"=="true" ( + rem Cannot "exit /B" inside an if block because %ERRORLEVEL% will be wrong. + rem We just skip everything after the first "errorlevel 1" and then exit after + if defined BUILDX86 call "%PCBUILD%build.bat" -p Win32 -d -e %REBUILD% %BUILDTEST% --disable-gil + if not errorlevel 1 if defined BUILDX86 call "%PCBUILD%build.bat" -p Win32 -e %REBUILD% %BUILDTEST% --disable-gil + + if not errorlevel 1 if defined BUILDX64 call "%PCBUILD%build.bat" -p x64 -d -e %REBUILD% %BUILDTEST% --disable-gil + if not errorlevel 1 if defined BUILDX64 call "%PCBUILD%build.bat" -p x64 -e %REBUILD% %BUILDTEST% --disable-gil + + if not errorlevel 1 if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -d -e %REBUILD% %BUILDTEST% --disable-gil + if not errorlevel 1 if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -e %REBUILD% %BUILDTEST% --disable-gil +) +if errorlevel 1 exit /B %ERRORLEVEL% + if defined BUILDDOC call "%PCBUILD%..\Doc\make.bat" html if errorlevel 1 exit /B %ERRORLEVEL% diff --git a/Tools/msi/bundle/Default.thm b/Tools/msi/bundle/Default.thm index d1b0f5b..471d37a 100644 --- a/Tools/msi/bundle/Default.thm +++ b/Tools/msi/bundle/Default.thm @@ -83,6 +83,7 @@ #(loc.PrecompileLabel) #(loc.Include_symbolsLabel) #(loc.Include_debugLabel) + #(loc.Include_freethreadedLabel) #(loc.CustomLocationLabel) diff --git a/Tools/msi/bundle/Default.wxl b/Tools/msi/bundle/Default.wxl index 6f8befb..1540f05 100644 --- a/Tools/msi/bundle/Default.wxl +++ b/Tools/msi/bundle/Default.wxl @@ -91,6 +91,7 @@ Select Customize to review current options. &Precompile standard library Download debugging &symbols Download debu&g binaries (requires VS 2017 or later) + Download &free-threaded binaries (experimental) [ActionLikeInstallation] Progress [ActionLikeInstalling]: diff --git a/Tools/msi/bundle/bundle.targets b/Tools/msi/bundle/bundle.targets index 9c7410f..cb3effb 100644 --- a/Tools/msi/bundle/bundle.targets +++ b/Tools/msi/bundle/bundle.targets @@ -72,6 +72,7 @@ + diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs index 8b12baa..9b4f072 100644 --- a/Tools/msi/bundle/bundle.wxs +++ b/Tools/msi/bundle/bundle.wxs @@ -82,6 +82,13 @@ + + + + + + + @@ -104,6 +111,9 @@ + + + diff --git a/Tools/msi/bundle/packagegroups/freethreaded.wxs b/Tools/msi/bundle/packagegroups/freethreaded.wxs new file mode 100644 index 0000000..121ca34 --- /dev/null +++ b/Tools/msi/bundle/packagegroups/freethreaded.wxs @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded.wixproj b/Tools/msi/freethreaded/freethreaded.wixproj new file mode 100644 index 0000000..0b4bd05 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded.wixproj @@ -0,0 +1,20 @@ + + + + {1B4502D5-B627-4F50-ABEA-4CC5A8E88265} + 2.0 + freethreaded + Package + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded.wxs b/Tools/msi/freethreaded/freethreaded.wxs new file mode 100644 index 0000000..063aa28 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded.wxs @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/Tools/msi/freethreaded/freethreaded_d.wixproj b/Tools/msi/freethreaded/freethreaded_d.wixproj new file mode 100644 index 0000000..e1563d4 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_d.wixproj @@ -0,0 +1,20 @@ + + + + {D3677DCF-098A-4398-9FA5-8E74AC37E0DF} + 2.0 + freethreaded_d + Package + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded_d.wxs b/Tools/msi/freethreaded/freethreaded_d.wxs new file mode 100644 index 0000000..cddf22a --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_d.wxs @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/Tools/msi/freethreaded/freethreaded_en-US.wxl_template b/Tools/msi/freethreaded/freethreaded_en-US.wxl_template new file mode 100644 index 0000000..b9747eb --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_en-US.wxl_template @@ -0,0 +1,8 @@ + + + Freethreaded Interpreter + freethreaded + Python {{ShortVersion}} ({{Bitness}}, freethreaded) + Launches the !(loc.ProductName) freethreaded interpreter. + https://www.python.org/ + diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs new file mode 100644 index 0000000..adaf63c --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -0,0 +1,175 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Tools/msi/freethreaded/freethreaded_pdb.wixproj b/Tools/msi/freethreaded/freethreaded_pdb.wixproj new file mode 100644 index 0000000..789a4f5 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_pdb.wixproj @@ -0,0 +1,20 @@ + + + + {E98E7539-64E7-4DCE-AACD-01E3ADE40EFD} + 2.0 + freethreaded_pdb + Package + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded_pdb.wxs b/Tools/msi/freethreaded/freethreaded_pdb.wxs new file mode 100644 index 0000000..302ac41 --- /dev/null +++ b/Tools/msi/freethreaded/freethreaded_pdb.wxs @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index a82cad5..b8e16b5 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -26,10 +26,10 @@ - + - + @@ -63,10 +63,10 @@ - + - + @@ -100,10 +100,10 @@ - + - + diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props index cfb3ca9..372c482 100644 --- a/Tools/msi/msi.props +++ b/Tools/msi/msi.props @@ -24,14 +24,14 @@ This URI is used to generate the various GUIDs used by the installer. Installers built with the same URI will upgrade each other or block when attempting to downgrade. - + By default, this is the local computer name, which will produce installers that do not interfere with other installers. Products that intend to bundle Python should rebuild these modules with their own URI to avoid conflicting with the official releases. - + The official releases use "https://www.python.org/$(ArchName)" - + This is not the same as the DownloadUrl property used in the bundle projects. --> @@ -39,7 +39,7 @@ $(ReleaseUri)/ - + @@ -63,13 +63,17 @@ $(MajorVersionNumber).$(MinorVersionNumber).$(Field3Value).0 - + + + false + + $([System.Math]::Floor($([System.DateTime]::Now.Subtract($([System.DateTime]::new(2001, 1, 1))).TotalDays))) $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)dev$(RevisionNumber) $(MajorVersionNumber).$(MinorVersionNumber).$(RevisionNumber).0 - + 32-bit 64-bit @@ -91,9 +95,12 @@ PyDebugExt=$(PyDebugExt); PyArchExt=$(PyArchExt); PyTestExt=$(PyTestExt); + PydTag=$(PydTag); + FreethreadedPydTag=$(FreethreadedPydTag); OptionalFeatureName=$(OutputName); ssltag=$(OpenSSLDLLSuffix); Suffix32=$(PyArchExt); + IncludeFreethreaded=$(IncludeFreethreaded); $(DefineConstants);CRTRedist=$(CRTRedist); @@ -139,7 +146,7 @@ - + <_Uuid Include="CoreUpgradeCode"> upgradecode @@ -162,6 +169,12 @@ <_Uuid Include="PythonRegComponentGuid"> registry/$(OutputName) + <_Uuid Include="FreethreadedPythonExeComponentGuid" Condition="$(IncludeFreethreaded)"> + freethreaded/python.exe + + <_Uuid Include="FreethreadedPythonwExeComponentGuid" Condition="$(IncludeFreethreaded)"> + freethreaded/pythonw.exe + @(_Uuid->'("%(Identity)", "$(MajorVersionNumber).$(MinorVersionNumber)/%(Uri)")',',') <_GenerateCommand>import uuid; print('\n'.join('{}={}'.format(i, uuid.uuid5(uuid.UUID('c8d9733e-a70c-43ff-ab0c-e26456f11083'), '$(ReleaseUri.Replace(`{arch}`, `$(ArchName)`))' + j)) for i,j in [$(_Uuids.Replace(`"`,`'`))])) - + - + - + $(DefineConstants);@(_UuidValue,';'); -- cgit v0.12