gh-112984 Update Windows build and installer for free-threaded builds (GH-113129)

5 files changed, 199 insertions, 121 deletions

diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py
index 22db97b..9f0547f 100644
--- a/Lib/test/test_ctypes/test_loading.py
+++ b/Lib/test/test_ctypes/test_loading.py
@@ -141,7 +141,7 @@ class LoaderTest(unittest.TestCase):
     def test_load_dll_with_flags(self):
         _sqlite3 = import_helper.import_module("_sqlite3")
         src = _sqlite3.__file__
-        if src.lower().endswith("_d.pyd"):
+        if src.partition(".")[0].lower().endswith("_d"):
             ext = "_d.dll"
         else:
             ext = ".dll"
diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py
index bcd4ed6..3da6173 100644
--- a/Lib/test/test_launcher.py
+++ b/Lib/test/test_launcher.py
@@ -19,8 +19,10 @@ import winreg
 
 
 PY_EXE = "py.exe"
+DEBUG_BUILD = False
 if sys.executable.casefold().endswith("_d.exe".casefold()):
     PY_EXE = "py_d.exe"
+    DEBUG_BUILD = True
 
 # Registry data to create. On removal, everything beneath top-level names will
 # be deleted.
@@ -232,7 +234,7 @@ class RunPyMixin:
             p.stdin.close()
             p.wait(10)
             out = p.stdout.read().decode("utf-8", "replace")
-            err = p.stderr.read().decode("ascii", "replace")
+            err = p.stderr.read().decode("ascii", "replace").replace("\uFFFD", "?")
         if p.returncode != expect_returncode and support.verbose and not allow_fail:
             print("++ COMMAND ++")
             print([self.py_exe, *args])
@@ -273,7 +275,7 @@ class RunPyMixin:
     def fake_venv(self):
         venv = Path.cwd() / "Scripts"
         venv.mkdir(exist_ok=True, parents=True)
-        venv_exe = (venv / Path(sys.executable).name)
+        venv_exe = (venv / ("python_d.exe" if DEBUG_BUILD else "python.exe"))
         venv_exe.touch()
         try:
             yield venv_exe, {"VIRTUAL_ENV": str(venv.parent)}
@@ -521,6 +523,9 @@ class TestLauncher(unittest.TestCase, RunPyMixin):
                     self.assertEqual(str(venv_exe), m.group(1))
                     break
             else:
+                if support.verbose:
+                    print(data["stdout"])
+                    print(data["stderr"])
                 self.fail("did not find active venv path")
 
             data = self.run_py(["-0"], env=env)
@@ -616,25 +621,29 @@ class TestLauncher(unittest.TestCase, RunPyMixin):
         self.assertEqual("True", data["SearchInfo.oldStyleTag"])
 
     def test_search_path(self):
-        stem = Path(sys.executable).stem
+        exe = Path("arbitrary-exe-name.exe").absolute()
+        exe.touch()
+        self.addCleanup(exe.unlink)
         with self.py_ini(TEST_PY_DEFAULTS):
-            with self.script(f"#! /usr/bin/env {stem} -prearg") as script:
+            with self.script(f"#! /usr/bin/env {exe.stem} -prearg") as script:
                 data = self.run_py(
                     [script, "-postarg"],
-                    env={"PATH": f"{Path(sys.executable).parent};{os.getenv('PATH')}"},
+                    env={"PATH": f"{exe.parent};{os.getenv('PATH')}"},
                 )
-        self.assertEqual(f"{sys.executable} -prearg {script} -postarg", data["stdout"].strip())
+        self.assertEqual(f"{exe} -prearg {script} -postarg", data["stdout"].strip())
 
     def test_search_path_exe(self):
         # Leave the .exe on the name to ensure we don't add it a second time
-        name = Path(sys.executable).name
+        exe = Path("arbitrary-exe-name.exe").absolute()
+        exe.touch()
+        self.addCleanup(exe.unlink)
         with self.py_ini(TEST_PY_DEFAULTS):
-            with self.script(f"#! /usr/bin/env {name} -prearg") as script:
+            with self.script(f"#! /usr/bin/env {exe.name} -prearg") as script:
                 data = self.run_py(
                     [script, "-postarg"],
-                    env={"PATH": f"{Path(sys.executable).parent};{os.getenv('PATH')}"},
+                    env={"PATH": f"{exe.parent};{os.getenv('PATH')}"},
                 )
-        self.assertEqual(f"{sys.executable} -prearg {script} -postarg", data["stdout"].strip())
+        self.assertEqual(f"{exe} -prearg {script} -postarg", data["stdout"].strip())
 
     def test_recursive_search_path(self):
         stem = self.get_py_exe().stem
@@ -727,15 +736,18 @@ class TestLauncher(unittest.TestCase, RunPyMixin):
             data = self.run_py([script], expect_returncode=103)
 
         with self.fake_venv() as (venv_exe, env):
-            # Put a real Python (ourselves) on PATH as a distraction.
+            # Put a "normal" Python on PATH as a distraction.
             # The active VIRTUAL_ENV should be preferred when the name isn't an
             # exact match.
-            env["PATH"] = f"{Path(sys.executable).parent};{os.environ['PATH']}"
+            exe = Path(Path(venv_exe).name).absolute()
+            exe.touch()
+            self.addCleanup(exe.unlink)
+            env["PATH"] = f"{exe.parent};{os.environ['PATH']}"
 
             with self.script(f'#! /usr/bin/env {stem} arg1') as script:
                 data = self.run_py([script], env=env)
             self.assertEqual(data["stdout"].strip(), f"{venv_exe} arg1 {script}")
 
-            with self.script(f'#! /usr/bin/env {Path(sys.executable).stem} arg1') as script:
+            with self.script(f'#! /usr/bin/env {exe.stem} arg1') as script:
                 data = self.run_py([script], env=env)
-            self.assertEqual(data["stdout"].strip(), f"{sys.executable} arg1 {script}")
+            self.assertEqual(data["stdout"].strip(), f"{exe} arg1 {script}")
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index e828941..89562fa 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -845,6 +845,8 @@ class ProgramsTestCase(BaseTestCase):
             test_args.append('-x64')   # 64-bit build
         if not support.Py_DEBUG:
             test_args.append('+d')     # Release build, use python.exe
+        if sysconfig.get_config_var("Py_GIL_DISABLED"):
+            test_args.append('--disable-gil')
         self.run_batch(script, *test_args, *self.tests)
 
     @unittest.skipUnless(sys.platform == 'win32', 'Windows only')
@@ -862,6 +864,8 @@ class ProgramsTestCase(BaseTestCase):
             rt_args.append('-x64')   # 64-bit build
         if support.Py_DEBUG:
             rt_args.append('-d')     # Debug build, use python_d.exe
+        if sysconfig.get_config_var("Py_GIL_DISABLED"):
+            rt_args.append('--disable-gil')
         self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests)
 
 
diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
index 218e756..6852625 100644
--- a/Lib/test/test_venv.py
+++ b/Lib/test/test_venv.py
@@ -223,8 +223,14 @@ class BasicTest(BaseTest):
 
     def test_upgrade_dependencies(self):
         builder = venv.EnvBuilder()
-        bin_path = 'Scripts' if sys.platform == 'win32' else 'bin'
+        bin_path = 'bin'
         python_exe = os.path.split(sys.executable)[1]
+        if sys.platform == 'win32':
+            bin_path = 'Scripts'
+            if os.path.normcase(os.path.splitext(python_exe)[0]).endswith('_d'):
+                python_exe = 'python_d.exe'
+            else:
+                python_exe = 'python.exe'
         with tempfile.TemporaryDirectory() as fake_env_dir:
             expect_exe = os.path.normcase(
                 os.path.join(fake_env_dir, bin_path, python_exe)
@@ -283,7 +289,9 @@ class BasicTest(BaseTest):
             # build environment
             ('is_python_build()', str(sysconfig.is_python_build())),
             ('get_makefile_filename()', sysconfig.get_makefile_filename()),
-            ('get_config_h_filename()', sysconfig.get_config_h_filename())):
+            ('get_config_h_filename()', sysconfig.get_config_h_filename()),
+            ('get_config_var("Py_GIL_DISABLED")',
+             str(sysconfig.get_config_var("Py_GIL_DISABLED")))):
             with self.subTest(call):
                 cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call
                 out, err = check_output(cmd, encoding='utf-8')
@@ -315,7 +323,9 @@ class BasicTest(BaseTest):
             # build environment
             ('is_python_build()', str(sysconfig.is_python_build())),
             ('get_makefile_filename()', sysconfig.get_makefile_filename()),
-            ('get_config_h_filename()', sysconfig.get_config_h_filename())):
+            ('get_config_h_filename()', sysconfig.get_config_h_filename()),
+            ('get_config_var("Py_GIL_DISABLED")',
+             str(sysconfig.get_config_var("Py_GIL_DISABLED")))):
             with self.subTest(call):
                 cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call
                 out, err = check_output(cmd, encoding='utf-8')
diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py
index d960bf3..f04ca8f 100644
--- a/Lib/venv/__init__.py
+++ b/Lib/venv/__init__.py
@@ -139,6 +139,11 @@ class EnvBuilder:
                              'check that your PATH environment variable is '
                              'correctly set.')
         dirname, exename = os.path.split(os.path.abspath(executable))
+        if sys.platform == 'win32':
+            # Always create the simplest name in the venv. It will either be a
+            # link back to executable, or a copy of the appropriate launcher
+            _d = '_d' if os.path.splitext(exename)[0].endswith('_d') else ''
+            exename = f'python{_d}.exe'
         context.executable = executable
         context.python_dir = dirname
         context.python_exe = exename
@@ -222,67 +227,26 @@ class EnvBuilder:
             args = ' '.join(args)
             f.write(f'command = {sys.executable} -m venv {args}\n')
 
-    if os.name != 'nt':
-        def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
-            """
-            Try symlinking a file, and if that fails, fall back to copying.
-            """
-            force_copy = not self.symlinks
-            if not force_copy:
-                try:
-                    if not os.path.islink(dst):  # can't link to itself!
-                        if relative_symlinks_ok:
-                            assert os.path.dirname(src) == os.path.dirname(dst)
-                            os.symlink(os.path.basename(src), dst)
-                        else:
-                            os.symlink(src, dst)
-                except Exception:   # may need to use a more specific exception
-                    logger.warning('Unable to symlink %r to %r', src, dst)
-                    force_copy = True
-            if force_copy:
-                shutil.copyfile(src, dst)
-    else:
-        def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
-            """
-            Try symlinking a file, and if that fails, fall back to copying.
-            """
-            bad_src = os.path.lexists(src) and not os.path.exists(src)
-            if self.symlinks and not bad_src and not os.path.islink(dst):
-                try:
+    def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
+        """
+        Try symlinking a file, and if that fails, fall back to copying.
+        (Unused on Windows, because we can't just copy a failed symlink file: we
+        switch to a different set of files instead.)
+        """
+        assert os.name != 'nt'
+        force_copy = not self.symlinks
+        if not force_copy:
+            try:
+                if not os.path.islink(dst):  # can't link to itself!
                     if relative_symlinks_ok:
                         assert os.path.dirname(src) == os.path.dirname(dst)
                         os.symlink(os.path.basename(src), dst)
                     else:
                         os.symlink(src, dst)
-                    return
-                except Exception:   # may need to use a more specific exception
-                    logger.warning('Unable to symlink %r to %r', src, dst)
-
-            # On Windows, we rewrite symlinks to our base python.exe into
-            # copies of venvlauncher.exe
-            basename, ext = os.path.splitext(os.path.basename(src))
-            srcfn = os.path.join(os.path.dirname(__file__),
-                                 "scripts",
-                                 "nt",
-                                 basename + ext)
-            # Builds or venv's from builds need to remap source file
-            # locations, as we do not put them into Lib/venv/scripts
-            if sysconfig.is_python_build() or not os.path.isfile(srcfn):
-                if basename.endswith('_d'):
-                    ext = '_d' + ext
-                    basename = basename[:-2]
-                if basename == 'python':
-                    basename = 'venvlauncher'
-                elif basename == 'pythonw':
-                    basename = 'venvwlauncher'
-                src = os.path.join(os.path.dirname(src), basename + ext)
-            else:
-                src = srcfn
-            if not os.path.exists(src):
-                if not bad_src:
-                    logger.warning('Unable to copy %r', src)
-                return
-
+            except Exception:   # may need to use a more specific exception
+                logger.warning('Unable to symlink %r to %r', src, dst)
+                force_copy = True
+        if force_copy:
             shutil.copyfile(src, dst)
 
     def create_git_ignore_file(self, context):
@@ -298,22 +262,23 @@ class EnvBuilder:
                        'see https://docs.python.org/3/library/venv.html\n')
             file.write('*\n')
 
-    def setup_python(self, context):
-        """
-        Set up a Python executable in the environment.
+    if os.name != 'nt':
+        def setup_python(self, context):
+            """
+            Set up a Python executable in the environment.
 
-        :param context: The information for the environment creation request
-                        being processed.
-        """
-        binpath = context.bin_path
-        path = context.env_exe
-        copier = self.symlink_or_copy
-        dirname = context.python_dir
-        if os.name != 'nt':
+            :param context: The information for the environment creation request
+                            being processed.
+            """
+            binpath = context.bin_path
+            path = context.env_exe
+            copier = self.symlink_or_copy
+            dirname = context.python_dir
             copier(context.executable, path)
             if not os.path.islink(path):
                 os.chmod(path, 0o755)
-            for suffix in ('python', 'python3', f'python3.{sys.version_info[1]}'):
+            for suffix in ('python', 'python3',
+                           f'python3.{sys.version_info[1]}'):
                 path = os.path.join(binpath, suffix)
                 if not os.path.exists(path):
                     # Issue 18807: make copies if
@@ -321,30 +286,105 @@ class EnvBuilder:
                     copier(context.env_exe, path, relative_symlinks_ok=True)
                     if not os.path.islink(path):
                         os.chmod(path, 0o755)
-        else:
-            if self.symlinks:
-                # For symlinking, we need a complete copy of the root directory
-                # If symlinks fail, you'll get unnecessary copies of files, but
-                # we assume that if you've opted into symlinks on Windows then
-                # you know what you're doing.
-                suffixes = [
-                    f for f in os.listdir(dirname) if
-                    os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll')
-                ]
-                if sysconfig.is_python_build():
-                    suffixes = [
-                        f for f in suffixes if
-                        os.path.normcase(f).startswith(('python', 'vcruntime'))
-                    ]
+
+    else:
+        def setup_python(self, context):
+            """
+            Set up a Python executable in the environment.
+
+            :param context: The information for the environment creation request
+                            being processed.
+            """
+            binpath = context.bin_path
+            dirname = context.python_dir
+            exename = os.path.basename(context.env_exe)
+            exe_stem = os.path.splitext(exename)[0]
+            exe_d = '_d' if os.path.normcase(exe_stem).endswith('_d') else ''
+            if sysconfig.is_python_build():
+                scripts = dirname
+            else:
+                scripts = os.path.join(os.path.dirname(__file__),
+                                       'scripts', 'nt')
+            if not sysconfig.get_config_var("Py_GIL_DISABLED"):
+                python_exe = os.path.join(dirname, f'python{exe_d}.exe')
+                pythonw_exe = os.path.join(dirname, f'pythonw{exe_d}.exe')
+                link_sources = {
+                    'python.exe': python_exe,
+                    f'python{exe_d}.exe': python_exe,
+                    'pythonw.exe': pythonw_exe,
+                    f'pythonw{exe_d}.exe': pythonw_exe,
+                }
+                python_exe = os.path.join(scripts, f'venvlauncher{exe_d}.exe')
+                pythonw_exe = os.path.join(scripts, f'venvwlauncher{exe_d}.exe')
+                copy_sources = {
+                    'python.exe': python_exe,
+                    f'python{exe_d}.exe': python_exe,
+                    'pythonw.exe': pythonw_exe,
+                    f'pythonw{exe_d}.exe': pythonw_exe,
+                }
             else:
-                suffixes = {'python.exe', 'python_d.exe', 'pythonw.exe', 'pythonw_d.exe'}
-                base_exe = os.path.basename(context.env_exe)
-                suffixes.add(base_exe)
+                exe_t = f'3.{sys.version_info[1]}t'
+                python_exe = os.path.join(dirname, f'python{exe_t}{exe_d}.exe')
+                pythonw_exe = os.path.join(dirname, f'pythonw{exe_t}{exe_d}.exe')
+                link_sources = {
+                    'python.exe': python_exe,
+                    f'python{exe_d}.exe': python_exe,
+                    f'python{exe_t}.exe': python_exe,
+                    f'python{exe_t}{exe_d}.exe': python_exe,
+                    'pythonw.exe': pythonw_exe,
+                    f'pythonw{exe_d}.exe': pythonw_exe,
+                    f'pythonw{exe_t}.exe': pythonw_exe,
+                    f'pythonw{exe_t}{exe_d}.exe': pythonw_exe,
+                }
+                python_exe = os.path.join(scripts, f'venvlaunchert{exe_d}.exe')
+                pythonw_exe = os.path.join(scripts, f'venvwlaunchert{exe_d}.exe')
+                copy_sources = {
+                    'python.exe': python_exe,
+                    f'python{exe_d}.exe': python_exe,
+                    f'python{exe_t}.exe': python_exe,
+                    f'python{exe_t}{exe_d}.exe': python_exe,
+                    'pythonw.exe': pythonw_exe,
+                    f'pythonw{exe_d}.exe': pythonw_exe,
+                    f'pythonw{exe_t}.exe': pythonw_exe,
+                    f'pythonw{exe_t}{exe_d}.exe': pythonw_exe,
+                }
+
+            do_copies = True
+            if self.symlinks:
+                do_copies = False
+                # For symlinking, we need all the DLLs to be available alongside
+                # the executables.
+                link_sources.update({
+                    f: os.path.join(dirname, f) for f in os.listdir(dirname)
+                    if os.path.normcase(f).startswith(('python', 'vcruntime'))
+                    and os.path.normcase(os.path.splitext(f)[1]) == '.dll'
+                })
+
+                to_unlink = []
+                for dest, src in link_sources.items():
+                    dest = os.path.join(binpath, dest)
+                    try:
+                        os.symlink(src, dest)
+                        to_unlink.append(dest)
+                    except OSError:
+                        logger.warning('Unable to symlink %r to %r', src, dst)
+                        do_copies = True
+                        for f in to_unlink:
+                            try:
+                                os.unlink(f)
+                            except OSError:
+                                logger.warning('Failed to clean up symlink %r',
+                                               f)
+                        logger.warning('Retrying with copies')
+                        break
 
-            for suffix in suffixes:
-                src = os.path.join(dirname, suffix)
-                if os.path.lexists(src):
-                    copier(src, os.path.join(binpath, suffix))
+            if do_copies:
+                for dest, src in copy_sources.items():
+                    dest = os.path.join(binpath, dest)
+                    try:
+                        shutil.copy2(src, dest)
+                    except OSError:
+                        logger.warning('Unable to copy %r to %r', src, dest)
 
             if sysconfig.is_python_build():
                 # copy init.tcl
@@ -437,6 +477,14 @@ class EnvBuilder:
         """
         binpath = context.bin_path
         plen = len(path)
+        if os.name == 'nt':
+            def skip_file(f):
+                f = os.path.normcase(f)
+                return (f.startswith(('python', 'venv'))
+                        and f.endswith(('.exe', '.pdb')))
+        else:
+            def skip_file(f):
+                return False
         for root, dirs, files in os.walk(path):
             if root == path:  # at top-level, remove irrelevant dirs
                 for d in dirs[:]:
@@ -444,8 +492,7 @@ class EnvBuilder:
                         dirs.remove(d)
                 continue  # ignore files in top level
             for f in files:
-                if (os.name == 'nt' and f.startswith('python')
-                        and f.endswith(('.exe', '.pdb'))):
+                if skip_file(f):
                     continue
                 srcfile = os.path.join(root, f)
                 suffix = root[plen:].split(os.sep)[2:]
@@ -456,20 +503,25 @@ class EnvBuilder:
                 if not os.path.exists(dstdir):
                     os.makedirs(dstdir)
                 dstfile = os.path.join(dstdir, f)
+                if os.name == 'nt' and srcfile.endswith(('.exe', '.pdb')):
+                    shutil.copy2(srcfile, dstfile)
+                    continue
                 with open(srcfile, 'rb') as f:
                     data = f.read()
-                if not srcfile.endswith(('.exe', '.pdb')):
-                    try:
-                        data = data.decode('utf-8')
-                        data = self.replace_variables(data, context)
-                        data = data.encode('utf-8')
-                    except UnicodeError as e:
-                        data = None
-                        logger.warning('unable to copy script %r, '
-                                       'may be binary: %s', srcfile, e)
-                if data is not None:
+                try:
+                    new_data = (
+                        self.replace_variables(data.decode('utf-8'), context)
+                            .encode('utf-8')
+                    )
+                except UnicodeError as e:
+                    logger.warning('unable to copy script %r, '
+                                   'may be binary: %s', srcfile, e)
+                    continue
+                if new_data == data:
+                    shutil.copy2(srcfile, dstfile)
+                else:
                     with open(dstfile, 'wb') as f:
-                        f.write(data)
+                        f.write(new_data)
                     shutil.copymode(srcfile, dstfile)
 
     def upgrade_dependencies(self, context):