From d13f782a181d579fc3c23ea6059ff352ec9fab93 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Sep 2023 15:13:29 +0200 Subject: gh-109276: libregrtest: fix worker working dir (#109313) Fix Emscripten and WASI: start the test worker process in the Python source code directory, where 'python.js' and 'python.wasm' can be found. Then worker_process() changes to a temporary directory created to run tests. * create_worker_process() uses os_helper.SAVEDCWD as cwd. * worker_process() uses get_temp_dir() as the parent directory for get_work_dir(). * Don't use plural but singual for "test" in "Run 1 test ..." message. * Remove unused imports. * Add WORK_DIR_PREFIX and WORKER_WORK_DIR_PREFIX constants. --- Lib/test/libregrtest/main.py | 4 ++-- Lib/test/libregrtest/run_workers.py | 8 ++++---- Lib/test/libregrtest/utils.py | 15 ++++++++------- Lib/test/libregrtest/worker.py | 13 ++++++++++--- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index a89e3c6..dd9a107 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -297,7 +297,7 @@ class Regrtest: jobs = runtests.get_jobs() if jobs is not None: - tests = f'{jobs} tests' + tests = count(jobs, 'test') else: tests = 'tests' msg = f"Run {tests} sequentially" @@ -458,7 +458,7 @@ class Regrtest: def run_tests(self, selected: TestTuple, tests: TestList | None) -> int: os.makedirs(self.tmp_dir, exist_ok=True) - work_dir = get_work_dir(parent_dir=self.tmp_dir) + work_dir = get_work_dir(self.tmp_dir) # Put a timeout on Python exit with exit_timeout(): diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index cfa36f7..eaca0af 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -9,7 +9,7 @@ import tempfile import threading import time import traceback -from typing import Literal, TextIO +from typing import Literal from test import support from test.support import os_helper @@ -21,7 +21,7 @@ from .runtests import RunTests from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, StrJSON, TestName, MS_WINDOWS, - format_duration, print_warning, plural) + format_duration, print_warning, count, plural) from .worker import create_worker_process, USE_PROCESS_GROUP if MS_WINDOWS: @@ -280,7 +280,7 @@ class WorkerThread(threading.Thread): if worker_json: result = TestResult.from_json(worker_json) else: - err_msg = f"empty JSON" + err_msg = "empty JSON" except Exception as exc: # gh-101634: Catch UnicodeDecodeError if stdout cannot be # decoded from encoding @@ -412,7 +412,7 @@ class RunWorkers: for index in range(1, self.num_workers + 1)] jobs = self.runtests.get_jobs() if jobs is not None: - tests = f'{jobs} tests' + tests = count(jobs, 'test') else: tests = 'tests' nworkers = len(self.workers) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index ce1b108..03c27b9 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -17,6 +17,8 @@ from test.support import threading_helper MS_WINDOWS = (sys.platform == 'win32') +WORK_DIR_PREFIX = 'test_python_' +WORKER_WORK_DIR_PREFIX = f'{WORK_DIR_PREFIX}worker_' # bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()). # Used to protect against threading._shutdown() hang. @@ -346,7 +348,7 @@ def get_build_info(): return build -def get_temp_dir(tmp_dir): +def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: if tmp_dir: tmp_dir = os.path.expanduser(tmp_dir) else: @@ -379,7 +381,7 @@ def fix_umask(): os.umask(old_mask) -def get_work_dir(*, parent_dir: StrPath = '', worker: bool = False): +def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath: # Define a writable temp dir that will be used as cwd while running # the tests. The name of the dir includes the pid to allow parallel # testing (see the -j option). @@ -391,12 +393,11 @@ def get_work_dir(*, parent_dir: StrPath = '', worker: bool = False): nounce = os.getpid() if worker: - work_dir = 'test_python_worker_{}'.format(nounce) + work_dir = WORK_DIR_PREFIX + str(nounce) else: - work_dir = 'test_python_{}'.format(nounce) + work_dir = WORKER_WORK_DIR_PREFIX + str(nounce) work_dir += os_helper.FS_NONASCII - if parent_dir: - work_dir = os.path.join(parent_dir, work_dir) + work_dir = os.path.join(parent_dir, work_dir) return work_dir @@ -579,7 +580,7 @@ def display_header(): def cleanup_temp_dir(tmp_dir: StrPath): import glob - path = os.path.join(glob.escape(tmp_dir), 'test_python_*') + path = os.path.join(glob.escape(tmp_dir), WORK_DIR_PREFIX + '*') print("Cleanup %s directory" % tmp_dir) for name in glob.glob(path): if os.path.isdir(name): diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index b3b204f..0963faa 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -1,7 +1,7 @@ import subprocess import sys import os -from typing import TextIO, NoReturn +from typing import NoReturn from test import support from test.support import os_helper @@ -11,7 +11,7 @@ from .runtests import RunTests from .single import run_single_test from .utils import ( StrPath, StrJSON, FilterTuple, MS_WINDOWS, - get_work_dir, exit_timeout) + get_temp_dir, get_work_dir, exit_timeout) USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) @@ -38,6 +38,11 @@ def create_worker_process(runtests: RunTests, env['TEMP'] = tmp_dir env['TMP'] = tmp_dir + # Emscripten and WASI Python must start in the Python source code directory + # to get 'python.js' or 'python.wasm' file. Then worker_process() changes + # to a temporary directory created to run tests. + work_dir = os_helper.SAVEDCWD + # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. @@ -48,6 +53,7 @@ def create_worker_process(runtests: RunTests, stderr=output_fd, text=True, close_fds=True, + cwd=work_dir, ) if not MS_WINDOWS: kwargs['pass_fds'] = [json_fd] @@ -102,7 +108,8 @@ def main(): sys.exit(1) worker_json = sys.argv[1] - work_dir = get_work_dir(worker=True) + tmp_dir = get_temp_dir() + work_dir = get_work_dir(tmp_dir, worker=True) with exit_timeout(): with os_helper.temp_cwd(work_dir, quiet=True): -- cgit v0.12