summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Doc/c-api/init_config.rst9
-rw-r--r--Doc/using/ios.rst53
-rw-r--r--Doc/whatsnew/3.14.rst7
-rw-r--r--Include/cpython/initconfig.h3
-rw-r--r--Lib/_apple_support.py66
-rw-r--r--Lib/test/test_apple.py155
-rw-r--r--Lib/test/test_capi/test_config.py4
-rw-r--r--Lib/test/test_embed.py2
-rw-r--r--Makefile.pre.in26
-rw-r--r--Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst2
-rw-r--r--Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst2
-rw-r--r--Python/initconfig.c15
-rw-r--r--Python/pylifecycle.c82
-rw-r--r--Python/stdlib_module_names.h1
-rw-r--r--iOS/README.rst54
-rw-r--r--iOS/testbed/__main__.py365
-rw-r--r--iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj2
-rw-r--r--iOS/testbed/iOSTestbedTests/iOSTestbedTests.m2
18 files changed, 792 insertions, 58 deletions
diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index d6569dd..7497bf2 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -1281,6 +1281,15 @@ PyConfig
Default: ``1`` in Python config and ``0`` in isolated config.
+ .. c:member:: int use_system_logger
+
+ If non-zero, ``stdout`` and ``stderr`` will be redirected to the system
+ log.
+
+ Only available on macOS 10.12 and later, and on iOS.
+
+ Default: ``0`` (don't use system log).
+
.. c:member:: int user_site_directory
If non-zero, add the user site directory to :data:`sys.path`.
diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst
index 4d4eb20..aa43f75 100644
--- a/Doc/using/ios.rst
+++ b/Doc/using/ios.rst
@@ -292,10 +292,12 @@ To add Python to an iOS Xcode project:
10. Add Objective C code to initialize and use a Python interpreter in embedded
mode. You should ensure that:
- * :c:member:`UTF-8 mode <PyPreConfig.utf8_mode>` is *enabled*;
- * :c:member:`Buffered stdio <PyConfig.buffered_stdio>` is *disabled*;
- * :c:member:`Writing bytecode <PyConfig.write_bytecode>` is *disabled*;
- * :c:member:`Signal handlers <PyConfig.install_signal_handlers>` are *enabled*;
+ * UTF-8 mode (:c:member:`PyPreConfig.utf8_mode`) is *enabled*;
+ * Buffered stdio (:c:member:`PyConfig.buffered_stdio`) is *disabled*;
+ * Writing bytecode (:c:member:`PyConfig.write_bytecode`) is *disabled*;
+ * Signal handlers (:c:member:`PyConfig.install_signal_handlers`) are *enabled*;
+ * System logging (:c:member:`PyConfig.use_system_logger`) is *enabled*
+ (optional, but strongly recommended);
* ``PYTHONHOME`` for the interpreter is configured to point at the
``python`` subfolder of your app's bundle; and
* The ``PYTHONPATH`` for the interpreter includes:
@@ -324,6 +326,49 @@ modules in your app, some additional steps will be required:
* If you're using a separate folder for third-party packages, ensure that folder
is included as part of the ``PYTHONPATH`` configuration in step 10.
+Testing a Python package
+------------------------
+
+The CPython source tree contains :source:`a testbed project <iOS/testbed>` that
+is used to run the CPython test suite on the iOS simulator. This testbed can also
+be used as a testbed project for running your Python library's test suite on iOS.
+
+After building or obtaining an iOS XCFramework (See :source:`iOS/README.rst`
+for details), create a clone of the Python iOS testbed project by running:
+
+.. code-block:: bash
+
+ $ python iOS/testbed clone --framework <path/to/Python.xcframework> --app <path/to/module1> --app <path/to/module2> app-testbed
+
+You will need to modify the ``iOS/testbed`` reference to point to that
+directory in the CPython source tree; any folders specified with the ``--app``
+flag will be copied into the cloned testbed project. The resulting testbed will
+be created in the ``app-testbed`` folder. In this example, the ``module1`` and
+``module2`` would be importable modules at runtime. If your project has
+additional dependencies, they can be installed into the
+``app-testbed/iOSTestbed/app_packages`` folder (using ``pip install --target
+app-testbed/iOSTestbed/app_packages`` or similar).
+
+You can then use the ``app-testbed`` folder to run the test suite for your app,
+For example, if ``module1.tests`` was the entry point to your test suite, you
+could run:
+
+.. code-block:: bash
+
+ $ python app-testbed run -- module1.tests
+
+This is the equivalent of running ``python -m module1.tests`` on a desktop
+Python build. Any arguments after the ``--`` will be passed to the testbed as
+if they were arguments to ``python -m`` on a desktop machine.
+
+You can also open the testbed project in Xcode by running:
+
+.. code-block:: bash
+
+ $ open app-testbed/iOSTestbed.xcodeproj
+
+This will allow you to use the full Xcode suite of tools for debugging.
+
App Store Compliance
====================
diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst
index b300e34..b71d31f 100644
--- a/Doc/whatsnew/3.14.rst
+++ b/Doc/whatsnew/3.14.rst
@@ -245,6 +245,13 @@ Other language changes
making it a :term:`generic type`.
(Contributed by Brian Schubert in :gh:`126012`.)
+* iOS and macOS apps can now be configured to redirect ``stdout`` and
+ ``stderr`` content to the system log. (Contributed by Russell Keith-Magee in
+ :gh:`127592`.)
+
+* The iOS testbed is now able to stream test output while the test is running.
+ The testbed can also be used to run the test suite of projects other than
+ CPython itself. (Contributed by Russell Keith-Magee in :gh:`127592`.)
New modules
===========
diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
index f69c586..8ef19f6 100644
--- a/Include/cpython/initconfig.h
+++ b/Include/cpython/initconfig.h
@@ -179,6 +179,9 @@ typedef struct PyConfig {
int use_frozen_modules;
int safe_path;
int int_max_str_digits;
+#ifdef __APPLE__
+ int use_system_logger;
+#endif
int cpu_count;
#ifdef Py_GIL_DISABLED
diff --git a/Lib/_apple_support.py b/Lib/_apple_support.py
new file mode 100644
index 0000000..92febdc
--- /dev/null
+++ b/Lib/_apple_support.py
@@ -0,0 +1,66 @@
+import io
+import sys
+
+
+def init_streams(log_write, stdout_level, stderr_level):
+ # Redirect stdout and stderr to the Apple system log. This method is
+ # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger
+ # is enabled.
+ sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors)
+ sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors)
+
+
+class SystemLog(io.TextIOWrapper):
+ def __init__(self, log_write, level, **kwargs):
+ kwargs.setdefault("encoding", "UTF-8")
+ kwargs.setdefault("line_buffering", True)
+ super().__init__(LogStream(log_write, level), **kwargs)
+
+ def __repr__(self):
+ return f"<SystemLog (level {self.buffer.level})>"
+
+ def write(self, s):
+ if not isinstance(s, str):
+ raise TypeError(
+ f"write() argument must be str, not {type(s).__name__}")
+
+ # In case `s` is a str subclass that writes itself to stdout or stderr
+ # when we call its methods, convert it to an actual str.
+ s = str.__str__(s)
+
+ # We want to emit one log message per line, so split
+ # the string before sending it to the superclass.
+ for line in s.splitlines(keepends=True):
+ super().write(line)
+
+ return len(s)
+
+
+class LogStream(io.RawIOBase):
+ def __init__(self, log_write, level):
+ self.log_write = log_write
+ self.level = level
+
+ def __repr__(self):
+ return f"<LogStream (level {self.level!r})>"
+
+ def writable(self):
+ return True
+
+ def write(self, b):
+ if type(b) is not bytes:
+ try:
+ b = bytes(memoryview(b))
+ except TypeError:
+ raise TypeError(
+ f"write() argument must be bytes-like, not {type(b).__name__}"
+ ) from None
+
+ # Writing an empty string to the stream should have no effect.
+ if b:
+ # Encode null bytes using "modified UTF-8" to avoid truncating the
+ # message. This should not affect the return value, as the caller
+ # may be expecting it to match the length of the input.
+ self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80"))
+
+ return len(b)
diff --git a/Lib/test/test_apple.py b/Lib/test/test_apple.py
new file mode 100644
index 0000000..ab5296a
--- /dev/null
+++ b/Lib/test/test_apple.py
@@ -0,0 +1,155 @@
+import unittest
+from _apple_support import SystemLog
+from test.support import is_apple
+from unittest.mock import Mock, call
+
+if not is_apple:
+ raise unittest.SkipTest("Apple-specific")
+
+
+# Test redirection of stdout and stderr to the Apple system log.
+class TestAppleSystemLogOutput(unittest.TestCase):
+ maxDiff = None
+
+ def assert_writes(self, output):
+ self.assertEqual(
+ self.log_write.mock_calls,
+ [
+ call(self.log_level, line)
+ for line in output
+ ]
+ )
+
+ self.log_write.reset_mock()
+
+ def setUp(self):
+ self.log_write = Mock()
+ self.log_level = 42
+ self.log = SystemLog(self.log_write, self.log_level, errors="replace")
+
+ def test_repr(self):
+ self.assertEqual(repr(self.log), "<SystemLog (level 42)>")
+ self.assertEqual(repr(self.log.buffer), "<LogStream (level 42)>")
+
+ def test_log_config(self):
+ self.assertIs(self.log.writable(), True)
+ self.assertIs(self.log.readable(), False)
+
+ self.assertEqual("UTF-8", self.log.encoding)
+ self.assertEqual("replace", self.log.errors)
+
+ self.assertIs(self.log.line_buffering, True)
+ self.assertIs(self.log.write_through, False)
+
+ def test_empty_str(self):
+ self.log.write("")
+ self.log.flush()
+
+ self.assert_writes([])
+
+ def test_simple_str(self):
+ self.log.write("hello world\n")
+
+ self.assert_writes([b"hello world\n"])
+
+ def test_buffered_str(self):
+ self.log.write("h")
+ self.log.write("ello")
+ self.log.write(" ")
+ self.log.write("world\n")
+ self.log.write("goodbye.")
+ self.log.flush()
+
+ self.assert_writes([b"hello world\n", b"goodbye."])
+
+ def test_manual_flush(self):
+ self.log.write("Hello")
+
+ self.assert_writes([])
+
+ self.log.write(" world\nHere for a while...\nGoodbye")
+ self.assert_writes([b"Hello world\n", b"Here for a while...\n"])
+
+ self.log.write(" world\nHello again")
+ self.assert_writes([b"Goodbye world\n"])
+
+ self.log.flush()
+ self.assert_writes([b"Hello again"])
+
+ def test_non_ascii(self):
+ # Spanish
+ self.log.write("ol\u00e9\n")
+ self.assert_writes([b"ol\xc3\xa9\n"])
+
+ # Chinese
+ self.log.write("\u4e2d\u6587\n")
+ self.assert_writes([b"\xe4\xb8\xad\xe6\x96\x87\n"])
+
+ # Printing Non-BMP emoji
+ self.log.write("\U0001f600\n")
+ self.assert_writes([b"\xf0\x9f\x98\x80\n"])
+
+ # Non-encodable surrogates are replaced
+ self.log.write("\ud800\udc00\n")
+ self.assert_writes([b"??\n"])
+
+ def test_modified_null(self):
+ # Null characters are logged using "modified UTF-8".
+ self.log.write("\u0000\n")
+ self.assert_writes([b"\xc0\x80\n"])
+ self.log.write("a\u0000\n")
+ self.assert_writes([b"a\xc0\x80\n"])
+ self.log.write("\u0000b\n")
+ self.assert_writes([b"\xc0\x80b\n"])
+ self.log.write("a\u0000b\n")
+ self.assert_writes([b"a\xc0\x80b\n"])
+
+ def test_nonstandard_str(self):
+ # String subclasses are accepted, but they should be converted
+ # to a standard str without calling any of their methods.
+ class CustomStr(str):
+ def splitlines(self, *args, **kwargs):
+ raise AssertionError()
+
+ def __len__(self):
+ raise AssertionError()
+
+ def __str__(self):
+ raise AssertionError()
+
+ self.log.write(CustomStr("custom\n"))
+ self.assert_writes([b"custom\n"])
+
+ def test_non_str(self):
+ # Non-string classes are not accepted.
+ for obj in [b"", b"hello", None, 42]:
+ with self.subTest(obj=obj):
+ with self.assertRaisesRegex(
+ TypeError,
+ fr"write\(\) argument must be str, not "
+ fr"{type(obj).__name__}"
+ ):
+ self.log.write(obj)
+
+ def test_byteslike_in_buffer(self):
+ # The underlying buffer *can* accept bytes-like objects
+ self.log.buffer.write(bytearray(b"hello"))
+ self.log.flush()
+
+ self.log.buffer.write(b"")
+ self.log.flush()
+
+ self.log.buffer.write(b"goodbye")
+ self.log.flush()
+
+ self.assert_writes([b"hello", b"goodbye"])
+
+ def test_non_byteslike_in_buffer(self):
+ for obj in ["hello", None, 42]:
+ with self.subTest(obj=obj):
+ with self.assertRaisesRegex(
+ TypeError,
+ fr"write\(\) argument must be bytes-like, not "
+ fr"{type(obj).__name__}"
+ ):
+ self.log.buffer.write(obj)
diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py
index 77730ad..a3179ef 100644
--- a/Lib/test/test_capi/test_config.py
+++ b/Lib/test/test_capi/test_config.py
@@ -110,6 +110,10 @@ class CAPITests(unittest.TestCase):
options.extend((
("_pystats", bool, None),
))
+ if support.is_apple:
+ options.extend((
+ ("use_system_logger", bool, None),
+ ))
for name, option_type, sys_attr in options:
with self.subTest(name=name, option_type=option_type,
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 5c38b28..7110fb8 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -649,6 +649,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
CONFIG_COMPAT.update({
'legacy_windows_stdio': False,
})
+ if support.is_apple:
+ CONFIG_COMPAT['use_system_logger'] = False
CONFIG_PYTHON = dict(CONFIG_COMPAT,
_config_init=API_PYTHON,
diff --git a/Makefile.pre.in b/Makefile.pre.in
index dd8a3ab..7b66802 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -2146,7 +2146,6 @@ testuniversal: all
# This must be run *after* a `make install` has completed the build. The
# `--with-framework-name` argument *cannot* be used when configuring the build.
XCFOLDER:=iOSTestbed.$(MULTIARCH).$(shell date +%s)
-XCRESULT=$(XCFOLDER)/$(MULTIARCH).xcresult
.PHONY: testios
testios:
@if test "$(MACHDEP)" != "ios"; then \
@@ -2165,29 +2164,12 @@ testios:
echo "Cannot find a finalized iOS Python.framework. Have you run 'make install' to finalize the framework build?"; \
exit 1;\
fi
- # Copy the testbed project into the build folder
- cp -r $(srcdir)/iOS/testbed $(XCFOLDER)
- # Copy the framework from the install location to the testbed project.
- cp -r $(PYTHONFRAMEWORKPREFIX)/* $(XCFOLDER)/Python.xcframework/ios-arm64_x86_64-simulator
-
- # Run the test suite for the Xcode project, targeting the iOS simulator.
- # If the suite fails, touch a file in the test folder as a marker
- if ! xcodebuild test -project $(XCFOLDER)/iOSTestbed.xcodeproj -scheme "iOSTestbed" -destination "platform=iOS Simulator,name=iPhone SE (3rd Generation)" -resultBundlePath $(XCRESULT) -derivedDataPath $(XCFOLDER)/DerivedData ; then \
- touch $(XCFOLDER)/failed; \
- fi
- # Regardless of success or failure, extract and print the test output
- xcrun xcresulttool get --path $(XCRESULT) \
- --id $$( \
- xcrun xcresulttool get --path $(XCRESULT) --format json | \
- $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['actions']['_values'][0]['actionResult']['logRef']['id']['_value'])" \
- ) \
- --format json | \
- $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['subsections']['_values'][1]['subsections']['_values'][0]['emittedOutput']['_value'])"
+ # Clone the testbed project into the XCFOLDER
+ $(PYTHON_FOR_BUILD) $(srcdir)/iOS/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)"
- @if test -e $(XCFOLDER)/failed ; then \
- exit 1; \
- fi
+ # Run the testbed project
+ $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run -- test -uall --single-process --rerun -W
# Like test, but using --slow-ci which enables all test resources and use
# longer timeout. Run an optional pybuildbot.identify script to include
diff --git a/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst b/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst
new file mode 100644
index 0000000..677acf5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst
@@ -0,0 +1,2 @@
+macOS and iOS apps can now choose to redirect stdout and stderr to the
+system log during interpreter configuration.
diff --git a/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst b/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst
new file mode 100644
index 0000000..fb307c7
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst
@@ -0,0 +1,2 @@
+iOS test results are now streamed during test execution, and the deprecated
+xcresulttool is no longer used.
diff --git a/Python/initconfig.c b/Python/initconfig.c
index 438f8a5..7851b86 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -168,6 +168,9 @@ static const PyConfigSpec PYCONFIG_SPEC[] = {
SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS),
SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS),
SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS),
+#ifdef __APPLE__
+ SPEC(use_system_logger, BOOL, PUBLIC, NO_SYS),
+#endif
SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_user_site
SPEC(warn_default_encoding, BOOL, READ_ONLY, NO_SYS),
@@ -884,6 +887,9 @@ config_check_consistency(const PyConfig *config)
assert(config->cpu_count != 0);
// config->use_frozen_modules is initialized later
// by _PyConfig_InitImportConfig().
+#ifdef __APPLE__
+ assert(config->use_system_logger >= 0);
+#endif
#ifdef Py_STATS
assert(config->_pystats >= 0);
#endif
@@ -986,6 +992,9 @@ _PyConfig_InitCompatConfig(PyConfig *config)
config->_is_python_build = 0;
config->code_debug_ranges = 1;
config->cpu_count = -1;
+#ifdef __APPLE__
+ config->use_system_logger = 0;
+#endif
#ifdef Py_GIL_DISABLED
config->enable_gil = _PyConfig_GIL_DEFAULT;
config->tlbc_enabled = 1;
@@ -1015,6 +1024,9 @@ config_init_defaults(PyConfig *config)
#ifdef MS_WINDOWS
config->legacy_windows_stdio = 0;
#endif
+#ifdef __APPLE__
+ config->use_system_logger = 0;
+#endif
}
@@ -1049,6 +1061,9 @@ PyConfig_InitIsolatedConfig(PyConfig *config)
#ifdef MS_WINDOWS
config->legacy_windows_stdio = 0;
#endif
+#ifdef __APPLE__
+ config->use_system_logger = 0;
+#endif
}
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index ceb30e9..0641812 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -45,7 +45,9 @@
#endif
#if defined(__APPLE__)
+# include <AvailabilityMacros.h>
# include <mach-o/loader.h>
+# include <os/log.h>
#endif
#ifdef HAVE_SIGNAL_H
@@ -75,6 +77,9 @@ static PyStatus init_sys_streams(PyThreadState *tstate);
#ifdef __ANDROID__
static PyStatus init_android_streams(PyThreadState *tstate);
#endif
+#if defined(__APPLE__)
+static PyStatus init_apple_streams(PyThreadState *tstate);
+#endif
static void wait_for_thread_shutdown(PyThreadState *tstate);
static void finalize_subinterpreters(void);
static void call_ll_exitfuncs(_PyRuntimeState *runtime);
@@ -1257,6 +1262,14 @@ init_interp_main(PyThreadState *tstate)
return status;
}
#endif
+#if defined(__APPLE__)
+ if (config->use_system_logger) {
+ status = init_apple_streams(tstate);
+ if (_PyStatus_EXCEPTION(status)) {
+ return status;
+ }
+ }
+#endif
#ifdef Py_DEBUG
run_presite(tstate);
@@ -2933,6 +2946,75 @@ done:
#endif // __ANDROID__
+#if defined(__APPLE__)
+
+static PyObject *
+apple_log_write_impl(PyObject *self, PyObject *args)
+{
+ int logtype = 0;
+ const char *text = NULL;
+ if (!PyArg_ParseTuple(args, "iy", &logtype, &text)) {
+ return NULL;
+ }
+
+ // Call the underlying Apple logging API. The os_log unified logging APIs
+ // were introduced in macOS 10.12, iOS 10.0, tvOS 10.0, and watchOS 3.0;
+ // this call is a no-op on older versions.
+ #if TARGET_OS_IPHONE || (TARGET_OS_OSX && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12)
+ // Pass the user-provided text through explicit %s formatting
+ // to avoid % literals being interpreted as a formatting directive.
+ os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text);
+ #endif
+ Py_RETURN_NONE;
+}
+
+
+static PyMethodDef apple_log_write_method = {
+ "apple_log_write", apple_log_write_impl, METH_VARARGS
+};
+
+
+static PyStatus
+init_apple_streams(PyThreadState *tstate)
+{
+ PyStatus status = _PyStatus_OK();
+ PyObject *_apple_support = NULL;
+ PyObject *apple_log_write = NULL;
+ PyObject *result = NULL;
+
+ _apple_support = PyImport_ImportModule("_apple_support");
+ if (_apple_support == NULL) {
+ goto error;
+ }
+
+ apple_log_write = PyCFunction_New(&apple_log_write_method, NULL);
+ if (apple_log_write == NULL) {
+ goto error;
+ }
+
+ // Initialize the logging streams, sending stdout -> Default; stderr -> Error
+ result = PyObject_CallMethod(
+ _apple_support, "init_streams", "Oii",
+ apple_log_write, OS_LOG_TYPE_DEFAULT, OS_LOG_TYPE_ERROR);
+ if (result == NULL) {
+ goto error;
+ }
+
+ goto done;
+
+error:
+ _PyErr_Print(tstate);
+ status = _PyStatus_ERR("failed to initialize Apple log streams");
+
+done:
+ Py_XDECREF(result);
+ Py_XDECREF(apple_log_write);
+ Py_XDECREF(_apple_support);
+ return status;
+}
+
+#endif // __APPLE__
+
static void
_Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp,
diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h
index c8cdb93..584b050 100644
--- a/Python/stdlib_module_names.h
+++ b/Python/stdlib_module_names.h
@@ -6,6 +6,7 @@ static const char* _Py_stdlib_module_names[] = {
"_abc",
"_aix_support",
"_android_support",
+"_apple_support",
"_ast",
"_asyncio",
"_bisect",
diff --git a/iOS/README.rst b/iOS/README.rst
index e33455e..9cea98c 100644
--- a/iOS/README.rst
+++ b/iOS/README.rst
@@ -285,52 +285,42 @@ This will:
* Install the Python iOS framework into the copy of the testbed project; and
* Run the test suite on an "iPhone SE (3rd generation)" simulator.
-While the test suite is running, Xcode does not display any console output.
-After showing some Xcode build commands, the console output will print ``Testing
-started``, and then appear to stop. It will remain in this state until the test
-suite completes. On a 2022 M1 MacBook Pro, the test suite takes approximately 12
-minutes to run; a couple of extra minutes is required to boot and prepare the
-iOS simulator.
-
On success, the test suite will exit and report successful completion of the
-test suite. No output of the Python test suite will be displayed.
-
-On failure, the output of the Python test suite *will* be displayed. This will
-show the details of the tests that failed.
+test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 12
+minutes to run; a couple of extra minutes is required to compile the testbed
+project, and then boot and prepare the iOS simulator.
Debugging test failures
-----------------------
-The easiest way to diagnose a single test failure is to open the testbed project
-in Xcode and run the tests from there using the "Product > Test" menu item.
-
-To test in Xcode, you must ensure the testbed project has a copy of a compiled
-framework. If you've configured your build with the default install location of
-``iOS/Frameworks``, you can copy from that location into the test project. To
-test on an ARM64 simulator, run::
-
- $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/*
- $ cp -r iOS/Frameworks/arm64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator
+Running ``make test`` generates a standalone version of the ``iOS/testbed``
+project, and runs the full test suite. It does this using ``iOS/testbed``
+itself - the folder is an executable module that can be used to create and run
+a clone of the testbed project.
-To test on an x86-64 simulator, run::
+You can generate your own standalone testbed instance by running::
- $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/*
- $ cp -r iOS/Frameworks/x86_64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator
+ $ python iOS/testbed clone --framework iOS/Frameworks/arm64-iphonesimulator my-testbed
-To test on a physical device::
+This invocation assumes that ``iOS/Frameworks/arm64-iphonesimulator`` is the
+path to the iOS simulator framework for your platform (ARM64 in this case);
+``my-testbed`` is the name of the folder for the new testbed clone.
- $ rm -rf iOS/testbed/Python.xcframework/ios-arm64/*
- $ cp -r iOS/Frameworks/arm64-iphoneos/* iOS/testbed/Python.xcframework/ios-arm64
+You can then use the ``my-testbed`` folder to run the Python test suite,
+passing in any command line arguments you may require. For example, if you're
+trying to diagnose a failure in the ``os`` module, you might run::
-Alternatively, you can configure your build to install directly into the
-testbed project. For a simulator, use::
+ $ python my-testbed run -- test -W test_os
- --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator
+This is the equivalent of running ``python -m test -W test_os`` on a desktop
+Python build. Any arguments after the ``--`` will be passed to testbed as if
+they were arguments to ``python -m`` on a desktop machine.
-For a physical device, use::
+You can also open the testbed project in Xcode by running::
- --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64
+ $ open my-testbed/iOSTestbed.xcodeproj
+This will allow you to use the full Xcode suite of tools for debugging.
Testing on an iOS device
^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py
new file mode 100644
index 0000000..22570ee
--- /dev/null
+++ b/iOS/testbed/__main__.py
@@ -0,0 +1,365 @@
+import argparse
+import asyncio
+import json
+import plistlib
+import shutil
+import subprocess
+import sys
+from contextlib import asynccontextmanager
+from datetime import datetime
+from pathlib import Path
+
+
+DECODE_ARGS = ("UTF-8", "backslashreplace")
+
+
+# Work around a bug involving sys.exit and TaskGroups
+# (https://github.com/python/cpython/issues/101515).
+def exit(*args):
+ raise MySystemExit(*args)
+
+
+class MySystemExit(Exception):
+ pass
+
+
+# All subprocesses are executed through this context manager so that no matter
+# what happens, they can always be cancelled from another task, and they will
+# always be cleaned up on exit.
+@asynccontextmanager
+async def async_process(*args, **kwargs):
+ process = await asyncio.create_subprocess_exec(*args, **kwargs)
+ try:
+ yield process
+ finally:
+ if process.returncode is None:
+ # Allow a reasonably long time for Xcode to clean itself up,
+ # because we don't want stale emulators left behind.
+ timeout = 10
+ process.terminate()
+ try:
+ await asyncio.wait_for(process.wait(), timeout)
+ except TimeoutError:
+ print(
+ f"Command {args} did not terminate after {timeout} seconds "
+ f" - sending SIGKILL"
+ )
+ process.kill()
+
+ # Even after killing the process we must still wait for it,
+ # otherwise we'll get the warning "Exception ignored in __del__".
+ await asyncio.wait_for(process.wait(), timeout=1)
+
+
+async def async_check_output(*args, **kwargs):
+ async with async_process(
+ *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs
+ ) as process:
+ stdout, stderr = await process.communicate()
+ if process.returncode == 0:
+ return stdout.decode(*DECODE_ARGS)
+ else:
+ raise subprocess.CalledProcessError(
+ process.returncode,
+ args,
+ stdout.decode(*DECODE_ARGS),
+ stderr.decode(*DECODE_ARGS),
+ )
+
+
+# Return a list of UDIDs associated with booted simulators
+async def list_devices():
+ # List the testing simulators, in JSON format
+ raw_json = await async_check_output(
+ "xcrun", "simctl", "--set", "testing", "list", "-j"
+ )
+ json_data = json.loads(raw_json)
+
+ # Filter out the booted iOS simulators
+ return [
+ simulator["udid"]
+ for runtime, simulators in json_data["devices"].items()
+ for simulator in simulators
+ if runtime.split(".")[-1].startswith("iOS") and simulator["state"] == "Booted"
+ ]
+
+
+async def find_device(initial_devices):
+ while True:
+ new_devices = set(await list_devices()).difference(initial_devices)
+ if len(new_devices) == 0:
+ await asyncio.sleep(1)
+ elif len(new_devices) == 1:
+ udid = new_devices.pop()
+ print(f"{datetime.now():%Y-%m-%d %H:%M:%S}: New test simulator detected")
+ print(f"UDID: {udid}")
+ return udid
+ else:
+ exit(f"Found more than one new device: {new_devices}")
+
+
+async def log_stream_task(initial_devices):
+ # Wait up to 5 minutes for the build to complete and the simulator to boot.
+ udid = await asyncio.wait_for(find_device(initial_devices), 5 * 60)
+
+ # Stream the iOS device's logs, filtering out messages that come from the
+ # XCTest test suite (catching NSLog messages from the test method), or
+ # Python itself (catching stdout/stderr content routed to the system log
+ # with config->use_system_logger).
+ args = [
+ "xcrun",
+ "simctl",
+ "--set",
+ "testing",
+ "spawn",
+ udid,
+ "log",
+ "stream",
+ "--style",
+ "compact",
+ "--predicate",
+ (
+ 'senderImagePath ENDSWITH "/iOSTestbedTests.xctest/iOSTestbedTests"'
+ ' OR senderImagePath ENDSWITH "/Python.framework/Python"'
+ ),
+ ]
+
+ async with async_process(
+ *args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ ) as process:
+ suppress_dupes = False
+ while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
+ # The iOS log streamer can sometimes lag; when it does, it outputs
+ # a warning about messages being dropped... often multiple times.
+ # Only print the first of these duplicated warnings.
+ if line.startswith("=== Messages dropped "):
+ if not suppress_dupes:
+ suppress_dupes = True
+ sys.stdout.write(line)
+ else:
+ suppress_dupes = False
+ sys.stdout.write(line)
+
+
+async def xcode_test(location, simulator):
+ # Run the test suite on the named simulator
+ args = [
+ "xcodebuild",
+ "test",
+ "-project",
+ str(location / "iOSTestbed.xcodeproj"),
+ "-scheme",
+ "iOSTestbed",
+ "-destination",
+ f"platform=iOS Simulator,name={simulator}",
+ "-resultBundlePath",
+ str(location / f"{datetime.now():%Y%m%d-%H%M%S}.xcresult"),
+ "-derivedDataPath",
+ str(location / "DerivedData"),
+ ]
+ async with async_process(
+ *args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ ) as process:
+ while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
+ sys.stdout.write(line)
+
+ status = await asyncio.wait_for(process.wait(), timeout=1)
+ exit(status)
+
+
+def clone_testbed(
+ source: Path,
+ target: Path,
+ framework: Path,
+ apps: list[Path],
+) -> None:
+ if target.exists():
+ print(f"{target} already exists; aborting without creating project.")
+ sys.exit(10)
+
+ if framework is None:
+ if not (source / "Python.xcframework/ios-arm64_x86_64-simulator/bin").is_dir():
+ print(
+ f"The testbed being cloned ({source}) does not contain "
+ f"a simulator framework. Re-run with --framework"
+ )
+ sys.exit(11)
+ else:
+ if not framework.is_dir():
+ print(f"{framework} does not exist.")
+ sys.exit(12)
+ elif not (
+ framework.suffix == ".xcframework"
+ or (framework / "Python.framework").is_dir()
+ ):
+ print(
+ f"{framework} is not an XCframework, "
+ f"or a simulator slice of a framework build."
+ )
+ sys.exit(13)
+
+ print("Cloning testbed project...")
+ shutil.copytree(source, target)
+
+ if framework is not None:
+ if framework.suffix == ".xcframework":
+ print("Installing XCFramework...")
+ xc_framework_path = target / "Python.xcframework"
+ shutil.rmtree(xc_framework_path)
+ shutil.copytree(framework, xc_framework_path)
+ else:
+ print("Installing simulator Framework...")
+ sim_framework_path = (
+ target / "Python.xcframework" / "ios-arm64_x86_64-simulator"
+ )
+ shutil.rmtree(sim_framework_path)
+ shutil.copytree(framework, sim_framework_path)
+ else:
+ print("Using pre-existing iOS framework.")
+
+ for app_src in apps:
+ print(f"Installing app {app_src.name!r}...")
+ app_target = target / f"iOSTestbed/app/{app_src.name}"
+ if app_target.is_dir():
+ shutil.rmtree(app_target)
+ shutil.copytree(app_src, app_target)
+
+ print(f"Testbed project created in {target}")
+
+
+def update_plist(testbed_path, args):
+ # Add the test runner arguments to the testbed's Info.plist file.
+ info_plist = testbed_path / "iOSTestbed" / "iOSTestbed-Info.plist"
+ with info_plist.open("rb") as f:
+ info = plistlib.load(f)
+
+ info["TestArgs"] = args
+
+ with info_plist.open("wb") as f:
+ plistlib.dump(info, f)
+
+
+async def run_testbed(simulator: str, args: list[str]):
+ location = Path(__file__).parent
+ print("Updating plist...")
+ update_plist(location, args)
+
+ # Get the list of devices that are booted at the start of the test run.
+ # The simulator started by the test suite will be detected as the new
+ # entry that appears on the device list.
+ initial_devices = await list_devices()
+
+ try:
+ async with asyncio.TaskGroup() as tg:
+ tg.create_task(log_stream_task(initial_devices))
+ tg.create_task(xcode_test(location, simulator))
+ except* MySystemExit as e:
+ raise SystemExit(*e.exceptions[0].args) from None
+ except* subprocess.CalledProcessError as e:
+ # Extract it from the ExceptionGroup so it can be handled by `main`.
+ raise e.exceptions[0]
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description=(
+ "Manages the process of testing a Python project in the iOS simulator."
+ ),
+ )
+
+ subcommands = parser.add_subparsers(dest="subcommand")
+
+ clone = subcommands.add_parser(
+ "clone",
+ description=(
+ "Clone the testbed project, copying in an iOS Python framework and"
+ "any specified application code."
+ ),
+ help="Clone a testbed project to a new location.",
+ )
+ clone.add_argument(
+ "--framework",
+ help=(
+ "The location of the XCFramework (or simulator-only slice of an "
+ "XCFramework) to use when running the testbed"
+ ),
+ )
+ clone.add_argument(
+ "--app",
+ dest="apps",
+ action="append",
+ default=[],
+ help="The location of any code to include in the testbed project",
+ )
+ clone.add_argument(
+ "location",
+ help="The path where the testbed will be cloned.",
+ )
+
+ run = subcommands.add_parser(
+ "run",
+ usage="%(prog)s [-h] [--simulator SIMULATOR] -- <test arg> [<test arg> ...]",
+ description=(
+ "Run a testbed project. The arguments provided after `--` will be "
+ "passed to the running iOS process as if they were arguments to "
+ "`python -m`."
+ ),
+ help="Run a testbed project",
+ )
+ run.add_argument(
+ "--simulator",
+ default="iPhone SE (3rd Generation)",
+ help="The name of the simulator to use (default: 'iPhone SE (3rd Generation)')",
+ )
+
+ try:
+ pos = sys.argv.index("--")
+ testbed_args = sys.argv[1:pos]
+ test_args = sys.argv[pos + 1 :]
+ except ValueError:
+ testbed_args = sys.argv[1:]
+ test_args = []
+
+ context = parser.parse_args(testbed_args)
+
+ if context.subcommand == "clone":
+ clone_testbed(
+ source=Path(__file__).parent,
+ target=Path(context.location),
+ framework=Path(context.framework) if context.framework else None,
+ apps=[Path(app) for app in context.apps],
+ )
+ elif context.subcommand == "run":
+ if test_args:
+ if not (
+ Path(__file__).parent / "Python.xcframework/ios-arm64_x86_64-simulator/bin"
+ ).is_dir():
+ print(
+ f"Testbed does not contain a compiled iOS framework. Use "
+ f"`python {sys.argv[0]} clone ...` to create a runnable "
+ f"clone of this testbed."
+ )
+ sys.exit(20)
+
+ asyncio.run(
+ run_testbed(
+ simulator=context.simulator,
+ args=test_args,
+ )
+ )
+ else:
+ print(f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)")
+ print()
+ parser.print_help(sys.stderr)
+ sys.exit(21)
+ else:
+ parser.print_help(sys.stderr)
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj
index 6819ac0..c7d6390 100644
--- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj
+++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj
@@ -263,6 +263,7 @@
runOnlyForDeploymentPostprocessing = 0;
shellPath = /bin/sh;
shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n echo \"Installing Python modules for iOS Simulator\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n echo \"Installing Python modules for iOS Device\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n";
+ showEnvVarsInLog = 0;
};
607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = {
isa = PBXShellScriptBuildPhase;
@@ -282,6 +283,7 @@
runOnlyForDeploymentPostprocessing = 0;
shellPath = /bin/sh;
shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\necho \"Install app package extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app_packages\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app_packages/ \"$FULL_EXT\"\ndone\necho \"Install app extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n";
+ showEnvVarsInLog = 0;
};
/* End PBXShellScriptBuildPhase section */
diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m
index db00d43..ac78456 100644
--- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m
+++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m
@@ -50,6 +50,8 @@
// Enforce UTF-8 encoding for stderr, stdout, file-system encoding and locale.
// See https://docs.python.org/3/library/os.html#python-utf-8-mode.
preconfig.utf8_mode = 1;
+ // Use the system logger for stdout/err
+ config.use_system_logger = 1;
// Don't buffer stdio. We want output to appears in the log immediately
config.buffered_stdio = 0;
// Don't write bytecode; we can't modify the app bundle