gh-116622: Redirect stdout and stderr to system log when embedded in an Android app (#118063)

7 files changed, 511 insertions, 0 deletions

diff --git a/Lib/_android_support.py b/Lib/_android_support.py
new file mode 100644
index 0000000..590e85e
--- /dev/null
+++ b/Lib/_android_support.py
@@ -0,0 +1,94 @@
+import io
+import sys
+
+
+# The maximum length of a log message in bytes, including the level marker and
+# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in
+# platform/system/logging/liblog/include/log/log.h. As of API level 30, messages
+# longer than this will be be truncated by logcat. This limit has already been
+# reduced at least once in the history of Android (from 4076 to 4068 between API
+# level 23 and 26), so leave some headroom.
+MAX_BYTES_PER_WRITE = 4000
+
+# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this
+# size ensures that TextIOWrapper can always avoid exceeding MAX_BYTES_PER_WRITE.
+# However, if the actual number of bytes per character is smaller than that,
+# then TextIOWrapper may still join multiple consecutive text writes into binary
+# writes containing a larger number of characters.
+MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4
+
+
+# When embedded in an app on current versions of Android, there's no easy way to
+# monitor the C-level stdout and stderr. The testbed comes with a .c file to
+# redirect them to the system log using a pipe, but that wouldn't be convenient
+# or appropriate for all apps. So we redirect at the Python level instead.
+def init_streams(android_log_write, stdout_prio, stderr_prio):
+    if sys.executable:
+        return  # Not embedded in an app.
+
+    sys.stdout = TextLogStream(
+        android_log_write, stdout_prio, "python.stdout", errors=sys.stdout.errors)
+    sys.stderr = TextLogStream(
+        android_log_write, stderr_prio, "python.stderr", errors=sys.stderr.errors)
+
+
+class TextLogStream(io.TextIOWrapper):
+    def __init__(self, android_log_write, prio, tag, **kwargs):
+        kwargs.setdefault("encoding", "UTF-8")
+        kwargs.setdefault("line_buffering", True)
+        super().__init__(BinaryLogStream(android_log_write, prio, tag), **kwargs)
+        self._CHUNK_SIZE = MAX_BYTES_PER_WRITE
+
+    def __repr__(self):
+        return f"<TextLogStream {self.buffer.tag!r}>"
+
+    def write(self, s):
+        if not isinstance(s, str):
+            raise TypeError(
+                f"write() argument must be str, not {type(s).__name__}")
+
+        # In case `s` is a str subclass that writes itself to stdout or stderr
+        # when we call its methods, convert it to an actual str.
+        s = str.__str__(s)
+
+        # We want to emit one log message per line wherever possible, so split
+        # the string before sending it to the superclass. Note that
+        # "".splitlines() == [], so nothing will be logged for an empty string.
+        for line in s.splitlines(keepends=True):
+            while line:
+                super().write(line[:MAX_CHARS_PER_WRITE])
+                line = line[MAX_CHARS_PER_WRITE:]
+
+        return len(s)
+
+
+class BinaryLogStream(io.RawIOBase):
+    def __init__(self, android_log_write, prio, tag):
+        self.android_log_write = android_log_write
+        self.prio = prio
+        self.tag = tag
+
+    def __repr__(self):
+        return f"<BinaryLogStream {self.tag!r}>"
+
+    def writable(self):
+        return True
+
+    def write(self, b):
+        if type(b) is not bytes:
+            try:
+                b = bytes(memoryview(b))
+            except TypeError:
+                raise TypeError(
+                    f"write() argument must be bytes-like, not {type(b).__name__}"
+                ) from None
+
+        # Writing an empty string to the stream should have no effect.
+        if b:
+            # Encode null bytes using "modified UTF-8" to avoid truncating the
+            # message. This should not affect the return value, as the caller
+            # may be expecting it to match the length of the input.
+            self.android_log_write(self.prio, self.tag,
+                                   b.replace(b"\x00", b"\xc0\x80"))
+
+        return len(b)
diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py
new file mode 100644
index 0000000..115882a
--- /dev/null
+++ b/Lib/test/test_android.py
@@ -0,0 +1,332 @@
+import platform
+import queue
+import re
+import subprocess
+import sys
+import unittest
+from array import array
+from contextlib import contextmanager
+from threading import Thread
+from test.support import LOOPBACK_TIMEOUT
+from time import time
+
+
+if sys.platform != "android":
+    raise unittest.SkipTest("Android-specific")
+
+api_level = platform.android_ver().api_level
+
+
+# Test redirection of stdout and stderr to the Android log.
+@unittest.skipIf(
+    api_level < 23 and platform.machine() == "aarch64",
+    "SELinux blocks reading logs on older ARM64 emulators"
+)
+class TestAndroidOutput(unittest.TestCase):
+    maxDiff = None
+
+    def setUp(self):
+        self.logcat_process = subprocess.Popen(
+            ["logcat", "-v", "tag"], stdout=subprocess.PIPE,
+            errors="backslashreplace"
+        )
+        self.logcat_queue = queue.Queue()
+
+        def logcat_thread():
+            for line in self.logcat_process.stdout:
+                self.logcat_queue.put(line.rstrip("\n"))
+            self.logcat_process.stdout.close()
+        Thread(target=logcat_thread).start()
+
+        from ctypes import CDLL, c_char_p, c_int
+        android_log_write = getattr(CDLL("liblog.so"), "__android_log_write")
+        android_log_write.argtypes = (c_int, c_char_p, c_char_p)
+        ANDROID_LOG_INFO = 4
+
+        # Separate tests using a marker line with a different tag.
+        tag, message = "python.test", f"{self.id()} {time()}"
+        android_log_write(
+            ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8"))
+        self.assert_log("I", tag, message, skip=True, timeout=5)
+
+    def assert_logs(self, level, tag, expected, **kwargs):
+        for line in expected:
+            self.assert_log(level, tag, line, **kwargs)
+
+    def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5):
+        deadline = time() + timeout
+        while True:
+            try:
+                line = self.logcat_queue.get(timeout=(deadline - time()))
+            except queue.Empty:
+                self.fail(f"line not found: {expected!r}")
+            if match := re.fullmatch(fr"(.)/{tag}: (.*)", line):
+                try:
+                    self.assertEqual(level, match[1])
+                    self.assertEqual(expected, match[2])
+                    break
+                except AssertionError:
+                    if not skip:
+                        raise
+
+    def tearDown(self):
+        self.logcat_process.terminate()
+        self.logcat_process.wait(LOOPBACK_TIMEOUT)
+
+    @contextmanager
+    def unbuffered(self, stream):
+        stream.reconfigure(write_through=True)
+        try:
+            yield
+        finally:
+            stream.reconfigure(write_through=False)
+
+    def test_str(self):
+        for stream_name, level in [("stdout", "I"), ("stderr", "W")]:
+            with self.subTest(stream=stream_name):
+                stream = getattr(sys, stream_name)
+                tag = f"python.{stream_name}"
+                self.assertEqual(f"<TextLogStream '{tag}'>", repr(stream))
+
+                self.assertTrue(stream.writable())
+                self.assertFalse(stream.readable())
+                self.assertEqual("UTF-8", stream.encoding)
+                self.assertTrue(stream.line_buffering)
+                self.assertFalse(stream.write_through)
+
+                # stderr is backslashreplace by default; stdout is configured
+                # that way by libregrtest.main.
+                self.assertEqual("backslashreplace", stream.errors)
+
+                def write(s, lines=None, *, write_len=None):
+                    if write_len is None:
+                        write_len = len(s)
+                    self.assertEqual(write_len, stream.write(s))
+                    if lines is None:
+                        lines = [s]
+                    self.assert_logs(level, tag, lines)
+
+                # Single-line messages,
+                with self.unbuffered(stream):
+                    write("", [])
+
+                    write("a")
+                    write("Hello")
+                    write("Hello world")
+                    write(" ")
+                    write("  ")
+
+                    # Non-ASCII text
+                    write("ol\u00e9")  # Spanish
+                    write("\u4e2d\u6587")  # Chinese
+
+                    # Non-BMP emoji
+                    write("\U0001f600")
+
+                    # Non-encodable surrogates
+                    write("\ud800\udc00", [r"\ud800\udc00"])
+
+                    # Code used by surrogateescape (which isn't enabled here)
+                    write("\udc80", [r"\udc80"])
+
+                    # Null characters are logged using "modified UTF-8".
+                    write("\u0000", [r"\xc0\x80"])
+                    write("a\u0000", [r"a\xc0\x80"])
+                    write("\u0000b", [r"\xc0\x80b"])
+                    write("a\u0000b", [r"a\xc0\x80b"])
+
+                # Multi-line messages. Avoid identical consecutive lines, as
+                # they may activate "chatty" filtering and break the tests.
+                write("\nx", [""])
+                write("\na\n", ["x", "a"])
+                write("\n", [""])
+                write("b\n", ["b"])
+                write("c\n\n", ["c", ""])
+                write("d\ne", ["d"])
+                write("xx", [])
+                write("f\n\ng", ["exxf", ""])
+                write("\n", ["g"])
+
+                with self.unbuffered(stream):
+                    write("\nx", ["", "x"])
+                    write("\na\n", ["", "a"])
+                    write("\n", [""])
+                    write("b\n", ["b"])
+                    write("c\n\n", ["c", ""])
+                    write("d\ne", ["d", "e"])
+                    write("xx", ["xx"])
+                    write("f\n\ng", ["f", "", "g"])
+                    write("\n", [""])
+
+                # "\r\n" should be translated into "\n".
+                write("hello\r\n", ["hello"])
+                write("hello\r\nworld\r\n", ["hello", "world"])
+                write("\r\n", [""])
+
+                # Non-standard line separators should be preserved.
+                write("before form feed\x0cafter form feed\n",
+                      ["before form feed\x0cafter form feed"])
+                write("before line separator\u2028after line separator\n",
+                      ["before line separator\u2028after line separator"])
+
+                # String subclasses are accepted, but they should be converted
+                # to a standard str without calling any of their methods.
+                class CustomStr(str):
+                    def splitlines(self, *args, **kwargs):
+                        raise AssertionError()
+
+                    def __len__(self):
+                        raise AssertionError()
+
+                    def __str__(self):
+                        raise AssertionError()
+
+                write(CustomStr("custom\n"), ["custom"], write_len=7)
+
+                # Non-string classes are not accepted.
+                for obj in [b"", b"hello", None, 42]:
+                    with self.subTest(obj=obj):
+                        with self.assertRaisesRegex(
+                            TypeError,
+                            fr"write\(\) argument must be str, not "
+                            fr"{type(obj).__name__}"
+                        ):
+                            stream.write(obj)
+
+                # Manual flushing is supported.
+                write("hello", [])
+                stream.flush()
+                self.assert_log(level, tag, "hello")
+                write("hello", [])
+                write("world", [])
+                stream.flush()
+                self.assert_log(level, tag, "helloworld")
+
+                # Long lines are split into blocks of 1000 characters
+                # (MAX_CHARS_PER_WRITE in _android_support.py), but
+                # TextIOWrapper should then join them back together as much as
+                # possible without exceeding 4000 UTF-8 bytes
+                # (MAX_BYTES_PER_WRITE).
+                #
+                # ASCII (1 byte per character)
+                write(("foobar" * 700) + "\n",
+                      [("foobar" * 666) + "foob",  # 4000 bytes
+                       "ar" + ("foobar" * 33)])  # 200 bytes
+
+                # "Full-width" digits 0-9 (3 bytes per character)
+                s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19"
+                write((s * 150) + "\n",
+                      [s * 100,  # 3000 bytes
+                       s * 50])  # 1500 bytes
+
+                s = "0123456789"
+                write(s * 200, [])
+                write(s * 150, [])
+                write(s * 51, [s * 350])  # 3500 bytes
+                write("\n", [s * 51])  # 510 bytes
+
+    def test_bytes(self):
+        for stream_name, level in [("stdout", "I"), ("stderr", "W")]:
+            with self.subTest(stream=stream_name):
+                stream = getattr(sys, stream_name).buffer
+                tag = f"python.{stream_name}"
+                self.assertEqual(f"<BinaryLogStream '{tag}'>", repr(stream))
+                self.assertTrue(stream.writable())
+                self.assertFalse(stream.readable())
+
+                def write(b, lines=None, *, write_len=None):
+                    if write_len is None:
+                        write_len = len(b)
+                    self.assertEqual(write_len, stream.write(b))
+                    if lines is None:
+                        lines = [b.decode()]
+                    self.assert_logs(level, tag, lines)
+
+                # Single-line messages,
+                write(b"", [])
+
+                write(b"a")
+                write(b"Hello")
+                write(b"Hello world")
+                write(b" ")
+                write(b"  ")
+
+                # Non-ASCII text
+                write(b"ol\xc3\xa9")  # Spanish
+                write(b"\xe4\xb8\xad\xe6\x96\x87")  # Chinese
+
+                # Non-BMP emoji
+                write(b"\xf0\x9f\x98\x80")
+
+                # Null bytes are logged using "modified UTF-8".
+                write(b"\x00", [r"\xc0\x80"])
+                write(b"a\x00", [r"a\xc0\x80"])
+                write(b"\x00b", [r"\xc0\x80b"])
+                write(b"a\x00b", [r"a\xc0\x80b"])
+
+                # Invalid UTF-8
+                write(b"\xff", [r"\xff"])
+                write(b"a\xff", [r"a\xff"])
+                write(b"\xffb", [r"\xffb"])
+                write(b"a\xffb", [r"a\xffb"])
+
+                # Log entries containing newlines are shown differently by
+                # `logcat -v tag`, `logcat -v long`, and Android Studio. We
+                # currently use `logcat -v tag`, which shows each line as if it
+                # was a separate log entry, but strips a single trailing
+                # newline.
+                #
+                # On newer versions of Android, all three of the above tools (or
+                # maybe Logcat itself) will also strip any number of leading
+                # newlines.
+                write(b"\nx", ["", "x"] if api_level < 30 else ["x"])
+                write(b"\na\n", ["", "a"] if api_level < 30 else ["a"])
+                write(b"\n", [""])
+                write(b"b\n", ["b"])
+                write(b"c\n\n", ["c", ""])
+                write(b"d\ne", ["d", "e"])
+                write(b"xx", ["xx"])
+                write(b"f\n\ng", ["f", "", "g"])
+                write(b"\n", [""])
+
+                # "\r\n" should be translated into "\n".
+                write(b"hello\r\n", ["hello"])
+                write(b"hello\r\nworld\r\n", ["hello", "world"])
+                write(b"\r\n", [""])
+
+                # Other bytes-like objects are accepted.
+                write(bytearray(b"bytearray"))
+
+                mv = memoryview(b"memoryview")
+                write(mv, ["memoryview"])  # Continuous
+                write(mv[::2], ["mmrve"])  # Discontinuous
+
+                write(
+                    # Android only supports little-endian architectures, so the
+                    # bytes representation is as follows:
+                    array("H", [
+                        0,      # 00 00
+                        1,      # 01 00
+                        65534,  # FE FF
+                        65535,  # FF FF
+                    ]),
+
+                    # After encoding null bytes with modified UTF-8, the only
+                    # valid UTF-8 sequence is \x01. All other bytes are handled
+                    # by backslashreplace.
+                    ["\\xc0\\x80\\xc0\\x80"
+                     "\x01\\xc0\\x80"
+                     "\\xfe\\xff"
+                     "\\xff\\xff"],
+                    write_len=8,
+                )
+
+                # Non-bytes-like classes are not accepted.
+                for obj in ["", "hello", None, 42]:
+                    with self.subTest(obj=obj):
+                        with self.assertRaisesRegex(
+                            TypeError,
+                            fr"write\(\) argument must be bytes-like, not "
+                            fr"{type(obj).__name__}"
+                        ):
+                            stream.write(obj)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst
new file mode 100644
index 0000000..04f8479
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst
@@ -0,0 +1 @@
+Redirect stdout and stderr to system log when embedded in an Android app.
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 0f3ca4a..a672d8c 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -71,6 +71,9 @@ static PyStatus add_main_module(PyInterpreterState *interp);
 static PyStatus init_import_site(void);
 static PyStatus init_set_builtins_open(void);
 static PyStatus init_sys_streams(PyThreadState *tstate);
+#ifdef __ANDROID__
+static PyStatus init_android_streams(PyThreadState *tstate);
+#endif
 static void wait_for_thread_shutdown(PyThreadState *tstate);
 static void call_ll_exitfuncs(_PyRuntimeState *runtime);
 
@@ -1223,6 +1226,13 @@ init_interp_main(PyThreadState *tstate)
         return status;
     }
 
+#ifdef __ANDROID__
+    status = init_android_streams(tstate);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+#endif
+
 #ifdef Py_DEBUG
     run_presite(tstate);
 #endif
@@ -2719,6 +2729,73 @@ done:
 }
 
 
+#ifdef __ANDROID__
+#include <android/log.h>
+
+static PyObject *
+android_log_write_impl(PyObject *self, PyObject *args)
+{
+    int prio = 0;
+    const char *tag = NULL;
+    const char *text = NULL;
+    if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) {
+        return NULL;
+    }
+
+    // Despite its name, this function is part of the public API
+    // (https://developer.android.com/ndk/reference/group/logging).
+    __android_log_write(prio, tag, text);
+    Py_RETURN_NONE;
+}
+
+
+static PyMethodDef android_log_write_method = {
+    "android_log_write", android_log_write_impl, METH_VARARGS
+};
+
+
+static PyStatus
+init_android_streams(PyThreadState *tstate)
+{
+    PyStatus status = _PyStatus_OK();
+    PyObject *_android_support = NULL;
+    PyObject *android_log_write = NULL;
+    PyObject *result = NULL;
+
+    _android_support = PyImport_ImportModule("_android_support");
+    if (_android_support == NULL) {
+        goto error;
+    }
+
+    android_log_write = PyCFunction_New(&android_log_write_method, NULL);
+    if (android_log_write == NULL) {
+        goto error;
+    }
+
+    // These log priorities match those used by Java's System.out and System.err.
+    result = PyObject_CallMethod(
+        _android_support, "init_streams", "Oii",
+        android_log_write, ANDROID_LOG_INFO, ANDROID_LOG_WARN);
+    if (result == NULL) {
+        goto error;
+    }
+
+    goto done;
+
+error:
+    _PyErr_Print(tstate);
+    status = _PyStatus_ERR("failed to initialize Android streams");
+
+done:
+    Py_XDECREF(result);
+    Py_XDECREF(android_log_write);
+    Py_XDECREF(_android_support);
+    return status;
+}
+
+#endif  // __ANDROID__
+
+
 static void
 _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp,
                               PyThreadState *tstate)
diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h
index 08a66f4..f44abf1 100644
--- a/Python/stdlib_module_names.h
+++ b/Python/stdlib_module_names.h
@@ -5,6 +5,7 @@ static const char* _Py_stdlib_module_names[] = {
 "__future__",
 "_abc",
 "_aix_support",
+"_android_support",
 "_ast",
 "_asyncio",
 "_bisect",
diff --git a/configure b/configure
index 571ab8c..e1d0f36 100755
--- a/configure
+++ b/configure
@@ -7103,6 +7103,9 @@ printf "%s\n" "$ANDROID_API_LEVEL" >&6; }
 printf "%s\n" "#define ANDROID_API_LEVEL $ANDROID_API_LEVEL" >>confdefs.h
 
 
+  # For __android_log_write() in Python/pylifecycle.c.
+  LIBS="$LIBS -llog"
+
   { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the Android arm ABI" >&5
 printf %s "checking for the Android arm ABI... " >&6; }
   { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $_arm_arch" >&5
diff --git a/configure.ac b/configure.ac
index a2d6b13..7681ea3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1192,6 +1192,9 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then
   AC_DEFINE_UNQUOTED([ANDROID_API_LEVEL], [$ANDROID_API_LEVEL],
                      [The Android API level.])
 
+  # For __android_log_write() in Python/pylifecycle.c.
+  LIBS="$LIBS -llog"
+
   AC_MSG_CHECKING([for the Android arm ABI])
   AC_MSG_RESULT([$_arm_arch])
   if test "$_arm_arch" = 7; then