Closes #22258: Fix the the internal function set_inheritable() on Illumos.

This platform exposes the function ioctl(FIOCLEX), but calling it fails with
errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable() now falls
back to the slower fcntl() (F_GETFD and then F_SETFD).

2 files changed, 39 insertions, 14 deletions

diff --git a/Misc/NEWS b/Misc/NEWS
index 5111b8a..9d35acf 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,11 @@ Release date: XXXX-XX-XX
 Core and Builtins
 -----------------
 
+- Issue #22258: Fix the the internal function set_inheritable() on Illumos.
+  This platform exposes the function ``ioctl(FIOCLEX)``, but calling it fails
+  with errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable()
+  now falls back to the slower ``fcntl()`` (``F_GETFD`` and then ``F_SETFD``).
+
 - Issue #21669: With the aid of heuristics in SyntaxError.__init__, the
   parser now attempts to generate more meaningful (or at least more search
   engine friendly) error messages when "exec" and "print" are used as
diff --git a/Python/fileutils.c b/Python/fileutils.c
index a55064f..f5000e5 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -622,10 +622,12 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
 #ifdef MS_WINDOWS
     HANDLE handle;
     DWORD flags;
-#elif defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX)
+#else
+#if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX)
+    static int ioctl_works = -1;
     int request;
     int err;
-#elif defined(HAVE_FCNTL_H)
+#endif
     int flags;
     int res;
 #endif
@@ -671,20 +673,38 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
     }
     return 0;
 
-#elif defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX)
-    if (inheritable)
-        request = FIONCLEX;
-    else
-        request = FIOCLEX;
-    err = ioctl(fd, request, NULL);
-    if (err) {
-        if (raise)
-            PyErr_SetFromErrno(PyExc_OSError);
-        return -1;
+#else
+
+#if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX)
+    if (ioctl_works != 0) {
+        /* fast-path: ioctl() only requires one syscall */
+        if (inheritable)
+            request = FIONCLEX;
+        else
+            request = FIOCLEX;
+        err = ioctl(fd, request, NULL);
+        if (!err) {
+            ioctl_works = 1;
+            return 0;
+        }
+
+        if (errno != ENOTTY) {
+            if (raise)
+                PyErr_SetFromErrno(PyExc_OSError);
+            return -1;
+        }
+        else {
+            /* Issue #22258: Here, ENOTTY means "Inappropriate ioctl for
+               device". The ioctl is declared but not supported by the kernel.
+               Remember that ioctl() doesn't work. It is the case on
+               Illumos-based OS for example. */
+            ioctl_works = 0;
+        }
+        /* fallback to fcntl() if ioctl() does not work */
     }
-    return 0;
+#endif
 
-#else
+    /* slow-path: fcntl() requires two syscalls */
     flags = fcntl(fd, F_GETFD);
     if (flags < 0) {
         if (raise)