gh-126554: ctypes: Correctly handle NULL dlsym values (GH-126555)

For dlsym(), a return value of NULL does not necessarily indicate
an error [1].

Therefore, to avoid using stale (or NULL) dlerror() values, we must:

 1. clear the previous error state by calling dlerror()
 2. call dlsym()
 3. call dlerror()

If the return value of dlerror() is not NULL, an error occured.

In ctypes we choose to treat a NULL return value from dlsym()
as a "not found" error. This is the same as the fallback
message we use on Windows, Cygwin or when getting/formatting
the error reason fails.

[1]: https://man7.org/linux/man-pages/man3/dlsym.3.html

Signed-off-by: Georgios Alexopoulos <grgalex42@gmail.com>
Signed-off-by: Georgios Alexopoulos <grgalex@ba.uoa.gr>
Co-authored-by: Peter Bierma <zintensitydev@gmail.com>
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
Co-authored-by: Petr Viktorin <encukou@gmail.com>

2 files changed, 95 insertions, 31 deletions

diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index eae69e4..34529bc 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -956,32 +956,48 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll,
         return NULL;
     }
 
+#undef USE_DLERROR
 #ifdef MS_WIN32
     Py_BEGIN_ALLOW_THREADS
     address = (void *)GetProcAddress(handle, name);
     Py_END_ALLOW_THREADS
-    if (!address) {
-        PyErr_Format(PyExc_ValueError,
-                     "symbol '%s' not found",
-                     name);
-        return NULL;
-    }
 #else
+    #ifdef __CYGWIN__
+        // dlerror() isn't very helpful on cygwin
+    #else
+        #define USE_DLERROR
+        /* dlerror() always returns the latest error.
+         *
+         * Clear the previous value before calling dlsym(),
+         * to ensure we can tell if our call resulted in an error.
+         */
+        (void)dlerror();
+    #endif
     address = (void *)dlsym(handle, name);
-    if (!address) {
-#ifdef __CYGWIN__
-/* dlerror() isn't very helpful on cygwin */
-        PyErr_Format(PyExc_ValueError,
-                     "symbol '%s' not found",
-                     name);
-#else
-        PyErr_SetString(PyExc_ValueError, dlerror());
 #endif
-        return NULL;
+
+    if (address) {
+        ctypes_state *st = get_module_state_by_def(Py_TYPE(type));
+        return PyCData_AtAddress(st, type, address);
     }
-#endif
-    ctypes_state *st = get_module_state_by_def(Py_TYPE(type));
-    return PyCData_AtAddress(st, type, address);
+
+    #ifdef USE_DLERROR
+    const char *dlerr = dlerror();
+    if (dlerr) {
+        PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape");
+        if (message) {
+            PyErr_SetObject(PyExc_ValueError, message);
+            Py_DECREF(message);
+            return NULL;
+        }
+        // Ignore errors from PyUnicode_DecodeLocale,
+        // fall back to the generic error below.
+        PyErr_Clear();
+    }
+    #endif
+#undef USE_DLERROR
+    PyErr_Format(PyExc_ValueError, "symbol '%s' not found", name);
+    return NULL;
 }
 
 /*[clinic input]
@@ -3759,6 +3775,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
+#undef USE_DLERROR
 #ifdef MS_WIN32
     address = FindAddress(handle, name, (PyObject *)type);
     if (!address) {
@@ -3774,20 +3791,41 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds)
         return NULL;
     }
 #else
+    #ifdef __CYGWIN__
+        //dlerror() isn't very helpful on cygwin */
+    #else
+        #define USE_DLERROR
+        /* dlerror() always returns the latest error.
+         *
+         * Clear the previous value before calling dlsym(),
+         * to ensure we can tell if our call resulted in an error.
+         */
+        (void)dlerror();
+    #endif
     address = (PPROC)dlsym(handle, name);
+
     if (!address) {
-#ifdef __CYGWIN__
-/* dlerror() isn't very helpful on cygwin */
-        PyErr_Format(PyExc_AttributeError,
-                     "function '%s' not found",
-                     name);
-#else
-        PyErr_SetString(PyExc_AttributeError, dlerror());
-#endif
+	#ifdef USE_DLERROR
+        const char *dlerr = dlerror();
+        if (dlerr) {
+            PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape");
+            if (message) {
+                PyErr_SetObject(PyExc_AttributeError, message);
+                Py_DECREF(ftuple);
+                Py_DECREF(message);
+                return NULL;
+            }
+            // Ignore errors from PyUnicode_DecodeLocale,
+            // fall back to the generic error below.
+            PyErr_Clear();
+        }
+	#endif
+        PyErr_Format(PyExc_AttributeError, "function '%s' not found", name);
         Py_DECREF(ftuple);
         return NULL;
     }
 #endif
+#undef USE_DLERROR
     ctypes_state *st = get_module_state_by_def(Py_TYPE(type));
     if (!_validate_paramflags(st, type, paramflags)) {
         Py_DECREF(ftuple);
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
index 5ac9cf1..218c3a9 100644
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -1623,13 +1623,39 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args)
     if (PySys_Audit("ctypes.dlsym/handle", "O", args) < 0) {
         return NULL;
     }
+#undef USE_DLERROR
+    #ifdef __CYGWIN__
+        // dlerror() isn't very helpful on cygwin
+    #else
+        #define USE_DLERROR
+        /* dlerror() always returns the latest error.
+         *
+         * Clear the previous value before calling dlsym(),
+         * to ensure we can tell if our call resulted in an error.
+         */
+        (void)dlerror();
+    #endif
     ptr = dlsym((void*)handle, name);
-    if (!ptr) {
-        PyErr_SetString(PyExc_OSError,
-                               dlerror());
-        return NULL;
+    if (ptr) {
+        return PyLong_FromVoidPtr(ptr);
+    }
+	#ifdef USE_DLERROR
+    const char *dlerr = dlerror();
+    if (dlerr) {
+        PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape");
+        if (message) {
+            PyErr_SetObject(PyExc_OSError, message);
+            Py_DECREF(message);
+            return NULL;
+        }
+        // Ignore errors from PyUnicode_DecodeLocale,
+        // fall back to the generic error below.
+        PyErr_Clear();
     }
-    return PyLong_FromVoidPtr(ptr);
+	#endif
+	#undef USE_DLERROR
+    PyErr_Format(PyExc_OSError, "symbol '%s' not found", name);
+    return NULL;
 }
 #endif