bpo-40943: PY_SSIZE_T_CLEAN required for '#' formats (GH-20784)

The PY_SSIZE_T_CLEAN macro must now be defined to use
PyArg_ParseTuple() and Py_BuildValue() "#" formats: "es#", "et#",
"s#", "u#", "y#", "z#", "U#" and "Z#". See the PEP 353.

Update _testcapi.test_buildvalue_issue38913().

2 files changed, 45 insertions, 77 deletions

diff --git a/Python/getargs.c b/Python/getargs.c
index cf0cc07..aaf687a 100644
--- a/Python/getargs.c
+++ b/Python/getargs.c
@@ -656,27 +656,12 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
               char *msgbuf, size_t bufsize, freelist_t *freelist)
 {
     /* For # codes */
-#define FETCH_SIZE      int *q=NULL;Py_ssize_t *q2=NULL;\
-    if (flags & FLAG_SIZE_T) q2=va_arg(*p_va, Py_ssize_t*); \
-    else { \
-        if (PyErr_WarnEx(PyExc_DeprecationWarning, \
-                    "PY_SSIZE_T_CLEAN will be required for '#' formats", 1)) { \
-            return NULL; \
-        } \
-        q=va_arg(*p_va, int*); \
-    }
-#define STORE_SIZE(s)   \
-    if (flags & FLAG_SIZE_T) \
-        *q2=s; \
-    else { \
-        if (INT_MAX < s) { \
-            PyErr_SetString(PyExc_OverflowError, \
-                "size does not fit in an int"); \
-            return converterr("", arg, msgbuf, bufsize); \
-        } \
-        *q = (int)s; \
-    }
-#define BUFFER_LEN      ((flags & FLAG_SIZE_T) ? *q2:*q)
+#define REQUIRE_PY_SSIZE_T_CLEAN \
+    if (!(flags & FLAG_SIZE_T)) { \
+        PyErr_SetString(PyExc_SystemError, \
+                        "PY_SSIZE_T_CLEAN macro must be defined for '#' formats"); \
+        return NULL; \
+    }
 #define RETURN_ERR_OCCURRED return msgbuf
 
     const char *format = *p_format;
@@ -931,8 +916,9 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
         if (count < 0)
             return converterr(buf, arg, msgbuf, bufsize);
         if (*format == '#') {
-            FETCH_SIZE;
-            STORE_SIZE(count);
+            REQUIRE_PY_SSIZE_T_CLEAN;
+            Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*);
+            *psize = count;
             format++;
         } else {
             if (strlen(*p) != (size_t)count) {
@@ -974,11 +960,12 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
         } else if (*format == '#') { /* a string or read-only bytes-like object */
             /* "s#" or "z#" */
             const void **p = (const void **)va_arg(*p_va, const char **);
-            FETCH_SIZE;
+            REQUIRE_PY_SSIZE_T_CLEAN;
+            Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*);
 
             if (c == 'z' && arg == Py_None) {
                 *p = NULL;
-                STORE_SIZE(0);
+                *psize = 0;
             }
             else if (PyUnicode_Check(arg)) {
                 Py_ssize_t len;
@@ -987,7 +974,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
                     return converterr(CONV_UNICODE,
                                       arg, msgbuf, bufsize);
                 *p = sarg;
-                STORE_SIZE(len);
+                *psize = len;
             }
             else { /* read-only bytes-like object */
                 /* XXX Really? */
@@ -995,7 +982,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
                 Py_ssize_t count = convertbuffer(arg, p, &buf);
                 if (count < 0)
                     return converterr(buf, arg, msgbuf, bufsize);
-                STORE_SIZE(count);
+                *psize = count;
             }
             format++;
         } else {
@@ -1034,18 +1021,19 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS
 
         if (*format == '#') {
             /* "u#" or "Z#" */
-            FETCH_SIZE;
+            REQUIRE_PY_SSIZE_T_CLEAN;
+            Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*);
 
             if (c == 'Z' && arg == Py_None) {
                 *p = NULL;
-                STORE_SIZE(0);
+                *psize = 0;
             }
             else if (PyUnicode_Check(arg)) {
                 Py_ssize_t len;
                 *p = PyUnicode_AsUnicodeAndSize(arg, &len);
                 if (*p == NULL)
                     RETURN_ERR_OCCURRED;
-                STORE_SIZE(len);
+                *psize = len;
             }
             else
                 return converterr(c == 'Z' ? "str or None" : "str",
@@ -1160,22 +1148,11 @@ _Py_COMP_DIAG_POP
                trailing 0-byte
 
             */
-            int *q = NULL; Py_ssize_t *q2 = NULL;
-            if (flags & FLAG_SIZE_T) {
-                q2 = va_arg(*p_va, Py_ssize_t*);
-            }
-            else {
-                if (PyErr_WarnEx(PyExc_DeprecationWarning,
-                            "PY_SSIZE_T_CLEAN will be required for '#' formats", 1))
-                {
-                    Py_DECREF(s);
-                    return NULL;
-                }
-                q = va_arg(*p_va, int*);
-            }
+            REQUIRE_PY_SSIZE_T_CLEAN;
+            Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*);
 
             format++;
-            if (q == NULL && q2 == NULL) {
+            if (psize == NULL) {
                 Py_DECREF(s);
                 return converterr(
                     "(buffer_len is NULL)",
@@ -1195,30 +1172,20 @@ _Py_COMP_DIAG_POP
                         arg, msgbuf, bufsize);
                 }
             } else {
-                if (size + 1 > BUFFER_LEN) {
+                if (size + 1 > *psize) {
                     Py_DECREF(s);
                     PyErr_Format(PyExc_ValueError,
                                  "encoded string too long "
                                  "(%zd, maximum length %zd)",
-                                 (Py_ssize_t)size, (Py_ssize_t)(BUFFER_LEN-1));
+                                 (Py_ssize_t)size, (Py_ssize_t)(*psize - 1));
                     RETURN_ERR_OCCURRED;
                 }
             }
             memcpy(*buffer, ptr, size+1);
 
-            if (flags & FLAG_SIZE_T) {
-                *q2 = size;
-            }
-            else {
-                if (INT_MAX < size) {
-                    Py_DECREF(s);
-                    PyErr_SetString(PyExc_OverflowError,
-                                    "size does not fit in an int");
-                    return converterr("", arg, msgbuf, bufsize);
-                }
-                *q = (int)size;
-            }
-        } else {
+            *psize = size;
+        }
+        else {
             /* Using a 0-terminated buffer:
 
                - the encoded string has to be 0-terminated
@@ -1356,9 +1323,7 @@ _Py_COMP_DIAG_POP
     *p_format = format;
     return NULL;
 
-#undef FETCH_SIZE
-#undef STORE_SIZE
-#undef BUFFER_LEN
+#undef REQUIRE_PY_SSIZE_T_CLEAN
 #undef RETURN_ERR_OCCURRED
 }
 
diff --git a/Python/modsupport.c b/Python/modsupport.c
index 845bdcb..2637039 100644
--- a/Python/modsupport.c
+++ b/Python/modsupport.c
@@ -283,6 +283,13 @@ do_mktuple(const char **p_format, va_list *p_va, char endchar, Py_ssize_t n, int
 static PyObject *
 do_mkvalue(const char **p_format, va_list *p_va, int flags)
 {
+#define ERROR_NEED_PY_SSIZE_T_CLEAN \
+    { \
+        PyErr_SetString(PyExc_SystemError, \
+                        "PY_SSIZE_T_CLEAN macro must be defined for '#' formats"); \
+        return NULL; \
+    }
+
     for (;;) {
         switch (*(*p_format)++) {
         case '(':
@@ -341,14 +348,12 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags)
             Py_ssize_t n;
             if (**p_format == '#') {
                 ++*p_format;
-                if (flags & FLAG_SIZE_T)
+                if (flags & FLAG_SIZE_T) {
                     n = va_arg(*p_va, Py_ssize_t);
+                }
                 else {
                     n = va_arg(*p_va, int);
-                    if (PyErr_WarnEx(PyExc_DeprecationWarning,
-                                "PY_SSIZE_T_CLEAN will be required for '#' formats", 1)) {
-                        return NULL;
-                    }
+                    ERROR_NEED_PY_SSIZE_T_CLEAN;
                 }
             }
             else
@@ -394,14 +399,12 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags)
             Py_ssize_t n;
             if (**p_format == '#') {
                 ++*p_format;
-                if (flags & FLAG_SIZE_T)
+                if (flags & FLAG_SIZE_T) {
                     n = va_arg(*p_va, Py_ssize_t);
+                }
                 else {
                     n = va_arg(*p_va, int);
-                    if (PyErr_WarnEx(PyExc_DeprecationWarning,
-                                "PY_SSIZE_T_CLEAN will be required for '#' formats", 1)) {
-                        return NULL;
-                    }
+                    ERROR_NEED_PY_SSIZE_T_CLEAN;
                 }
             }
             else
@@ -432,14 +435,12 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags)
             Py_ssize_t n;
             if (**p_format == '#') {
                 ++*p_format;
-                if (flags & FLAG_SIZE_T)
+                if (flags & FLAG_SIZE_T) {
                     n = va_arg(*p_va, Py_ssize_t);
+                }
                 else {
                     n = va_arg(*p_va, int);
-                    if (PyErr_WarnEx(PyExc_DeprecationWarning,
-                                "PY_SSIZE_T_CLEAN will be required for '#' formats", 1)) {
-                        return NULL;
-                    }
+                    ERROR_NEED_PY_SSIZE_T_CLEAN;
                 }
             }
             else
@@ -507,6 +508,8 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags)
 
         }
     }
+
+#undef ERROR_NEED_PY_SSIZE_T_CLEAN
 }