Issue #16182: Fix readline begidx, endidx, and use locale encoding

Based on patch by Serhiy Storchaka.

1 files changed, 94 insertions, 38 deletions

diff --git a/Modules/readline.c b/Modules/readline.c
index 8c00dec..91f7cca 100644
--- a/Modules/readline.c
+++ b/Modules/readline.c
@@ -128,20 +128,40 @@ static PyModuleDef readlinemodule;
 #define readlinestate_global ((readlinestate *)PyModule_GetState(PyState_FindModule(&readlinemodule)))
 
 
+/* Convert to/from multibyte C strings */
+
+static PyObject *
+encode(PyObject *b)
+{
+    return PyUnicode_EncodeLocale(b, "surrogateescape");
+}
+
+static PyObject *
+decode(const char *s)
+{
+    return PyUnicode_DecodeLocale(s, "surrogateescape");
+}
+
+
 /* Exported function to send one line to readline's init file parser */
 
 static PyObject *
-parse_and_bind(PyObject *self, PyObject *args)
+parse_and_bind(PyObject *self, PyObject *string)
 {
-    char *s, *copy;
-    if (!PyArg_ParseTuple(args, "s:parse_and_bind", &s))
+    char *copy;
+    PyObject *encoded = encode(string);
+    if (encoded == NULL) {
         return NULL;
+    }
     /* Make a copy -- rl_parse_and_bind() modifies its argument */
     /* Bernard Herzog */
-    copy = PyMem_Malloc(1 + strlen(s));
-    if (copy == NULL)
+    copy = PyMem_Malloc(1 + PyBytes_GET_SIZE(encoded));
+    if (copy == NULL) {
+        Py_DECREF(encoded);
         return PyErr_NoMemory();
-    strcpy(copy, s);
+    }
+    strcpy(copy, PyBytes_AS_STRING(encoded));
+    Py_DECREF(encoded);
     rl_parse_and_bind(copy);
     PyMem_Free(copy); /* Free the copy */
     Py_RETURN_NONE;
@@ -441,17 +461,18 @@ get the ending index of the completion scope");
 /* Set the tab-completion word-delimiters that readline uses */
 
 static PyObject *
-set_completer_delims(PyObject *self, PyObject *args)
+set_completer_delims(PyObject *self, PyObject *string)
 {
     char *break_chars;
-
-    if (!PyArg_ParseTuple(args, "s:set_completer_delims", &break_chars)) {
+    PyObject *encoded = encode(string);
+    if (encoded == NULL) {
         return NULL;
     }
     /* Keep a reference to the allocated memory in the module state in case
        some other module modifies rl_completer_word_break_characters
        (see issue #17289). */
-    break_chars = strdup(break_chars);
+    break_chars = strdup(PyBytes_AS_STRING(encoded));
+    Py_DECREF(encoded);
     if (break_chars) {
         free(completer_word_break_characters);
         completer_word_break_characters = break_chars;
@@ -531,10 +552,11 @@ static PyObject *
 py_replace_history(PyObject *self, PyObject *args)
 {
     int entry_number;
-    char *line;
+    PyObject *line;
+    PyObject *encoded;
     HIST_ENTRY *old_entry;
 
-    if (!PyArg_ParseTuple(args, "is:replace_history_item", &entry_number,
+    if (!PyArg_ParseTuple(args, "iU:replace_history_item", &entry_number,
                           &line)) {
         return NULL;
     }
@@ -543,7 +565,12 @@ py_replace_history(PyObject *self, PyObject *args)
                         "History index cannot be negative");
         return NULL;
     }
-    old_entry = replace_history_entry(entry_number, line, (void *)NULL);
+    encoded = encode(line);
+    if (encoded == NULL) {
+        return NULL;
+    }
+    old_entry = replace_history_entry(entry_number, PyBytes_AS_STRING(encoded), (void *)NULL);
+    Py_DECREF(encoded);
     if (!old_entry) {
         PyErr_Format(PyExc_ValueError,
                      "No history item at position %d",
@@ -562,14 +589,14 @@ replaces history item given by its position with contents of line");
 /* Add a line to the history buffer */
 
 static PyObject *
-py_add_history(PyObject *self, PyObject *args)
+py_add_history(PyObject *self, PyObject *string)
 {
-    char *line;
-
-    if(!PyArg_ParseTuple(args, "s:add_history", &line)) {
+    PyObject *encoded = encode(string);
+    if (encoded == NULL) {
         return NULL;
     }
-    add_history(line);
+    add_history(PyBytes_AS_STRING(encoded));
+    Py_DECREF(encoded);
     Py_RETURN_NONE;
 }
 
@@ -583,7 +610,7 @@ add an item to the history buffer");
 static PyObject *
 get_completer_delims(PyObject *self, PyObject *noarg)
 {
-    return PyUnicode_FromString(rl_completer_word_break_characters);
+    return decode(rl_completer_word_break_characters);
 }
 
 PyDoc_STRVAR(doc_get_completer_delims,
@@ -673,7 +700,7 @@ get_history_item(PyObject *self, PyObject *args)
     }
 #endif /* __APPLE__ */
     if ((hist_ent = history_get(idx)))
-        return PyUnicode_FromString(hist_ent->line);
+        return decode(hist_ent->line);
     else {
         Py_RETURN_NONE;
     }
@@ -702,7 +729,7 @@ return the current (not the maximum) length of history.");
 static PyObject *
 get_line_buffer(PyObject *self, PyObject *noarg)
 {
-    return PyUnicode_FromString(rl_line_buffer);
+    return decode(rl_line_buffer);
 }
 
 PyDoc_STRVAR(doc_get_line_buffer,
@@ -730,12 +757,14 @@ Clear the current readline history.");
 /* Exported function to insert text into the line buffer */
 
 static PyObject *
-insert_text(PyObject *self, PyObject *args)
+insert_text(PyObject *self, PyObject *string)
 {
-    char *s;
-    if (!PyArg_ParseTuple(args, "s:insert_text", &s))
+    PyObject *encoded = encode(string);
+    if (encoded == NULL) {
         return NULL;
-    rl_insert_text(s);
+    }
+    rl_insert_text(PyBytes_AS_STRING(encoded));
+    Py_DECREF(encoded);
     Py_RETURN_NONE;
 }
 
@@ -763,9 +792,9 @@ contents of the line buffer.");
 
 static struct PyMethodDef readline_methods[] =
 {
-    {"parse_and_bind", parse_and_bind, METH_VARARGS, doc_parse_and_bind},
+    {"parse_and_bind", parse_and_bind, METH_O, doc_parse_and_bind},
     {"get_line_buffer", get_line_buffer, METH_NOARGS, doc_get_line_buffer},
-    {"insert_text", insert_text, METH_VARARGS, doc_insert_text},
+    {"insert_text", insert_text, METH_O, doc_insert_text},
     {"redisplay", redisplay, METH_NOARGS, doc_redisplay},
     {"read_init_file", read_init_file, METH_VARARGS, doc_read_init_file},
     {"read_history_file", read_history_file,
@@ -792,8 +821,8 @@ static struct PyMethodDef readline_methods[] =
     {"get_endidx", get_endidx, METH_NOARGS, doc_get_endidx},
 
     {"set_completer_delims", set_completer_delims,
-     METH_VARARGS, doc_set_completer_delims},
-    {"add_history", py_add_history, METH_VARARGS, doc_add_history},
+     METH_O, doc_set_completer_delims},
+    {"add_history", py_add_history, METH_O, doc_add_history},
     {"remove_history_item", py_remove_history, METH_VARARGS, doc_remove_history},
     {"replace_history_item", py_replace_history, METH_VARARGS, doc_replace_history},
     {"get_completer_delims", get_completer_delims,
@@ -890,7 +919,7 @@ on_completion_display_matches_hook(char **matches,
                                    int num_matches, int max_length)
 {
     int i;
-    PyObject *m=NULL, *s=NULL, *r=NULL;
+    PyObject *sub, *m=NULL, *s=NULL, *r=NULL;
 #ifdef WITH_THREAD
     PyGILState_STATE gilstate = PyGILState_Ensure();
 #endif
@@ -898,16 +927,17 @@ on_completion_display_matches_hook(char **matches,
     if (m == NULL)
         goto error;
     for (i = 0; i < num_matches; i++) {
-        s = PyUnicode_FromString(matches[i+1]);
+        s = decode(matches[i+1]);
         if (s == NULL)
             goto error;
         if (PyList_SetItem(m, i, s) == -1)
             goto error;
     }
+    sub = decode(matches[0]);
     r = PyObject_CallFunction(readlinestate_global->completion_display_matches_hook,
-                              "sOi", matches[0], m, max_length);
+                              "NNi", sub, m, max_length);
 
-    Py_DECREF(m); m=NULL;
+    m=NULL;
 
     if (r == NULL ||
         (r != Py_None && PyLong_AsLong(r) == -1 && PyErr_Occurred())) {
@@ -955,22 +985,24 @@ on_completion(const char *text, int state)
 {
     char *result = NULL;
     if (readlinestate_global->completer != NULL) {
-        PyObject *r;
+        PyObject *r = NULL, *t;
 #ifdef WITH_THREAD
         PyGILState_STATE gilstate = PyGILState_Ensure();
 #endif
         rl_attempted_completion_over = 1;
-        r = PyObject_CallFunction(readlinestate_global->completer, "si", text, state);
+        t = decode(text);
+        r = PyObject_CallFunction(readlinestate_global->completer, "Ni", t, state);
         if (r == NULL)
             goto error;
         if (r == Py_None) {
             result = NULL;
         }
         else {
-            char *s = _PyUnicode_AsString(r);
-            if (s == NULL)
+            PyObject *encoded = encode(r);
+            if (encoded == NULL)
                 goto error;
-            result = strdup(s);
+            result = strdup(PyBytes_AS_STRING(encoded));
+            Py_DECREF(encoded);
         }
         Py_DECREF(r);
         goto done;
@@ -994,6 +1026,9 @@ static char **
 flex_complete(const char *text, int start, int end)
 {
     char **result;
+    char saved;
+    size_t start_size, end_size;
+    wchar_t *s;
 #ifdef WITH_THREAD
     PyGILState_STATE gilstate = PyGILState_Ensure();
 #endif
@@ -1003,6 +1038,27 @@ flex_complete(const char *text, int start, int end)
 #ifdef HAVE_RL_COMPLETION_SUPPRESS_APPEND
     rl_completion_suppress_append = 0;
 #endif
+
+    saved = rl_line_buffer[start];
+    rl_line_buffer[start] = 0;
+    s = Py_DecodeLocale(rl_line_buffer, &start_size);
+    rl_line_buffer[start] = saved;
+    if (s == NULL) {
+        goto done;
+    }
+    PyMem_RawFree(s);
+    saved = rl_line_buffer[end];
+    rl_line_buffer[end] = 0;
+    s = Py_DecodeLocale(rl_line_buffer + start, &end_size);
+    rl_line_buffer[end] = saved;
+    if (s == NULL) {
+        goto done;
+    }
+    PyMem_RawFree(s);
+    start = (int)start_size;
+    end = start + (int)end_size;
+
+done:
     Py_XDECREF(readlinestate_global->begidx);
     Py_XDECREF(readlinestate_global->endidx);
     readlinestate_global->begidx = PyLong_FromLong((long) start);