Close #11619: The parser and the import machinery do not encode Unicode

filenames anymore on Windows.

1 files changed, 65 insertions, 14 deletions

diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index d61ba92..2df9159 100644
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -13,7 +13,7 @@
 
 /* Forward */
 static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
-static int initerr(perrdetail *err_ret, const char* filename);
+static int initerr(perrdetail *err_ret, PyObject * filename);
 
 /* Parse input coming from a string.  Return error code, print some errors. */
 node *
@@ -41,9 +41,9 @@ PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
 }
 
 node *
-PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
-                          grammar *g, int start,
-                          perrdetail *err_ret, int *flags)
+PyParser_ParseStringObject(const char *s, PyObject *filename,
+                           grammar *g, int start,
+                           perrdetail *err_ret, int *flags)
 {
     struct tok_state *tok;
     int exec_input = start == file_input;
@@ -67,6 +67,29 @@ PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
     return parsetok(tok, g, start, err_ret, flags);
 }
 
+node *
+PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename_str,
+                          grammar *g, int start,
+                          perrdetail *err_ret, int *flags)
+{
+    node *n;
+    PyObject *filename = NULL;
+#ifndef PGEN
+    if (filename_str != NULL) {
+        filename = PyUnicode_DecodeFSDefault(filename_str);
+        if (filename == NULL) {
+            err_ret->error = E_ERROR;
+            return NULL;
+        }
+    }
+#endif
+    n = PyParser_ParseStringObject(s, filename, g, start, err_ret, flags);
+#ifndef PGEN
+    Py_XDECREF(filename);
+#endif
+    return n;
+}
+
 /* Parse input coming from a file.  Return error code, print some errors. */
 
 node *
@@ -88,9 +111,10 @@ PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc,
 }
 
 node *
-PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
-                          const char *enc, grammar *g, int start,
-                          char *ps1, char *ps2, perrdetail *err_ret, int *flags)
+PyParser_ParseFileObject(FILE *fp, PyObject *filename,
+                         const char *enc, grammar *g, int start,
+                         char *ps1, char *ps2, perrdetail *err_ret,
+                         int *flags)
 {
     struct tok_state *tok;
 
@@ -108,6 +132,30 @@ PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
     return parsetok(tok, g, start, err_ret, flags);
 }
 
+node *
+PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
+                          const char *enc, grammar *g, int start,
+                          char *ps1, char *ps2, perrdetail *err_ret, int *flags)
+{
+    node *n;
+    PyObject *fileobj = NULL;
+#ifndef PGEN
+    if (filename != NULL) {
+        fileobj = PyUnicode_DecodeFSDefault(filename);
+        if (fileobj == NULL) {
+            err_ret->error = E_ERROR;
+            return NULL;
+        }
+    }
+#endif
+    n = PyParser_ParseFileObject(fp, fileobj, enc, g,
+                                 start, ps1, ps2, err_ret, flags);
+#ifndef PGEN
+    Py_XDECREF(fileobj);
+#endif
+    return n;
+}
+
 #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
 #if 0
 static char with_msg[] =
@@ -306,7 +354,7 @@ done:
 }
 
 static int
-initerr(perrdetail *err_ret, const char *filename)
+initerr(perrdetail *err_ret, PyObject *filename)
 {
     err_ret->error = E_OK;
     err_ret->lineno = 0;
@@ -315,13 +363,16 @@ initerr(perrdetail *err_ret, const char *filename)
     err_ret->token = -1;
     err_ret->expected = -1;
 #ifndef PGEN
-    if (filename)
-        err_ret->filename = PyUnicode_DecodeFSDefault(filename);
-    else
+    if (filename) {
+        Py_INCREF(filename);
+        err_ret->filename = filename;
+    }
+    else {
         err_ret->filename = PyUnicode_FromString("<string>");
-    if (err_ret->filename == NULL) {
-        err_ret->error = E_ERROR;
-        return -1;
+        if (err_ret->filename == NULL) {
+            err_ret->error = E_ERROR;
+            return -1;
+        }
     }
 #endif
     return 0;