Issue #9738: Document encodings of AST, compiler, parser and PyRun functions

6 files changed, 111 insertions, 49 deletions

diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst
index 3e41ec7..26e0716 100644
--- a/Doc/c-api/veryhigh.rst
+++ b/Doc/c-api/veryhigh.rst
@@ -66,8 +66,9 @@ the same library that the Python runtime is using.
    If *fp* refers to a file associated with an interactive device (console or
    terminal input or Unix pseudo-terminal), return the value of
    :c:func:`PyRun_InteractiveLoop`, otherwise return the result of
-   :c:func:`PyRun_SimpleFile`.  If *filename* is *NULL*, this function uses
-   ``"???"`` as the filename.
+   :c:func:`PyRun_SimpleFile`.  *filename* is decoded from the filesystem
+   encoding (:func:`sys.getfilesystemencoding`).  If *filename* is *NULL*, this
+   function uses ``"???"`` as the filename.
 
 
 .. c:function:: int PyRun_SimpleString(const char *command)
@@ -110,9 +111,10 @@ the same library that the Python runtime is using.
 .. c:function:: int PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags)
 
    Similar to :c:func:`PyRun_SimpleStringFlags`, but the Python source code is read
-   from *fp* instead of an in-memory string. *filename* should be the name of the
-   file.  If *closeit* is true, the file is closed before PyRun_SimpleFileExFlags
-   returns.
+   from *fp* instead of an in-memory string. *filename* should be the name of
+   the file, it is decoded from the filesystem encoding
+   (:func:`sys.getfilesystemencoding`).  If *closeit* is true, the file is
+   closed before PyRun_SimpleFileExFlags returns.
 
 
 .. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename)
@@ -125,7 +127,10 @@ the same library that the Python runtime is using.
 
    Read and execute a single statement from a file associated with an
    interactive device according to the *flags* argument.  The user will be
-   prompted using ``sys.ps1`` and ``sys.ps2``.  Returns ``0`` when the input was
+   prompted using ``sys.ps1`` and ``sys.ps2``.  *filename* is decoded from the
+   filesystem encoding (:func:`sys.getfilesystemencoding`).
+
+   Returns ``0`` when the input was
    executed successfully, ``-1`` if there was an exception, or an error code
    from the :file:`errcode.h` include file distributed as part of Python if
    there was a parse error.  (Note that :file:`errcode.h` is not included by
@@ -142,7 +147,8 @@ the same library that the Python runtime is using.
 
    Read and execute statements from a file associated with an interactive device
    until EOF is reached.  The user will be prompted using ``sys.ps1`` and
-   ``sys.ps2``.  Returns ``0`` at EOF.
+   ``sys.ps2``.  *filename* is decoded from the filesystem encoding
+   (:func:`sys.getfilesystemencoding`).  Returns ``0`` at EOF.
 
 
 .. c:function:: struct _node* PyParser_SimpleParseString(const char *str, int start)
@@ -164,7 +170,8 @@ the same library that the Python runtime is using.
    Parse Python source code from *str* using the start token *start* according to
    the *flags* argument.  The result can be used to create a code object which can
    be evaluated efficiently. This is useful if a code fragment must be evaluated
-   many times.
+   many times. *filename* is decoded from the filesystem encoding
+   (:func:`sys.getfilesystemencoding`).
 
 
 .. c:function:: struct _node* PyParser_SimpleParseFile(FILE *fp, const char *filename, int start)
@@ -217,7 +224,8 @@ the same library that the Python runtime is using.
 .. c:function:: PyObject* PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags)
 
    Similar to :c:func:`PyRun_StringFlags`, but the Python source code is read from
-   *fp* instead of an in-memory string. *filename* should be the name of the file.
+   *fp* instead of an in-memory string. *filename* should be the name of the file,
+   it is decoded from the filesystem encoding (:func:`sys.getfilesystemencoding`).
    If *closeit* is true, the file is closed before :c:func:`PyRun_FileExFlags`
    returns.
 
@@ -241,8 +249,9 @@ the same library that the Python runtime is using.
    code which can be compiled and should be :const:`Py_eval_input`,
    :const:`Py_file_input`, or :const:`Py_single_input`.  The filename specified by
    *filename* is used to construct the code object and may appear in tracebacks or
-   :exc:`SyntaxError` exception messages.  This returns *NULL* if the code cannot
-   be parsed or compiled.
+   :exc:`SyntaxError` exception messages, it is decoded from the filesystem
+   encoding (:func:`sys.getfilesystemencoding`).  This returns *NULL* if the
+   code cannot be parsed or compiled.
 
    The integer *optimize* specifies the optimization level of the compiler; a
    value of ``-1`` selects the optimization level of the interpreter as given by
diff --git a/Include/ast.h b/Include/ast.h
index cc14b7f..a015336 100644
--- a/Include/ast.h
+++ b/Include/ast.h
@@ -4,8 +4,11 @@
 extern "C" {
 #endif
 
-PyAPI_FUNC(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags,
-				  const char *, PyArena *);
+PyAPI_FUNC(mod_ty) PyAST_FromNode(
+    const node *n,
+    PyCompilerFlags *flags,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyArena *arena);
 
 #ifdef __cplusplus
 }
diff --git a/Include/compile.h b/Include/compile.h
index 456a494..5ce5b77 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -30,8 +30,12 @@ typedef struct {
 
 struct _mod; /* Declare the existence of this type */
 #define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar)
-PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(struct _mod *, const char *,
-					   PyCompilerFlags *, int, PyArena *);
+PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(
+    mod_ty mod,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyCompilerFlags *flags,
+    int optimize,
+    PyArena *arena);
 PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
 
 
diff --git a/Include/parsetok.h b/Include/parsetok.h
index d183784..4b7694f 100644
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -9,10 +9,10 @@ extern "C" {
 
 typedef struct {
     int error;
-    const char *filename;
+    const char *filename;       /* decoded from the filesystem encoding */
     int lineno;
     int offset;
-    char *text;
+    char *text;                 /* UTF-8-encoded string */
     int token;
     int expected;
 } perrdetail;
@@ -39,23 +39,32 @@ PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
 
 PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
                                               perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, 
+PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *,
 					   const char*, grammar *,
 						 int, char *, char *,
 						 perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(FILE *, const char *,
-					   const char*, grammar *,
-						 int, char *, char *,
-						 perrdetail *, int *);
+PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    const char *enc,
+    grammar *g,
+    int start,
+    char *ps1,
+    char *ps2,
+    perrdetail *err_ret,
+    int *flags);
 
 PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
 					      const char *,
 					      grammar *, int,
                                               perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(const char *,
-					      const char *,
-					      grammar *, int,
-                                              perrdetail *, int *);
+PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(
+    const char *s,
+    const char *filename,       /* decoded from the filesystem encoding */
+    grammar *g,
+    int start,
+    perrdetail *err_ret,
+    int *flags);
 
 /* Note that he following function is defined in pythonrun.c not parsetok.c. */
 PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index f290687..86da6e0 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -38,19 +38,41 @@ PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *);
 PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_SimpleFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveOneFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags *);
-
-PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *,
-                                                 int, PyCompilerFlags *flags,
-                                                 PyArena *);
-PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *,
-                                               const char*, int,
-                                               char *, char *,
-                                               PyCompilerFlags *, int *,
-                                               PyArena *);
+PyAPI_FUNC(int) PyRun_AnyFileExFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int closeit,
+    PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_SimpleFileExFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int closeit,
+    PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_InteractiveOneFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyCompilerFlags *flags);
+
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(
+    const char *s,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start,
+    PyCompilerFlags *flags,
+    PyArena *arena);
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    const char* enc,
+    int start,
+    char *ps1,
+    char *ps2,
+    PyCompilerFlags *flags,
+    int *errcode,
+    PyArena *arena);
 #endif
 
 #ifndef PyParser_SimpleParseString
@@ -68,9 +90,14 @@ PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *,
 PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *,
                                          PyObject *, PyCompilerFlags *);
 
-PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int,
-                                         PyObject *, PyObject *, int,
-                                         PyCompilerFlags *);
+PyAPI_FUNC(PyObject *) PyRun_FileExFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start,
+    PyObject *globals,
+    PyObject *locals,
+    int closeit,
+    PyCompilerFlags *flags);
 #endif
 
 #ifdef Py_LIMITED_API
@@ -78,10 +105,17 @@ PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int);
 #else
 #define Py_CompileString(str, p, s) Py_CompileStringExFlags(str, p, s, NULL, -1)
 #define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags(str, p, s, f, -1)
-PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(const char *, const char *, int,
-                                               PyCompilerFlags *, int);
-#endif
-PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int);
+PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(
+    const char *str,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start,
+    PyCompilerFlags *flags,
+    int optimize);
+#endif
+PyAPI_FUNC(struct symtable *) Py_SymtableString(
+    const char *str,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start);
 
 PyAPI_FUNC(void) PyErr_Print(void);
 PyAPI_FUNC(void) PyErr_PrintEx(int);
diff --git a/Include/symtable.h b/Include/symtable.h
index 2ad204d..fd7de04 100644
--- a/Include/symtable.h
+++ b/Include/symtable.h
@@ -16,7 +16,8 @@ typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
 struct _symtable_entry;
 
 struct symtable {
-    const char *st_filename;        /* name of file being compiled */
+    const char *st_filename;        /* name of file being compiled,
+                                       decoded from the filesystem encoding */
     struct _symtable_entry *st_cur; /* current symbol table entry */
     struct _symtable_entry *st_top; /* symbol table entry for module */
     PyObject *st_blocks;            /* dict: map AST node addresses
@@ -60,8 +61,10 @@ PyAPI_DATA(PyTypeObject) PySTEntry_Type;
 
 PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *);
 
-PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *,
-                                              PyFutureFeatures *);
+PyAPI_FUNC(struct symtable *) PySymtable_Build(
+    mod_ty mod,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyFutureFeatures *future);
 PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
 
 PyAPI_FUNC(void) PySymtable_Free(struct symtable *);