Issue #9738: Document encodings of AST, compiler, parser and PyRun functions

5 files changed, 91 insertions, 38 deletions

diff --git a/Include/ast.h b/Include/ast.h
index cc14b7f..a015336 100644
--- a/Include/ast.h
+++ b/Include/ast.h
@@ -4,8 +4,11 @@
 extern "C" {
 #endif
 
-PyAPI_FUNC(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags,
-				  const char *, PyArena *);
+PyAPI_FUNC(mod_ty) PyAST_FromNode(
+    const node *n,
+    PyCompilerFlags *flags,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyArena *arena);
 
 #ifdef __cplusplus
 }
diff --git a/Include/compile.h b/Include/compile.h
index 456a494..5ce5b77 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -30,8 +30,12 @@ typedef struct {
 
 struct _mod; /* Declare the existence of this type */
 #define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar)
-PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(struct _mod *, const char *,
-					   PyCompilerFlags *, int, PyArena *);
+PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(
+    mod_ty mod,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyCompilerFlags *flags,
+    int optimize,
+    PyArena *arena);
 PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
 
 
diff --git a/Include/parsetok.h b/Include/parsetok.h
index d183784..4b7694f 100644
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -9,10 +9,10 @@ extern "C" {
 
 typedef struct {
     int error;
-    const char *filename;
+    const char *filename;       /* decoded from the filesystem encoding */
     int lineno;
     int offset;
-    char *text;
+    char *text;                 /* UTF-8-encoded string */
     int token;
     int expected;
 } perrdetail;
@@ -39,23 +39,32 @@ PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
 
 PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
                                               perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, 
+PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *,
 					   const char*, grammar *,
 						 int, char *, char *,
 						 perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(FILE *, const char *,
-					   const char*, grammar *,
-						 int, char *, char *,
-						 perrdetail *, int *);
+PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    const char *enc,
+    grammar *g,
+    int start,
+    char *ps1,
+    char *ps2,
+    perrdetail *err_ret,
+    int *flags);
 
 PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
 					      const char *,
 					      grammar *, int,
                                               perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(const char *,
-					      const char *,
-					      grammar *, int,
-                                              perrdetail *, int *);
+PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(
+    const char *s,
+    const char *filename,       /* decoded from the filesystem encoding */
+    grammar *g,
+    int start,
+    perrdetail *err_ret,
+    int *flags);
 
 /* Note that he following function is defined in pythonrun.c not parsetok.c. */
 PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index f290687..86da6e0 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -38,19 +38,41 @@ PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *);
 PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_SimpleFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveOneFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags *);
-
-PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *,
-                                                 int, PyCompilerFlags *flags,
-                                                 PyArena *);
-PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *,
-                                               const char*, int,
-                                               char *, char *,
-                                               PyCompilerFlags *, int *,
-                                               PyArena *);
+PyAPI_FUNC(int) PyRun_AnyFileExFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int closeit,
+    PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_SimpleFileExFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int closeit,
+    PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_InteractiveOneFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyCompilerFlags *flags);
+
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(
+    const char *s,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start,
+    PyCompilerFlags *flags,
+    PyArena *arena);
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    const char* enc,
+    int start,
+    char *ps1,
+    char *ps2,
+    PyCompilerFlags *flags,
+    int *errcode,
+    PyArena *arena);
 #endif
 
 #ifndef PyParser_SimpleParseString
@@ -68,9 +90,14 @@ PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *,
 PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *,
                                          PyObject *, PyCompilerFlags *);
 
-PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int,
-                                         PyObject *, PyObject *, int,
-                                         PyCompilerFlags *);
+PyAPI_FUNC(PyObject *) PyRun_FileExFlags(
+    FILE *fp,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start,
+    PyObject *globals,
+    PyObject *locals,
+    int closeit,
+    PyCompilerFlags *flags);
 #endif
 
 #ifdef Py_LIMITED_API
@@ -78,10 +105,17 @@ PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int);
 #else
 #define Py_CompileString(str, p, s) Py_CompileStringExFlags(str, p, s, NULL, -1)
 #define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags(str, p, s, f, -1)
-PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(const char *, const char *, int,
-                                               PyCompilerFlags *, int);
-#endif
-PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int);
+PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(
+    const char *str,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start,
+    PyCompilerFlags *flags,
+    int optimize);
+#endif
+PyAPI_FUNC(struct symtable *) Py_SymtableString(
+    const char *str,
+    const char *filename,       /* decoded from the filesystem encoding */
+    int start);
 
 PyAPI_FUNC(void) PyErr_Print(void);
 PyAPI_FUNC(void) PyErr_PrintEx(int);
diff --git a/Include/symtable.h b/Include/symtable.h
index 2ad204d..fd7de04 100644
--- a/Include/symtable.h
+++ b/Include/symtable.h
@@ -16,7 +16,8 @@ typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
 struct _symtable_entry;
 
 struct symtable {
-    const char *st_filename;        /* name of file being compiled */
+    const char *st_filename;        /* name of file being compiled,
+                                       decoded from the filesystem encoding */
     struct _symtable_entry *st_cur; /* current symbol table entry */
     struct _symtable_entry *st_top; /* symbol table entry for module */
     PyObject *st_blocks;            /* dict: map AST node addresses
@@ -60,8 +61,10 @@ PyAPI_DATA(PyTypeObject) PySTEntry_Type;
 
 PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *);
 
-PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *,
-                                              PyFutureFeatures *);
+PyAPI_FUNC(struct symtable *) PySymtable_Build(
+    mod_ty mod,
+    const char *filename,       /* decoded from the filesystem encoding */
+    PyFutureFeatures *future);
 PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
 
 PyAPI_FUNC(void) PySymtable_Free(struct symtable *);