From 00676d143626dbbb7e1ad456ed013afba4420978 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 27 Dec 2010 01:49:31 +0000 Subject: Issue #9738: Document encodings of AST, compiler, parser and PyRun functions --- Doc/c-api/veryhigh.rst | 31 +++++++++++++-------- Include/ast.h | 7 +++-- Include/compile.h | 8 ++++-- Include/parsetok.h | 31 +++++++++++++-------- Include/pythonrun.h | 74 ++++++++++++++++++++++++++++++++++++-------------- Include/symtable.h | 9 ++++-- 6 files changed, 111 insertions(+), 49 deletions(-) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 3e41ec7..26e0716 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -66,8 +66,9 @@ the same library that the Python runtime is using. If *fp* refers to a file associated with an interactive device (console or terminal input or Unix pseudo-terminal), return the value of :c:func:`PyRun_InteractiveLoop`, otherwise return the result of - :c:func:`PyRun_SimpleFile`. If *filename* is *NULL*, this function uses - ``"???"`` as the filename. + :c:func:`PyRun_SimpleFile`. *filename* is decoded from the filesystem + encoding (:func:`sys.getfilesystemencoding`). If *filename* is *NULL*, this + function uses ``"???"`` as the filename. .. c:function:: int PyRun_SimpleString(const char *command) @@ -110,9 +111,10 @@ the same library that the Python runtime is using. .. c:function:: int PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) Similar to :c:func:`PyRun_SimpleStringFlags`, but the Python source code is read - from *fp* instead of an in-memory string. *filename* should be the name of the - file. If *closeit* is true, the file is closed before PyRun_SimpleFileExFlags - returns. + from *fp* instead of an in-memory string. *filename* should be the name of + the file, it is decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). If *closeit* is true, the file is + closed before PyRun_SimpleFileExFlags returns. .. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) @@ -125,7 +127,10 @@ the same library that the Python runtime is using. Read and execute a single statement from a file associated with an interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. Returns ``0`` when the input was + prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the + filesystem encoding (:func:`sys.getfilesystemencoding`). + + Returns ``0`` when the input was executed successfully, ``-1`` if there was an exception, or an error code from the :file:`errcode.h` include file distributed as part of Python if there was a parse error. (Note that :file:`errcode.h` is not included by @@ -142,7 +147,8 @@ the same library that the Python runtime is using. Read and execute statements from a file associated with an interactive device until EOF is reached. The user will be prompted using ``sys.ps1`` and - ``sys.ps2``. Returns ``0`` at EOF. + ``sys.ps2``. *filename* is decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF. .. c:function:: struct _node* PyParser_SimpleParseString(const char *str, int start) @@ -164,7 +170,8 @@ the same library that the Python runtime is using. Parse Python source code from *str* using the start token *start* according to the *flags* argument. The result can be used to create a code object which can be evaluated efficiently. This is useful if a code fragment must be evaluated - many times. + many times. *filename* is decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). .. c:function:: struct _node* PyParser_SimpleParseFile(FILE *fp, const char *filename, int start) @@ -217,7 +224,8 @@ the same library that the Python runtime is using. .. c:function:: PyObject* PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags) Similar to :c:func:`PyRun_StringFlags`, but the Python source code is read from - *fp* instead of an in-memory string. *filename* should be the name of the file. + *fp* instead of an in-memory string. *filename* should be the name of the file, + it is decoded from the filesystem encoding (:func:`sys.getfilesystemencoding`). If *closeit* is true, the file is closed before :c:func:`PyRun_FileExFlags` returns. @@ -241,8 +249,9 @@ the same library that the Python runtime is using. code which can be compiled and should be :const:`Py_eval_input`, :const:`Py_file_input`, or :const:`Py_single_input`. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or - :exc:`SyntaxError` exception messages. This returns *NULL* if the code cannot - be parsed or compiled. + :exc:`SyntaxError` exception messages, it is decoded from the filesystem + encoding (:func:`sys.getfilesystemencoding`). This returns *NULL* if the + code cannot be parsed or compiled. The integer *optimize* specifies the optimization level of the compiler; a value of ``-1`` selects the optimization level of the interpreter as given by diff --git a/Include/ast.h b/Include/ast.h index cc14b7f..a015336 100644 --- a/Include/ast.h +++ b/Include/ast.h @@ -4,8 +4,11 @@ extern "C" { #endif -PyAPI_FUNC(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags, - const char *, PyArena *); +PyAPI_FUNC(mod_ty) PyAST_FromNode( + const node *n, + PyCompilerFlags *flags, + const char *filename, /* decoded from the filesystem encoding */ + PyArena *arena); #ifdef __cplusplus } diff --git a/Include/compile.h b/Include/compile.h index 456a494..5ce5b77 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -30,8 +30,12 @@ typedef struct { struct _mod; /* Declare the existence of this type */ #define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar) -PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(struct _mod *, const char *, - PyCompilerFlags *, int, PyArena *); +PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx( + mod_ty mod, + const char *filename, /* decoded from the filesystem encoding */ + PyCompilerFlags *flags, + int optimize, + PyArena *arena); PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *); diff --git a/Include/parsetok.h b/Include/parsetok.h index d183784..4b7694f 100644 --- a/Include/parsetok.h +++ b/Include/parsetok.h @@ -9,10 +9,10 @@ extern "C" { typedef struct { int error; - const char *filename; + const char *filename; /* decoded from the filesystem encoding */ int lineno; int offset; - char *text; + char *text; /* UTF-8-encoded string */ int token; int expected; } perrdetail; @@ -39,23 +39,32 @@ PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int, PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int, perrdetail *, int); -PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, +PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, const char*, grammar *, int, char *, char *, perrdetail *, int); -PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(FILE *, const char *, - const char*, grammar *, - int, char *, char *, - perrdetail *, int *); +PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + const char *enc, + grammar *g, + int start, + char *ps1, + char *ps2, + perrdetail *err_ret, + int *flags); PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *, const char *, grammar *, int, perrdetail *, int); -PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(const char *, - const char *, - grammar *, int, - perrdetail *, int *); +PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx( + const char *s, + const char *filename, /* decoded from the filesystem encoding */ + grammar *g, + int start, + perrdetail *err_ret, + int *flags); /* Note that he following function is defined in pythonrun.c not parsetok.c. */ PyAPI_FUNC(void) PyParser_SetError(perrdetail *); diff --git a/Include/pythonrun.h b/Include/pythonrun.h index f290687..86da6e0 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -38,19 +38,41 @@ PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); #ifndef Py_LIMITED_API PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *); PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *); -PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, const char *, int, PyCompilerFlags *); -PyAPI_FUNC(int) PyRun_SimpleFileExFlags(FILE *, const char *, int, PyCompilerFlags *); -PyAPI_FUNC(int) PyRun_InteractiveOneFlags(FILE *, const char *, PyCompilerFlags *); -PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags *); - -PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *, - int, PyCompilerFlags *flags, - PyArena *); -PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, - const char*, int, - char *, char *, - PyCompilerFlags *, int *, - PyArena *); +PyAPI_FUNC(int) PyRun_AnyFileExFlags( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + int closeit, + PyCompilerFlags *flags); +PyAPI_FUNC(int) PyRun_SimpleFileExFlags( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + int closeit, + PyCompilerFlags *flags); +PyAPI_FUNC(int) PyRun_InteractiveOneFlags( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + PyCompilerFlags *flags); +PyAPI_FUNC(int) PyRun_InteractiveLoopFlags( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + PyCompilerFlags *flags); + +PyAPI_FUNC(struct _mod *) PyParser_ASTFromString( + const char *s, + const char *filename, /* decoded from the filesystem encoding */ + int start, + PyCompilerFlags *flags, + PyArena *arena); +PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + const char* enc, + int start, + char *ps1, + char *ps2, + PyCompilerFlags *flags, + int *errcode, + PyArena *arena); #endif #ifndef PyParser_SimpleParseString @@ -68,9 +90,14 @@ PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *, PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, PyObject *, PyCompilerFlags *); -PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int, - PyObject *, PyObject *, int, - PyCompilerFlags *); +PyAPI_FUNC(PyObject *) PyRun_FileExFlags( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + int start, + PyObject *globals, + PyObject *locals, + int closeit, + PyCompilerFlags *flags); #endif #ifdef Py_LIMITED_API @@ -78,10 +105,17 @@ PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int); #else #define Py_CompileString(str, p, s) Py_CompileStringExFlags(str, p, s, NULL, -1) #define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags(str, p, s, f, -1) -PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(const char *, const char *, int, - PyCompilerFlags *, int); -#endif -PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int); +PyAPI_FUNC(PyObject *) Py_CompileStringExFlags( + const char *str, + const char *filename, /* decoded from the filesystem encoding */ + int start, + PyCompilerFlags *flags, + int optimize); +#endif +PyAPI_FUNC(struct symtable *) Py_SymtableString( + const char *str, + const char *filename, /* decoded from the filesystem encoding */ + int start); PyAPI_FUNC(void) PyErr_Print(void); PyAPI_FUNC(void) PyErr_PrintEx(int); diff --git a/Include/symtable.h b/Include/symtable.h index 2ad204d..fd7de04 100644 --- a/Include/symtable.h +++ b/Include/symtable.h @@ -16,7 +16,8 @@ typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock } struct _symtable_entry; struct symtable { - const char *st_filename; /* name of file being compiled */ + const char *st_filename; /* name of file being compiled, + decoded from the filesystem encoding */ struct _symtable_entry *st_cur; /* current symbol table entry */ struct _symtable_entry *st_top; /* symbol table entry for module */ PyObject *st_blocks; /* dict: map AST node addresses @@ -60,8 +61,10 @@ PyAPI_DATA(PyTypeObject) PySTEntry_Type; PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *); -PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *, - PyFutureFeatures *); +PyAPI_FUNC(struct symtable *) PySymtable_Build( + mod_ty mod, + const char *filename, /* decoded from the filesystem encoding */ + PyFutureFeatures *future); PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *); PyAPI_FUNC(void) PySymtable_Free(struct symtable *); -- cgit v0.12