diff options
| -rw-r--r-- | Include/pythonrun.h | 13 | ||||
| -rw-r--r-- | Lib/test/test_symtable.py | 9 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst | 2 | ||||
| -rw-r--r-- | Modules/clinic/symtablemodule.c.h | 24 | ||||
| -rw-r--r-- | Modules/symtablemodule.c | 23 | ||||
| -rw-r--r-- | Python/bltinmodule.c | 55 | ||||
| -rw-r--r-- | Python/pythonrun.c | 64 |
7 files changed, 111 insertions, 79 deletions
diff --git a/Include/pythonrun.h b/Include/pythonrun.h index e83846a..196355c 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -119,10 +119,23 @@ PyAPI_FUNC(struct symtable *) Py_SymtableString( const char *filename, /* decoded from the filesystem encoding */ int start); #ifndef Py_LIMITED_API +PyAPI_FUNC(const char *) _Py_SourceAsString( + PyObject *cmd, + const char *funcname, + const char *what, + PyCompilerFlags *cf, + PyObject **cmd_copy); + PyAPI_FUNC(struct symtable *) Py_SymtableStringObject( const char *str, PyObject *filename, int start); + +PyAPI_FUNC(struct symtable *) _Py_SymtableStringObjectFlags( + const char *str, + PyObject *filename, + int start, + PyCompilerFlags *flags); #endif PyAPI_FUNC(void) PyErr_Print(void); diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 0a1cb8d..bea2ce1 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -215,6 +215,15 @@ class SymtableTest(unittest.TestCase): def test_exec(self): symbols = symtable.symtable("def f(x): return x", "?", "exec") + def test_bytes(self): + top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec") + self.assertIsNotNone(find_block(top, "Mine")) + + code = b'# -*- coding: iso8859-15 -*-\nclass \xb4: pass\n' + + top = symtable.symtable(code, "?", "exec") + self.assertIsNotNone(find_block(top, "\u017d")) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst b/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst new file mode 100644 index 0000000..5bcd7a9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst @@ -0,0 +1,2 @@ +:func:`symtable.symtable` now accepts the same input types for source code as the +built-in :func:`compile` function. Patch by Dino Viehland. diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index 73e340b..7d8b0ad 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -3,7 +3,7 @@ preserve [clinic start generated code]*/ PyDoc_STRVAR(_symtable_symtable__doc__, -"symtable($module, str, filename, startstr, /)\n" +"symtable($module, source, filename, startstr, /)\n" "--\n" "\n" "Return symbol and scope dictionaries used internally by compiler."); @@ -12,33 +12,21 @@ PyDoc_STRVAR(_symtable_symtable__doc__, {"symtable", (PyCFunction)(void(*)(void))_symtable_symtable, METH_FASTCALL, _symtable_symtable__doc__}, static PyObject * -_symtable_symtable_impl(PyObject *module, const char *str, +_symtable_symtable_impl(PyObject *module, PyObject *source, PyObject *filename, const char *startstr); static PyObject * _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - const char *str; + PyObject *source; PyObject *filename; const char *startstr; if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) { goto exit; } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("symtable", 1, "str", args[0]); - goto exit; - } - Py_ssize_t str_length; - str = PyUnicode_AsUTF8AndSize(args[0], &str_length); - if (str == NULL) { - goto exit; - } - if (strlen(str) != (size_t)str_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + source = args[0]; if (!PyUnicode_FSDecoder(args[1], &filename)) { goto exit; } @@ -55,9 +43,9 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _symtable_symtable_impl(module, str, filename, startstr); + return_value = _symtable_symtable_impl(module, source, filename, startstr); exit: return return_value; } -/*[clinic end generated code: output=be1cca59de019984 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=de655625eee705f4 input=a9049054013a1b77]*/ diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index e8d2f5b..d66cb44 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -14,7 +14,7 @@ module _symtable /*[clinic input] _symtable.symtable - str: str + source: object filename: object(converter='PyUnicode_FSDecoder') startstr: str / @@ -23,13 +23,23 @@ Return symbol and scope dictionaries used internally by compiler. [clinic start generated code]*/ static PyObject * -_symtable_symtable_impl(PyObject *module, const char *str, +_symtable_symtable_impl(PyObject *module, PyObject *source, PyObject *filename, const char *startstr) -/*[clinic end generated code: output=914b369c9b785956 input=6c615e84d5f408e3]*/ +/*[clinic end generated code: output=59eb0d5fc7285ac4 input=9dd8a50c0c36a4d7]*/ { struct symtable *st; PyObject *t; int start; + PyCompilerFlags cf; + PyObject *source_copy = NULL; + + cf.cf_flags = PyCF_SOURCE_IS_UTF8; + cf.cf_feature_version = PY_MINOR_VERSION; + + const char *str = _Py_SourceAsString(source, "symtable", "string or bytes", &cf, &source_copy); + if (str == NULL) { + return NULL; + } if (strcmp(startstr, "exec") == 0) start = Py_file_input; @@ -41,12 +51,15 @@ _symtable_symtable_impl(PyObject *module, const char *str, PyErr_SetString(PyExc_ValueError, "symtable() arg 3 must be 'exec' or 'eval' or 'single'"); Py_DECREF(filename); + Py_XDECREF(source_copy); return NULL; } - st = Py_SymtableStringObject(str, filename, start); + st = _Py_SymtableStringObjectFlags(str, filename, start, &cf); Py_DECREF(filename); - if (st == NULL) + Py_XDECREF(source_copy); + if (st == NULL) { return NULL; + } t = (PyObject *)st->st_top; Py_INCREF(t); PyMem_Free((void *)st->st_future); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 5d58085..065ad95 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -687,55 +687,6 @@ builtin_chr_impl(PyObject *module, int i) } -static const char * -source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy) -{ - const char *str; - Py_ssize_t size; - Py_buffer view; - - *cmd_copy = NULL; - if (PyUnicode_Check(cmd)) { - cf->cf_flags |= PyCF_IGNORE_COOKIE; - str = PyUnicode_AsUTF8AndSize(cmd, &size); - if (str == NULL) - return NULL; - } - else if (PyBytes_Check(cmd)) { - str = PyBytes_AS_STRING(cmd); - size = PyBytes_GET_SIZE(cmd); - } - else if (PyByteArray_Check(cmd)) { - str = PyByteArray_AS_STRING(cmd); - size = PyByteArray_GET_SIZE(cmd); - } - else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) { - /* Copy to NUL-terminated buffer. */ - *cmd_copy = PyBytes_FromStringAndSize( - (const char *)view.buf, view.len); - PyBuffer_Release(&view); - if (*cmd_copy == NULL) { - return NULL; - } - str = PyBytes_AS_STRING(*cmd_copy); - size = PyBytes_GET_SIZE(*cmd_copy); - } - else { - PyErr_Format(PyExc_TypeError, - "%s() arg 1 must be a %s object", - funcname, what); - return NULL; - } - - if (strlen(str) != (size_t)size) { - PyErr_SetString(PyExc_ValueError, - "source code string cannot contain null bytes"); - Py_CLEAR(*cmd_copy); - return NULL; - } - return str; -} - /*[clinic input] compile as builtin_compile @@ -855,7 +806,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto finally; } - str = source_as_string(source, "compile", "string, bytes or AST", &cf, &source_copy); + str = _Py_SourceAsString(source, "compile", "string, bytes or AST", &cf, &source_copy); if (str == NULL) goto error; @@ -991,7 +942,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, cf.cf_flags = PyCF_SOURCE_IS_UTF8; cf.cf_feature_version = PY_MINOR_VERSION; - str = source_as_string(source, "eval", "string, bytes or code", &cf, &source_copy); + str = _Py_SourceAsString(source, "eval", "string, bytes or code", &cf, &source_copy); if (str == NULL) return NULL; @@ -1083,7 +1034,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyCompilerFlags cf; cf.cf_flags = PyCF_SOURCE_IS_UTF8; cf.cf_feature_version = PY_MINOR_VERSION; - str = source_as_string(source, "exec", + str = _Py_SourceAsString(source, "exec", "string, bytes or code", &cf, &source_copy); if (str == NULL) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index ace9f2f..784c15b 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1231,21 +1231,77 @@ PyCompileString(const char *str, const char *filename, int start) return Py_CompileStringFlags(str, filename, start, NULL); } +const char * +_Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy) +{ + const char *str; + Py_ssize_t size; + Py_buffer view; + + *cmd_copy = NULL; + if (PyUnicode_Check(cmd)) { + cf->cf_flags |= PyCF_IGNORE_COOKIE; + str = PyUnicode_AsUTF8AndSize(cmd, &size); + if (str == NULL) + return NULL; + } + else if (PyBytes_Check(cmd)) { + str = PyBytes_AS_STRING(cmd); + size = PyBytes_GET_SIZE(cmd); + } + else if (PyByteArray_Check(cmd)) { + str = PyByteArray_AS_STRING(cmd); + size = PyByteArray_GET_SIZE(cmd); + } + else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) { + /* Copy to NUL-terminated buffer. */ + *cmd_copy = PyBytes_FromStringAndSize( + (const char *)view.buf, view.len); + PyBuffer_Release(&view); + if (*cmd_copy == NULL) { + return NULL; + } + str = PyBytes_AS_STRING(*cmd_copy); + size = PyBytes_GET_SIZE(*cmd_copy); + } + else { + PyErr_Format(PyExc_TypeError, + "%s() arg 1 must be a %s object", + funcname, what); + return NULL; + } + + if (strlen(str) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + Py_CLEAR(*cmd_copy); + return NULL; + } + return str; +} + struct symtable * Py_SymtableStringObject(const char *str, PyObject *filename, int start) { + PyCompilerFlags flags; + + flags.cf_flags = 0; + flags.cf_feature_version = PY_MINOR_VERSION; + return _Py_SymtableStringObjectFlags(str, filename, start, &flags); +} + +struct symtable * +_Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, PyCompilerFlags *flags) +{ struct symtable *st; mod_ty mod; - PyCompilerFlags flags; PyArena *arena; arena = PyArena_New(); if (arena == NULL) return NULL; - flags.cf_flags = 0; - flags.cf_feature_version = PY_MINOR_VERSION; - mod = PyParser_ASTFromStringObject(str, filename, start, &flags, arena); + mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); if (mod == NULL) { PyArena_Free(arena); return NULL; |