bpo-37001: Makes symtable.symtable have parity with compile for input (#13483)

* Makes symtable.symtable have parity for accepted datatypes
for source code as compile()

* Add NEWS blurb

7 files changed, 111 insertions, 79 deletions

diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index e83846a..196355c 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -119,10 +119,23 @@ PyAPI_FUNC(struct symtable *) Py_SymtableString(
     const char *filename,       /* decoded from the filesystem encoding */
     int start);
 #ifndef Py_LIMITED_API
+PyAPI_FUNC(const char *) _Py_SourceAsString(
+    PyObject *cmd,
+    const char *funcname,
+    const char *what, 
+    PyCompilerFlags *cf,
+    PyObject **cmd_copy);
+
 PyAPI_FUNC(struct symtable *) Py_SymtableStringObject(
     const char *str,
     PyObject *filename,
     int start);
+
+PyAPI_FUNC(struct symtable *) _Py_SymtableStringObjectFlags(
+    const char *str,
+    PyObject *filename,
+    int start,
+    PyCompilerFlags *flags);
 #endif
 
 PyAPI_FUNC(void) PyErr_Print(void);
diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py
index 0a1cb8d..bea2ce1 100644
--- a/Lib/test/test_symtable.py
+++ b/Lib/test/test_symtable.py
@@ -215,6 +215,15 @@ class SymtableTest(unittest.TestCase):
     def test_exec(self):
         symbols = symtable.symtable("def f(x): return x", "?", "exec")
 
+    def test_bytes(self):
+        top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec")
+        self.assertIsNotNone(find_block(top, "Mine"))
+
+        code = b'# -*- coding: iso8859-15 -*-\nclass \xb4: pass\n'
+
+        top = symtable.symtable(code, "?", "exec")
+        self.assertIsNotNone(find_block(top, "\u017d"))
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst b/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst
new file mode 100644
index 0000000..5bcd7a9
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst
@@ -0,0 +1,2 @@
+:func:`symtable.symtable` now accepts the same input types for source code as the
+built-in :func:`compile` function. Patch by Dino Viehland.
diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h
index 73e340b..7d8b0ad 100644
--- a/Modules/clinic/symtablemodule.c.h
+++ b/Modules/clinic/symtablemodule.c.h
@@ -3,7 +3,7 @@ preserve
 [clinic start generated code]*/
 
 PyDoc_STRVAR(_symtable_symtable__doc__,
-"symtable($module, str, filename, startstr, /)\n"
+"symtable($module, source, filename, startstr, /)\n"
 "--\n"
 "\n"
 "Return symbol and scope dictionaries used internally by compiler.");
@@ -12,33 +12,21 @@ PyDoc_STRVAR(_symtable_symtable__doc__,
     {"symtable", (PyCFunction)(void(*)(void))_symtable_symtable, METH_FASTCALL, _symtable_symtable__doc__},
 
 static PyObject *
-_symtable_symtable_impl(PyObject *module, const char *str,
+_symtable_symtable_impl(PyObject *module, PyObject *source,
                         PyObject *filename, const char *startstr);
 
 static PyObject *
 _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
 {
     PyObject *return_value = NULL;
-    const char *str;
+    PyObject *source;
     PyObject *filename;
     const char *startstr;
 
     if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) {
         goto exit;
     }
-    if (!PyUnicode_Check(args[0])) {
-        _PyArg_BadArgument("symtable", 1, "str", args[0]);
-        goto exit;
-    }
-    Py_ssize_t str_length;
-    str = PyUnicode_AsUTF8AndSize(args[0], &str_length);
-    if (str == NULL) {
-        goto exit;
-    }
-    if (strlen(str) != (size_t)str_length) {
-        PyErr_SetString(PyExc_ValueError, "embedded null character");
-        goto exit;
-    }
+    source = args[0];
     if (!PyUnicode_FSDecoder(args[1], &filename)) {
         goto exit;
     }
@@ -55,9 +43,9 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
         PyErr_SetString(PyExc_ValueError, "embedded null character");
         goto exit;
     }
-    return_value = _symtable_symtable_impl(module, str, filename, startstr);
+    return_value = _symtable_symtable_impl(module, source, filename, startstr);
 
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=be1cca59de019984 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=de655625eee705f4 input=a9049054013a1b77]*/
diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c
index e8d2f5b..d66cb44 100644
--- a/Modules/symtablemodule.c
+++ b/Modules/symtablemodule.c
@@ -14,7 +14,7 @@ module _symtable
 /*[clinic input]
 _symtable.symtable
 
-    str:       str
+    source:    object
     filename:  object(converter='PyUnicode_FSDecoder')
     startstr:  str
     /
@@ -23,13 +23,23 @@ Return symbol and scope dictionaries used internally by compiler.
 [clinic start generated code]*/
 
 static PyObject *
-_symtable_symtable_impl(PyObject *module, const char *str,
+_symtable_symtable_impl(PyObject *module, PyObject *source,
                         PyObject *filename, const char *startstr)
-/*[clinic end generated code: output=914b369c9b785956 input=6c615e84d5f408e3]*/
+/*[clinic end generated code: output=59eb0d5fc7285ac4 input=9dd8a50c0c36a4d7]*/
 {
     struct symtable *st;
     PyObject *t;
     int start;
+    PyCompilerFlags cf;
+    PyObject *source_copy = NULL;
+
+    cf.cf_flags = PyCF_SOURCE_IS_UTF8;
+    cf.cf_feature_version = PY_MINOR_VERSION;
+
+    const char *str = _Py_SourceAsString(source, "symtable", "string or bytes", &cf, &source_copy);
+    if (str == NULL) {
+        return NULL;
+    }
 
     if (strcmp(startstr, "exec") == 0)
         start = Py_file_input;
@@ -41,12 +51,15 @@ _symtable_symtable_impl(PyObject *module, const char *str,
         PyErr_SetString(PyExc_ValueError,
            "symtable() arg 3 must be 'exec' or 'eval' or 'single'");
         Py_DECREF(filename);
+        Py_XDECREF(source_copy);
         return NULL;
     }
-    st = Py_SymtableStringObject(str, filename, start);
+    st = _Py_SymtableStringObjectFlags(str, filename, start, &cf);
     Py_DECREF(filename);
-    if (st == NULL)
+    Py_XDECREF(source_copy);
+    if (st == NULL) {
         return NULL;
+    }
     t = (PyObject *)st->st_top;
     Py_INCREF(t);
     PyMem_Free((void *)st->st_future);
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 5d58085..065ad95 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -687,55 +687,6 @@ builtin_chr_impl(PyObject *module, int i)
 }
 
 
-static const char *
-source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
-{
-    const char *str;
-    Py_ssize_t size;
-    Py_buffer view;
-
-    *cmd_copy = NULL;
-    if (PyUnicode_Check(cmd)) {
-        cf->cf_flags |= PyCF_IGNORE_COOKIE;
-        str = PyUnicode_AsUTF8AndSize(cmd, &size);
-        if (str == NULL)
-            return NULL;
-    }
-    else if (PyBytes_Check(cmd)) {
-        str = PyBytes_AS_STRING(cmd);
-        size = PyBytes_GET_SIZE(cmd);
-    }
-    else if (PyByteArray_Check(cmd)) {
-        str = PyByteArray_AS_STRING(cmd);
-        size = PyByteArray_GET_SIZE(cmd);
-    }
-    else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
-        /* Copy to NUL-terminated buffer. */
-        *cmd_copy = PyBytes_FromStringAndSize(
-            (const char *)view.buf, view.len);
-        PyBuffer_Release(&view);
-        if (*cmd_copy == NULL) {
-            return NULL;
-        }
-        str = PyBytes_AS_STRING(*cmd_copy);
-        size = PyBytes_GET_SIZE(*cmd_copy);
-    }
-    else {
-        PyErr_Format(PyExc_TypeError,
-          "%s() arg 1 must be a %s object",
-          funcname, what);
-        return NULL;
-    }
-
-    if (strlen(str) != (size_t)size) {
-        PyErr_SetString(PyExc_ValueError,
-                        "source code string cannot contain null bytes");
-        Py_CLEAR(*cmd_copy);
-        return NULL;
-    }
-    return str;
-}
-
 /*[clinic input]
 compile as builtin_compile
 
@@ -855,7 +806,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
         goto finally;
     }
 
-    str = source_as_string(source, "compile", "string, bytes or AST", &cf, &source_copy);
+    str = _Py_SourceAsString(source, "compile", "string, bytes or AST", &cf, &source_copy);
     if (str == NULL)
         goto error;
 
@@ -991,7 +942,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals,
 
     cf.cf_flags = PyCF_SOURCE_IS_UTF8;
     cf.cf_feature_version = PY_MINOR_VERSION;
-    str = source_as_string(source, "eval", "string, bytes or code", &cf, &source_copy);
+    str = _Py_SourceAsString(source, "eval", "string, bytes or code", &cf, &source_copy);
     if (str == NULL)
         return NULL;
 
@@ -1083,7 +1034,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals,
         PyCompilerFlags cf;
         cf.cf_flags = PyCF_SOURCE_IS_UTF8;
         cf.cf_feature_version = PY_MINOR_VERSION;
-        str = source_as_string(source, "exec",
+        str = _Py_SourceAsString(source, "exec",
                                        "string, bytes or code", &cf,
                                        &source_copy);
         if (str == NULL)
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index ace9f2f..784c15b 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -1231,21 +1231,77 @@ PyCompileString(const char *str, const char *filename, int start)
     return Py_CompileStringFlags(str, filename, start, NULL);
 }
 
+const char *
+_Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
+{
+    const char *str;
+    Py_ssize_t size;
+    Py_buffer view;
+
+    *cmd_copy = NULL;
+    if (PyUnicode_Check(cmd)) {
+        cf->cf_flags |= PyCF_IGNORE_COOKIE;
+        str = PyUnicode_AsUTF8AndSize(cmd, &size);
+        if (str == NULL)
+            return NULL;
+    }
+    else if (PyBytes_Check(cmd)) {
+        str = PyBytes_AS_STRING(cmd);
+        size = PyBytes_GET_SIZE(cmd);
+    }
+    else if (PyByteArray_Check(cmd)) {
+        str = PyByteArray_AS_STRING(cmd);
+        size = PyByteArray_GET_SIZE(cmd);
+    }
+    else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
+        /* Copy to NUL-terminated buffer. */
+        *cmd_copy = PyBytes_FromStringAndSize(
+            (const char *)view.buf, view.len);
+        PyBuffer_Release(&view);
+        if (*cmd_copy == NULL) {
+            return NULL;
+        }
+        str = PyBytes_AS_STRING(*cmd_copy);
+        size = PyBytes_GET_SIZE(*cmd_copy);
+    }
+    else {
+        PyErr_Format(PyExc_TypeError,
+            "%s() arg 1 must be a %s object",
+            funcname, what);
+        return NULL;
+    }
+
+    if (strlen(str) != (size_t)size) {
+        PyErr_SetString(PyExc_ValueError,
+            "source code string cannot contain null bytes");
+        Py_CLEAR(*cmd_copy);
+        return NULL;
+    }
+    return str;
+}
+
 struct symtable *
 Py_SymtableStringObject(const char *str, PyObject *filename, int start)
 {
+    PyCompilerFlags flags;
+
+    flags.cf_flags = 0;
+    flags.cf_feature_version = PY_MINOR_VERSION;
+    return _Py_SymtableStringObjectFlags(str, filename, start, &flags);
+}
+
+struct symtable *
+_Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, PyCompilerFlags *flags)
+{
     struct symtable *st;
     mod_ty mod;
-    PyCompilerFlags flags;
     PyArena *arena;
 
     arena = PyArena_New();
     if (arena == NULL)
         return NULL;
 
-    flags.cf_flags = 0;
-    flags.cf_feature_version = PY_MINOR_VERSION;
-    mod = PyParser_ASTFromStringObject(str, filename, start, &flags, arena);
+    mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
     if (mod == NULL) {
         PyArena_Free(arena);
         return NULL;