More coding by random modification.

Encoding now return bytes instead of str8.
eval(), exec(), compile() now accept unicode or bytes.

5 files changed, 78 insertions, 73 deletions

diff --git a/Python/ast.c b/Python/ast.c
index 821a5ad..5845076 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -3101,8 +3101,9 @@ decode_unicode(const char *s, size_t len, int rawmode, const char *encoding)
                     Py_DECREF(u);
                     return NULL;
                 }
-                r = PyString_AsString(w);
-                rn = PyString_Size(w);
+                assert(PyBytes_Check(w));
+                r = PyBytes_AsString(w);
+                rn = PyBytes_Size(w);
                 assert(rn % 2 == 0);
                 for (i = 0; i < rn; i += 2) {
                     sprintf(p, "\\u%02x%02x",
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 828cb5d..7d52e2b 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -412,6 +412,36 @@ PyDoc_STRVAR(cmp_doc,
 \n\
 Return negative if x<y, zero if x==y, positive if x>y.");
 
+
+static char *
+source_as_string(PyObject *cmd)
+{
+	char *str;
+	Py_ssize_t size;
+
+	if (!PyObject_CheckReadBuffer(cmd) &&
+	    !PyUnicode_Check(cmd)) {
+		PyErr_SetString(PyExc_TypeError,
+			   "eval()/exec() arg 1 must be a string, bytes or code object");
+		return NULL;
+	}
+
+	if (PyUnicode_Check(cmd)) {
+		cmd = _PyUnicode_AsDefaultEncodedString(cmd, NULL);
+		if (cmd == NULL)
+			return NULL;
+	}
+	if (PyObject_AsReadBuffer(cmd, (const void **)&str, &size) < 0) {
+		return NULL;
+	}
+	if (strlen(str) != size) {
+		PyErr_SetString(PyExc_TypeError,
+				"source code string cannot contain null bytes");
+		return NULL;
+	}
+	return str;
+}
+
 static PyObject *
 builtin_compile(PyObject *self, PyObject *args, PyObject *kwds)
 {
@@ -422,8 +452,7 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds)
 	int dont_inherit = 0;
 	int supplied_flags = 0;
 	PyCompilerFlags cf;
-	PyObject *result = NULL, *cmd, *tmp = NULL;
-	Py_ssize_t length;
+	PyObject *cmd;
 	static char *kwlist[] = {"source", "filename", "mode", "flags",
 				 "dont_inherit", NULL};
 
@@ -432,22 +461,11 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds)
 					 &supplied_flags, &dont_inherit))
 		return NULL;
 
-	cf.cf_flags = supplied_flags;
+	cf.cf_flags = supplied_flags | PyCF_SOURCE_IS_UTF8;
 
-	if (PyUnicode_Check(cmd)) {
-		tmp = PyUnicode_AsUTF8String(cmd);
-		if (tmp == NULL)
-			return NULL;
-		cmd = tmp;
-		cf.cf_flags |= PyCF_SOURCE_IS_UTF8;
-	}
-	if (PyObject_AsReadBuffer(cmd, (const void **)&str, &length))
+	str = source_as_string(cmd);
+	if (str == NULL)
 		return NULL;
-	if ((size_t)length != strlen(str)) {
-		PyErr_SetString(PyExc_TypeError,
-				"compile() expected string without null bytes");
-		goto cleanup;
-	}
 
 	if (strcmp(startstr, "exec") == 0)
 		start = Py_file_input;
@@ -458,7 +476,7 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds)
 	else {
 		PyErr_SetString(PyExc_ValueError,
 		   "compile() arg 3 must be 'exec' or 'eval' or 'single'");
-		goto cleanup;
+		return NULL;
 	}
 
 	if (supplied_flags &
@@ -466,17 +484,14 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds)
 	{
 		PyErr_SetString(PyExc_ValueError,
 				"compile(): unrecognised flags");
-		goto cleanup;
+		return NULL;
 	}
 	/* XXX Warn if (supplied_flags & PyCF_MASK_OBSOLETE) != 0? */
 
 	if (!dont_inherit) {
 		PyEval_MergeCompilerFlags(&cf);
 	}
-	result = Py_CompileStringFlags(str, filename, start, &cf);
-cleanup:
-	Py_XDECREF(tmp);
-	return result;
+	return Py_CompileStringFlags(str, filename, start, &cf);
 }
 
 PyDoc_STRVAR(compile_doc,
@@ -584,28 +599,14 @@ builtin_eval(PyObject *self, PyObject *args)
 		return PyEval_EvalCode((PyCodeObject *) cmd, globals, locals);
 	}
 
-	if (!PyString_Check(cmd) &&
-	    !PyUnicode_Check(cmd)) {
-		PyErr_SetString(PyExc_TypeError,
-			   "eval() arg 1 must be a string or code object");
+	str = source_as_string(cmd);
+	if (str == NULL)
 		return NULL;
-	}
-	cf.cf_flags = 0;
 
-	if (PyUnicode_Check(cmd)) {
-		tmp = PyUnicode_AsUTF8String(cmd);
-		if (tmp == NULL)
-			return NULL;
-		cmd = tmp;
-		cf.cf_flags |= PyCF_SOURCE_IS_UTF8;
-	}
-	if (PyString_AsStringAndSize(cmd, &str, NULL)) {
-		Py_XDECREF(tmp);
-		return NULL;
-	}
 	while (*str == ' ' || *str == '\t')
 		str++;
 
+	cf.cf_flags = PyCF_SOURCE_IS_UTF8;
 	(void)PyEval_MergeCompilerFlags(&cf);
 	result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf);
 	Py_XDECREF(tmp);
@@ -694,25 +695,16 @@ builtin_exec(PyObject *self, PyObject *args)
 				       locals);
 	}
 	else {
-		PyObject *tmp = NULL;
-		char *str;
+		char *str = source_as_string(prog);
 		PyCompilerFlags cf;
-		cf.cf_flags = 0;
-		if (PyUnicode_Check(prog)) {
-			tmp = PyUnicode_AsUTF8String(prog);
-			if (tmp == NULL)
-				return NULL;
-			prog = tmp;
-			cf.cf_flags |= PyCF_SOURCE_IS_UTF8;
-		}
-		if (PyString_AsStringAndSize(prog, &str, NULL))
+		if (str == NULL)
 			return NULL;
+		cf.cf_flags = PyCF_SOURCE_IS_UTF8;
 		if (PyEval_MergeCompilerFlags(&cf))
 			v = PyRun_StringFlags(str, Py_file_input, globals,
 					      locals, &cf);
 		else
 			v = PyRun_String(str, Py_file_input, globals, locals);
-		Py_XDECREF(tmp);
 	}
 	if (v == NULL)
 		return NULL;
diff --git a/Python/getargs.c b/Python/getargs.c
index 2a02a89..7d55bae 100644
--- a/Python/getargs.c
+++ b/Python/getargs.c
@@ -883,7 +883,9 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
 		char **buffer;
 		const char *encoding;
 		PyObject *s;
-		int size, recode_strings;
+		int recode_strings;
+		Py_ssize_t size;
+		char *ptr;
 
 		/* Get 'e' parameter: the encoding name */
 		encoding = (const char *)va_arg(*p_va, const char *);
@@ -912,6 +914,8 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
 		if (!recode_strings && PyString_Check(arg)) {
 			s = arg;
 			Py_INCREF(s);
+			size = PyString_GET_SIZE(s);
+			ptr = PyString_AS_STRING(s);
 		}
 		else {
 		    	PyObject *u;
@@ -931,14 +935,15 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
 			if (s == NULL)
 				return converterr("(encoding failed)",
 						  arg, msgbuf, bufsize);
-			if (!PyString_Check(s)) {
+			if (!PyBytes_Check(s)) {
 				Py_DECREF(s);
 				return converterr(
-					"(encoder failed to return a string)",
+					"(encoder failed to return bytes)",
 					arg, msgbuf, bufsize);
 			}
+			size = PyBytes_GET_SIZE(s);
+			ptr = PyBytes_AS_STRING(s);
 		}
-		size = PyString_GET_SIZE(s);
 
 		/* Write output; output is guaranteed to be 0-terminated */
 		if (*format == '#') { 
@@ -994,9 +999,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
 						arg, msgbuf, bufsize);
 				}
 			}
-			memcpy(*buffer,
-			       PyString_AS_STRING(s),
-			       size + 1);
+			memcpy(*buffer, ptr, size+1);
 			STORE_SIZE(size);
 		} else {
 			/* Using a 0-terminated buffer:
@@ -1012,8 +1015,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
 			   PyMem_Free()ing it after usage
 
 			*/
-			if ((Py_ssize_t)strlen(PyString_AS_STRING(s))
-								!= size) {
+			if ((Py_ssize_t)strlen(ptr) != size) {
 				Py_DECREF(s);
 				return converterr(
 					"(encoded string without NULL bytes)",
@@ -1030,9 +1032,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags,
 				return converterr("(cleanup problem)",
 						arg, msgbuf, bufsize);
 			}
-			memcpy(*buffer,
-			       PyString_AS_STRING(s),
-			       size + 1);
+			memcpy(*buffer, ptr, size+1);
 		}
 		Py_DECREF(s);
 		break;
diff --git a/Python/import.c b/Python/import.c
index 7e3d2f4..2e1f894 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -1254,6 +1254,9 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
 	for (i = 0; i < npath; i++) {
 		PyObject *copy = NULL;
 		PyObject *v = PyList_GetItem(path, i);
+		PyObject *origv = v;
+		char *base;
+		Py_ssize_t size;
 		if (!v)
 			return NULL;
 		if (PyUnicode_Check(v)) {
@@ -1263,15 +1266,24 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
 				return NULL;
 			v = copy;
 		}
-		else
-		if (!PyString_Check(v))
+		if (PyString_Check(v)) {
+			base = PyString_AS_STRING(v);
+			size = PyString_GET_SIZE(v);
+		}
+		else if (PyBytes_Check(v)) {
+			base = PyBytes_AS_STRING(v);
+			size = PyBytes_GET_SIZE(v);
+		}
+		else {
+			Py_XDECREF(copy);
 			continue;
-		len = PyString_GET_SIZE(v);
+		}
+		len = size;
 		if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) {
 			Py_XDECREF(copy);
 			continue; /* Too long */
 		}
-		strcpy(buf, PyString_AS_STRING(v));
+		strcpy(buf, base);
 		if (strlen(buf) != len) {
 			Py_XDECREF(copy);
 			continue; /* v contains '\0' */
@@ -1282,7 +1294,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
 			PyObject *importer;
 
 			importer = get_path_importer(path_importer_cache,
-						     path_hooks, v);
+						     path_hooks, origv);
 			if (importer == NULL) {
 				Py_XDECREF(copy);
 				return NULL;
diff --git a/Python/marshal.c b/Python/marshal.c
index 94d73a0..9243798 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -263,14 +263,14 @@ w_object(PyObject *v, WFILE *p)
 			return;
 		}
 		w_byte(TYPE_UNICODE, p);
-		n = PyString_GET_SIZE(utf8);
+		n = PyBytes_GET_SIZE(utf8);
 		if (n > INT_MAX) {
 			p->depth--;
 			p->error = 1;
 			return;
 		}
 		w_long((long)n, p);
-		w_string(PyString_AS_STRING(utf8), (int)n, p);
+		w_string(PyBytes_AS_STRING(utf8), (int)n, p);
 		Py_DECREF(utf8);
 	}
 	else if (PyTuple_Check(v)) {
@@ -1031,7 +1031,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version)
 		if (wf.ptr - base > PY_SSIZE_T_MAX) {
 			Py_DECREF(wf.str);
 			PyErr_SetString(PyExc_OverflowError,
-					"too much marshall data for a string");
+					"too much marshal data for a string");
 			return NULL;
 		}
 		_PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base));