Patch #1272, by Christian Heimes and Alexandre Vassalotti.

Changes to make __file__ a proper Unicode object, using the default
filesystem encoding.
This is a bit tricky because the default filesystem encoding isn't
set by the time we import the first modules; at that point we fudge
things a bit.  This is okay since __file__ isn't really used much
except for error reporting.
Tested on OSX and Linux only so far.

8 files changed, 26 insertions, 19 deletions

diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index cafffdc..338c424 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -10,6 +10,9 @@
 
 /* The default encoding used by the platform file system APIs
    Can remain NULL for all platforms that don't have such a concept
+
+   Don't forget to modify PyUnicode_DecodeFSDefault() if you touch any of the
+   values for Py_FileSystemDefaultEncoding!
 */
 #if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
 const char *Py_FileSystemDefaultEncoding = "mbcs";
diff --git a/Python/ceval.c b/Python/ceval.c
index dd6f6c4..ae8434d 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -767,7 +767,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 	lltrace = PyDict_GetItemString(f->f_globals, "__lltrace__") != NULL;
 #endif
 #if defined(Py_DEBUG) || defined(LLTRACE)
-	filename = PyString_AsString(co->co_filename);
+	filename = PyUnicode_AsString(co->co_filename);
 #endif
 
 	why = WHY_NOT;
@@ -2565,7 +2565,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
 		if (argcount > co->co_argcount) {
 			if (!(co->co_flags & CO_VARARGS)) {
 				PyErr_Format(PyExc_TypeError,
-				    "%S() takes %s %d "
+				    "%U() takes %s %d "
 				    "%spositional argument%s (%d given)",
 				    co->co_name,
 				    defcount ? "at most" : "exactly",
@@ -2599,7 +2599,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
 			int j;
 			if (keyword == NULL || !PyUnicode_Check(keyword)) {
 				PyErr_Format(PyExc_TypeError,
-				    "%S() keywords must be strings",
+				    "%U() keywords must be strings",
 				    co->co_name);
 				goto fail;
 			}
@@ -2622,7 +2622,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
 			if (j >= co->co_argcount + co->co_kwonlyargcount) {
 				if (kwdict == NULL) {
 					PyErr_Format(PyExc_TypeError,
-					    "%S() got an unexpected "
+					    "%U() got an unexpected "
 					    "keyword argument '%S'",
 					    co->co_name,
 					    keyword);
@@ -2633,7 +2633,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
 			else {
 				if (GETLOCAL(j) != NULL) {
 					PyErr_Format(PyExc_TypeError,
-					     "%S() got multiple "
+					     "%U() got multiple "
 					     "values for keyword "
 					     "argument '%S'",
 					     co->co_name,
@@ -2661,7 +2661,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
 					continue;
 				}
 				PyErr_Format(PyExc_TypeError,
-					"%S() needs keyword-only argument %S",
+					"%U() needs keyword-only argument %S",
 					co->co_name, name);
 				goto fail;
 			}
@@ -2671,7 +2671,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
 			for (i = argcount; i < m; i++) {
 				if (GETLOCAL(i) == NULL) {
 					PyErr_Format(PyExc_TypeError,
-					    "%S() takes %s %d "
+					    "%U() takes %s %d "
 					    "%spositional argument%s "
 					    "(%d given)",
 					    co->co_name,
@@ -2699,7 +2699,7 @@ PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
 	else {
 		if (argcount > 0 || kwcount > 0) {
 			PyErr_Format(PyExc_TypeError,
-				     "%S() takes no arguments (%d given)",
+				     "%U() takes no arguments (%d given)",
 				     co->co_name,
 				     argcount + kwcount);
 			goto fail;
diff --git a/Python/compile.c b/Python/compile.c
index d20da0a..93087db 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -1247,7 +1247,7 @@ compiler_make_closure(struct compiler *c, PyCodeObject *co, int args)
 				PyObject_REPR(name), 
 				PyString_AS_STRING(c->u->u_name), 
 				reftype, arg,
-				PyString_AS_STRING(co->co_name),
+				PyUnicode_AsString(co->co_name),
 				PyObject_REPR(co->co_freevars));
 			Py_FatalError("compiler_make_closure()");
 		}
@@ -4001,7 +4001,7 @@ makecode(struct compiler *c, struct assembler *a)
 	freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars));
 	if (!freevars)
 	    goto error;
-	filename = PyString_FromString(c->c_filename);
+	filename = PyUnicode_DecodeFSDefault(c->c_filename);
 	if (!filename)
 		goto error;
 
diff --git a/Python/frozen.c b/Python/frozen.c
index d404562..ee06c35 100644
--- a/Python/frozen.c
+++ b/Python/frozen.c
@@ -17,7 +17,7 @@ static unsigned char M___hello__[] = {
 	131,1,0,1,100,1,0,83,40,2,0,0,0,117,14,0,
 	0,0,72,101,108,108,111,32,119,111,114,108,100,46,46,46,
 	78,40,1,0,0,0,117,5,0,0,0,112,114,105,110,116,
-	40,0,0,0,0,40,0,0,0,0,40,0,0,0,0,115,
+	40,0,0,0,0,40,0,0,0,0,40,0,0,0,0,117,
 	8,0,0,0,104,101,108,108,111,46,112,121,117,8,0,0,
 	0,60,109,111,100,117,108,101,62,1,0,0,0,115,0,0,
 	0,0,
diff --git a/Python/import.c b/Python/import.c
index c2f42e9..21dcbd4 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -74,10 +74,11 @@ extern time_t PyOS_GetLastModificationTime(char *, FILE *);
 		      3040 (added signature annotations)
 		      3050 (print becomes a function)
 		      3060 (PEP 3115 metaclass syntax)
-          3070 (PEP 3109 raise changes)
+		      3070 (PEP 3109 raise changes)
+		      3080 (PEP 3137 make __file__ and __name__ unicode)
 .
 */
-#define MAGIC (3070 | ((long)'\r'<<16) | ((long)'\n'<<24))
+#define MAGIC (3080 | ((long)'\r'<<16) | ((long)'\n'<<24))
 
 /* Magic word as global; note that _PyImport_Init() can change the
    value of this global to accommodate for alterations of how the
@@ -652,7 +653,7 @@ PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname)
 	/* Remember the filename as the __file__ attribute */
 	v = NULL;
 	if (pathname != NULL) {
-		v = PyString_FromString(pathname);
+		v = PyUnicode_DecodeFSDefault(pathname);
 		if (v == NULL)
 			PyErr_Clear();
 	}
@@ -983,7 +984,7 @@ load_package(char *name, char *pathname)
 		PySys_WriteStderr("import %s # directory %s\n",
 			name, pathname);
 	d = PyModule_GetDict(m);
-	file = PyString_FromString(pathname);
+	file = PyUnicode_DecodeFSDefault(pathname);
 	if (file == NULL)
 		goto error;
 	path = Py_BuildValue("[O]", file);
diff --git a/Python/importdl.c b/Python/importdl.c
index 9c325e4..7978c48 100644
--- a/Python/importdl.c
+++ b/Python/importdl.c
@@ -62,7 +62,9 @@ _PyImport_LoadDynamicModule(char *name, char *pathname, FILE *fp)
 		return NULL;
 	}
 	/* Remember the filename as the __file__ attribute */
-	if (PyModule_AddStringConstant(m, "__file__", pathname) < 0)
+	PyObject *path;
+	path = PyUnicode_DecodeFSDefault(pathname);
+	if (PyModule_AddObject(m, "__file__", path) < 0)
 		PyErr_Clear(); /* Not important enough to report */
 
 	if (_PyImport_FixupExtension(name, pathname) == NULL)
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index a37a3e4..4e239c9 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -867,7 +867,8 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit,
 		return -1;
 	d = PyModule_GetDict(m);
 	if (PyDict_GetItemString(d, "__file__") == NULL) {
-		PyObject *f = PyString_FromString(filename);
+		PyObject *f;
+		f = PyUnicode_DecodeFSDefault(filename);
 		if (f == NULL)
 			return -1;
 		if (PyDict_SetItemString(d, "__file__", f) < 0) {
diff --git a/Python/traceback.c b/Python/traceback.c
index 5bb8841..9d7a2e0 100644
--- a/Python/traceback.c
+++ b/Python/traceback.c
@@ -229,10 +229,10 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, int limit)
 	while (tb != NULL && err == 0) {
 		if (depth <= limit) {
 			err = tb_displayline(f,
-			    PyString_AsString(
+			    PyUnicode_AsString(
 				    tb->tb_frame->f_code->co_filename),
 			    tb->tb_lineno,
-			    PyString_AsString(tb->tb_frame->f_code->co_name));
+			    PyUnicode_AsString(tb->tb_frame->f_code->co_name));
 		}
 		depth--;
 		tb = tb->tb_next;