Issue #6543: Write the traceback in the terminal encoding instead of utf-8.

Fix the encoding of the modules filename.

Reindent also traceback.h, just because I hate tabs :-)

6 files changed, 70 insertions, 57 deletions

diff --git a/Include/traceback.h b/Include/traceback.h
index e7943da..fc0c586 100644
--- a/Include/traceback.h
+++ b/Include/traceback.h
@@ -10,16 +10,16 @@ struct _frame;
 /* Traceback interface */
 
 typedef struct _traceback {
-	PyObject_HEAD
-	struct _traceback *tb_next;
-	struct _frame *tb_frame;
-	int tb_lasti;
-	int tb_lineno;
+    PyObject_HEAD
+    struct _traceback *tb_next;
+    struct _frame *tb_frame;
+    int tb_lasti;
+    int tb_lineno;
 } PyTracebackObject;
 
 PyAPI_FUNC(int) PyTraceBack_Here(struct _frame *);
 PyAPI_FUNC(int) PyTraceBack_Print(PyObject *, PyObject *);
-PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, const char *, int, int);
+PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int);
 
 /* Reveal traceback type so we can typecheck traceback objects */
 PyAPI_DATA(PyTypeObject) PyTraceBack_Type;
diff --git a/Misc/NEWS b/Misc/NEWS
index 121e92d..4f46268 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,9 @@ What's New in Python 3.2 Alpha 1?
 Core and Builtins
 -----------------
 
+- Issue #6543: Write the traceback in the terminal encoding instead of utf-8.
+  Fix the encoding of the modules filename.
+
 - Issue #9011: Remove buggy and unnecessary (in 3.x) ST->AST
   compilation code dealing with unary minus applied to a constant.
   The removed code was mutating the ST, causing a second compilation
diff --git a/Python/_warnings.c b/Python/_warnings.c
index b1b2b71..55f7fda 100644
--- a/Python/_warnings.c
+++ b/Python/_warnings.c
@@ -282,8 +282,7 @@ show_warning(PyObject *filename, int lineno, PyObject *text, PyObject
         PyFile_WriteString("\n", f_stderr);
     }
     else
-        if (_Py_DisplaySourceLine(f_stderr, _PyUnicode_AsString(filename),
-                              lineno, 2) < 0)
+        if (_Py_DisplaySourceLine(f_stderr, filename, lineno, 2) < 0)
                 return;
     PyErr_Clear();
 }
diff --git a/Python/compile.c b/Python/compile.c
index fa7dcef..c9f6e8d 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -3942,7 +3942,7 @@ makecode(struct compiler *c, struct assembler *a)
     freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars));
     if (!freevars)
         goto error;
-    filename = PyUnicode_DecodeFSDefault(c->c_filename);
+    filename = PyUnicode_FromString(c->c_filename);
     if (!filename)
         goto error;
 
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 5b7cd20..f45d7dc 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -1190,7 +1190,7 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit,
     d = PyModule_GetDict(m);
     if (PyDict_GetItemString(d, "__file__") == NULL) {
         PyObject *f;
-        f = PyUnicode_DecodeFSDefault(filename);
+        f = PyUnicode_FromString(filename);
         if (f == NULL)
             return -1;
         if (PyDict_SetItemString(d, "__file__", f) < 0) {
diff --git a/Python/traceback.c b/Python/traceback.c
index 51a9226..9c9c357 100644
--- a/Python/traceback.c
+++ b/Python/traceback.c
@@ -133,33 +133,38 @@ PyTraceBack_Here(PyFrameObject *frame)
     return 0;
 }
 
-static int
-_Py_FindSourceFile(const char* filename, char* namebuf, size_t namelen, int open_flags)
+static PyObject *
+_Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject *io)
 {
-    int i;
-    int fd = -1;
+    Py_ssize_t i;
+    PyObject *binary;
     PyObject *v;
-    Py_ssize_t _npath;
-    int npath;
+    Py_ssize_t npath;
     size_t taillen;
     PyObject *syspath;
     const char* path;
     const char* tail;
+    const char* filepath;
     Py_ssize_t len;
 
+    filepath = _PyUnicode_AsString(filename);
+    if (filepath == NULL) {
+        PyErr_Clear();
+        return NULL;
+    }
+
     /* Search tail of filename in sys.path before giving up */
-    tail = strrchr(filename, SEP);
+    tail = strrchr(filepath, SEP);
     if (tail == NULL)
-        tail = filename;
+        tail = filepath;
     else
         tail++;
     taillen = strlen(tail);
 
     syspath = PySys_GetObject("path");
     if (syspath == NULL || !PyList_Check(syspath))
-        return -1;
-    _npath = PyList_Size(syspath);
-    npath = Py_SAFE_DOWNCAST(_npath, Py_ssize_t, int);
+        return NULL;
+    npath = PyList_Size(syspath);
 
     for (i = 0; i < npath; i++) {
         v = PyList_GetItem(syspath, i);
@@ -170,6 +175,10 @@ _Py_FindSourceFile(const char* filename, char* namebuf, size_t namelen, int open
         if (!PyUnicode_Check(v))
             continue;
         path = _PyUnicode_AsStringAndSize(v, &len);
+        if (path == NULL) {
+            PyErr_Clear();
+            continue;
+        }
         if (len + 1 + (Py_ssize_t)taillen >= (Py_ssize_t)namelen - 1)
             continue; /* Too long */
         strcpy(namebuf, path);
@@ -178,31 +187,27 @@ _Py_FindSourceFile(const char* filename, char* namebuf, size_t namelen, int open
         if (len > 0 && namebuf[len-1] != SEP)
             namebuf[len++] = SEP;
         strcpy(namebuf+len, tail);
-        Py_BEGIN_ALLOW_THREADS
-        fd = open(namebuf, open_flags);
-        Py_END_ALLOW_THREADS
-        if (0 <= fd) {
-            return fd;
-        }
+
+        binary = PyObject_CallMethod(io, "open", "ss", namebuf, "rb");
+        if (binary != NULL)
+            return binary;
+        PyErr_Clear();
     }
-    return -1;
+    return NULL;
 }
 
 int
-_Py_DisplaySourceLine(PyObject *f, const char *filename, int lineno, int indent)
+_Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent)
 {
     int err = 0;
     int fd;
     int i;
     char *found_encoding;
     char *encoding;
+    PyObject *io;
+    PyObject *binary;
     PyObject *fob = NULL;
     PyObject *lineobj = NULL;
-#ifdef O_BINARY
-    const int open_flags = O_RDONLY | O_BINARY;   /* necessary for Windows */
-#else
-    const int open_flags = O_RDONLY;
-#endif
     char buf[MAXPATHLEN+1];
     Py_UNICODE *u, *p;
     Py_ssize_t len;
@@ -210,27 +215,32 @@ _Py_DisplaySourceLine(PyObject *f, const char *filename, int lineno, int indent)
     /* open the file */
     if (filename == NULL)
         return 0;
-    Py_BEGIN_ALLOW_THREADS
-    fd = open(filename, open_flags);
-    Py_END_ALLOW_THREADS
-    if (fd < 0) {
-        fd = _Py_FindSourceFile(filename, buf, sizeof(buf), open_flags);
-        if (fd < 0)
+
+    io = PyImport_ImportModuleNoBlock("io");
+    if (io == NULL)
+        return -1;
+    binary = PyObject_CallMethod(io, "open", "Os", filename, "rb");
+
+    if (binary == NULL) {
+        binary = _Py_FindSourceFile(filename, buf, sizeof(buf), io);
+        if (binary == NULL) {
+            Py_DECREF(io);
             return 0;
-        filename = buf;
+        }
     }
 
     /* use the right encoding to decode the file as unicode */
+    fd = PyObject_AsFileDescriptor(binary);
     found_encoding = PyTokenizer_FindEncoding(fd);
-    encoding = (found_encoding != NULL) ? found_encoding :
-        (char*)PyUnicode_GetDefaultEncoding();
+    encoding = (found_encoding != NULL) ? found_encoding : "utf-8";
     lseek(fd, 0, 0); /* Reset position */
-    fob = PyFile_FromFd(fd, (char*)filename, "r", -1, (char*)encoding,
-        NULL, NULL, 1);
+    fob = PyObject_CallMethod(io, "TextIOWrapper", "Os", binary, encoding);
+    Py_DECREF(io);
+    Py_DECREF(binary);
     PyMem_FREE(found_encoding);
+
     if (fob == NULL) {
         PyErr_Clear();
-        close(fd);
         return 0;
     }
 
@@ -287,17 +297,19 @@ _Py_DisplaySourceLine(PyObject *f, const char *filename, int lineno, int indent)
 }
 
 static int
-tb_displayline(PyObject *f, const char *filename, int lineno, const char *name)
+tb_displayline(PyObject *f, PyObject *filename, int lineno, PyObject *name)
 {
-    int err = 0;
-    char linebuf[2000];
+    int err;
+    PyObject *line;
 
     if (filename == NULL || name == NULL)
         return -1;
-    /* This is needed by Emacs' compile command */
-#define FMT "  File \"%.500s\", line %d, in %.500s\n"
-    PyOS_snprintf(linebuf, sizeof(linebuf), FMT, filename, lineno, name);
-    err = PyFile_WriteString(linebuf, f);
+    line = PyUnicode_FromFormat("  File \"%U\", line %d, in %U\n",
+                                filename, lineno, name);
+    if (line == NULL)
+        return -1;
+    err = PyFile_WriteObject(line, f, Py_PRINT_RAW);
+    Py_DECREF(line);
     if (err != 0)
         return err;
     return _Py_DisplaySourceLine(f, filename, lineno, 4);
@@ -316,10 +328,9 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit)
     while (tb != NULL && err == 0) {
         if (depth <= limit) {
             err = tb_displayline(f,
-                _PyUnicode_AsString(
-                    tb->tb_frame->f_code->co_filename),
-                tb->tb_lineno,
-                _PyUnicode_AsString(tb->tb_frame->f_code->co_name));
+                                 tb->tb_frame->f_code->co_filename,
+                                 tb->tb_lineno,
+                                 tb->tb_frame->f_code->co_name);
         }
         depth--;
         tb = tb->tb_next;